Update README.md

README.md

@@ -18,13 +18,40 @@ It's so funny that the huggingface hub lets you do this
 
 multiprocessing is suuuper weird so make sure you dont have the variables "p" or "calculate_ncd_row" in your code anywhere..
 
-pruning:
+### Usage
+
+
 ```python
-import random
-target_dim = 8192
+# Requirements
+%pip install -qq transformers
+
+# Download Model
+from transformers import AutoModel
+model = AutoModel.from_pretrained("crumb/gzip-openhermes", trust_remote_code=True)
+
+# Prune model
+model.config.update({
+    "corpus": model.config.corpus[:1024]
+})
+model.dimensionality() # 1024
+
+# Inference
+model(["this is a test sequence"], num_procs=16).shape # [1, 1024]
+
+# Finetuning
+from tqdm.auto import tqdm
+
+new_data = ["i love GZIP! it is my favorite!", "i HATE transformers!"]
+normalized_data = [
+    model.normalize(i) for i in tqdm(new_data)
+]
+print(f"Input: '{new_data[0]}'\nTransformed: '{normalized_data[0]}'")
+
 model.config.update({
-    "corpus": random.sample(model.config.corpus, target_dim)
+    "corpus": model.config.corpus + normalized_data
 })
+model.dimensionality()
+model.save_pretrained("my-finetuned-gzip-model")
 ```
 
 config: