Add the model files

.gitattributes

@@ -0,0 +1,18 @@
+embeddings filter=lfs diff=lfs merge=lfs -text
+model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.128.fp32.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.512.fp16.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/tokenizer.json.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.1024.fp16.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.256.fp16.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.384.fp16.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.512.fp32.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.1024.fp32.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.128.fp16.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.128.int8.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.384.fp32.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.1024.int8.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.256.fp32.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.256.int8.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.384.int8.npy.zst filter=lfs diff=lfs merge=lfs -text
+model/static-embeddings.512.int8.npy.zst filter=lfs diff=lfs merge=lfs -text

js/example.mjs

@@ -1,4 +1,4 @@
-import { pipeline, AutoTokenizer, AutoModel, TokenizerModel } from '@huggingface/transformers';
+import { pipeline, AutoTokenizer, AutoModel, TokenizerModel, PreTrainedTokenizer } from '@huggingface/transformers';
 import fs from 'node:fs/promises';
 import { constants } from 'node:fs';
 import path from 'path';
@@ -12,9 +12,7 @@ async function main() {
   const url = "https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1/resolve/main/0_StaticEmbedding/tokenizer.json"
 
   const config = await ensureTokenizerJson(url)
-  // const tokenizer = TokenizerModel.fromConfig(config.model)
-  
-  const tokenizer = await AutoTokenizer.from_pretrained("./")
+  const tokenizer = new PreTrainedTokenizer(config, {})
   
   const examples = [
     "This is an example of encoding",

js/tokenizer_config.json

@@ -0,0 +1 @@
+{}

main.py

@@ -1,6 +0,0 @@
-def main():
-    print("Hello from sentence-embeddings!")
-
-
-if __name__ == "__main__":
-    main()

model/static-embeddings.1024.fp16.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b67bc3307eeb0387866325ff33a41f8bdd673e309c18224e1415f65b9873eba
+size 200900906

model/static-embeddings.1024.fp32.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef55fc97d834f27413b63d9bba3b4832b8c66b2a27bfaba7afc8d2db4c236e68
+size 402617664

model/static-embeddings.1024.int8.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5337ce683b6db454feeb8842deb46807e3e6d989cb46b1b660fbfa0d4506c7
+size 74935075

model/static-embeddings.128.fp16.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b67bc3307eeb0387866325ff33a41f8bdd673e309c18224e1415f65b9873eba
+size 200900906

model/static-embeddings.128.fp32.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef55fc97d834f27413b63d9bba3b4832b8c66b2a27bfaba7afc8d2db4c236e68
+size 402617664

model/static-embeddings.128.int8.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5337ce683b6db454feeb8842deb46807e3e6d989cb46b1b660fbfa0d4506c7
+size 74935075

model/static-embeddings.256.fp16.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b67bc3307eeb0387866325ff33a41f8bdd673e309c18224e1415f65b9873eba
+size 200900906

model/static-embeddings.256.fp32.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef55fc97d834f27413b63d9bba3b4832b8c66b2a27bfaba7afc8d2db4c236e68
+size 402617664

model/static-embeddings.256.int8.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5337ce683b6db454feeb8842deb46807e3e6d989cb46b1b660fbfa0d4506c7
+size 74935075

model/static-embeddings.384.fp16.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b67bc3307eeb0387866325ff33a41f8bdd673e309c18224e1415f65b9873eba
+size 200900906

model/static-embeddings.384.fp32.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef55fc97d834f27413b63d9bba3b4832b8c66b2a27bfaba7afc8d2db4c236e68
+size 402617664

model/static-embeddings.384.int8.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5337ce683b6db454feeb8842deb46807e3e6d989cb46b1b660fbfa0d4506c7
+size 74935075

model/static-embeddings.512.fp16.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b67bc3307eeb0387866325ff33a41f8bdd673e309c18224e1415f65b9873eba
+size 200900906

model/static-embeddings.512.fp32.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef55fc97d834f27413b63d9bba3b4832b8c66b2a27bfaba7afc8d2db4c236e68
+size 402617664

model/static-embeddings.512.int8.npy.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5337ce683b6db454feeb8842deb46807e3e6d989cb46b1b660fbfa0d4506c7
+size 74935075

model/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11aaf894a4ccf3d95e8830e27c0f8152791fbbff2b988e29a265580b86edd216
+size 2563370

model/tokenizer.json.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e18e87ed3f9053c05694c40fa520cd13ca7a7c9003009d890a14a5ab0aafd9d6
+size 829644

build_models.py → scripts/build_models.py

@@ -11,7 +11,7 @@ def save_data(path: Path, tensor: torch.Tensor):
     """Writes out the static embeddings to a .npy.zst file"""
     assert str(path).endswith(".npy.zst")
     buffer = io.BytesIO()
-    np.save(buffer, tensor.numpy())
+    np.save(buffer, tensor.detach().numpy())
 
     with (
         open(path, "wb") as outfile,
@@ -20,15 +20,14 @@ def save_data(path: Path, tensor: torch.Tensor):
         writer.write(buffer.getvalue())
 
 
-data_path = Path("embeddings")
-
+model_path = Path("model")
 model_name = "sentence-transformers/static-similarity-mrl-multilingual-v1"
 vocab_size = 105_879
 dimensions = 1024
 
 
 def load_embeddings():
-    model = SentenceTransformer(model_name)
+    model = SentenceTransformer(model_name, device="cpu")
     embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
     embeddings = torch.Tensor(embedding_bag.weight)
 
@@ -49,13 +48,13 @@ def load_embeddings():
         truncated = embeddings[:, :dim]
         assert truncated.shape == torch.Size([vocab_size, dim])
 
-        save_data(data_path / f"static-embeddings.{dim}.fp32.npy.zst", embeddings)
+        save_data(model_path / f"static-embeddings.{dim}.fp32.npy.zst", embeddings)
         save_data(
-            data_path / f"static-embeddings.{dim}.fp16.npy.zst",
+            model_path / f"static-embeddings.{dim}.fp16.npy.zst",
             embeddings.to(dtype=torch.float16),
         )
         save_data(
-            data_path / f"static-embeddings.{dim}.int8.npy.zst",
+            model_path / f"static-embeddings.{dim}.int8.npy.zst",
             embeddings.to(dtype=torch.int8),
         )
 

multilingual.py → scripts/experiments/multilingual.py

@@ -3,46 +3,44 @@ from tokenizers import Encoding, Tokenizer
 from torch.nn import EmbeddingBag
 import torch
 
-examples = [
-    "This is an example of encoding",
-    "The quick brown fox jumps over the lazy dog.",
-    "Curaçao, naïve fiancé, jalapeño, déjà vu.",
-    "Привет, как дела?",
-    "Бързата кафява лисица прескача мързеливото куче.",
-    "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
-    "اللغة العربية جميلة وغنية بالتاريخ.",
-    "مرحبا بالعالم!",
-    "Simplified: 快速的棕色狐狸跳过懒狗。",
-    "Traditional: 快速的棕色狐狸跳過懶狗。",
-    "素早い茶色の狐が怠け者の犬を飛び越える。",
-    "コンピュータープログラミング",
-    "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
-    "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
-    "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
-    "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
-    "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
-    "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
-    "Hello 世界 مرحبا 🌍",
-    "123, αβγ, абв, العربية, 中文, हिन्दी.",
-]
-
-tokenizer: Tokenizer = Tokenizer.from_file("js/tokenizer.json")
-
-for example in examples:
-    encoding: Encoding = tokenizer.encode(example)
-    tokenizer.decode
-    print(example)
-    print(encoding.tokens)
-    print()
-
-
-print("!!! tokenizer", tokenizer)
-assert False
+
+def test_tokenizer():
+    examples = [
+        "This is an example of encoding",
+        "The quick brown fox jumps over the lazy dog.",
+        "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+        "Привет, как дела?",
+        "Бързата кафява лисица прескача мързеливото куче.",
+        "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+        "اللغة العربية جميلة وغنية بالتاريخ.",
+        "مرحبا بالعالم!",
+        "Simplified: 快速的棕色狐狸跳过懒狗。",
+        "Traditional: 快速的棕色狐狸跳過懶狗。",
+        "素早い茶色の狐が怠け者の犬を飛び越える。",
+        "コンピュータープログラミング",
+        "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+        "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+        "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+        "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+        "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+        "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+        "Hello 世界 مرحبا 🌍",
+        "123, αβγ, абв, العربية, 中文, हिन्दी.",
+    ]
+
+    tokenizer: Tokenizer = Tokenizer.from_file("js/tokenizer.json")
+
+    for example in examples:
+        encoding: Encoding = tokenizer.encode(example)
+        print(example)
+        print(encoding.tokens)
+        print()
+
+
 # https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1
 model = SentenceTransformer(
     "sentence-transformers/static-similarity-mrl-multilingual-v1", device="cpu"
 )
-embeddings = model.encode(examples)
 
 embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
 embeddings = torch.Tensor(embedding_bag.weight)