feat: add static-embeddings like F16 safetensors lookup table

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa798c355d9bc602f8314d7ca411c27467787cee19ad15e25390c2f9aaec37d0
+size 31376776

scripts/convert-to-safensors.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env -S uv run --script
+#
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["safetensors", "numpy"]
+# ///
+
+import numpy as np
+from safetensors.numpy import save_file
+
+d = np.load("model.npz")
+W = np.ascontiguousarray(d["rows_int8"])  # (30522, 1024) int8, the released table
+scale = d["int8_scale"].astype(np.float32)  # (30522,) per-row dequantization scale
+assert W.shape == (30522, 1024) and W.dtype == np.int8 and scale.shape == (30522,)
+
+# FastEmbed's static models multiply each token's row by `weights`, which dequantizes it.
+save_file({"embeddings": W, "weights": scale}, "model.safetensors")

scripts/test-safentesor.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env -S uv run --script
+#
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["safetensors", "numpy", "tokenizers"]
+# ///
+
+# Checks that model.safetensors, pooled by the model card's rule, gives the released vectors.
+
+import sys
+from pathlib import Path
+
+import numpy as np
+from safetensors.numpy import load_file
+
+model_dir = Path(__file__).parents[1]
+sys.path.insert(0, str(model_dir))
+
+from zero_encoder import ZeroQueryEncoder
+
+model = ZeroQueryEncoder(model_dir, variant="int8")
+t = load_file(model_dir / "model.safetensors")
+texts = ["hello world", "data data data pipeline", "🤗 emoji query", ""]
+
+for text, released in zip(texts, model.encode(texts)):
+    # special and unknown tokens included, a token seen c times weighs sqrt(c)
+    ids, counts = np.unique(model.tokenizer.encode(text).ids, return_counts=True)
+    v = (np.sqrt(counts) * t["weights"][ids]) @ t["embeddings"][ids].astype(np.float32)
+    v /= np.linalg.norm(v)
+    assert np.abs(v - released).max() < 1e-6, text
+print("model.safetensors matches the released int8 encoder")