Initial public release 1.0

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 AtomicoLabs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,79 @@
+---
+license: mit
+language:
+  - en
+  - es
+library_name: onnxruntime
+pipeline_tag: sentence-similarity
+tags:
+  - embeddings
+  - onnx
+  - retrieval
+  - sts
+  - on-device
+base_model: intfloat/multilingual-e5-large-instruct
+---
+
+# ALF-emb-micro 1.0
+
+Bilingual EN+ES embeddings. ALF is the AtomicoLabs model family. **Micro** = under 1B parameters.
+
+Vocab-pruned finetune of multilingual-e5-large-instruct. Shipping format is per-channel int8 ONNX (mean-pool, L2-normalized, 1024-d).
+
+Weights: GitHub Release [`v1.0`](https://github.com/AtomicoLabs/ALF-emb-micro/releases/tag/v1.0) and this Hugging Face revision `v1.0`.
+
+| | Composite | EN retrieval | ES retrieval | STS | Domain |
+|---|---|---|---|---|---|
+| ALF-emb-micro 1.0 (int8) | 79.99 | 57.25 | 77.80 | 89.07 | 95.85 |
+| OpenAI text-embedding-3-small | 80.08 | 59.74 | 79.52 | 88.66 | 92.39 |
+
+370M params · vocab 64.5k · cosine parity vs fp32 0.984.
+
+This tokenizer is **not** drop-in e5. You must remap ids with `keep_ids.npy` / `remap.py`.
+
+## Use
+
+```python
+from pathlib import Path
+import sys
+import numpy as np
+import onnxruntime as ort
+from huggingface_hub import snapshot_download
+
+repo = Path(snapshot_download("AtomicoLabs/ALF-emb-micro", revision="v1.0"))
+sys.path.insert(0, str(repo))
+from remap import RemapTokenizer
+
+QUERY = (
+    "Instruct: Given a web search query, retrieve relevant passages that answer the query\n"
+    "Query: "
+)
+
+tok = RemapTokenizer(repo, np.load(repo / "keep_ids.npy").tolist())
+sess = ort.InferenceSession(str(repo / "model.onnx"), providers=["CPUExecutionProvider"])
+
+def embed(texts, *, is_query=False):
+    batch = [(QUERY + t if is_query else t) for t in texts]
+    enc = tok(batch, padding=True, truncation=True, max_length=512, return_tensors="np")
+    vec = sess.run(None, {
+        "input_ids": enc["input_ids"].astype(np.int64),
+        "attention_mask": enc["attention_mask"].astype(np.int64),
+    })[0]
+    return vec.astype(np.float32)
+```
+
+Documents take an empty prefix. Queries need the instruct prefix above.
+
+## Limits
+
+- English retrieval is a bit under the OpenAI small bar; Spanish and domain are close or ahead.
+- Max 512 tokens. int8, not fp16/fp32.
+- Not a general instruction model.
+
+## Training
+
+Finetune of [`intfloat/multilingual-e5-large-instruct`](https://huggingface.co/intfloat/multilingual-e5-large-instruct) (MIT), then vocab prune 560M → 370M, then dynamic per-channel int8 ONNX.
+
+## License
+
+MIT. Include this notice and the e5-large-instruct MIT notice when you redistribute.

keep_ids.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fda54c0a5cd1da4326af3614982a3edbc4b19e708e46244cac83314a25b5a4a
+size 258376

manifest.json

@@ -0,0 +1,33 @@
+{
+  "id": "ALF-emb-micro-1.0",
+  "family": "ALF",
+  "track": "emb",
+  "tier": "micro",
+  "version": "1.0",
+  "description": "AtomicoLabs model family EN+ES embeddings. Vocab-pruned e5-large-instruct finetune, per-channel int8 ONNX.",
+  "files": {
+    "model": "model.onnx",
+    "tokenizer": "tokenizer.json",
+    "keep_ids": "keep_ids.npy",
+    "remap": "remap.py"
+  },
+  "format": "onnx-int8-dynamic-per-channel",
+  "size_bytes": 371768932,
+  "params": 369999872,
+  "vocab_size": 64562,
+  "dim": 1024,
+  "max_length": 512,
+  "pooling": "mean",
+  "normalize": true,
+  "query_prefix": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: ",
+  "doc_prefix": "",
+  "inputs": ["input_ids", "attention_mask"],
+  "output": "embedding",
+  "base": "intfloat/multilingual-e5-large-instruct",
+  "finetune": "e5l-560",
+  "scores": {
+    "bar": {"name": "openai-text-embedding-3-small", "composite": 80.08, "en": 59.74, "es": 79.52, "sts": 88.66, "domain": 92.39},
+    "fp32": {"composite": 80.65, "en": 58.56, "es": 78.63, "sts": 88.97, "domain": 96.45},
+    "int8": {"composite": 79.99, "en": 57.25, "es": 77.80, "sts": 89.07, "domain": 95.85, "cosine_parity": 0.984}
+  }
+}

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82028ab4f0b13aa76a0837e3b15898fbd6061023348a9b0f89b3cc18e77cfa0e
+size 371768932

prune_meta.json

@@ -0,0 +1,6 @@
+{
+  "vocab_size": 64562,
+  "coverage": 1.0,
+  "n_params": 369999872,
+  "base": "embeddings/checkpoints/e5l-560/model"
+}

remap.py

@@ -0,0 +1,41 @@
+"""Pruned-vocab id remap for ALF-emb-micro."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class RemapTokenizer:
+    def __init__(self, tok_dir: str | Path, keep_ids: list[int], unk_new: int | None = None):
+        from transformers import AutoTokenizer
+        import numpy as np
+        import torch
+
+        self.base = AutoTokenizer.from_pretrained(str(tok_dir))
+        self.map = {old: new for new, old in enumerate(keep_ids)}
+        unk_old = self.base.unk_token_id
+        self.unk_new = unk_new if unk_new is not None else self.map.get(unk_old, 0)
+        vocab = int(getattr(self.base, "vocab_size", 0) or 0)
+        size = max(vocab, max(keep_ids) + 1 if keep_ids else 1)
+        lut = np.full(size, self.unk_new, dtype=np.int64)
+        for old, new in self.map.items():
+            if 0 <= old < size:
+                lut[old] = new
+        self._lut_np = lut
+        self._lut = torch.from_numpy(lut)
+
+    def __call__(self, texts, **kwargs):
+        enc = self.base(texts, **kwargs)
+        ids = enc["input_ids"]
+        if hasattr(ids, "clamp"):
+            lut = self._lut.to(device=ids.device)
+            safe = ids.clamp(0, lut.numel() - 1).long()
+            enc["input_ids"] = lut[safe].to(dtype=ids.dtype)
+        else:
+            import numpy as np
+
+            arr = np.asarray(ids, dtype=np.int64)
+            mapped = self._lut_np[np.clip(arr, 0, len(self._lut_np) - 1)]
+            tensors = kwargs.get("return_tensors")
+            enc["input_ids"] = mapped if tensors in ("np", "pt") or hasattr(ids, "shape") else mapped.tolist()
+        return enc

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57bd17004cf7bbc354c19884c97c0e8a0503d2dcf10dde17d6890fc90adc8cb6
+size 17082833

tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "is_local": true,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "max_length": 512,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "stride": 0,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<unk>"
+}