Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+onnx/model.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/model_fp16.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/model_q4.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/model_q4f16.onnx_data filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+onnx/model_quantized.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/model_no_gather_q4.onnx_data filter=lfs diff=lfs merge=lfs -text
+mteb_results_by_task.png filter=lfs diff=lfs merge=lfs -text
+mteb_total_scores.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,80 @@
+---
+license: gemma
+base_model:
+- google/embeddinggemma-300m
+pipeline_tag: sentence-similarity
+library_name: transformers.js
+tags:
+- text-embeddings-inference
+---
+
+# embeddinggemma-300m-ONNX-uint8
+
+This is based on https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX/blob/main/onnx/model_quantized.onnx, but it outputs a uint8 tensor instead of an f32 one.
+
+This model is compatible with Qdrant, but I'm not sure what other vector DBs it's compatible with.
+
+For calibration data I used my own multilingual dataset of around 1.5m tokens: https://github.com/electroglyph/dataset_build
+
+I ran all 1.5m tokens through the model and logged the highest/lowest values seen. I found a range of: -0.19112960994243622 to 0.22116543352603912
+
+So I hacked on the sentence_embedding output of the ONNX model and added QuantizeLinear node based on the range of -0.22116543352603912 to 0.22116543352603912 to keep it symmetric. It would be cool if Qdrant let me specify my own zero point for a little more accuracy, but symmetric will have to do.
+
+# Benchmarks
+
+For benchmarking with MTEB I dequantize the uint8 output to the f32 that MTEB expects.
+
+These retrieval benchmarks are a little wild. All the benchmarks used the `task: search result` query format. I have no idea why this model benchmarks better than the base model on most retrieval tasks, but I'll take it.
+
+![mteb retrieval results](.\mteb_results_by_task.png)
+
+![mteb totals](.\mteb_total_scores.png)
+
+# Benchmark example code
+
+```python
+import mteb
+from mteb.encoder_interface import PromptType
+import numpy as np
+import onnxruntime as rt
+from transformers import AutoTokenizer
+
+class CustomModel:
+    def __init__(self) -> None:
+        self.tokenizer = AutoTokenizer.from_pretrained("C:/LLM/embeddinggemma-300m-ONNX-uint8")
+        self.session = rt.InferenceSession("C:/LLM/embeddinggemma-300m-ONNX-uint8/onnx/model.onnx", providers=["CPUExecutionProvider"])
+        self.scale = 0.22116543352603912 / 127.0
+
+    def dequantize(self, quantized: list | np.ndarray, scale: float) -> np.ndarray:
+        quantized = np.array(quantized)
+        dequant = (quantized.astype(np.float32) - 128) * scale
+        if dequant.ndim == 3 and dequant.shape[0] == 1:
+            return np.squeeze(dequant, axis=0)
+        return dequant
+
+    def encode(
+        self,
+        sentences: list[str],
+        task_name: str,
+        prompt_type: PromptType | None = None,
+        **kwargs,
+    ) -> np.ndarray:
+        if prompt_type == PromptType.query:
+            sentences = [f"task: search result | query: {s}" for s in sentences]
+        inputs = self.tokenizer(sentences, padding=True, truncation=True, return_tensors="np")
+        q = self.session.run(["sentence_embedding"], dict(inputs))
+        return self.dequantize(q, self.scale)
+
+
+model = CustomModel()
+benchmark = mteb.get_benchmark("NanoBEIR")
+evaluation = mteb.MTEB(tasks=benchmark)
+results = evaluation.run(model, corpus_chunk_size=128)
+for r in results:
+    print(r)
+
+```
+
+# FastEmbed usage
+
+You should be able to use this as a custom model with no pooling and no normalization. The sentence_embedding output is ready to use.

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

config.json

@@ -0,0 +1,71 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3TextModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 2048,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 512,
+  "transformers_version": "4.57.0.dev0",
+  "use_bidirectional_attention": true,
+  "use_cache": true,
+  "vocab_size": 262144,
+  "transformers.js_config": {
+    "use_external_data_format": {
+      "model.onnx": 1,
+      "model_fp16.onnx": 1,
+      "model_quantized.onnx": 1,
+      "model_q4.onnx": 1,
+      "model_q4f16.onnx": 1,
+      "model_no_gather_q4.onnx": 1
+    },
+    "kv_cache_dtype": false
+  }
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.57.0.dev0"
+}

onnx/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21585443cf1ee0e87ba306ba9b1b97761d0aa3666f96947f8e65123dfee06688
+size 309435349

special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dda02faaf32bc91031dc8c88457ac272b00c1016cc679757d1c441b248b9c47
+size 20323312

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074