Duplicate from nomic-ai/nomic-embed-multimodal-7b

Co-authored-by: Zach Nussbaum <zpn@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,181 @@
+---
+base_model:
+- Qwen/Qwen2.5-VL-7B-Instruct
+library_name: peft
+datasets:
+- nomic-ai/colpali-queries-mined-20250321-by-source
+language:
+- en
+- it
+- fr
+- de
+- es
+pipeline_tag: visual-document-retrieval
+tags:
+- vidore
+- colpali
+- multimodal_embedding
+- multilingual_embedding
+- Text-to-Visual Document (T→VD) retrieval
+license: apache-2.0
+---
+
+# Nomic Embed Multimodal 7B: State-of-the-Art Visual Document Retrieval
+
+`nomic-embed-multimodal-7b` is a dense state-of-the-art multimodal embedding model that excels at visual document retrieval tasks:
+
+- **High Performance**: Achieves 58.8 NDCG@5 on Vidore-v2, outperforming all other dense multimodal embedding models.
+- **Unified Text-Image Encoding**: Directly encodes interleaved text and images without complex preprocessing
+- **Advanced Architecture**: 7B parameter multimodal embedding model
+- **Fully Open-Source**: Model weights, training data, and code available
+
+
+## Performance
+
+| Model | Avg. | ESG Restaurant Human | Econ Macro Multi. | AXA Multi. | MIT Bio | ESG Restaurant Synth. | ESG Restaurant Synth. Multi. | MIT Bio Multi. | AXA | Econ. Macro |
+|-------|------|----------------------|-------------------|------------|---------|----------------------|----------------------------|---------------|-----|------------|
+| [ColNomic Embed Multimodal 7B](https://huggingface.co/nomic-ai/colnomic-embed-multimodal-7b) | 62.7 | 73.9 | 54.7 | 61.3 | 66.1 | 57.3 | 56.7 | 64.2 | 68.3 | 61.6 |
+| [ColNomic Embed Multimodal 3B](https://huggingface.co/nomic-ai/colnomic-embed-multimodal-3b) | 61.2 | 65.8 | 55.4 | 61.0 | 63.5 | 56.6 | 57.2 | 62.5 | 68.8 | 60.2 |
+| T-Systems ColQwen2.5-3B | 59.9 | 72.1 | 51.2 | 60.0 | 65.3 | 51.7 | 53.3 | 61.7 | 69.3 | 54.8 |
+| **Nomic Embed Multimodal 7B** | 59.7 | 65.7 | 57.7 | 59.3 | 64.0 | 49.2 | 51.9 | 61.2 | 66.3 | 63.1 |
+| GME Qwen2 7B | 59.0 | 65.8 | 56.2 | 55.4 | 64.0 | 54.3 | 56.7 | 55.1 | 60.7 | 62.9 |
+| [Nomic Embed Multimodal 3B](https://huggingface.co/nomic-ai/nomic-embed-multimodal-3b)  | 58.8 | 59.8 | 57.5 | 58.8 | 62.5 | 49.4 | 49.4 | 58.6 | 69.6 | 63.5 |
+| Llama Index vdr-2b-multi-v1 | 58.4 | 63.1 | 52.8 | 61.0 | 60.6 | 50.3 | 51.2 | 56.9 | 68.8 | 61.2 |
+| Voyage Multimodal 3 | 55.0 | 56.1 | 55.0 | 59.5 | 56.4 | 47.2 | 46.2 | 51.5 | 64.1 | 58.8 |
+
+
+## Getting Started
+
+To use `nomic-embed-multimodal-7b`, please install `colpali` from source
+
+```bash
+pip install git+https://github.com/illuin-tech/colpali.git
+```
+
+
+```python
+import torch
+from PIL import Image
+from transformers.utils.import_utils import is_flash_attn_2_available
+
+from colpali_engine.models import BiQwen2_5, BiQwen2_5_Processor
+
+model_name = "nomic-ai/nomic-embed-multimodal-7b"
+
+model = BiQwen2_5.from_pretrained(
+    model_name,
+    torch_dtype=torch.bfloat16,
+    device_map="cuda:0",  # or "mps" if on Apple Silicon
+    attn_implementation="flash_attention_2" if is_flash_attn_2_available() else None,
+).eval()
+
+processor = BiQwen2_5_Processor.from_pretrained(model_name)
+
+# Your inputs
+images = [
+    Image.new("RGB", (128, 128), color="white"),
+    Image.new("RGB", (64, 32), color="black"),
+]
+queries = [
+    "What is the organizational structure for our R&D department?",
+    "Can you provide a breakdown of last year’s financial performance?",
+]
+
+# Process the inputs
+batch_images = processor.process_images(images).to(model.device)
+batch_queries = processor.process_queries(queries).to(model.device)
+
+# Forward pass
+with torch.no_grad():
+    image_embeddings = model(**batch_images)
+    query_embeddings = model(**batch_queries)
+
+scores = processor.score(list(torch.unbind(query_embeddings)), list(torch.unbind(image_embeddings)))
+```
+
+
+## Model Architecture
+
+- **Total Parameters**: 7B
+- **Training Approach**: Fine-tuned from Qwen2.5-VL 7B Instruct
+- **Architecture Type**: Vision-Language Model with unified text and image input processing
+- **Key Innovations**:
+  - Same-source sampling to create harder in-batch negatives
+  - Hard negative mining with positive-aware techniques
+
+## Integration with RAG Workflows
+
+Nomic Embed Multimodal 7B seamlessly integrates with Retrieval Augmented Generation (RAG) workflows:
+
+1. **Direct Document Embedding**: Skip OCR and complex processing by directly embedding document page images
+2. **Faster Processing**: Eliminate preprocessing steps for quicker indexing
+3. **More Complete Information**: Capture both textual and visual cues in a single embedding
+4. **Simple Implementation**: Use the same API for both text and images
+
+## Recommended Use Cases
+
+The model excels at handling real-world document retrieval scenarios that challenge traditional text-only systems:
+
+- **Research Papers**: Capture equations, diagrams, and tables
+- **Technical Documentation**: Encode code blocks, flowcharts, and screenshots
+- **Product Catalogs**: Represent images, specifications, and pricing tables
+- **Financial Reports**: Embed charts, graphs, and numerical data
+- **Visually Rich Content**: Where layout and visual information are important
+- **Multilingual Documents**: Where visual context provides important cues
+
+## Training Details
+
+Nomic Embed Multimodal 7B was developed through several key innovations:
+
+1. **Sampling From the Same Source**: Forcing sampling from the same dataset source creates harder in-batch negatives, preventing the model from learning dataset artifacts.
+
+2. **Hard Negative Mining**: Using an initial model to retrieve top-k nearest neighbors for each query, then incorporating these hard negatives into training.
+
+3. **Positive-aware Hard Negative Mining**: Reducing false negatives using techniques introduced in NV-Retriever.
+
+
+## Limitations
+
+- Performance may vary when processing documents with unconventional layouts or unusual visual elements
+- While it handles multiple languages, performance is strongest on English content
+- Processing very large or complex documents may require dividing them into smaller chunks
+- Performance on documents with handwriting or heavily stylized fonts may be reduced
+
+## Join the Nomic Community
+
+- Nomic Embed Ecosystem: [https://www.nomic.ai/embed](https://www.nomic.ai/embed)
+- Website: [https://nomic.ai](https://nomic.ai)
+- Twitter: [https://twitter.com/nomic_ai](https://twitter.com/nomic_ai)
+- Discord: [https://discord.gg/myY5YDR8z8](https://discord.gg/myY5YDR8z8)
+
+## Citation
+
+If you find this model useful in your research or applications, please consider citing:
+
+```bibtex
+@misc{faysse2024colpaliefficientdocumentretrieval,
+  title={ColPali: Efficient Document Retrieval with Vision Language Models}, 
+  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
+  year={2024},
+  eprint={2407.01449},
+  archivePrefix={arXiv},
+  primaryClass={cs.IR},
+  url={https://arxiv.org/abs/2407.01449}, 
+}
+@misc{ma2024unifyingmultimodalretrievaldocument,
+      title={Unifying Multimodal Retrieval via Document Screenshot Embedding}, 
+      author={Xueguang Ma and Sheng-Chieh Lin and Minghan Li and Wenhu Chen and Jimmy Lin},
+      year={2024},
+      eprint={2406.11251},
+      archivePrefix={arXiv},
+      primaryClass={cs.IR},
+      url={https://arxiv.org/abs/2406.11251}, 
+}
+@misc{nomicembedmultimodal2025,
+  title={Nomic Embed Multimodal: Interleaved Text, Image, and Screenshots for Visual Document Retrieval},
+  author={Nomic Team},
+  year={2025},
+  publisher={Nomic AI},
+  url={https://nomic.ai/blog/posts/nomic-embed-multimodal},
+}
+```

adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-VL-7B-Instruct",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f68069598612ca16989a1dab8d2e8ed7f3847ae3514fba571be3711d6e3190f2
+size 323014168

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.49.0"
+}

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 602112,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "BiQwen2_5_Processor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 602112,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "max_num_visual_tokens": 768,
+  "processor_class": "BiQwen2_5_Processor"
+}

results.json

@@ -0,0 +1,1104 @@
+{
+   "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": {
+      "ndcg_at_1": 0.63125,
+      "ndcg_at_3": 0.62606,
+      "ndcg_at_5": 0.64036,
+      "ndcg_at_10": 0.67252,
+      "ndcg_at_20": 0.70051,
+      "ndcg_at_50": 0.7225,
+      "ndcg_at_100": 0.73342,
+      "map_at_1": 0.38694,
+      "map_at_3": 0.5157,
+      "map_at_5": 0.55286,
+      "map_at_10": 0.58663,
+      "map_at_20": 0.60199,
+      "map_at_50": 0.61092,
+      "map_at_100": 0.61334,
+      "recall_at_1": 0.38694,
+      "recall_at_3": 0.58442,
+      "recall_at_5": 0.65538,
+      "recall_at_10": 0.75502,
+      "recall_at_20": 0.83375,
+      "recall_at_50": 0.89435,
+      "recall_at_100": 0.93847,
+      "precision_at_1": 0.63125,
+      "precision_at_3": 0.37917,
+      "precision_at_5": 0.28375,
+      "precision_at_10": 0.17875,
+      "precision_at_20": 0.10594,
+      "precision_at_50": 0.05062,
+      "precision_at_100": 0.02756,
+      "mrr_at_1": 0.6375,
+      "mrr_at_3": 0.7270833333333332,
+      "mrr_at_5": 0.7339583333333333,
+      "mrr_at_10": 0.7424727182539682,
+      "mrr_at_20": 0.7446067387473636,
+      "mrr_at_50": 0.7449988719519969,
+      "mrr_at_100": 0.7452357796521399,
+      "naucs_at_1_max": 0.2860227930855084,
+      "naucs_at_1_std": -0.3138417441324,
+      "naucs_at_1_diff1": 0.42744397436023296,
+      "naucs_at_3_max": 0.1892512065375088,
+      "naucs_at_3_std": -0.09854623333300405,
+      "naucs_at_3_diff1": 0.00578348450006455,
+      "naucs_at_5_max": 0.0985916495531507,
+      "naucs_at_5_std": -0.06582623582128855,
+      "naucs_at_5_diff1": -0.08828438926991423,
+      "naucs_at_10_max": 0.0933821578304943,
+      "naucs_at_10_std": 0.08155017778485431,
+      "naucs_at_10_diff1": -0.17980682481558072,
+      "naucs_at_20_max": 0.0020201283703657888,
+      "naucs_at_20_std": 0.10903975278089359,
+      "naucs_at_20_diff1": -0.2369554821180032,
+      "naucs_at_50_max": -0.05388593736433463,
+      "naucs_at_50_std": 0.21602702406052268,
+      "naucs_at_50_diff1": -0.25488066393739356,
+      "naucs_at_100_max": -0.09001262910229582,
+      "naucs_at_100_std": 0.2278313121002944,
+      "naucs_at_100_diff1": -0.25651206108102814
+   },
+   "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": {
+      "ndcg_at_1": 0.7069,
+      "ndcg_at_3": 0.6531,
+      "ndcg_at_5": 0.63125,
+      "ndcg_at_10": 0.60817,
+      "ndcg_at_20": 0.61254,
+      "ndcg_at_50": 0.67937,
+      "ndcg_at_100": 0.7167,
+      "map_at_1": 0.12782,
+      "map_at_3": 0.21562,
+      "map_at_5": 0.28161,
+      "map_at_10": 0.35497,
+      "map_at_20": 0.40334,
+      "map_at_50": 0.46201,
+      "map_at_100": 0.49032,
+      "recall_at_1": 0.12782,
+      "recall_at_3": 0.23164,
+      "recall_at_5": 0.33225,
+      "recall_at_10": 0.47034,
+      "recall_at_20": 0.59047,
+      "recall_at_50": 0.78316,
+      "recall_at_100": 0.89751,
+      "precision_at_1": 0.7069,
+      "precision_at_3": 0.59195,
+      "precision_at_5": 0.54138,
+      "precision_at_10": 0.42759,
+      "precision_at_20": 0.30345,
+      "precision_at_50": 0.1969,
+      "precision_at_100": 0.12741,
+      "mrr_at_1": 0.7241379310344828,
+      "mrr_at_3": 0.7729885057471265,
+      "mrr_at_5": 0.7850574712643678,
+      "mrr_at_10": 0.8005199781061849,
+      "mrr_at_20": 0.8015975643130815,
+      "mrr_at_50": 0.8015975643130815,
+      "mrr_at_100": 0.8015975643130815,
+      "naucs_at_1_max": 0.5414377044487941,
+      "naucs_at_1_std": 0.2885423494902651,
+      "naucs_at_1_diff1": 0.4517408404026178,
+      "naucs_at_3_max": 0.5103699198828265,
+      "naucs_at_3_std": 0.34891966684046116,
+      "naucs_at_3_diff1": 0.1460156103430603,
+      "naucs_at_5_max": 0.5399440580636293,
+      "naucs_at_5_std": 0.3199440767695221,
+      "naucs_at_5_diff1": 0.18068785350199154,
+      "naucs_at_10_max": 0.5255610151091152,
+      "naucs_at_10_std": 0.3925916787135598,
+      "naucs_at_10_diff1": 0.14785999421779913,
+      "naucs_at_20_max": 0.4516553789306758,
+      "naucs_at_20_std": 0.4790296057633737,
+      "naucs_at_20_diff1": 0.14584034106403623,
+      "naucs_at_50_max": 0.34044111140425515,
+      "naucs_at_50_std": 0.4292008781133996,
+      "naucs_at_50_diff1": 0.0657235966934423,
+      "naucs_at_100_max": 0.296959342986135,
+      "naucs_at_100_std": 0.4138623681388592,
+      "naucs_at_100_diff1": 0.06422150196714925
+   },
+   "vidore/synthetic_rse_restaurant_filtered_v1.0": {
+      "ndcg_at_1": 0.40351,
+      "ndcg_at_3": 0.47539,
+      "ndcg_at_5": 0.49152,
+      "ndcg_at_10": 0.553,
+      "ndcg_at_20": 0.59699,
+      "ndcg_at_50": 0.62258,
+      "ndcg_at_100": 0.63184,
+      "map_at_1": 0.2155,
+      "map_at_3": 0.35506,
+      "map_at_5": 0.38442,
+      "map_at_10": 0.43472,
+      "map_at_20": 0.45888,
+      "map_at_50": 0.46993,
+      "map_at_100": 0.47398,
+      "recall_at_1": 0.2155,
+      "recall_at_3": 0.46042,
+      "recall_at_5": 0.5245,
+      "recall_at_10": 0.69217,
+      "recall_at_20": 0.83089,
+      "recall_at_50": 0.9086,
+      "recall_at_100": 0.93039,
+      "precision_at_1": 0.40351,
+      "precision_at_3": 0.32749,
+      "precision_at_5": 0.25263,
+      "precision_at_10": 0.18421,
+      "precision_at_20": 0.12105,
+      "precision_at_50": 0.06105,
+      "precision_at_100": 0.03421,
+      "mrr_at_1": 0.40350877192982454,
+      "mrr_at_3": 0.5555555555555555,
+      "mrr_at_5": 0.5730994152046783,
+      "mrr_at_10": 0.5808966861598441,
+      "mrr_at_20": 0.586509202298676,
+      "mrr_at_50": 0.5877768876410641,
+      "mrr_at_100": 0.5877768876410641,
+      "naucs_at_1_max": 0.29648339825644304,
+      "naucs_at_1_std": 0.06641652874714649,
+      "naucs_at_1_diff1": 0.5045818569761475,
+      "naucs_at_3_max": 0.16818047225254976,
+      "naucs_at_3_std": 0.17028690578931582,
+      "naucs_at_3_diff1": 0.17461140240553258,
+      "naucs_at_5_max": 0.19459542279126757,
+      "naucs_at_5_std": 0.24823743512641647,
+      "naucs_at_5_diff1": 0.19749834680039569,
+      "naucs_at_10_max": 0.07835810840401428,
+      "naucs_at_10_std": 0.19882694823881913,
+      "naucs_at_10_diff1": 0.13857285069040404,
+      "naucs_at_20_max": -0.007433838800528796,
+      "naucs_at_20_std": 0.19538335810174498,
+      "naucs_at_20_diff1": 0.0012071916296247957,
+      "naucs_at_50_max": -0.0902754753027608,
+      "naucs_at_50_std": 0.1592772246389163,
+      "naucs_at_50_diff1": -0.02404431111361124,
+      "naucs_at_100_max": -0.12119070620203119,
+      "naucs_at_100_std": 0.13618966353380924,
+      "naucs_at_100_diff1": -0.04813882401945786
+   },
+   "vidore/synthetic_axa_filtered_v1.0": {
+      "ndcg_at_1": 0.72222,
+      "ndcg_at_3": 0.69497,
+      "ndcg_at_5": 0.66273,
+      "ndcg_at_10": 0.67378,
+      "ndcg_at_20": 0.69012,
+      "ndcg_at_50": 0.75042,
+      "ndcg_at_100": 0.77219,
+      "map_at_1": 0.36488,
+      "map_at_3": 0.47229,
+      "map_at_5": 0.49718,
+      "map_at_10": 0.54558,
+      "map_at_20": 0.56749,
+      "map_at_50": 0.60107,
+      "map_at_100": 0.60562,
+      "recall_at_1": 0.36488,
+      "recall_at_3": 0.54457,
+      "recall_at_5": 0.58939,
+      "recall_at_10": 0.68972,
+      "recall_at_20": 0.74479,
+      "recall_at_50": 0.89156,
+      "recall_at_100": 0.98955,
+      "precision_at_1": 0.72222,
+      "precision_at_3": 0.44444,
+      "precision_at_5": 0.33333,
+      "precision_at_10": 0.24444,
+      "precision_at_20": 0.15,
+      "precision_at_50": 0.08667,
+      "precision_at_100": 0.04667,
+      "mrr_at_1": 0.7222222222222222,
+      "mrr_at_3": 0.7962962962962963,
+      "mrr_at_5": 0.8101851851851852,
+      "mrr_at_10": 0.8101851851851852,
+      "mrr_at_20": 0.8101851851851852,
+      "mrr_at_50": 0.8101851851851852,
+      "mrr_at_100": 0.8109903381642511,
+      "naucs_at_1_max": 0.19079787602675777,
+      "naucs_at_1_std": 0.03533531122016283,
+      "naucs_at_1_diff1": 0.632680674681619,
+      "naucs_at_3_max": -0.31385535499287276,
+      "naucs_at_3_std": -0.04655749634420289,
+      "naucs_at_3_diff1": 0.155475737963548,
+      "naucs_at_5_max": -0.2472297825482412,
+      "naucs_at_5_std": 0.03761412944004308,
+      "naucs_at_5_diff1": -0.10924664115489778,
+      "naucs_at_10_max": -0.37765403672620185,
+      "naucs_at_10_std": 0.002000186439662032,
+      "naucs_at_10_diff1": -0.14440595994296845,
+      "naucs_at_20_max": -0.4699503134091192,
+      "naucs_at_20_std": 0.025147844173298965,
+      "naucs_at_20_diff1": -0.16389645563321997,
+      "naucs_at_50_max": -0.5068940056618596,
+      "naucs_at_50_std": -0.019754043372479577,
+      "naucs_at_50_diff1": -0.16850642524469236,
+      "naucs_at_100_max": -0.4767515120911255,
+      "naucs_at_100_std": 0.015763234490211366,
+      "naucs_at_100_diff1": -0.20731314235125625
+   },
+   "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": {
+      "ndcg_at_1": 0.4386,
+      "ndcg_at_3": 0.48147,
+      "ndcg_at_5": 0.51908,
+      "ndcg_at_10": 0.57159,
+      "ndcg_at_20": 0.60702,
+      "ndcg_at_50": 0.63417,
+      "ndcg_at_100": 0.64411,
+      "map_at_1": 0.23507,
+      "map_at_3": 0.35761,
+      "map_at_5": 0.40257,
+      "map_at_10": 0.44822,
+      "map_at_20": 0.47031,
+      "map_at_50": 0.48211,
+      "map_at_100": 0.48628,
+      "recall_at_1": 0.23507,
+      "recall_at_3": 0.4487,
+      "recall_at_5": 0.56129,
+      "recall_at_10": 0.71123,
+      "recall_at_20": 0.82755,
+      "recall_at_50": 0.90956,
+      "recall_at_100": 0.93439,
+      "precision_at_1": 0.4386,
+      "precision_at_3": 0.32895,
+      "precision_at_5": 0.27018,
+      "precision_at_10": 0.18991,
+      "precision_at_20": 0.11974,
+      "precision_at_50": 0.06123,
+      "precision_at_100": 0.03447,
+      "mrr_at_1": 0.4517543859649123,
+      "mrr_at_3": 0.5796783625730992,
+      "mrr_at_5": 0.6033625730994151,
+      "mrr_at_10": 0.6098318713450293,
+      "mrr_at_20": 0.6143413495606139,
+      "mrr_at_50": 0.6150114081677309,
+      "mrr_at_100": 0.6150114081677309,
+      "naucs_at_1_max": 0.1695609084486461,
+      "naucs_at_1_std": 0.06302260506626338,
+      "naucs_at_1_diff1": 0.34368925726365046,
+      "naucs_at_3_max": 0.07022225247354692,
+      "naucs_at_3_std": 0.10306674677084651,
+      "naucs_at_3_diff1": 0.14995084571990153,
+      "naucs_at_5_max": 0.07354175839710718,
+      "naucs_at_5_std": 0.16198320276362924,
+      "naucs_at_5_diff1": 0.06343317549781861,
+      "naucs_at_10_max": 0.022729360196430883,
+      "naucs_at_10_std": 0.1654593087070378,
+      "naucs_at_10_diff1": 0.04276127936743703,
+      "naucs_at_20_max": -0.031554725803849416,
+      "naucs_at_20_std": 0.1300165212363859,
+      "naucs_at_20_diff1": 0.006021945700144339,
+      "naucs_at_50_max": -0.06313785005592049,
+      "naucs_at_50_std": 0.13595654225029755,
+      "naucs_at_50_diff1": 0.008224218691189635,
+      "naucs_at_100_max": -0.08372981904605119,
+      "naucs_at_100_std": 0.12653852178262695,
+      "naucs_at_100_diff1": -0.012169561956816101
+   },
+   "vidore/synthetic_axa_filtered_v1.0_multilingual": {
+      "ndcg_at_1": 0.65278,
+      "ndcg_at_3": 0.61009,
+      "ndcg_at_5": 0.59294,
+      "ndcg_at_10": 0.61008,
+      "ndcg_at_20": 0.63877,
+      "ndcg_at_50": 0.69737,
+      "ndcg_at_100": 0.72109,
+      "map_at_1": 0.3313,
+      "map_at_3": 0.41787,
+      "map_at_5": 0.44884,
+      "map_at_10": 0.49024,
+      "map_at_20": 0.51424,
+      "map_at_50": 0.54276,
+      "map_at_100": 0.54899,
+      "recall_at_1": 0.3313,
+      "recall_at_3": 0.46646,
+      "recall_at_5": 0.52791,
+      "recall_at_10": 0.64008,
+      "recall_at_20": 0.73391,
+      "recall_at_50": 0.8845,
+      "recall_at_100": 0.97673,
+      "precision_at_1": 0.65278,
+      "precision_at_3": 0.39352,
+      "precision_at_5": 0.30833,
+      "precision_at_10": 0.22222,
+      "precision_at_20": 0.14028,
+      "precision_at_50": 0.08111,
+      "precision_at_100": 0.04556,
+      "mrr_at_1": 0.6527777777777778,
+      "mrr_at_3": 0.7175925925925928,
+      "mrr_at_5": 0.7307870370370372,
+      "mrr_at_10": 0.7397156084656086,
+      "mrr_at_20": 0.7417099104599107,
+      "mrr_at_50": 0.7421307858807862,
+      "mrr_at_100": 0.7426557166541077,
+      "naucs_at_1_max": 0.318303386786102,
+      "naucs_at_1_std": 0.23808943823013543,
+      "naucs_at_1_diff1": 0.6508710234226615,
+      "naucs_at_3_max": 0.13214248537450415,
+      "naucs_at_3_std": 0.16151259206691232,
+      "naucs_at_3_diff1": 0.22229988734788014,
+      "naucs_at_5_max": 0.05889008903334795,
+      "naucs_at_5_std": 0.11423669671153379,
+      "naucs_at_5_diff1": 0.11548423013320717,
+      "naucs_at_10_max": -0.037397142604183475,
+      "naucs_at_10_std": 0.09416088185519944,
+      "naucs_at_10_diff1": 0.008856567275955608,
+      "naucs_at_20_max": -0.11441594604958465,
+      "naucs_at_20_std": 0.04973867186679883,
+      "naucs_at_20_diff1": -0.05320557709786541,
+      "naucs_at_50_max": -0.18138441233720995,
+      "naucs_at_50_std": 0.010825689422469654,
+      "naucs_at_50_diff1": -0.10510638193999315,
+      "naucs_at_100_max": -0.202379290531666,
+      "naucs_at_100_std": -0.014979683943287803,
+      "naucs_at_100_diff1": -0.14558601228430124
+   },
+   "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": {
+      "ndcg_at_1": 0.57344,
+      "ndcg_at_3": 0.59236,
+      "ndcg_at_5": 0.61157,
+      "ndcg_at_10": 0.63866,
+      "ndcg_at_20": 0.66879,
+      "ndcg_at_50": 0.69522,
+      "ndcg_at_100": 0.70444,
+      "map_at_1": 0.36007,
+      "map_at_3": 0.48525,
+      "map_at_5": 0.52371,
+      "map_at_10": 0.5533,
+      "map_at_20": 0.5688,
+      "map_at_50": 0.57892,
+      "map_at_100": 0.58096,
+      "recall_at_1": 0.36007,
+      "recall_at_3": 0.55859,
+      "recall_at_5": 0.63871,
+      "recall_at_10": 0.7243,
+      "recall_at_20": 0.81097,
+      "recall_at_50": 0.88961,
+      "recall_at_100": 0.92417,
+      "precision_at_1": 0.57344,
+      "precision_at_3": 0.36302,
+      "precision_at_5": 0.27469,
+      "precision_at_10": 0.17016,
+      "precision_at_20": 0.10219,
+      "precision_at_50": 0.04997,
+      "precision_at_100": 0.02702,
+      "mrr_at_1": 0.5796875,
+      "mrr_at_3": 0.680729166666666,
+      "mrr_at_5": 0.6924479166666659,
+      "mrr_at_10": 0.6992057291666662,
+      "mrr_at_20": 0.7025276038138841,
+      "mrr_at_50": 0.7032994002394296,
+      "mrr_at_100": 0.7035081912324784,
+      "naucs_at_1_max": 0.2618446848882775,
+      "naucs_at_1_std": -0.170161905683693,
+      "naucs_at_1_diff1": 0.4753067384546821,
+      "naucs_at_3_max": 0.17827521235144483,
+      "naucs_at_3_std": -0.060198662170649925,
+      "naucs_at_3_diff1": 0.05418513753300305,
+      "naucs_at_5_max": 0.08757447272019533,
+      "naucs_at_5_std": -0.049911565737325504,
+      "naucs_at_5_diff1": -0.04360643940712858,
+      "naucs_at_10_max": 0.0797198239661374,
+      "naucs_at_10_std": 0.03949138307521818,
+      "naucs_at_10_diff1": -0.09762488661941034,
+      "naucs_at_20_max": 0.01961040143553897,
+      "naucs_at_20_std": 0.10087633582361409,
+      "naucs_at_20_diff1": -0.17144901884144179,
+      "naucs_at_50_max": -0.02239975053336602,
+      "naucs_at_50_std": 0.1699503781904292,
+      "naucs_at_50_diff1": -0.1722611533882454,
+      "naucs_at_100_max": -0.04512586039808127,
+      "naucs_at_100_std": 0.17386318414141583,
+      "naucs_at_100_diff1": -0.1871247841007667
+   },
+   "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": {
+      "ndcg_at_1": 0.66379,
+      "ndcg_at_3": 0.60737,
+      "ndcg_at_5": 0.57718,
+      "ndcg_at_10": 0.55599,
+      "ndcg_at_20": 0.57988,
+      "ndcg_at_50": 0.64999,
+      "ndcg_at_100": 0.68674,
+      "map_at_1": 0.12281,
+      "map_at_3": 0.20294,
+      "map_at_5": 0.25378,
+      "map_at_10": 0.31487,
+      "map_at_20": 0.36947,
+      "map_at_50": 0.42823,
+      "map_at_100": 0.45448,
+      "recall_at_1": 0.12281,
+      "recall_at_3": 0.2256,
+      "recall_at_5": 0.30659,
+      "recall_at_10": 0.42269,
+      "recall_at_20": 0.56687,
+      "recall_at_50": 0.76718,
+      "recall_at_100": 0.88492,
+      "precision_at_1": 0.66379,
+      "precision_at_3": 0.5431,
+      "precision_at_5": 0.48362,
+      "precision_at_10": 0.38836,
+      "precision_at_20": 0.29591,
+      "precision_at_50": 0.19345,
+      "precision_at_100": 0.12414,
+      "mrr_at_1": 0.6767241379310345,
+      "mrr_at_3": 0.7399425287356323,
+      "mrr_at_5": 0.7548132183908046,
+      "mrr_at_10": 0.7652247536945812,
+      "mrr_at_20": 0.7667345585449032,
+      "mrr_at_50": 0.7675009343641154,
+      "mrr_at_100": 0.7675472821579604,
+      "naucs_at_1_max": 0.3427760433833983,
+      "naucs_at_1_std": 0.17486041852136283,
+      "naucs_at_1_diff1": 0.3493033195022897,
+      "naucs_at_3_max": 0.29896675454636334,
+      "naucs_at_3_std": 0.2241328326881254,
+      "naucs_at_3_diff1": 0.07681853603265505,
+      "naucs_at_5_max": 0.3541130170379229,
+      "naucs_at_5_std": 0.27779731381034506,
+      "naucs_at_5_diff1": 0.04428119389439255,
+      "naucs_at_10_max": 0.3161718142368522,
+      "naucs_at_10_std": 0.2654928526689307,
+      "naucs_at_10_diff1": 0.014100490157475373,
+      "naucs_at_20_max": 0.2499680124975916,
+      "naucs_at_20_std": 0.2633550881217926,
+      "naucs_at_20_diff1": -0.02686064189005896,
+      "naucs_at_50_max": 0.1951500660706873,
+      "naucs_at_50_std": 0.23487608842256993,
+      "naucs_at_50_diff1": -0.0871206335340013,
+      "naucs_at_100_max": 0.16510434929270115,
+      "naucs_at_100_std": 0.21054693222984838,
+      "naucs_at_100_diff1": -0.10274771435027014
+   },
+   "vidore/restaurant_esg_reports_beir": {
+      "ndcg_at_1": 0.64103,
+      "ndcg_at_3": 0.62507,
+      "ndcg_at_5": 0.65665,
+      "ndcg_at_10": 0.67232,
+      "ndcg_at_20": 0.69863,
+      "ndcg_at_50": 0.72704,
+      "ndcg_at_100": 0.73746,
+      "map_at_1": 0.44583,
+      "map_at_3": 0.54321,
+      "map_at_5": 0.58695,
+      "map_at_10": 0.60061,
+      "map_at_20": 0.61379,
+      "map_at_50": 0.62209,
+      "map_at_100": 0.62409,
+      "recall_at_1": 0.44583,
+      "recall_at_3": 0.60335,
+      "recall_at_5": 0.69232,
+      "recall_at_10": 0.74186,
+      "recall_at_20": 0.81674,
+      "recall_at_50": 0.92301,
+      "recall_at_100": 0.9696,
+      "precision_at_1": 0.65385,
+      "precision_at_3": 0.33333,
+      "precision_at_5": 0.25385,
+      "precision_at_10": 0.14231,
+      "precision_at_20": 0.08558,
+      "precision_at_50": 0.04154,
+      "precision_at_100": 0.0225,
+      "mrr_at_1": 0.6538461538461539,
+      "mrr_at_3": 0.7211538461538461,
+      "mrr_at_5": 0.7288461538461538,
+      "mrr_at_10": 0.736935286935287,
+      "mrr_at_20": 0.738003663003663,
+      "mrr_at_50": 0.739894529945878,
+      "mrr_at_100": 0.7402260949326155,
+      "naucs_at_1_max": 0.18675644205833766,
+      "naucs_at_1_std": 0.12089377306440403,
+      "naucs_at_1_diff1": 0.23180938958754124,
+      "naucs_at_3_max": 0.1337570930336171,
+      "naucs_at_3_std": 0.2424272064488133,
+      "naucs_at_3_diff1": -0.21025078361612204,
+      "naucs_at_5_max": 0.005149088005913239,
+      "naucs_at_5_std": 0.30173104877190665,
+      "naucs_at_5_diff1": -0.3297309037027529,
+      "naucs_at_10_max": 0.058274590610789254,
+      "naucs_at_10_std": 0.32737391815331685,
+      "naucs_at_10_diff1": -0.390552006407966,
+      "naucs_at_20_max": -0.04962516887611918,
+      "naucs_at_20_std": 0.33292517052358606,
+      "naucs_at_20_diff1": -0.435159500418056,
+      "naucs_at_50_max": -0.0972258593819507,
+      "naucs_at_50_std": 0.3213589716904649,
+      "naucs_at_50_diff1": -0.40791385861358653,
+      "naucs_at_100_max": -0.16245636826248977,
+      "naucs_at_100_std": 0.261806614637912,
+      "naucs_at_100_diff1": -0.4069117709772787
+   },
+   "vidore/arxivqa_test_subsampled": {
+      "ndcg_at_1": 0.82,
+      "ndcg_at_3": 0.87317,
+      "ndcg_at_5": 0.88134,
+      "ndcg_at_10": 0.88917,
+      "ndcg_at_20": 0.89577,
+      "ndcg_at_50": 0.89856,
+      "ndcg_at_100": 0.89953,
+      "map_at_1": 0.82,
+      "map_at_3": 0.861,
+      "map_at_5": 0.8655,
+      "map_at_10": 0.86878,
+      "map_at_20": 0.8706,
+      "map_at_50": 0.87105,
+      "map_at_100": 0.87113,
+      "recall_at_1": 0.82,
+      "recall_at_3": 0.908,
+      "recall_at_5": 0.928,
+      "recall_at_10": 0.952,
+      "recall_at_20": 0.978,
+      "recall_at_50": 0.992,
+      "recall_at_100": 0.998,
+      "precision_at_1": 0.82,
+      "precision_at_3": 0.30267,
+      "precision_at_5": 0.1856,
+      "precision_at_10": 0.0952,
+      "precision_at_20": 0.0489,
+      "precision_at_50": 0.01984,
+      "precision_at_100": 0.00998,
+      "mrr_at_1": 0.818,
+      "mrr_at_3": 0.8596666666666666,
+      "mrr_at_5": 0.8641666666666665,
+      "mrr_at_10": 0.8674912698412698,
+      "mrr_at_20": 0.8692320429570428,
+      "mrr_at_50": 0.8697783157643562,
+      "mrr_at_100": 0.8698595552455958,
+      "naucs_at_1_max": 0.8542625839850205,
+      "naucs_at_1_std": -0.15451591584976296,
+      "naucs_at_1_diff1": 0.9423945368432642,
+      "naucs_at_3_max": 0.8243413307351934,
+      "naucs_at_3_std": -0.019983355661106324,
+      "naucs_at_3_diff1": 0.8796330126253413,
+      "naucs_at_5_max": 0.8361733582321816,
+      "naucs_at_5_std": 0.04758014316837583,
+      "naucs_at_5_diff1": 0.8694755680049819,
+      "naucs_at_10_max": 0.8481948334889503,
+      "naucs_at_10_std": 0.00797541238717203,
+      "naucs_at_10_diff1": 0.8792989417989403,
+      "naucs_at_20_max": 0.9123588829471215,
+      "naucs_at_20_std": 0.13352007469653804,
+      "naucs_at_20_diff1": 0.9019607843137273,
+      "naucs_at_50_max": 0.9673202614378978,
+      "naucs_at_50_std": 0.24684873949577324,
+      "naucs_at_50_diff1": 0.9305555555555584,
+      "naucs_at_100_max": 1.0,
+      "naucs_at_100_std": 1.0,
+      "naucs_at_100_diff1": 1.0
+   },
+   "vidore/docvqa_test_subsampled": {
+      "ndcg_at_1": 0.51663,
+      "ndcg_at_3": 0.59278,
+      "ndcg_at_5": 0.61529,
+      "ndcg_at_10": 0.63354,
+      "ndcg_at_20": 0.64688,
+      "ndcg_at_50": 0.66214,
+      "ndcg_at_100": 0.67157,
+      "map_at_1": 0.51663,
+      "map_at_3": 0.57391,
+      "map_at_5": 0.58622,
+      "map_at_10": 0.59393,
+      "map_at_20": 0.59753,
+      "map_at_50": 0.60011,
+      "map_at_100": 0.60096,
+      "recall_at_1": 0.51663,
+      "recall_at_3": 0.64745,
+      "recall_at_5": 0.70288,
+      "recall_at_10": 0.75831,
+      "recall_at_20": 0.81153,
+      "recall_at_50": 0.88692,
+      "recall_at_100": 0.94457,
+      "precision_at_1": 0.51663,
+      "precision_at_3": 0.21582,
+      "precision_at_5": 0.14058,
+      "precision_at_10": 0.07583,
+      "precision_at_20": 0.04058,
+      "precision_at_50": 0.01774,
+      "precision_at_100": 0.00945,
+      "mrr_at_1": 0.5144124168514412,
+      "mrr_at_3": 0.5731707317073172,
+      "mrr_at_5": 0.586031042128603,
+      "mrr_at_10": 0.5933419561468339,
+      "mrr_at_20": 0.5967957449683643,
+      "mrr_at_50": 0.599471176191027,
+      "mrr_at_100": 0.600328115684025,
+      "naucs_at_1_max": 0.6741589793839163,
+      "naucs_at_1_std": -0.16205001702916214,
+      "naucs_at_1_diff1": 0.8162930079294569,
+      "naucs_at_3_max": 0.7465129968490366,
+      "naucs_at_3_std": -0.09544436940181997,
+      "naucs_at_3_diff1": 0.7350663423580653,
+      "naucs_at_5_max": 0.7629182852125305,
+      "naucs_at_5_std": -0.050579687432907604,
+      "naucs_at_5_diff1": 0.7012350138123477,
+      "naucs_at_10_max": 0.7889385100226093,
+      "naucs_at_10_std": 0.04757918573005526,
+      "naucs_at_10_diff1": 0.6915024085662332,
+      "naucs_at_20_max": 0.8147748548808955,
+      "naucs_at_20_std": 0.10152398884541157,
+      "naucs_at_20_diff1": 0.704970590799652,
+      "naucs_at_50_max": 0.8657229702927914,
+      "naucs_at_50_std": 0.3246550245384335,
+      "naucs_at_50_diff1": 0.6989609170252572,
+      "naucs_at_100_max": 0.9055488370856275,
+      "naucs_at_100_std": 0.49971793394423536,
+      "naucs_at_100_diff1": 0.6936703826495957
+   },
+   "vidore/infovqa_test_subsampled": {
+      "ndcg_at_1": 0.86842,
+      "ndcg_at_3": 0.90307,
+      "ndcg_at_5": 0.91204,
+      "ndcg_at_10": 0.91773,
+      "ndcg_at_20": 0.92031,
+      "ndcg_at_50": 0.92201,
+      "ndcg_at_100": 0.92265,
+      "map_at_1": 0.86842,
+      "map_at_3": 0.89474,
+      "map_at_5": 0.8996,
+      "map_at_10": 0.90182,
+      "map_at_20": 0.90254,
+      "map_at_50": 0.90285,
+      "map_at_100": 0.90291,
+      "recall_at_1": 0.86842,
+      "recall_at_3": 0.92713,
+      "recall_at_5": 0.94939,
+      "recall_at_10": 0.96761,
+      "recall_at_20": 0.97773,
+      "recall_at_50": 0.98583,
+      "recall_at_100": 0.98988,
+      "precision_at_1": 0.86842,
+      "precision_at_3": 0.30904,
+      "precision_at_5": 0.18988,
+      "precision_at_10": 0.09676,
+      "precision_at_20": 0.04889,
+      "precision_at_50": 0.01972,
+      "precision_at_100": 0.0099,
+      "mrr_at_1": 0.868421052631579,
+      "mrr_at_3": 0.894736842105263,
+      "mrr_at_5": 0.8996963562753034,
+      "mrr_at_10": 0.9018740762161812,
+      "mrr_at_20": 0.9025973885538661,
+      "mrr_at_50": 0.9029005381313663,
+      "mrr_at_100": 0.9029517941275169,
+      "naucs_at_1_max": 0.6108584962344086,
+      "naucs_at_1_std": -0.1359018313249994,
+      "naucs_at_1_diff1": 0.9136811065149164,
+      "naucs_at_3_max": 0.6840687823130331,
+      "naucs_at_3_std": 0.03370197684645164,
+      "naucs_at_3_diff1": 0.886059733600837,
+      "naucs_at_5_max": 0.7765209527928315,
+      "naucs_at_5_std": 0.2228524148946178,
+      "naucs_at_5_diff1": 0.8620463071981626,
+      "naucs_at_10_max": 0.8392814749886633,
+      "naucs_at_10_std": 0.5798638053062866,
+      "naucs_at_10_diff1": 0.8728108205086554,
+      "naucs_at_20_max": 1.0,
+      "naucs_at_20_std": 0.8268704165639359,
+      "naucs_at_20_diff1": 0.8986141894622774,
+      "naucs_at_50_max": 1.0,
+      "naucs_at_50_std": 0.8102686931758233,
+      "naucs_at_50_diff1": 0.8593367911642417,
+      "naucs_at_100_max": 1.0,
+      "naucs_at_100_std": 0.7899231179977639,
+      "naucs_at_100_diff1": 0.9183327616354471
+   },
+   "vidore/tabfquad_test_subsampled": {
+      "ndcg_at_1": 0.89286,
+      "ndcg_at_3": 0.92836,
+      "ndcg_at_5": 0.93112,
+      "ndcg_at_10": 0.93804,
+      "ndcg_at_20": 0.94089,
+      "ndcg_at_50": 0.94232,
+      "ndcg_at_100": 0.94232,
+      "map_at_1": 0.89286,
+      "map_at_3": 0.91964,
+      "map_at_5": 0.92107,
+      "map_at_10": 0.92391,
+      "map_at_20": 0.92477,
+      "map_at_50": 0.925,
+      "map_at_100": 0.925,
+      "recall_at_1": 0.89286,
+      "recall_at_3": 0.95357,
+      "recall_at_5": 0.96071,
+      "recall_at_10": 0.98214,
+      "recall_at_20": 0.99286,
+      "recall_at_50": 1.0,
+      "recall_at_100": 1.0,
+      "precision_at_1": 0.89286,
+      "precision_at_3": 0.31786,
+      "precision_at_5": 0.19214,
+      "precision_at_10": 0.09821,
+      "precision_at_20": 0.04964,
+      "precision_at_50": 0.02,
+      "precision_at_100": 0.01,
+      "mrr_at_1": 0.8928571428571429,
+      "mrr_at_3": 0.9196428571428571,
+      "mrr_at_5": 0.9210714285714285,
+      "mrr_at_10": 0.9239129818594106,
+      "mrr_at_20": 0.9247733714698002,
+      "mrr_at_50": 0.9250078037042323,
+      "mrr_at_100": 0.9250078037042323,
+      "naucs_at_1_max": 0.8354945054945048,
+      "naucs_at_1_std": 0.37345368916797433,
+      "naucs_at_1_diff1": 0.9218210361067499,
+      "naucs_at_3_max": 0.9657042304101127,
+      "naucs_at_3_std": 0.7062773827479698,
+      "naucs_at_3_diff1": 0.9396681749622927,
+      "naucs_at_5_max": 0.9594686359392222,
+      "naucs_at_5_std": 0.6647568118156384,
+      "naucs_at_5_diff1": 0.9286987522281656,
+      "naucs_at_10_max": 1.0,
+      "naucs_at_10_std": 0.9738562091503306,
+      "naucs_at_10_diff1": 0.9477124183006519,
+      "naucs_at_20_max": 1.0,
+      "naucs_at_20_std": 1.0,
+      "naucs_at_20_diff1": 1.0,
+      "naucs_at_50_max": 1.0,
+      "naucs_at_50_std": 1.0,
+      "naucs_at_50_diff1": 1.0,
+      "naucs_at_100_max": 1.0,
+      "naucs_at_100_std": 1.0,
+      "naucs_at_100_diff1": 1.0
+   },
+   "vidore/tatdqa_test": {
+      "ndcg_at_1": 0.60571,
+      "ndcg_at_3": 0.71371,
+      "ndcg_at_5": 0.73981,
+      "ndcg_at_10": 0.7601,
+      "ndcg_at_20": 0.76893,
+      "ndcg_at_50": 0.7733,
+      "ndcg_at_100": 0.77517,
+      "map_at_1": 0.60571,
+      "map_at_3": 0.68702,
+      "map_at_5": 0.70154,
+      "map_at_10": 0.71006,
+      "map_at_20": 0.71253,
+      "map_at_50": 0.71329,
+      "map_at_100": 0.71345,
+      "recall_at_1": 0.60571,
+      "recall_at_3": 0.79101,
+      "recall_at_5": 0.85419,
+      "recall_at_10": 0.91616,
+      "recall_at_20": 0.95079,
+      "recall_at_50": 0.97205,
+      "recall_at_100": 0.9836,
+      "precision_at_1": 0.60571,
+      "precision_at_3": 0.26367,
+      "precision_at_5": 0.17084,
+      "precision_at_10": 0.09162,
+      "precision_at_20": 0.04754,
+      "precision_at_50": 0.01944,
+      "precision_at_100": 0.00984,
+      "mrr_at_1": 0.6038882138517618,
+      "mrr_at_3": 0.6857027136492516,
+      "mrr_at_5": 0.7003138922640758,
+      "mrr_at_10": 0.7090893749156212,
+      "mrr_at_20": 0.7114126192954364,
+      "mrr_at_50": 0.7121742514967643,
+      "mrr_at_100": 0.7123364657648046,
+      "naucs_at_1_max": 0.3821214020038102,
+      "naucs_at_1_std": -0.13489407631885159,
+      "naucs_at_1_diff1": 0.7298383859421139,
+      "naucs_at_3_max": 0.45115702284602416,
+      "naucs_at_3_std": -0.04154958812127681,
+      "naucs_at_3_diff1": 0.6171010959328003,
+      "naucs_at_5_max": 0.4834741295717716,
+      "naucs_at_5_std": -0.03652161720284896,
+      "naucs_at_5_diff1": 0.6133480640518033,
+      "naucs_at_10_max": 0.5818884435678058,
+      "naucs_at_10_std": 0.1493431453981439,
+      "naucs_at_10_diff1": 0.6125109504507562,
+      "naucs_at_20_max": 0.7030343359941152,
+      "naucs_at_20_std": 0.3539243062200548,
+      "naucs_at_20_diff1": 0.6751362244435348,
+      "naucs_at_50_max": 0.7344845360513886,
+      "naucs_at_50_std": 0.5609137914009904,
+      "naucs_at_50_diff1": 0.6435577411690651,
+      "naucs_at_100_max": 0.7291790977931831,
+      "naucs_at_100_std": 0.7412505638179597,
+      "naucs_at_100_diff1": 0.6088543824874393
+   },
+   "vidore/shiftproject_test": {
+      "ndcg_at_1": 0.74,
+      "ndcg_at_3": 0.85333,
+      "ndcg_at_5": 0.85764,
+      "ndcg_at_10": 0.87455,
+      "ndcg_at_20": 0.8769,
+      "ndcg_at_50": 0.8769,
+      "ndcg_at_100": 0.8769,
+      "map_at_1": 0.74,
+      "map_at_3": 0.82667,
+      "map_at_5": 0.82917,
+      "map_at_10": 0.8366,
+      "map_at_20": 0.83715,
+      "map_at_50": 0.83715,
+      "map_at_100": 0.83715,
+      "recall_at_1": 0.74,
+      "recall_at_3": 0.93,
+      "recall_at_5": 0.94,
+      "recall_at_10": 0.99,
+      "recall_at_20": 1.0,
+      "recall_at_50": 1.0,
+      "recall_at_100": 1.0,
+      "precision_at_1": 0.74,
+      "precision_at_3": 0.31,
+      "precision_at_5": 0.188,
+      "precision_at_10": 0.099,
+      "precision_at_20": 0.05,
+      "precision_at_50": 0.02,
+      "precision_at_100": 0.01,
+      "mrr_at_1": 0.74,
+      "mrr_at_3": 0.8266666666666665,
+      "mrr_at_5": 0.8291666666666666,
+      "mrr_at_10": 0.8365952380952381,
+      "mrr_at_20": 0.8371834733893557,
+      "mrr_at_50": 0.8371834733893557,
+      "mrr_at_100": 0.8371834733893557,
+      "naucs_at_1_max": 0.36659097360204357,
+      "naucs_at_1_std": -0.13924820566886958,
+      "naucs_at_1_diff1": 0.7847208142411088,
+      "naucs_at_3_max": 0.7129518474056293,
+      "naucs_at_3_std": -0.039482459650526366,
+      "naucs_at_3_diff1": 0.8102574363078549,
+      "naucs_at_5_max": 0.8603330220977291,
+      "naucs_at_5_std": 0.24393090569561146,
+      "naucs_at_5_diff1": 0.8004201680672299,
+      "naucs_at_10_max": 0.5541549953314738,
+      "naucs_at_10_std": 0.12278244631185926,
+      "naucs_at_10_diff1": 0.35807656395891135,
+      "naucs_at_20_max": 1.0,
+      "naucs_at_20_std": 1.0,
+      "naucs_at_20_diff1": 1.0,
+      "naucs_at_50_max": null,
+      "naucs_at_50_std": null,
+      "naucs_at_50_diff1": null,
+      "naucs_at_100_max": null,
+      "naucs_at_100_std": null,
+      "naucs_at_100_diff1": null
+   },
+   "vidore/syntheticDocQA_artificial_intelligence_test": {
+      "ndcg_at_1": 0.96,
+      "ndcg_at_3": 0.98393,
+      "ndcg_at_5": 0.98393,
+      "ndcg_at_10": 0.98393,
+      "ndcg_at_20": 0.98393,
+      "ndcg_at_50": 0.98393,
+      "ndcg_at_100": 0.98393,
+      "map_at_1": 0.96,
+      "map_at_3": 0.97833,
+      "map_at_5": 0.97833,
+      "map_at_10": 0.97833,
+      "map_at_20": 0.97833,
+      "map_at_50": 0.97833,
+      "map_at_100": 0.97833,
+      "recall_at_1": 0.96,
+      "recall_at_3": 1.0,
+      "recall_at_5": 1.0,
+      "recall_at_10": 1.0,
+      "recall_at_20": 1.0,
+      "recall_at_50": 1.0,
+      "recall_at_100": 1.0,
+      "precision_at_1": 0.96,
+      "precision_at_3": 0.33333,
+      "precision_at_5": 0.2,
+      "precision_at_10": 0.1,
+      "precision_at_20": 0.05,
+      "precision_at_50": 0.02,
+      "precision_at_100": 0.01,
+      "mrr_at_1": 0.96,
+      "mrr_at_3": 0.9783333333333333,
+      "mrr_at_5": 0.9783333333333333,
+      "mrr_at_10": 0.9783333333333333,
+      "mrr_at_20": 0.9783333333333333,
+      "mrr_at_50": 0.9783333333333333,
+      "mrr_at_100": 0.9783333333333333,
+      "naucs_at_1_max": 0.7373949579831944,
+      "naucs_at_1_std": -0.32784780578897976,
+      "naucs_at_1_diff1": 0.9346405228758167,
+      "naucs_at_3_max": 1.0,
+      "naucs_at_3_std": 1.0,
+      "naucs_at_3_diff1": 1.0,
+      "naucs_at_5_max": 1.0,
+      "naucs_at_5_std": 1.0,
+      "naucs_at_5_diff1": 1.0,
+      "naucs_at_10_max": 1.0,
+      "naucs_at_10_std": 1.0,
+      "naucs_at_10_diff1": 1.0,
+      "naucs_at_20_max": 1.0,
+      "naucs_at_20_std": 1.0,
+      "naucs_at_20_diff1": 1.0,
+      "naucs_at_50_max": null,
+      "naucs_at_50_std": null,
+      "naucs_at_50_diff1": null,
+      "naucs_at_100_max": null,
+      "naucs_at_100_std": null,
+      "naucs_at_100_diff1": null
+   },
+   "vidore/syntheticDocQA_energy_test": {
+      "ndcg_at_1": 0.94,
+      "ndcg_at_3": 0.95262,
+      "ndcg_at_5": 0.95693,
+      "ndcg_at_10": 0.96008,
+      "ndcg_at_20": 0.96287,
+      "ndcg_at_50": 0.96511,
+      "ndcg_at_100": 0.96511,
+      "map_at_1": 0.94,
+      "map_at_3": 0.95,
+      "map_at_5": 0.9525,
+      "map_at_10": 0.95375,
+      "map_at_20": 0.95466,
+      "map_at_50": 0.95514,
+      "map_at_100": 0.95514,
+      "recall_at_1": 0.94,
+      "recall_at_3": 0.96,
+      "recall_at_5": 0.97,
+      "recall_at_10": 0.98,
+      "recall_at_20": 0.99,
+      "recall_at_50": 1.0,
+      "recall_at_100": 1.0,
+      "precision_at_1": 0.94,
+      "precision_at_3": 0.32,
+      "precision_at_5": 0.194,
+      "precision_at_10": 0.098,
+      "precision_at_20": 0.0495,
+      "precision_at_50": 0.02,
+      "precision_at_100": 0.01,
+      "mrr_at_1": 0.94,
+      "mrr_at_3": 0.95,
+      "mrr_at_5": 0.9525,
+      "mrr_at_10": 0.95375,
+      "mrr_at_20": 0.9551590909090909,
+      "mrr_at_50": 0.9551590909090909,
+      "mrr_at_100": 0.9551590909090909,
+      "naucs_at_1_max": 0.16293183940242884,
+      "naucs_at_1_std": -1.0354808590102687,
+      "naucs_at_1_diff1": 0.9782135076252712,
+      "naucs_at_3_max": 0.31500933706816353,
+      "naucs_at_3_std": -0.9773576097105482,
+      "naucs_at_3_diff1": 1.0,
+      "naucs_at_5_max": 0.08667911609088073,
+      "naucs_at_5_std": -1.5438842203547984,
+      "naucs_at_5_diff1": 1.0,
+      "naucs_at_10_max": -0.3699813258636707,
+      "naucs_at_10_std": -1.7399626517273414,
+      "naucs_at_10_diff1": 1.0,
+      "naucs_at_20_max": 1.0,
+      "naucs_at_20_std": -1.7399626517273863,
+      "naucs_at_20_diff1": 1.0,
+      "naucs_at_50_max": null,
+      "naucs_at_50_std": null,
+      "naucs_at_50_diff1": null,
+      "naucs_at_100_max": null,
+      "naucs_at_100_std": null,
+      "naucs_at_100_diff1": null
+   },
+   "vidore/syntheticDocQA_government_reports_test": {
+      "ndcg_at_1": 0.9,
+      "ndcg_at_3": 0.95678,
+      "ndcg_at_5": 0.95678,
+      "ndcg_at_10": 0.95994,
+      "ndcg_at_20": 0.95994,
+      "ndcg_at_50": 0.95994,
+      "ndcg_at_100": 0.95994,
+      "map_at_1": 0.9,
+      "map_at_3": 0.945,
+      "map_at_5": 0.945,
+      "map_at_10": 0.94625,
+      "map_at_20": 0.94625,
+      "map_at_50": 0.94625,
+      "map_at_100": 0.94625,
+      "recall_at_1": 0.9,
+      "recall_at_3": 0.99,
+      "recall_at_5": 0.99,
+      "recall_at_10": 1.0,
+      "recall_at_20": 1.0,
+      "recall_at_50": 1.0,
+      "recall_at_100": 1.0,
+      "precision_at_1": 0.9,
+      "precision_at_3": 0.33,
+      "precision_at_5": 0.198,
+      "precision_at_10": 0.1,
+      "precision_at_20": 0.05,
+      "precision_at_50": 0.02,
+      "precision_at_100": 0.01,
+      "mrr_at_1": 0.9,
+      "mrr_at_3": 0.945,
+      "mrr_at_5": 0.945,
+      "mrr_at_10": 0.9464285714285714,
+      "mrr_at_20": 0.9464285714285714,
+      "mrr_at_50": 0.9464285714285714,
+      "mrr_at_100": 0.9464285714285714,
+      "naucs_at_1_max": 0.5815126050420164,
+      "naucs_at_1_std": -0.42105508870214864,
+      "naucs_at_1_diff1": 0.9330065359477123,
+      "naucs_at_3_max": 0.5541549953314585,
+      "naucs_at_3_std": -1.7399626517274398,
+      "naucs_at_3_diff1": 0.8692810457516356,
+      "naucs_at_5_max": 0.5541549953314738,
+      "naucs_at_5_std": -1.7399626517273863,
+      "naucs_at_5_diff1": 0.8692810457516413,
+      "naucs_at_10_max": 1.0,
+      "naucs_at_10_std": 1.0,
+      "naucs_at_10_diff1": 1.0,
+      "naucs_at_20_max": 1.0,
+      "naucs_at_20_std": 1.0,
+      "naucs_at_20_diff1": 1.0,
+      "naucs_at_50_max": null,
+      "naucs_at_50_std": null,
+      "naucs_at_50_diff1": null,
+      "naucs_at_100_max": null,
+      "naucs_at_100_std": null,
+      "naucs_at_100_diff1": null
+   },
+   "vidore/syntheticDocQA_healthcare_industry_test": {
+      "ndcg_at_1": 0.96,
+      "ndcg_at_3": 0.97893,
+      "ndcg_at_5": 0.98323,
+      "ndcg_at_10": 0.98323,
+      "ndcg_at_20": 0.98323,
+      "ndcg_at_50": 0.98323,
+      "ndcg_at_100": 0.98323,
+      "map_at_1": 0.96,
+      "map_at_3": 0.975,
+      "map_at_5": 0.9775,
+      "map_at_10": 0.9775,
+      "map_at_20": 0.9775,
+      "map_at_50": 0.9775,
+      "map_at_100": 0.9775,
+      "recall_at_1": 0.96,
+      "recall_at_3": 0.99,
+      "recall_at_5": 1.0,
+      "recall_at_10": 1.0,
+      "recall_at_20": 1.0,
+      "recall_at_50": 1.0,
+      "recall_at_100": 1.0,
+      "precision_at_1": 0.96,
+      "precision_at_3": 0.33,
+      "precision_at_5": 0.2,
+      "precision_at_10": 0.1,
+      "precision_at_20": 0.05,
+      "precision_at_50": 0.02,
+      "precision_at_100": 0.01,
+      "mrr_at_1": 0.96,
+      "mrr_at_3": 0.975,
+      "mrr_at_5": 0.9775,
+      "mrr_at_10": 0.9775,
+      "mrr_at_20": 0.9775,
+      "mrr_at_50": 0.9775,
+      "mrr_at_100": 0.9775,
+      "naucs_at_1_max": 0.7006302521008423,
+      "naucs_at_1_std": -0.7350606909430405,
+      "naucs_at_1_diff1": 0.9305555555555578,
+      "naucs_at_3_max": 0.7222222222222157,
+      "naucs_at_3_std": -1.1517273576097802,
+      "naucs_at_3_diff1": 1.0,
+      "naucs_at_5_max": 1.0,
+      "naucs_at_5_std": 1.0,
+      "naucs_at_5_diff1": 1.0,
+      "naucs_at_10_max": 1.0,
+      "naucs_at_10_std": 1.0,
+      "naucs_at_10_diff1": 1.0,
+      "naucs_at_20_max": 1.0,
+      "naucs_at_20_std": 1.0,
+      "naucs_at_20_diff1": 1.0,
+      "naucs_at_50_max": null,
+      "naucs_at_50_std": null,
+      "naucs_at_50_diff1": null,
+      "naucs_at_100_max": null,
+      "naucs_at_100_std": null,
+      "naucs_at_100_diff1": null
+   }
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
+size 11422063

tokenizer_config.json

@@ -0,0 +1,213 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "max_length": null,
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
+  "processor_class": "BiQwen2_5_Processor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_config.yml

@@ -0,0 +1,78 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/biqwen2_5_1ep_colpali_train_5neg_128bs_7b_2e4
+  processor:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.BiQwen2_5_Processor
+    pretrained_model_name_or_path:  "Qwen/Qwen2.5-VL-7B-Instruct" # "./models/paligemma-3b-mix-448"
+    # num_image_tokens: 2048
+    # max_length: 50
+  num_negatives: 5 
+
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.BiQwen2_5
+    pretrained_model_name_or_path: "Qwen/Qwen2.5-VL-7B-Instruct"
+    torch_dtype:  !ext torch.bfloat16
+    use_cache: false
+    attn_implementation: "flash_attention_2"
+#    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_train_set_ir_negs
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  # max_length: 50
+  run_eval: true
+  loss_func:
+    (): colpali_engine.loss.bi_encoder_losses.BiEncoderLoss
+  tr_args:
+    (): transformers.training_args.TrainingArguments
+    output_dir: null
+    overwrite_output_dir: true
+    num_train_epochs: 1 
+    # this is global batch size 
+    per_device_train_batch_size: 128 
+    gradient_checkpointing: true
+    gradient_checkpointing_kwargs: { "use_reentrant": false }
+    ddp_find_unused_parameters: false
+    resume_from_checkpoint: false
+    # gradient_checkpointing: true
+    # 6 x 8 gpus = 48 batch size
+    # gradient_accumulation_steps: 4
+    per_device_eval_batch_size: 16
+    eval_strategy: "no"
+    dataloader_num_workers: 0
+    log_level: "info"
+    accelerator_config:
+      split_batches: true
+    # bf16: true
+    save_steps: 500
+    logging_steps: 10
+    eval_steps: 100
+    warmup_ratio: 0.01
+    learning_rate: 2e-4
+    save_total_limit: 1
+    # resume_from_checkpoint: true
+    optim: "paged_adamw_8bit"
+    # fsdp: "shard_grad_op"
+    # wandb logging
+    run_name: "biqwen2_5_1ep_colpali_train_5neg_128bs_7b_2e4"
+    report_to: "wandb"
+
+
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'