Hugging Face's logo

lightonai
/
ModernColBERT-embed-base-kd-only

Sentence Similarity
Safetensors
sentence-transformers
English
PyLate
modernbert
ColBERT
embeddings
retrieval
feature-extraction
Generated from Trainer
dataset_size:640000
loss:Distillation
Eval Results (legacy)
text-embeddings-inference
🇪🇺 Region: EU
Model card Files
xet
Community
ModernColBERT-embed-base-kd-only / README.md
NohTow's picture
NohTow
Update README.md
d6c8db5 verified
preview code
|
raw
history blame contribute delete
152 kB
 ---
tags:
- ColBERT
- PyLate
- sentence-transformers
- sentence-similarity
- embeddings
- retrieval
- feature-extraction
- generated_from_trainer
- dataset_size:640000
- loss:Distillation
pipeline_tag: sentence-similarity
library_name: PyLate
license: apache-2.0
language:
- en
metrics:
- MaxSim_accuracy@1
- MaxSim_accuracy@3
- MaxSim_accuracy@5
- MaxSim_accuracy@10
- MaxSim_precision@1
- MaxSim_precision@3
- MaxSim_precision@5
- MaxSim_precision@10
- MaxSim_recall@1
- MaxSim_recall@3
- MaxSim_recall@5
- MaxSim_recall@10
- MaxSim_ndcg@10
- MaxSim_mrr@10
- MaxSim_map@100
model-index:
- name: PyLate
results:
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoClimateFEVER
type: NanoClimateFEVER
metrics:
- type: MaxSim_accuracy@1
value: 0.34
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.6
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.7
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.84
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.34
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.24666666666666667
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.19199999999999995
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.12799999999999997
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.18333333333333332
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.30333333333333334
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.3899999999999999
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.4933333333333333
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.4063363730066463
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.4916031746031746
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.3303819327927656
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoDBPedia
type: NanoDBPedia
metrics:
- type: MaxSim_accuracy@1
value: 0.86
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.94
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.94
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.96
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.86
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.7199999999999999
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.66
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.5720000000000001
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.12659835318654536
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.21845761987893375
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.2938340415477099
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.4105335585789726
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.7283036112199561
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.8991666666666666
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.5925340100852293
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoFEVER
type: NanoFEVER
metrics:
- type: MaxSim_accuracy@1
value: 0.94
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 1.0
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 1.0
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 1.0
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.94
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.3666666666666666
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.21999999999999997
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.10999999999999999
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.8766666666666667
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.98
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.98
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.98
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.953933314347975
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.9633333333333333
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.9375757575757575
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoFiQA2018
type: NanoFiQA2018
metrics:
- type: MaxSim_accuracy@1
value: 0.5
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.72
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.74
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.76
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.5
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.3466666666666666
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.24799999999999997
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.13599999999999998
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.2725793650793651
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.520904761904762
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.5646507936507936
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.5870079365079365
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.5309299781460816
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.6011904761904762
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.47808334745931363
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoHotpotQA
type: NanoHotpotQA
metrics:
- type: MaxSim_accuracy@1
value: 1.0
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 1.0
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 1.0
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 1.0
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 1.0
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.6
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.3679999999999999
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.18599999999999994
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.5
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.9
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.92
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.93
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.9222921452583728
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 1.0
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.8846838161838161
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoMSMARCO
type: NanoMSMARCO
metrics:
- type: MaxSim_accuracy@1
value: 0.54
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.68
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.76
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.86
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.54
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.22666666666666666
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.15200000000000002
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.08599999999999998
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.54
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.68
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.76
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.86
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.6888194232849568
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.6348809523809523
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.6440971195471196
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoNFCorpus
type: NanoNFCorpus
metrics:
- type: MaxSim_accuracy@1
value: 0.54
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.64
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.72
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.74
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.54
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.42
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.38400000000000006
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.28600000000000003
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.04566162692796489
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.08179125516090964
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.12712273647364136
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.15300844432718616
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.37177379221071954
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.6048333333333332
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.16751658822280646
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoNQ
type: NanoNQ
metrics:
- type: MaxSim_accuracy@1
value: 0.68
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.82
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.86
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.9
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.68
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.28
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.176
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.09799999999999998
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.64
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.77
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.8
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.87
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.7688812490759633
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.7574126984126984
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.7299065569552858
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoQuoraRetrieval
type: NanoQuoraRetrieval
metrics:
- type: MaxSim_accuracy@1
value: 0.98
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 1.0
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 1.0
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 1.0
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.98
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.3999999999999999
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.256
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.13799999999999998
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.8706666666666666
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.9520000000000001
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.9726666666666667
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.9966666666666666
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.981385502951296
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.99
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.9665185185185184
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoSCIDOCS
type: NanoSCIDOCS
metrics:
- type: MaxSim_accuracy@1
value: 0.46
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.68
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.76
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.88
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.46
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.35333333333333333
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.292
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.19599999999999998
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.09766666666666665
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.21766666666666665
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.2986666666666667
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.4006666666666666
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.3925816517049085
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.5987380952380952
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.30497643441660005
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoArguAna
type: NanoArguAna
metrics:
- type: MaxSim_accuracy@1
value: 0.24
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.58
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.7
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.88
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.24
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.19333333333333336
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.14
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.088
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.24
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.58
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.7
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.88
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.558015137444458
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.45589682539682536
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.4586809163059163
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoSciFact
type: NanoSciFact
metrics:
- type: MaxSim_accuracy@1
value: 0.74
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.84
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.88
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.92
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.74
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.29333333333333333
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.19599999999999998
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.10199999999999998
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.715
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.81
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.87
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.91
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.8249697859180465
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.8
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.7959520905923344
name: Maxsim Map@100
- task:
type: py-late-information-retrieval
name: Py Late Information Retrieval
dataset:
name: NanoTouche2020
type: NanoTouche2020
metrics:
- type: MaxSim_accuracy@1
value: 0.7959183673469388
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.9387755102040817
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.9591836734693877
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 1.0
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.7959183673469388
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.7142857142857143
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.6653061224489795
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.5142857142857142
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.052193001619842895
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.14293338708352385
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.21678776156605578
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.3275908393694154
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.5977067950547461
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.8719630709426628
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.42650930096472894
name: Maxsim Map@100
- task:
type: nano-beir
name: Nano BEIR
dataset:
name: NanoBEIR mean
type: NanoBEIR_mean
metrics:
- type: MaxSim_accuracy@1
value: 0.6627629513343799
name: Maxsim Accuracy@1
- type: MaxSim_accuracy@3
value: 0.8029827315541601
name: Maxsim Accuracy@3
- type: MaxSim_accuracy@5
value: 0.8476295133437991
name: Maxsim Accuracy@5
- type: MaxSim_accuracy@10
value: 0.9030769230769231
name: Maxsim Accuracy@10
- type: MaxSim_precision@1
value: 0.6627629513343799
name: Maxsim Precision@1
- type: MaxSim_precision@3
value: 0.3969963369963369
name: Maxsim Precision@3
- type: MaxSim_precision@5
value: 0.3037927786499215
name: Maxsim Precision@5
- type: MaxSim_precision@10
value: 0.20309890109890105
name: Maxsim Precision@10
- type: MaxSim_recall@1
value: 0.39695120616515783
name: Maxsim Recall@1
- type: MaxSim_recall@3
value: 0.5505451556944715
name: Maxsim Recall@3
- type: MaxSim_recall@5
value: 0.6072098974285796
name: Maxsim Recall@5
- type: MaxSim_recall@10
value: 0.6768313419577059
name: Maxsim Recall@10
- type: MaxSim_ndcg@10
value: 0.6712252892018559
name: Maxsim Ndcg@10
- type: MaxSim_mrr@10
value: 0.743770663576786
name: Maxsim Mrr@10
- type: MaxSim_map@100
value: 0.5936474145861687
name: Maxsim Map@100
---
<div align="center">
<img src="https://cdn-uploads.huggingface.co/production/uploads/609bbe2f4932693ca2009d6a/xn21ll7YRj0ZftBli3-T5.jpeg" width="600" height="auto">
[![Website](https://img.shields.io/badge/LightOn-Website-blue?logo=google-chrome)](https://lighton.ai)
[![LinkedIn](https://img.shields.io/badge/LightOn-LinkedIn-0A66C2?logo=linkedin)](https://www.linkedin.com/company/lighton/)
[![X](https://img.shields.io/badge/@LightOnIO-X-black?logo=x)](https://x.com/LightOnIO)
📄 [Paper](https://arxiv.org/abs/2602.16609) | 📝 [Blog](https://huggingface.co/blog/lightonai/colbert-zero) | 📚 [Collection](https://huggingface.co/collections/lightonai/colbert-zero)
</div>
# ColBERT-Zero
> 🎯 **TL;DR**: First large-scale fully pre-trained ColBERT model using only public data. Achieves **55.43 nDCG@10** on BEIR benchmark, outperforming GTE-ModernColBERT and GTE-ModernBERT trained on closed and stronger data. **New SOTA on BEIR for models <150M parameters**.
## Why ColBERT-Zero?
Late interaction (ColBERT / multi-vector) models have clear advantages in out-of-domain generalization, long-context handling, and reasoning-intensive retrieval. Yet they remain undertrained: current state-of-the-art ColBERT models (e.g, [GTE-ModernColBERT](https://huggingface.co/Alibaba-NLP/gte-modernbert-colbert) and [ColBERT-small](https://huggingface.co)) are simply built by bolting a small knowledge distillation step onto a strong dense (single-vector) model. Even recent efforts like [mxbai-edge-colbert-v0](https://huggingface.co/collections/mixedbread-ai/mxbai-edge-colbert-v0-series) perform all early training stages in a single-vector setting, only switching to the multi-vector objective at the very end.
**This leaves a lot of performance on the table.** ColBERT-Zero demonstrates that performing contrastive pre-training directly in the multi-vector setting, rather than treating it as an afterthought, unlocks a significantly higher performance ceiling. Trained exclusively on public data ([Nomic-embed](https://arxiv.org/abs/2402.01613) dataset mixture), [ColBERT-Zero](https://huggingface.co/lightonai/ColBERT-Zero) overcomes a 2.4-point data quality disadvantage to outperform models trained on proprietary, closed-source data. For detailed results, please have a look at our [blogpost](https://huggingface.co/blog/lightonai/colbert-zero/) and the [paper](https://arxiv.org/abs/2602.16609). All the [models](https://huggingface.co/collections/lightonai/colbert-zero) (including intermediate checkpoints) as well [training code](https://github.com/lightonai/pylate/tree/main/examples/train/ColBERT-zero) are released under an Apache 2.0 license.
## Controlled Comparison Design
We deliberately trained on the public [Nomic-embed](https://arxiv.org/abs/2402.01613) data mixture for a strategic reason: Nomic has already trained a dense ModernBERT model ([ModernBERT-embed](https://huggingface.co/nomic-ai/modernbert-embed-base)) on this exact data. This lets us compare dense vs. multi-vector training with the **same data, same base model ([ModernBERT](https://huggingface.co/answerdotai/ModernBERT-base)), and same pipeline**. The only variable is whether the contrastive phases are performed in the dense or multi-vector setting.
This design reveals a striking result: the dense baseline trained on Nomic data scores 52.89, while the one trained on GTE's proprietary data scores 55.33: a 2.4-point data quality gap. Despite this disadvantage, ColBERT-Zero's full multi-vector pre-training pipeline closes and surpasses this gap, reaching **55.43 nDCG@10**.
## The Three-Phase Training Pipeline
The development followed a three-phase pipeline, each providing a different type of learning signal:
### Phase 1 - Unsupervised Contrastive Pre-training
We began with the [nomic-embed-unsupervised-data](https://huggingface.co/datasets/nomic-ai/nomic-embed-unsupervised-data) dataset. Using [PyLate](https://lightonai.github.io/pylate/)'s **GradCache** implementation to scale per-GPU batch size without VRAM constraints, combined with **cross-GPU gathering** of representations, we reached effective batch sizes of **~16k**, required for unsupervised training to produce plausible in-batch hard negatives. Unlike dense training, the multi-vector objective allows the encoder to learn fine-grained token importance from the very first phase.
### Phase 2 - Supervised Contrastive Fine-tuning
We refined the model using the [nomic-embed-supervised-data](https://huggingface.co/datasets/nomic-ai/nomic-embed-supervised-data). This stage introduced mined hard negatives: documents that are superficially similar to the query but not actually relevant. This allows teaching the model to handle nuance by prioritizing specific keywords and contextual tokens most indicative of a true match.
### Phase 3 - Knowledge Distillation (KD)
The final stage used the [ms-marco-en-bge](https://huggingface.co/datasets/lightonai/ms-marco-en-bge) dataset. We leveraged a powerful Gemma-based model as a teacher, allowing our student models to learn to replicate complex reasoning scores via the efficient MaxSim operator.
## Key Findings
### 1. The Standard Recipe Leaves Performance on the Table
The KD-only approach (the current industry standard) scores 54.09, lagging behind full pre-training by **1.3 points**. A simple distillation step is insufficient for optimal multi-vector performance.
### 2. Supervised + KD Is the Efficiency Sweet Spot
By running a supervised contrastive step in the multi-vector setting before distillation, we reach **55.12 nDCG@10**, closing most of the gap with the fully pre-trained model (55.43). This costs **~40 GH200-hours instead of ~408**: roughly **10× cheaper for 99.4% of the performance**.
<div align="center">
<img src="https://cdn-uploads.huggingface.co/production/uploads/609bbe2f4932693ca2009d6a/V1_hTZ0VnJHldfd3Ip-Jm.png" width="600" height="auto">
</div>
### 3. Prompt Alignment Is Non-Negotiable
Nomic's base models are pre-trained with asymmetric prompts (`search_query:` and `search_document:`). While ColBERT has its own asymmetric mechanism via `[Q]` and `[D]` markers, we found:
- **Stripping pre-training prompts during fine-tuning** causes significant performance degradation.
- **Adding prompts to a model not pre-trained with them** also hurts performance.
- **Even with perfect alignment**, prompts provide an intrinsic benefit: full ColBERT pre-training with prompts (55.43) vs. without prompts (54.61), no mismatch in either case, shows a meaningful 0.82-point gap.
<div align="center">
<img src="https://cdn-uploads.huggingface.co/production/uploads/609bbe2f4932693ca2009d6a/uZoRA7SwisR-svi4lPDTi.png" width="600" height="auto">
</div>
**Why do prompts help?** Our leading hypothesis is that prompt tokens act as **implicit query expansion**: extra slots that don't carry specific meaning but let the model store global information about the sequence. The original ColBERT used `[PAD]` tokens for this purpose, but modern Flash Attention implementations broke this trick (masked tokens no longer produce usable embeddings). Explicit prompt tokens may be quietly re-enabling it.
**Practical takeaway:** Always align your prompts with the base model's pre-training setup. Misalignment is one of the easiest ways to silently lose performance. Note that this sensitivity decreases with stronger downstream fine-tuning: with enough training, the model can adapt to an initial mismatch.
## Model Lineup
### The Main Models (ColBERT-Zero)
`ColBERT-Zero` utilizes the full 3-phase pipeline with strict prompt alignment, **achieving 55.43 nDCG@10 on BEIR**, setting a new SOTA for models <150M parameters. We also provide `ColBERT-Zero-noprompts`, the same pipeline without asymmetric prompts, to study the impact of query expansion on multi-vector performance.
### The cheap-to-train ones (ModernColBERT-embed-base)
These models represent the practical sweet spot. By skipping the expensive unsupervised phase, `ModernColBERT-embed-base` (Supervised + KD) achieves ~97% of the flagship's performance at only ~10% of the compute cost. For reference, `ModernColBERT-embed-base-kd` performs only the distillation step on a supervised dense base.
### Intermediate Checkpoints
For researchers studying the incremental impact of each phase and prompt alignment, we release several ablation variants: `ColBERT-Zero-supervised`, `ColBERT-Zero-unsupervised` (and their `-noprompts` versions), and `ModernColBERT-embed-base-supervised`.
#### Full Performance on BEIR
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<style>
.beir-wrap { overflow-x: auto; font-family: system-ui, sans-serif; width: 100%; display: block; -webkit-overflow-scrolling: touch; }
.beir-wrap table { border-collapse: collapse; font-size: 0.70rem; white-space: nowrap; background: #fff; box-shadow: 0 1px 4px rgba(0,0,0,.1); border-radius: 8px; min-width: max-content; }
.beir-wrap th, .beir-wrap td { padding: 7px 10px; text-align: center; border-bottom: 1px solid #e9ecef; }
.beir-wrap td:first-child, .beir-wrap th:first-child { text-align: left; min-width: 260px; }
.beir-wrap th { background: #1e293b; color: #fff; font-weight: 600; }
.beir-wrap th.avg-col { background: #f59e0b; color: #1e293b; font-weight: 700; }
.beir-wrap td.avg-col { font-weight: 700; font-size: 0.78rem; color: #1e293b; background: #fef3c7; border-left: 2px solid #f59e0b; border-right: 2px solid #f59e0b; }
.beir-wrap tr:last-child td.avg-col { border-bottom: 2px solid #f59e0b; }
.beir-wrap .section-row td { background: #334155; color: #94a3b8; font-weight: 600; font-size: 0.72rem; letter-spacing: .05em; text-transform: uppercase; padding: 5px 10px; }
.beir-wrap strong { color: #0f172a; }
.beir-wrap tbody tr:not(.section-row):hover td { background: #f1f5f9; }
.beir-wrap tbody tr:not(.section-row):hover td.avg-col { background: #fde68a; }
.beir-wrap a { color: #3b82f6; text-decoration: none; }
.beir-wrap a:hover { text-decoration: underline; }
</style>
</head>
<body>
<div class="beir-wrap">
<table>
<thead>
<tr>
<th>Model</th>
<th class="avg-col">Avg</th>
<th>FiQA</th><th>NFCorpus</th><th>TREC-COVID</th><th>Touche</th><th>ArguAna</th><th>Quora</th><th>SCIDOCS</th><th>SciFact</th><th>NQ</th><th>ClimateFEVER</th><th>HotpotQA</th><th>DBPedia</th><th>CQADupstack</th><th>FEVER</th><th>MSMARCO</th>
</tr>
</thead>
<tbody>
<tr class="section-row"><td colspan="17">Baselines</td></tr>
<tr>
<td><a href="https://huggingface.co/nomic-ai/modernbert-embed-base-unsupervised">ModernBERT-embed-unsupervised</a></td>
<td class="avg-col">47.05</td>
<td>42.53</td><td>35.33</td><td>68.44</td><td>18.58</td><td>48.82</td><td>88.63</td><td>19.83</td><td>72.30</td><td>46.32</td><td>22.97</td><td>60.00</td><td>37.97</td><td>42.40</td><td>67.39</td><td>34.23</td>
</tr>
<tr>
<td><a href="https://huggingface.co/nomic-ai/modernbert-embed-base">ModernBERT-embed-supervised</a></td>
<td class="avg-col">52.89</td>
<td>40.59</td><td>33.40</td><td><strong>84.15</strong></td><td>31.91</td><td>48.96</td><td><strong>88.85</strong></td><td>18.59</td><td>69.63</td><td>62.15</td><td>35.67</td><td>67.11</td><td>41.50</td><td>42.08</td><td>87.35</td><td>41.47</td>
</tr>
<tr>
<td><a href="https://huggingface.co/lightonai/GTE-ModernColBERT-v1">GTE-ModernColBERT</a></td>
<td class="avg-col">54.67</td>
<td>45.28</td><td><strong>37.93</strong></td><td>83.59</td><td>31.23</td><td>48.51</td><td>86.61</td><td>19.06</td><td>76.34</td><td>61.80</td><td>30.62</td><td>77.32</td><td>48.03</td><td>41.00</td><td>87.44</td><td>45.32</td>
</tr>
<tr>
<td><a href="https://huggingface.co/Alibaba-NLP/gte-modernbert-base">gte-modernbert-base</a></td>
<td class="avg-col">55.33</td>
<td><strong>48.81</strong></td><td>36.44</td><td>81.95</td><td>21.68</td><td><strong>72.68</strong></td><td>88.55</td><td>21.29</td><td><strong>77.40</strong></td><td>57.62</td><td><strong>37.74</strong></td><td>69.47</td><td>41.79</td><td>42.63</td><td><strong>91.03</strong></td><td>40.90</td>
</tr>
<tr class="section-row"><td colspan="17">KD from dense supervised</td></tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ModernColBERT-embed-base-kd-only">ModernColBERT-embed-base-kd-only</a></td>
<td class="avg-col">54.09</td>
<td>42.51</td><td>37.01</td><td>79.52</td><td>34.58</td><td>51.75</td><td>87.67</td><td>18.15</td><td>75.04</td><td>61.45</td><td>28.31</td><td>76.70</td><td>47.54</td><td>40.68</td><td>84.82</td><td>45.57</td>
</tr>
<tr class="section-row"><td colspan="17">Supervised + KD from dense unsupervised</td></tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ModernColBERT-embed-base-supervised">ModernColBERT-embed-base-supervised</a></td>
<td class="avg-col">50.72</td>
<td>40.09</td><td>35.56</td><td>71.12</td><td>25.53</td><td>44.27</td><td>86.96</td><td>18.19</td><td>73.78</td><td>58.89</td><td>32.95</td><td>71.49</td><td>43.23</td><td>42.55</td><td>70.51</td><td>45.72</td>
</tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ModernColBERT-embed-base">ModernColBERT-embed-base</a></td>
<td class="avg-col">55.12</td>
<td>41.50</td><td>36.51</td><td>77.46</td><td>33.77</td><td>52.45</td><td>86.26</td><td>18.66</td><td>74.90</td><td>62.24</td><td>37.27</td><td><strong>80.07</strong></td><td><strong>48.27</strong></td><td>41.60</td><td>89.71</td><td><strong>46.17</strong></td>
</tr>
<tr class="section-row"><td colspan="17">ColBERT-Zero</td></tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ColBERT-Zero-unsupervised">Unsupervised</a></td>
<td class="avg-col">51.44</td>
<td>45.38</td><td>36.88</td><td>67.82</td><td>22.59</td><td>51.53</td><td>87.78</td><td>22.30</td><td>76.76</td><td>58.80</td><td>24.24</td><td>68.29</td><td>43.16</td><td><strong>45.76</strong></td><td>81.58</td><td>38.78</td>
</tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ColBERT-Zero-supervised">Supervised</a></td>
<td class="avg-col">51.81</td>
<td>42.45</td><td>35.60</td><td>74.72</td><td>23.83</td><td>41.81</td><td>87.19</td><td>19.85</td><td>73.71</td><td>61.95</td><td>35.01</td><td>71.37</td><td>46.20</td><td>45.16</td><td>72.61</td><td>45.68</td>
</tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ColBERT-Zero">Distilled</a></td>
<td class="avg-col"><strong>55.43</strong></td>
<td>42.62</td><td>37.28</td><td>78.69</td><td>36.13</td><td>53.07</td><td>85.24</td><td>19.88</td><td>76.50</td><td>61.66</td><td>35.72</td><td>79.41</td><td>47.48</td><td>41.34</td><td>90.59</td><td>45.80</td>
</tr>
<tr class="section-row"><td colspan="17">ColBERT-Zero-noprompts</td></tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ColBERT-Zero-unsupervised-noprompts">Unsupervised</a></td>
<td class="avg-col">51.70</td>
<td>45.31</td><td>34.72</td><td>73.55</td><td>23.26</td><td>52.56</td><td>88.15</td><td><strong>22.63</strong></td><td>76.10</td><td>59.18</td><td>24.24</td><td>66.66</td><td>42.61</td><td>45.56</td><td>81.88</td><td>39.15</td>
</tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ColBERT-Zero-supervised-noprompts">Supervised</a></td>
<td class="avg-col">52.39</td>
<td>43.36</td><td>36.01</td><td>72.42</td><td>23.79</td><td>47.42</td><td>87.79</td><td>21.30</td><td>73.85</td><td><strong>62.25</strong></td><td>31.61</td><td>70.32</td><td>44.07</td><td>44.03</td><td>85.54</td><td>42.11</td>
</tr>
<tr>
<td><a href="https://huggingface.co/lightonai/ColBERT-Zero-noprompts">Distilled</a></td>
<td class="avg-col">54.61</td>
<td>43.14</td><td>36.60</td><td>78.60</td><td><strong>36.36</strong></td><td>49.49</td><td>88.05</td><td>19.13</td><td>76.42</td><td>61.73</td><td>32.70</td><td>76.99</td><td>47.69</td><td>40.21</td><td>85.97</td><td>46.01</td>
</tr>
</tbody>
</table>
</div>
</body>
</html>
## Limitations & Discussion
- **Data-specific findings.** We deliberately used the Nomic Embed data mixture for controlled comparison. Some observations (particularly around prompt sensitivity) may not generalize to different or stronger training configurations.
- **Scale vs. objective.** The gains from multi-vector pre-training likely reflect *more training time* in the multi-vector setting, rather than the contrastive objective itself. Performing KD alone at a larger scale might yield similar or superior results due to the higher quality of the distillation signal. Our study uses the conventional setup where training scale is inversely proportional to signal quality, reflecting the higher cost of generating high-quality labels.
- **Prompt sensitivity decreases with stronger fine-tuning.** When experimenting with stronger fine-tuning data (e.g., NV-Retriever), adding prompts on top of a model pre-trained without them did not degrade results the way it did with ColBERT-Zero. With enough downstream training, the model can adapt to an initial mismatch.
## Serving at Scale
For production deployment of ColBERT-Zero and other multi-vector models, check out [NextPlaid](https://github.com/lightonai/nextplaid) and [FastPlaid](https://github.com/lightonai/fastplaid), our production-grade engines for multi-vector retrieval.
## Resources
- 📦 **All checkpoints:** [HF Collection](https://huggingface.co/collections/lightonai/colbert-zero) - every phase, with and without prompts
- 💻 **Code:** [Training boilerplates](https://github.com/lightonai/pylate/tree/main/examples/train/ColBERT-zero)
- 📄 **Paper:** [ArXiv](https://arxiv.org/abs/2602.16609)
## Model Details
### Model Description
- **Model Type:** PyLate model
<!-- - **Base model:** [Unknown](https://huggingface.co/unknown) -->
- **Document Length:** 519 tokens
- **Query Length:** 39 tokens
- **Output Dimensionality:** 128 tokens
- **Similarity Function:** MaxSim
- **Training Dataset:**
- train
<!-- - **Language:** Unknown -->
<!-- - **License:** Unknown -->
### Model Sources
- **Documentation:** [PyLate Documentation](https://lightonai.github.io/pylate/)
- **Repository:** [PyLate on GitHub](https://github.com/lightonai/pylate)
- **Hugging Face:** [PyLate models on Hugging Face](https://huggingface.co/models?library=PyLate)
### Full Model Architecture
```
ColBERT(
(0): Transformer({'max_seq_length': 518, 'do_lower_case': False, 'architecture': 'ModernBertModel'})
(1): Dense({'in_features': 768, 'out_features': 128, 'bias': False, 'activation_function': 'torch.nn.modules.linear.Identity', 'use_residual': False})
)
```
## Usage
First install the PyLate library:
```bash
pip install -U pylate
```
> [!WARNING]
> **Prompt alignment is critical for ColBERT-Zero models.** You **must** use `prompt_name="query"` when encoding queries and `prompt_name="document"` when encoding documents. ColBERT-Zero was pre-trained with asymmetric prompts (`search_query:` / `search_document:`), and stripping them causes significant performance.
### Retrieval
Use this model with PyLate to index and retrieve documents. The index uses [FastPLAID](https://github.com/lightonai/fast-plaid) for efficient similarity search.
#### Indexing documents
Load the ColBERT model and initialize the PLAID index, then encode and index your documents:
```python
from pylate import indexes, models, retrieve
# Step 1: Load the ColBERT model
model = models.ColBERT(
model_name_or_path="pylate_model_id",
)
# Step 2: Initialize the PLAID index
index = indexes.PLAID(
index_folder="pylate-index",
index_name="index",
override=True, # This overwrites the existing index if any
)
# Step 3: Encode the documents
documents_ids = ["1", "2", "3"]
documents = ["document 1 text", "document 2 text", "document 3 text"]
documents_embeddings = model.encode(
documents,
batch_size=32,
is_query=False, # Ensure that it is set to False to indicate that these are documents, not queries
prompt_name="document", # ⚠️ Required for ColBERT-Zero! Do not omit.
show_progress_bar=True,
)
# Step 4: Add document embeddings to the index by providing embeddings and corresponding ids
index.add_documents(
documents_ids=documents_ids,
documents_embeddings=documents_embeddings,
)
```
Note that you do not have to recreate the index and encode the documents every time. Once you have created an index and added the documents, you can re-use the index later by loading it:
```python
# To load an index, simply instantiate it with the correct folder/name and without overriding it
index = indexes.PLAID(
index_folder="pylate-index",
index_name="index",
)
```
#### Retrieving top-k documents for queries
Once the documents are indexed, you can retrieve the top-k most relevant documents for a given set of queries.
To do so, initialize the ColBERT retriever with the index you want to search in, encode the queries and then retrieve the top-k documents to get the top matches ids and relevance scores:
[!WARNING]
Always pass prompt_name="query" for queries and prompt_name="document" for documents. Omitting these prompts will silently degrade retrieval quality.
```python
# Step 1: Initialize the ColBERT retriever
retriever = retrieve.ColBERT(index=index)
# Step 2: Encode the queries
queries_embeddings = model.encode(
["query for document 3", "query for document 1"],
batch_size=32,
is_query=True, # # Ensure that it is set to False to indicate that these are queries
prompt_name="query", # ⚠️ Required for ColBERT-Zero! Do not omit.
show_progress_bar=True,
)
# Step 3: Retrieve top-k documents
scores = retriever.retrieve(
queries_embeddings=queries_embeddings,
k=10, # Retrieve the top 10 matches for each query
)
```
### Reranking
> [!WARNING]
> Always pass `prompt_name="query"` for queries and `prompt_name="document"` for documents. Omitting these prompts will silently degrade retrieval quality.
If you only want to use the ColBERT model to perform reranking on top of your first-stage retrieval pipeline without building an index, you can simply use rank function and pass the queries and documents to rerank:
```python
from pylate import rank, models
queries = [
"query A",
"query B",
]
documents = [
["document A", "document B"],
["document 1", "document C", "document B"],
]
documents_ids = [
[1, 2],
[1, 3, 2],
]
model = models.ColBERT(
model_name_or_path="pylate_model_id",
)
queries_embeddings = model.encode(
queries,
is_query=True,
prompt_name="query" # ⚠️ Required for ColBERT-Zero! Do not omit.
)
documents_embeddings = model.encode(
documents,
is_query=False,
prompt_name="document" # ⚠️ Required for ColBERT-Zero! Do not omit.
)
reranked_documents = rank.rerank(
documents_ids=documents_ids,
queries_embeddings=queries_embeddings,
documents_embeddings=documents_embeddings,
)
```
<!--
### Direct Usage (Transformers)
<details><summary>Click to see the direct usage in Transformers</summary>
</details>
-->
<!--
### Downstream Usage (Sentence Transformers)
You can finetune this model on your own dataset.
<details><summary>Click to expand</summary>
</details>
-->
<!--
### Out-of-Scope Use
*List how the model may foreseeably be misused and address what users ought not to do with the model.*
-->
## Evaluation
### Metrics
#### Py Late Information Retrieval
* Dataset: `['NanoClimateFEVER', 'NanoDBPedia', 'NanoFEVER', 'NanoFiQA2018', 'NanoHotpotQA', 'NanoMSMARCO', 'NanoNFCorpus', 'NanoNQ', 'NanoQuoraRetrieval', 'NanoSCIDOCS', 'NanoArguAna', 'NanoSciFact', 'NanoTouche2020']`
* Evaluated with <code>pylate.evaluation.pylate_information_retrieval_evaluator.PyLateInformationRetrievalEvaluator</code>
| Metric | NanoClimateFEVER | NanoDBPedia | NanoFEVER | NanoFiQA2018 | NanoHotpotQA | NanoMSMARCO | NanoNFCorpus | NanoNQ | NanoQuoraRetrieval | NanoSCIDOCS | NanoArguAna | NanoSciFact | NanoTouche2020 |
|:--------------------|:-----------------|:------------|:-----------|:-------------|:-------------|:------------|:-------------|:-----------|:-------------------|:------------|:------------|:------------|:---------------|
| MaxSim_accuracy@1 | 0.34 | 0.86 | 0.94 | 0.5 | 1.0 | 0.54 | 0.54 | 0.68 | 0.98 | 0.46 | 0.24 | 0.74 | 0.7959 |
| MaxSim_accuracy@3 | 0.6 | 0.94 | 1.0 | 0.72 | 1.0 | 0.68 | 0.64 | 0.82 | 1.0 | 0.68 | 0.58 | 0.84 | 0.9388 |
| MaxSim_accuracy@5 | 0.7 | 0.94 | 1.0 | 0.74 | 1.0 | 0.76 | 0.72 | 0.86 | 1.0 | 0.76 | 0.7 | 0.88 | 0.9592 |
| MaxSim_accuracy@10 | 0.84 | 0.96 | 1.0 | 0.76 | 1.0 | 0.86 | 0.74 | 0.9 | 1.0 | 0.88 | 0.88 | 0.92 | 1.0 |
| MaxSim_precision@1 | 0.34 | 0.86 | 0.94 | 0.5 | 1.0 | 0.54 | 0.54 | 0.68 | 0.98 | 0.46 | 0.24 | 0.74 | 0.7959 |
| MaxSim_precision@3 | 0.2467 | 0.72 | 0.3667 | 0.3467 | 0.6 | 0.2267 | 0.42 | 0.28 | 0.4 | 0.3533 | 0.1933 | 0.2933 | 0.7143 |
| MaxSim_precision@5 | 0.192 | 0.66 | 0.22 | 0.248 | 0.368 | 0.152 | 0.384 | 0.176 | 0.256 | 0.292 | 0.14 | 0.196 | 0.6653 |
| MaxSim_precision@10 | 0.128 | 0.572 | 0.11 | 0.136 | 0.186 | 0.086 | 0.286 | 0.098 | 0.138 | 0.196 | 0.088 | 0.102 | 0.5143 |
| MaxSim_recall@1 | 0.1833 | 0.1266 | 0.8767 | 0.2726 | 0.5 | 0.54 | 0.0457 | 0.64 | 0.8707 | 0.0977 | 0.24 | 0.715 | 0.0522 |
| MaxSim_recall@3 | 0.3033 | 0.2185 | 0.98 | 0.5209 | 0.9 | 0.68 | 0.0818 | 0.77 | 0.952 | 0.2177 | 0.58 | 0.81 | 0.1429 |
| MaxSim_recall@5 | 0.39 | 0.2938 | 0.98 | 0.5647 | 0.92 | 0.76 | 0.1271 | 0.8 | 0.9727 | 0.2987 | 0.7 | 0.87 | 0.2168 |
| MaxSim_recall@10 | 0.4933 | 0.4105 | 0.98 | 0.587 | 0.93 | 0.86 | 0.153 | 0.87 | 0.9967 | 0.4007 | 0.88 | 0.91 | 0.3276 |
| **MaxSim_ndcg@10** | **0.4063** | **0.7283** | **0.9539** | **0.5309** | **0.9223** | **0.6888** | **0.3718** | **0.7689** | **0.9814** | **0.3926** | **0.558** | **0.825** | **0.5977** |
| MaxSim_mrr@10 | 0.4916 | 0.8992 | 0.9633 | 0.6012 | 1.0 | 0.6349 | 0.6048 | 0.7574 | 0.99 | 0.5987 | 0.4559 | 0.8 | 0.872 |
| MaxSim_map@100 | 0.3304 | 0.5925 | 0.9376 | 0.4781 | 0.8847 | 0.6441 | 0.1675 | 0.7299 | 0.9665 | 0.305 | 0.4587 | 0.796 | 0.4265 |
#### Nano BEIR
* Dataset: `NanoBEIR_mean`
* Evaluated with <code>pylate.evaluation.nano_beir_evaluator.NanoBEIREvaluator</code>
| Metric | Value |
|:--------------------|:-----------|
| MaxSim_accuracy@1 | 0.6628 |
| MaxSim_accuracy@3 | 0.803 |
| MaxSim_accuracy@5 | 0.8476 |
| MaxSim_accuracy@10 | 0.9031 |
| MaxSim_precision@1 | 0.6628 |
| MaxSim_precision@3 | 0.397 |
| MaxSim_precision@5 | 0.3038 |
| MaxSim_precision@10 | 0.2031 |
| MaxSim_recall@1 | 0.397 |
| MaxSim_recall@3 | 0.5505 |
| MaxSim_recall@5 | 0.6072 |
| MaxSim_recall@10 | 0.6768 |
| **MaxSim_ndcg@10** | **0.6712** |
| MaxSim_mrr@10 | 0.7438 |
| MaxSim_map@100 | 0.5936 |
<!--
## Bias, Risks and Limitations
*What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*
-->
<!--
### Recommendations
*What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*
-->
## Training Details
### Training Dataset
#### train
* Dataset: train
* Size: 640,000 training samples
* Columns: <code>query_id</code>, <code>document_ids</code>, and <code>scores</code>
* Approximate statistics based on the first 1000 samples:
| | query_id | document_ids | scores |
|:--------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------|:------------------------------------|
| type | int | list | list |
| details | <ul><li>836: ~0.10%</li><li>3582: ~0.10%</li><li>4599: ~0.10%</li>...</ul> | <ul><li>size: 32 elements</li></ul> |
* Samples:
| query_id | document_ids | scores |
|:--------------------|:----------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
| <code>685613</code> | <code>[7546874, 1176459, 197677, 2306318, 8541504, ...]</code> | <code>[0.9999999992804947, 0.24845418756716053, 0.7594154013647826, 0.26644182105618575, 0.390668914839766, ...]</code> |
| <code>237784</code> | <code>[6366584, 4034101, 2325374, 6914618, 6042146, ...]</code> | <code>[0.9999999991784339, 0.42233632827946693, 0.5956354295491569, 0.12644415907455164, 0.6636713730105909, ...]</code> |
| <code>904294</code> | <code>[448408, 8743975, 49600, 7339401, 2714261, ...]</code> | <code>[0.9999999991841937, 0.877629062381539, 0.8330146583389045, 0.3116634796692611, 0.4633524534142185, ...]</code> |
* Loss: <code>pylate.losses.distillation.Distillation</code>
### Training Hyperparameters
#### Non-Default Hyperparameters
- `eval_strategy`: steps
- `per_device_train_batch_size`: 4
- `per_device_eval_batch_size`: 4
- `gradient_accumulation_steps`: 2
- `learning_rate`: 8e-05
- `num_train_epochs`: 1.0
- `bf16`: True
- `dataloader_num_workers`: 4
- `ddp_find_unused_parameters`: False
#### All Hyperparameters
<details><summary>Click to expand</summary>
- `overwrite_output_dir`: False
- `do_predict`: False
- `eval_strategy`: steps
- `prediction_loss_only`: True
- `per_device_train_batch_size`: 4
- `per_device_eval_batch_size`: 4
- `per_gpu_train_batch_size`: None
- `per_gpu_eval_batch_size`: None
- `gradient_accumulation_steps`: 2
- `eval_accumulation_steps`: None
- `torch_empty_cache_steps`: None
- `learning_rate`: 8e-05
- `weight_decay`: 0.0
- `adam_beta1`: 0.9
- `adam_beta2`: 0.999
- `adam_epsilon`: 1e-08
- `max_grad_norm`: 1.0
- `num_train_epochs`: 1.0
- `max_steps`: -1
- `lr_scheduler_type`: linear
- `lr_scheduler_kwargs`: {}
- `warmup_ratio`: 0.0
- `warmup_steps`: 0
- `log_level`: passive
- `log_level_replica`: warning
- `log_on_each_node`: True
- `logging_nan_inf_filter`: True
- `save_safetensors`: True
- `save_on_each_node`: False
- `save_only_model`: False
- `restore_callback_states_from_checkpoint`: False
- `no_cuda`: False
- `use_cpu`: False
- `use_mps_device`: False
- `seed`: 42
- `data_seed`: None
- `jit_mode_eval`: False
- `use_ipex`: False
- `bf16`: True
- `fp16`: False
- `fp16_opt_level`: O1
- `half_precision_backend`: auto
- `bf16_full_eval`: False
- `fp16_full_eval`: False
- `tf32`: None
- `local_rank`: 1
- `ddp_backend`: None
- `tpu_num_cores`: None
- `tpu_metrics_debug`: False
- `debug`: []
- `dataloader_drop_last`: True
- `dataloader_num_workers`: 4
- `dataloader_prefetch_factor`: None
- `past_index`: -1
- `disable_tqdm`: False
- `remove_unused_columns`: True
- `label_names`: None
- `load_best_model_at_end`: False
- `ignore_data_skip`: False
- `fsdp`: []
- `fsdp_min_num_params`: 0
- `fsdp_config`: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}
- `fsdp_transformer_layer_cls_to_wrap`: None
- `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
- `deepspeed`: None
- `label_smoothing_factor`: 0.0
- `optim`: adamw_torch
- `optim_args`: None
- `adafactor`: False
- `group_by_length`: False
- `length_column_name`: length
- `ddp_find_unused_parameters`: False
- `ddp_bucket_cap_mb`: None
- `ddp_broadcast_buffers`: False
- `dataloader_pin_memory`: True
- `dataloader_persistent_workers`: False
- `skip_memory_metrics`: True
- `use_legacy_prediction_loop`: False
- `push_to_hub`: False
- `resume_from_checkpoint`: None
- `hub_model_id`: None
- `hub_strategy`: every_save
- `hub_private_repo`: None
- `hub_always_push`: False
- `gradient_checkpointing`: False
- `gradient_checkpointing_kwargs`: None
- `include_inputs_for_metrics`: False
- `include_for_metrics`: []
- `eval_do_concat_batches`: True
- `fp16_backend`: auto
- `push_to_hub_model_id`: None
- `push_to_hub_organization`: None
- `mp_parameters`:
- `auto_find_batch_size`: False
- `full_determinism`: False
- `torchdynamo`: None
- `ray_scope`: last
- `ddp_timeout`: 1800
- `torch_compile`: False
- `torch_compile_backend`: None
- `torch_compile_mode`: None
- `dispatch_batches`: None
- `split_batches`: None
- `include_tokens_per_second`: False
- `include_num_input_tokens_seen`: False
- `neftune_noise_alpha`: None
- `optim_target_modules`: None
- `batch_eval_metrics`: False
- `eval_on_start`: False
- `use_liger_kernel`: False
- `eval_use_gather_object`: False
- `average_tokens_across_devices`: False
- `prompts`: None
- `batch_sampler`: batch_sampler
- `multi_dataset_batch_sampler`: proportional
- `router_mapping`: {}
- `learning_rate_mapping`: {}
</details>
### Training Logs
<details><summary>Click to expand</summary>
| Epoch | Step | Training Loss | NanoClimateFEVER_MaxSim_ndcg@10 | NanoDBPedia_MaxSim_ndcg@10 | NanoFEVER_MaxSim_ndcg@10 | NanoFiQA2018_MaxSim_ndcg@10 | NanoHotpotQA_MaxSim_ndcg@10 | NanoMSMARCO_MaxSim_ndcg@10 | NanoNFCorpus_MaxSim_ndcg@10 | NanoNQ_MaxSim_ndcg@10 | NanoQuoraRetrieval_MaxSim_ndcg@10 | NanoSCIDOCS_MaxSim_ndcg@10 | NanoArguAna_MaxSim_ndcg@10 | NanoSciFact_MaxSim_ndcg@10 | NanoTouche2020_MaxSim_ndcg@10 | NanoBEIR_mean_MaxSim_ndcg@10 |
|:------:|:-----:|:-------------:|:-------------------------------:|:--------------------------:|:------------------------:|:---------------------------:|:---------------------------:|:--------------------------:|:---------------------------:|:---------------------:|:---------------------------------:|:--------------------------:|:--------------------------:|:--------------------------:|:-----------------------------:|:----------------------------:|
| 0.0025 | 50 | 0.0259 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.0275 | 550 | 0.019 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.0525 | 1050 | 0.0168 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.075 | 1500 | 0.0152 | 0.3530 | 0.6921 | 0.9345 | 0.5514 | 0.9121 | 0.6905 | 0.3714 | 0.7376 | 0.9617 | 0.3922 | 0.5317 | 0.7828 | 0.6200 | 0.6562 |
| 0.0775 | 1550 | 0.0147 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.1025 | 2050 | 0.0147 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.1275 | 2550 | 0.0141 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.15 | 3000 | 0.0135 | 0.3818 | 0.7111 | 0.9468 | 0.5737 | 0.8930 | 0.6751 | 0.3876 | 0.7329 | 0.9854 | 0.4004 | 0.5359 | 0.8012 | 0.6286 | 0.6656 |
| 0.1525 | 3050 | 0.0134 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.1775 | 3550 | 0.0132 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.2025 | 4050 | 0.0128 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.225 | 4500 | 0.0124 | 0.3586 | 0.7030 | 0.9472 | 0.5690 | 0.9114 | 0.6772 | 0.3946 | 0.7497 | 0.9750 | 0.3953 | 0.5223 | 0.8098 | 0.6158 | 0.6638 |
| 0.2275 | 4550 | 0.0122 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.2525 | 5050 | 0.0123 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.2775 | 5550 | 0.0118 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.3 | 6000 | 0.0119 | 0.3992 | 0.7115 | 0.9573 | 0.5612 | 0.9038 | 0.6984 | 0.3952 | 0.7582 | 0.9719 | 0.4023 | 0.5235 | 0.7987 | 0.6036 | 0.6681 |
| 0.3025 | 6050 | 0.0119 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.3275 | 6550 | 0.0116 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.3525 | 7050 | 0.0111 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.375 | 7500 | 0.0116 | 0.3920 | 0.7276 | 0.9523 | 0.5745 | 0.8960 | 0.6956 | 0.3928 | 0.7349 | 0.9779 | 0.3998 | 0.5397 | 0.8058 | 0.6309 | 0.6707 |
| 0.3775 | 7550 | 0.011 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.4025 | 8050 | 0.0107 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.4275 | 8550 | 0.0109 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.45 | 9000 | 0.0103 | 0.4048 | 0.7211 | 0.9469 | 0.5466 | 0.9155 | 0.6889 | 0.3713 | 0.7401 | 0.9806 | 0.4074 | 0.5507 | 0.8158 | 0.6125 | 0.6694 |
| 0.4525 | 9050 | 0.0107 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.4775 | 9550 | 0.0105 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.5025 | 10050 | 0.01 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.525 | 10500 | 0.0103 | 0.3875 | 0.7301 | 0.9445 | 0.5466 | 0.9113 | 0.6969 | 0.3752 | 0.7625 | 0.9795 | 0.4017 | 0.5424 | 0.8207 | 0.6067 | 0.6697 |
| 0.5275 | 10550 | 0.0103 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.5525 | 11050 | 0.0099 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.5775 | 11550 | 0.0096 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.6 | 12000 | 0.0098 | 0.4020 | 0.7211 | 0.9432 | 0.5410 | 0.9181 | 0.6831 | 0.3709 | 0.7479 | 0.9812 | 0.4049 | 0.5593 | 0.8293 | 0.5912 | 0.6687 |
| 0.6025 | 12050 | 0.0098 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.6275 | 12550 | 0.0095 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.6525 | 13050 | 0.0097 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.675 | 13500 | 0.0093 | 0.4119 | 0.7163 | 0.9522 | 0.5415 | 0.9182 | 0.7049 | 0.3714 | 0.7810 | 0.9827 | 0.3945 | 0.5462 | 0.8200 | 0.6176 | 0.6737 |
| 0.6775 | 13550 | 0.0095 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.7025 | 14050 | 0.0094 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.7275 | 14550 | 0.0092 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.75 | 15000 | 0.0092 | 0.4103 | 0.7132 | 0.9539 | 0.5326 | 0.9156 | 0.6947 | 0.3594 | 0.7590 | 0.9807 | 0.4009 | 0.5490 | 0.8321 | 0.6047 | 0.6697 |
| 0.7525 | 15050 | 0.0091 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.7775 | 15550 | 0.009 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.8025 | 16050 | 0.0084 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.825 | 16500 | 0.009 | 0.4041 | 0.7143 | 0.9555 | 0.5575 | 0.9165 | 0.6968 | 0.3698 | 0.7769 | 0.9812 | 0.3994 | 0.5557 | 0.8195 | 0.6004 | 0.6729 |
| 0.8275 | 16550 | 0.0086 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.8525 | 17050 | 0.0085 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.8775 | 17550 | 0.0086 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.9 | 18000 | 0.0086 | 0.4059 | 0.7210 | 0.9539 | 0.5391 | 0.9160 | 0.6962 | 0.3722 | 0.7770 | 0.9831 | 0.3985 | 0.5489 | 0.8330 | 0.6031 | 0.6729 |
| 0.9025 | 18050 | 0.0088 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.9275 | 18550 | 0.0085 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.9525 | 19050 | 0.0083 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| 0.975 | 19500 | 0.0086 | 0.4063 | 0.7283 | 0.9539 | 0.5309 | 0.9223 | 0.6888 | 0.3718 | 0.7689 | 0.9814 | 0.3926 | 0.5580 | 0.8250 | 0.5977 | 0.6712 |
| 0.9775 | 19550 | 0.0087 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
</details>
### Framework Versions
- Python: 3.13.0
- Sentence Transformers: 5.1.1
- PyLate: 1.3.4
- Transformers: 4.48.3
- PyTorch: 2.6.0
- Accelerate: 1.12.0
- Datasets: 4.4.1
- Tokenizers: 0.21.0
## Citation
### BibTeX
#### ColBERT-Zero
```bibtex
@misc{chaffin2026colbertzeropretrainpretraincolbert,
title = {ColBERT-Zero: To Pre-train Or Not To Pre-train ColBERT models},
author = {Antoine Chaffin and Luca Arnaboldi and Amélie Chatelain and Florent Krzakala},
year = {2026},
eprint = {2602.16609},
archivePrefix = {arXiv},
primaryClass = {cs.CL},
url = {https://arxiv.org/abs/2602.16609},
}
```
#### Sentence Transformers
```bibtex
@inproceedings{reimers-2019-sentence-bert,
title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
author = "Reimers, Nils and Gurevych, Iryna",
booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
month = "11",
year = "2019",
publisher = "Association for Computational Linguistics",
url = "https://arxiv.org/abs/1908.10084"
}
```
#### PyLate
```bibtex
@inproceedings{DBLP:conf/cikm/ChaffinS25,
author = {Antoine Chaffin and
Rapha{"{e}}l Sourty},
editor = {Meeyoung Cha and
Chanyoung Park and
Noseong Park and
Carl Yang and
Senjuti Basu Roy and
Jessie Li and
Jaap Kamps and
Kijung Shin and
Bryan Hooi and
Lifang He},
title = {PyLate: Flexible Training and Retrieval for Late Interaction Models},
booktitle = {Proceedings of the 34th {ACM} International Conference on Information
and Knowledge Management, {CIKM} 2025, Seoul, Republic of Korea, November
10-14, 2025},
pages = {6334--6339},
publisher = {{ACM}},
year = {2025},
url = {https://github.com/lightonai/pylate},
doi = {10.1145/3746252.3761608},
}
```
#### Nomic Embed
```bibtex
@article{DBLP:journals/tmlr/NussbaumMMD25,
author = {Zach Nussbaum and
John Xavier Morris and
Andriy Mulyar and
Brandon Duderstadt},
title = {Nomic Embed: Training a Reproducible Long Context Text Embedder},
journal = {Trans. Mach. Learn. Res.},
volume = {2025},
year = {2025},
url = {https://openreview.net/forum?id=IPmzyQSiQE},
timestamp = {Fri, 20 Jun 2025 14:19:48 +0200},
biburl = {https://dblp.org/rec/journals/tmlr/NussbaumMMD25.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
```
<!--
## Glossary
*Clearly define terms in order to be accessible across audiences.*
-->
<!--
## Model Card Authors
*Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*
-->
<!--
## Model Card Contact
*Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*
-->