{ "suite": "goktugtr-turkish-retrieval-suite-v1", "language_filter": "turkish_only_exclusive", "model": "GöktuğTR 270M v2 hard negatives", "model_source": "outputs/goktugtr-270m-v2/final", "model_revision": null, "parameters": 268098176, "embedding_dimension": 640, "inference_dtype": "bfloat16", "normalized_embeddings": true, "prompt_style": "harrier", "device": "NVIDIA GeForce RTX 5060 Laptop GPU", "mteb_version": "2.18.16", "task_main_scores": { "TurHistQuadRetrieval": 0.42198, "XQuADRetrieval": 0.86393, "WebFAQRetrieval": 0.56886, "MKQARetrieval": 0.10331, "BelebeleRetrieval": 0.88493 }, "macro_average": 0.568602, "tasks": [ { "name": "TurHistQuadRetrieval", "dataset": { "path": "asparius/TurHistQuAD", "revision": "2a2b8ddecf1189f530676244d0751e1d0a569e03" }, "license": "mit", "domains": [ "Encyclopaedic", "Non-fiction", "Academic", "Written" ], "eval_splits": [ "test" ], "subsets": [ "default" ] }, { "name": "XQuADRetrieval", "dataset": { "path": "google/xquad", "revision": "51adfef1c1287aab1d2d91b5bead9bcfb9c68583" }, "license": "cc-by-sa-4.0", "domains": [ "Web", "Written" ], "eval_splits": [ "validation" ], "subsets": [ "tr" ] }, { "name": "WebFAQRetrieval", "dataset": { "path": "mteb/WebFAQRetrieval", "revision": "f64f483ad0f31d2e78209d524c14a4a867965959" }, "license": "cc-by-4.0", "domains": [ "Web", "Written" ], "eval_splits": [ "test" ], "subsets": [ "tur" ] }, { "name": "MKQARetrieval", "dataset": { "path": "mteb/MKQARetrieval", "revision": "3e069e7a30079d214a859741f5a0de75cf878867" }, "license": "cc-by-3.0", "domains": [ "Written" ], "eval_splits": [ "train" ], "subsets": [ "tr" ] }, { "name": "BelebeleRetrieval", "dataset": { "path": "mteb/belebele", "revision": "979a211276faa22f671e69d096634193567cfd05" }, "license": "cc-by-sa-4.0", "domains": [ "Web", "News", "Written" ], "eval_splits": [ "test" ], "subsets": [ "tur_Latn-tur_Latn" ] } ], "raw_mteb_result": { "model_name": "no_model_name/available", "model_revision": "no_revision_available", "task_results": [ { "dataset_revision": "2a2b8ddecf1189f530676244d0751e1d0a569e03", "task_name": "TurHistQuadRetrieval", "mteb_version": "2.18.16", "scores": { "test": [ { "ndcg_at_1": 0.44727, "ndcg_at_3": 0.36818, "ndcg_at_5": 0.39742, "ndcg_at_10": 0.42198, "ndcg_at_20": 0.44086, "ndcg_at_100": 0.47466, "ndcg_at_1000": 0.51823, "map_at_1": 0.22363, "map_at_3": 0.29012, "map_at_5": 0.30726, "map_at_10": 0.31975, "map_at_20": 0.3268, "map_at_100": 0.33331, "map_at_1000": 0.33552, "recall_at_1": 0.22363, "recall_at_3": 0.35498, "recall_at_5": 0.41357, "recall_at_10": 0.4751, "recall_at_20": 0.53516, "recall_at_100": 0.68604, "recall_at_1000": 0.97266, "accuracy": 0.22363, "precision_at_1": 0.44727, "precision_at_3": 0.23665, "precision_at_5": 0.16543, "precision_at_10": 0.09502, "precision_at_20": 0.05352, "precision_at_100": 0.01372, "precision_at_1000": 0.00195, "mrr_at_1": 0.447265625, "mrr_at_3": 0.54541015625, "mrr_at_5": 0.563134765625, "mrr_at_10": 0.5715328156001984, "mrr_at_20": 0.5747740260525898, "mrr_at_100": 0.5764007840136673, "mrr_at_1000": 0.5767683869085801, "nauc_ndcg_at_1_max": 0.1947773993090055, "nauc_ndcg_at_1_std": -0.0910494007784227, "nauc_ndcg_at_1_diff1": 0.4073158485432013, "nauc_ndcg_at_3_max": 0.15459659971936998, "nauc_ndcg_at_3_std": -0.13572605317548517, "nauc_ndcg_at_3_diff1": 0.3590182901499481, "nauc_ndcg_at_5_max": 0.12351242622860217, "nauc_ndcg_at_5_std": -0.13862840806665486, "nauc_ndcg_at_5_diff1": 0.3411180215230726, "nauc_ndcg_at_10_max": 0.10508451932152987, "nauc_ndcg_at_10_std": -0.15917084194618117, "nauc_ndcg_at_10_diff1": 0.3456925788408306, "nauc_ndcg_at_20_max": 0.07934060994772754, "nauc_ndcg_at_20_std": -0.17556108096439801, "nauc_ndcg_at_20_diff1": 0.34005209604606523, "nauc_ndcg_at_100_max": 0.06263250772213183, "nauc_ndcg_at_100_std": -0.19644024854652758, "nauc_ndcg_at_100_diff1": 0.34964007250841367, "nauc_ndcg_at_1000_max": 0.06570815848059838, "nauc_ndcg_at_1000_std": -0.1926137221140858, "nauc_ndcg_at_1000_diff1": 0.36291691156660905, "nauc_map_at_1_max": 0.1947773993090055, "nauc_map_at_1_std": -0.0910494007784227, "nauc_map_at_1_diff1": 0.4073158485432013, "nauc_map_at_3_max": 0.15482569356800127, "nauc_map_at_3_std": -0.12953054928321617, "nauc_map_at_3_diff1": 0.3547572884337557, "nauc_map_at_5_max": 0.13400725043834277, "nauc_map_at_5_std": -0.13487559474480726, "nauc_map_at_5_diff1": 0.3411052428046666, "nauc_map_at_10_max": 0.12580634709979557, "nauc_map_at_10_std": -0.14482991442291338, "nauc_map_at_10_diff1": 0.3388736900349583, "nauc_map_at_20_max": 0.1171912993522551, "nauc_map_at_20_std": -0.15057501514134508, "nauc_map_at_20_diff1": 0.33579860813933154, "nauc_map_at_100_max": 0.11439780365587258, "nauc_map_at_100_std": -0.15406029446515598, "nauc_map_at_100_diff1": 0.3366239925867801, "nauc_map_at_1000_max": 0.11378034070804263, "nauc_map_at_1000_std": -0.15413650834976175, "nauc_map_at_1000_diff1": 0.33743800581253264, "nauc_recall_at_1_max": 0.1947773993090055, "nauc_recall_at_1_std": -0.0910494007784227, "nauc_recall_at_1_diff1": 0.4073158485432013, "nauc_recall_at_3_max": 0.11721165014618391, "nauc_recall_at_3_std": -0.16453793054408086, "nauc_recall_at_3_diff1": 0.31826402362729905, "nauc_recall_at_5_max": 0.032770031656111503, "nauc_recall_at_5_std": -0.16108032882961745, "nauc_recall_at_5_diff1": 0.24788015226976987, "nauc_recall_at_10_max": -0.015009736950870631, "nauc_recall_at_10_std": -0.2028271635328523, "nauc_recall_at_10_diff1": 0.24498908438888148, "nauc_recall_at_20_max": -0.09578925764875043, "nauc_recall_at_20_std": -0.22857085957680415, "nauc_recall_at_20_diff1": 0.19408264003786216, "nauc_recall_at_100_max": -0.16490966442677502, "nauc_recall_at_100_std": -0.3015537080276642, "nauc_recall_at_100_diff1": 0.21955302141734043, "nauc_recall_at_1000_max": -1.3782566762561899, "nauc_recall_at_1000_std": -1.2793437787675817, "nauc_recall_at_1000_diff1": 0.7032453889650138, "nauc_precision_at_1_max": 0.1947773993090055, "nauc_precision_at_1_std": -0.0910494007784227, "nauc_precision_at_1_diff1": 0.4073158485432013, "nauc_precision_at_3_max": 0.11721165014618445, "nauc_precision_at_3_std": -0.16453793054408086, "nauc_precision_at_3_diff1": 0.3182640236272993, "nauc_precision_at_5_max": 0.03277003165611099, "nauc_precision_at_5_std": -0.1610803288296178, "nauc_precision_at_5_diff1": 0.2478801522697694, "nauc_precision_at_10_max": -0.01500973695087072, "nauc_precision_at_10_std": -0.20282716353285224, "nauc_precision_at_10_diff1": 0.24498908438888103, "nauc_precision_at_20_max": -0.0957892576487511, "nauc_precision_at_20_std": -0.22857085957680465, "nauc_precision_at_20_diff1": 0.1940826400378614, "nauc_precision_at_100_max": -0.16490966442677507, "nauc_precision_at_100_std": -0.30155370802766424, "nauc_precision_at_100_diff1": 0.21955302141734045, "nauc_precision_at_1000_max": -1.378256676256199, "nauc_precision_at_1000_std": -1.279343778767596, "nauc_precision_at_1000_diff1": 0.7032453889650114, "nauc_mrr_at_1_max": 0.1947773993090055, "nauc_mrr_at_1_std": -0.0910494007784227, "nauc_mrr_at_1_diff1": 0.4073158485432013, "nauc_mrr_at_3_max": 0.17951015442495216, "nauc_mrr_at_3_std": -0.11954604918828698, "nauc_mrr_at_3_diff1": 0.3878193805370859, "nauc_mrr_at_5_max": 0.1690611363913176, "nauc_mrr_at_5_std": -0.11456851241986926, "nauc_mrr_at_5_diff1": 0.3854262100889673, "nauc_mrr_at_10_max": 0.16279109738166908, "nauc_mrr_at_10_std": -0.12140021284110425, "nauc_mrr_at_10_diff1": 0.39431158801201477, "nauc_mrr_at_20_max": 0.15854074365731471, "nauc_mrr_at_20_std": -0.12493806418765903, "nauc_mrr_at_20_diff1": 0.39652712372135157, "nauc_mrr_at_100_max": 0.15656075467937358, "nauc_mrr_at_100_std": -0.12717669984150143, "nauc_mrr_at_100_diff1": 0.39764245610734594, "nauc_mrr_at_1000_max": 0.15714128366655825, "nauc_mrr_at_1000_std": -0.1268302978503602, "nauc_mrr_at_1000_diff1": 0.3974845927958745, "hit_rate_at_1": 0.44727, "hit_rate_at_3": 0.66797, "hit_rate_at_5": 0.74805, "hit_rate_at_10": 0.80859, "hit_rate_at_20": 0.85352, "hit_rate_at_100": 0.91406, "hit_rate_at_1000": 1.0, "main_score": 0.42198, "hf_subset": "default", "languages": [ "tur-Latn" ], "mteb_version": "2.18.16" } ] }, "evaluation_time": 6.209755585761741, "kg_co2_emissions": null, "date": "2026-08-11T08:37:55.009831Z", "evaluation_phases": [ { "name": "Data loading", "start": 0.0, "end": 1.6432985588908195, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 1.6578817549161613, "end": 5.618854850064963, "split": "test", "subset": "default" }, { "name": "Scoring", "start": 5.618875534972176, "end": 6.201895288890228, "split": "test", "subset": "default" } ] }, { "dataset_revision": "51adfef1c1287aab1d2d91b5bead9bcfb9c68583", "task_name": "XQuADRetrieval", "mteb_version": "2.18.16", "scores": { "validation": [ { "ndcg_at_1": 0.81588, "ndcg_at_3": 0.85376, "ndcg_at_5": 0.85972, "ndcg_at_10": 0.86393, "ndcg_at_20": 0.87064, "ndcg_at_100": 0.88069, "ndcg_at_1000": 0.88277, "map_at_1": 0.81588, "map_at_3": 0.84516, "map_at_5": 0.84849, "map_at_10": 0.85014, "map_at_20": 0.85191, "map_at_100": 0.8535, "map_at_1000": 0.8536, "recall_at_1": 0.81588, "recall_at_3": 0.87838, "recall_at_5": 0.89274, "recall_at_10": 0.90625, "recall_at_20": 0.93328, "recall_at_100": 0.9848, "recall_at_1000": 1.0, "accuracy": 0.81588, "precision_at_1": 0.81588, "precision_at_3": 0.29279, "precision_at_5": 0.17855, "precision_at_10": 0.09063, "precision_at_20": 0.04666, "precision_at_100": 0.00985, "precision_at_1000": 0.001, "mrr_at_1": 0.8158783783783784, "mrr_at_3": 0.845579954954955, "mrr_at_5": 0.8487471846846847, "mrr_at_10": 0.8497372372372372, "mrr_at_20": 0.8519017309373831, "mrr_at_100": 0.853515089167124, "mrr_at_1000": 0.8536142784769822, "nauc_ndcg_at_1_max": 0.3284540665242737, "nauc_ndcg_at_1_std": -0.03350450468321942, "nauc_ndcg_at_1_diff1": 0.7757754213683679, "nauc_ndcg_at_3_max": 0.25818700964639063, "nauc_ndcg_at_3_std": -0.055404890207890786, "nauc_ndcg_at_3_diff1": 0.7357097193459545, "nauc_ndcg_at_5_max": 0.23532978864261622, "nauc_ndcg_at_5_std": -0.070773060535053, "nauc_ndcg_at_5_diff1": 0.7296829159511898, "nauc_ndcg_at_10_max": 0.2221519746395343, "nauc_ndcg_at_10_std": -0.0875650487509888, "nauc_ndcg_at_10_diff1": 0.732598352255786, "nauc_ndcg_at_20_max": 0.20773138536758215, "nauc_ndcg_at_20_std": -0.09989246220154115, "nauc_ndcg_at_20_diff1": 0.7340091998299748, "nauc_ndcg_at_100_max": 0.19515655222487252, "nauc_ndcg_at_100_std": -0.11460185139630633, "nauc_ndcg_at_100_diff1": 0.7352192500871594, "nauc_ndcg_at_1000_max": 0.22818231071148642, "nauc_ndcg_at_1000_std": -0.08733055810540781, "nauc_ndcg_at_1000_diff1": 0.7390905883324356, "nauc_map_at_1_max": 0.3284540665242737, "nauc_map_at_1_std": -0.03350450468321942, "nauc_map_at_1_diff1": 0.7757754213683679, "nauc_map_at_3_max": 0.2777399037391668, "nauc_map_at_3_std": -0.048568184058805995, "nauc_map_at_3_diff1": 0.7462289334451868, "nauc_map_at_5_max": 0.26622347518153544, "nauc_map_at_5_std": -0.0563052173192779, "nauc_map_at_5_diff1": 0.7433725065389168, "nauc_map_at_10_max": 0.26177082716700206, "nauc_map_at_10_std": -0.06208533201755686, "nauc_map_at_10_diff1": 0.7443466727140162, "nauc_map_at_20_max": 0.25872168437955684, "nauc_map_at_20_std": -0.06463743789758596, "nauc_map_at_20_diff1": 0.7448129266649355, "nauc_map_at_100_max": 0.2559148722096524, "nauc_map_at_100_std": -0.06773002736169367, "nauc_map_at_100_diff1": 0.7449903003740266, "nauc_map_at_1000_max": 0.2572009615963307, "nauc_map_at_1000_std": -0.06667363214343108, "nauc_map_at_1000_diff1": 0.7451457131512783, "nauc_recall_at_1_max": 0.3284540665242737, "nauc_recall_at_1_std": -0.03350450468321942, "nauc_recall_at_1_diff1": 0.7757754213683679, "nauc_recall_at_3_max": 0.18820039148395235, "nauc_recall_at_3_std": -0.08026325128177393, "nauc_recall_at_3_diff1": 0.6983862971238585, "nauc_recall_at_5_max": 0.10995206042485102, "nauc_recall_at_5_std": -0.13101453725155376, "nauc_recall_at_5_diff1": 0.6751281440235256, "nauc_recall_at_10_max": 0.03450281639611959, "nauc_recall_at_10_std": -0.21514187982604624, "nauc_recall_at_10_diff1": 0.6827023652581147, "nauc_recall_at_20_max": -0.14669192998797975, "nauc_recall_at_20_std": -0.35926833389834817, "nauc_recall_at_20_diff1": 0.6732122472101519, "nauc_recall_at_100_max": -1.5891802105217783, "nauc_recall_at_100_std": -1.5891802105217783, "nauc_recall_at_100_diff1": 0.5276547460163096, "nauc_recall_at_1000_max": null, "nauc_recall_at_1000_std": null, "nauc_recall_at_1000_diff1": null, "nauc_precision_at_1_max": 0.3284540665242737, "nauc_precision_at_1_std": -0.03350450468321942, "nauc_precision_at_1_diff1": 0.7757754213683679, "nauc_precision_at_3_max": 0.18820039148395123, "nauc_precision_at_3_std": -0.08026325128177557, "nauc_precision_at_3_diff1": 0.6983862971238574, "nauc_precision_at_5_max": 0.10995206042484995, "nauc_precision_at_5_std": -0.13101453725155207, "nauc_precision_at_5_diff1": 0.6751281440235273, "nauc_precision_at_10_max": 0.03450281639612019, "nauc_precision_at_10_std": -0.21514187982604183, "nauc_precision_at_10_diff1": 0.6827023652581166, "nauc_precision_at_20_max": -0.1466919299879782, "nauc_precision_at_20_std": -0.35926833389834945, "nauc_precision_at_20_diff1": 0.6732122472101516, "nauc_precision_at_100_max": -1.5891802105218076, "nauc_precision_at_100_std": -1.5891802105218076, "nauc_precision_at_100_diff1": 0.5276547460163015, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_1000_diff1": 1.0, "nauc_mrr_at_1_max": 0.3284540665242737, "nauc_mrr_at_1_std": -0.03350450468321942, "nauc_mrr_at_1_diff1": 0.7757754213683679, "nauc_mrr_at_3_max": 0.2755646757001353, "nauc_mrr_at_3_std": -0.051182154244866135, "nauc_mrr_at_3_diff1": 0.7452215992777843, "nauc_mrr_at_5_max": 0.2670252442185693, "nauc_mrr_at_5_std": -0.055120825642614404, "nauc_mrr_at_5_diff1": 0.7409265069774154, "nauc_mrr_at_10_max": 0.26289797765516126, "nauc_mrr_at_10_std": -0.05924126003203253, "nauc_mrr_at_10_diff1": 0.7415305795746361, "nauc_mrr_at_20_max": 0.2590123096058126, "nauc_mrr_at_20_std": -0.06499429407428235, "nauc_mrr_at_20_diff1": 0.7426366640770949, "nauc_mrr_at_100_max": 0.2560796971787312, "nauc_mrr_at_100_std": -0.06747575427838415, "nauc_mrr_at_100_diff1": 0.7427706567785499, "nauc_mrr_at_1000_max": 0.2573657793463667, "nauc_mrr_at_1000_std": -0.06641928062359208, "nauc_mrr_at_1000_diff1": 0.7429244793968778, "hit_rate_at_1": 0.81588, "hit_rate_at_3": 0.87838, "hit_rate_at_5": 0.89274, "hit_rate_at_10": 0.90625, "hit_rate_at_20": 0.93328, "hit_rate_at_100": 0.9848, "hit_rate_at_1000": 1.0, "main_score": 0.86393, "hf_subset": "tr", "languages": [ "tur-Latn" ], "mteb_version": "2.18.16" } ] }, "evaluation_time": 5.384912968846038, "kg_co2_emissions": null, "date": "2026-08-11T08:38:00.395989Z", "evaluation_phases": [ { "name": "Data loading", "start": 0.0, "end": 2.4216172359883785, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 2.435616716975346, "end": 5.167067898903042, "split": "validation", "subset": "tr" }, { "name": "Scoring", "start": 5.167083007982001, "end": 5.382447937969118, "split": "validation", "subset": "tr" } ] }, { "dataset_revision": "f64f483ad0f31d2e78209d524c14a4a867965959", "task_name": "WebFAQRetrieval", "mteb_version": "2.18.16", "scores": { "test": [ { "ndcg_at_1": 0.4783, "ndcg_at_3": 0.53775, "ndcg_at_5": 0.55368, "ndcg_at_10": 0.56886, "ndcg_at_20": 0.58072, "ndcg_at_100": 0.59863, "ndcg_at_1000": 0.60486, "map_at_1": 0.4783, "map_at_3": 0.52333, "map_at_5": 0.53216, "map_at_10": 0.53846, "map_at_20": 0.5417, "map_at_100": 0.54424, "map_at_1000": 0.54448, "recall_at_1": 0.4783, "recall_at_3": 0.5794, "recall_at_5": 0.6181, "recall_at_10": 0.6649, "recall_at_20": 0.7119, "recall_at_100": 0.8074, "recall_at_1000": 0.8563, "accuracy": 0.4783, "precision_at_1": 0.4783, "precision_at_3": 0.19313, "precision_at_5": 0.12362, "precision_at_10": 0.06649, "precision_at_20": 0.0356, "precision_at_100": 0.00807, "precision_at_1000": 0.00086, "mrr_at_1": 0.4782, "mrr_at_3": 0.5233166666666667, "mrr_at_5": 0.5320866666666667, "mrr_at_10": 0.5383844841269841, "mrr_at_20": 0.5416332922284973, "mrr_at_100": 0.544169475461704, "mrr_at_1000": 0.5444125970973864, "nauc_ndcg_at_1_max": 0.12378526993465135, "nauc_ndcg_at_1_std": 0.15914555681902476, "nauc_ndcg_at_1_diff1": 0.8371664323172846, "nauc_ndcg_at_3_max": 0.059497603641565644, "nauc_ndcg_at_3_std": 0.19280724617182773, "nauc_ndcg_at_3_diff1": 0.8106765381725193, "nauc_ndcg_at_5_max": 0.03830703968134529, "nauc_ndcg_at_5_std": 0.20203097792669134, "nauc_ndcg_at_5_diff1": 0.8069760545861215, "nauc_ndcg_at_10_max": 0.01692778814755956, "nauc_ndcg_at_10_std": 0.2100322549145765, "nauc_ndcg_at_10_diff1": 0.805655549489814, "nauc_ndcg_at_20_max": 0.0018299159845378435, "nauc_ndcg_at_20_std": 0.21686590866569472, "nauc_ndcg_at_20_diff1": 0.8035360516982142, "nauc_ndcg_at_100_max": -0.023569682566652816, "nauc_ndcg_at_100_std": 0.2278208198263667, "nauc_ndcg_at_100_diff1": 0.8051237611879092, "nauc_ndcg_at_1000_max": -0.03534215224230123, "nauc_ndcg_at_1000_std": 0.23034163818382103, "nauc_ndcg_at_1000_diff1": 0.8088858890505707, "nauc_map_at_1_max": 0.12378526993465135, "nauc_map_at_1_std": 0.15914555681902476, "nauc_map_at_1_diff1": 0.8371664323172846, "nauc_map_at_3_max": 0.07601436468829656, "nauc_map_at_3_std": 0.1844036695986002, "nauc_map_at_3_diff1": 0.8172367412045666, "nauc_map_at_5_max": 0.06473499918618558, "nauc_map_at_5_std": 0.1892017055047459, "nauc_map_at_5_diff1": 0.8152117908664681, "nauc_map_at_10_max": 0.05655810792894781, "nauc_map_at_10_std": 0.1922956559395659, "nauc_map_at_10_diff1": 0.8148003875474032, "nauc_map_at_20_max": 0.052812545064260606, "nauc_map_at_20_std": 0.1939204688449605, "nauc_map_at_20_diff1": 0.814304031703465, "nauc_map_at_100_max": 0.049870914558898784, "nauc_map_at_100_std": 0.19525523675361117, "nauc_map_at_100_diff1": 0.8145625414323227, "nauc_map_at_1000_max": 0.04951462246634232, "nauc_map_at_1000_std": 0.19535656764417045, "nauc_map_at_1000_diff1": 0.814689665486308, "nauc_recall_at_1_max": 0.12378526993465135, "nauc_recall_at_1_std": 0.15914555681902476, "nauc_recall_at_1_diff1": 0.8371664323172846, "nauc_recall_at_3_max": 0.00906800781090249, "nauc_recall_at_3_std": 0.21836130557966743, "nauc_recall_at_3_diff1": 0.7907510757682494, "nauc_recall_at_5_max": -0.049291960498243304, "nauc_recall_at_5_std": 0.244415239072271, "nauc_recall_at_5_diff1": 0.780370627517092, "nauc_recall_at_10_max": -0.1334658927813473, "nauc_recall_at_10_std": 0.2763889626664622, "nauc_recall_at_10_diff1": 0.7730445065609719, "nauc_recall_at_20_max": -0.22424014427676828, "nauc_recall_at_20_std": 0.3179950761211255, "nauc_recall_at_20_diff1": 0.7589578131314744, "nauc_recall_at_100_max": -0.5378719532193261, "nauc_recall_at_100_std": 0.4528372991444859, "nauc_recall_at_100_diff1": 0.756077007675729, "nauc_recall_at_1000_max": -0.8933631153969248, "nauc_recall_at_1000_std": 0.5591965880094494, "nauc_recall_at_1000_diff1": 0.8049804885079451, "nauc_precision_at_1_max": 0.12378526993465135, "nauc_precision_at_1_std": 0.15914555681902476, "nauc_precision_at_1_diff1": 0.8371664323172846, "nauc_precision_at_3_max": 0.009068007810902291, "nauc_precision_at_3_std": 0.21836130557966757, "nauc_precision_at_3_diff1": 0.7907510757682494, "nauc_precision_at_5_max": -0.0492919604982433, "nauc_precision_at_5_std": 0.24441523907227103, "nauc_precision_at_5_diff1": 0.7803706275170921, "nauc_precision_at_10_max": -0.13346589278134638, "nauc_precision_at_10_std": 0.27638896266646273, "nauc_precision_at_10_diff1": 0.773044506560973, "nauc_precision_at_20_max": -0.22424014427676833, "nauc_precision_at_20_std": 0.31799507612112565, "nauc_precision_at_20_diff1": 0.758957813131475, "nauc_precision_at_100_max": -0.5378719532193269, "nauc_precision_at_100_std": 0.4528372991444852, "nauc_precision_at_100_diff1": 0.7560770076757304, "nauc_precision_at_1000_max": -0.8933631153969233, "nauc_precision_at_1000_std": 0.5591965880094497, "nauc_precision_at_1000_diff1": 0.8049804885079432, "nauc_mrr_at_1_max": 0.1234547429404733, "nauc_mrr_at_1_std": 0.1586471848506029, "nauc_mrr_at_1_diff1": 0.8374205635455932, "nauc_mrr_at_3_max": 0.0759591156474296, "nauc_mrr_at_3_std": 0.18432139049652596, "nauc_mrr_at_3_diff1": 0.8172997943423154, "nauc_mrr_at_5_max": 0.06458798468556289, "nauc_mrr_at_5_std": 0.18879150925263186, "nauc_mrr_at_5_diff1": 0.8153588873994831, "nauc_mrr_at_10_max": 0.05640819131273918, "nauc_mrr_at_10_std": 0.19202416957177562, "nauc_mrr_at_10_diff1": 0.8149415213237646, "nauc_mrr_at_20_max": 0.05261900861603199, "nauc_mrr_at_20_std": 0.19368254224028988, "nauc_mrr_at_20_diff1": 0.8144435264474017, "nauc_mrr_at_100_max": 0.04969749121247066, "nauc_mrr_at_100_std": 0.19500516541332255, "nauc_mrr_at_100_diff1": 0.8147021280654135, "nauc_mrr_at_1000_max": 0.04934573213223441, "nauc_mrr_at_1000_std": 0.1951105366077533, "nauc_mrr_at_1000_diff1": 0.8148301808658873, "hit_rate_at_1": 0.4783, "hit_rate_at_3": 0.5794, "hit_rate_at_5": 0.6181, "hit_rate_at_10": 0.6649, "hit_rate_at_20": 0.7119, "hit_rate_at_100": 0.8074, "hit_rate_at_1000": 0.8563, "main_score": 0.56886, "hf_subset": "tur", "languages": [ "tur-Latn" ], "mteb_version": "2.18.16" } ] }, "evaluation_time": 308.66258307010867, "kg_co2_emissions": null, "date": "2026-08-11T08:43:09.059761Z", "evaluation_phases": [ { "name": "Data loading", "start": 4.1369348764419556e-06, "end": 8.95585033413954, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 8.955858692061156, "end": 8.955860669957474, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 9.023173480993137, "end": 301.69988178205676, "split": "test", "subset": "tur" }, { "name": "Scoring", "start": 301.69990209210664, "end": 308.4712615769822, "split": "test", "subset": "tur" } ] }, { "dataset_revision": "3e069e7a30079d214a859741f5a0de75cf878867", "task_name": "MKQARetrieval", "mteb_version": "2.18.16", "scores": { "train": [ { "ndcg_at_1": 0.06139, "ndcg_at_3": 0.07844, "ndcg_at_5": 0.08876, "ndcg_at_10": 0.10331, "ndcg_at_20": 0.11739, "ndcg_at_100": 0.14806, "ndcg_at_1000": 0.19718, "map_at_1": 0.04632, "map_at_3": 0.06645, "map_at_5": 0.07352, "map_at_10": 0.07996, "map_at_20": 0.08404, "map_at_100": 0.08831, "map_at_1000": 0.08992, "recall_at_1": 0.04632, "recall_at_3": 0.08999, "recall_at_5": 0.11716, "recall_at_10": 0.15855, "recall_at_20": 0.20969, "recall_at_100": 0.36572, "recall_at_1000": 0.7445, "accuracy": 0.04632, "precision_at_1": 0.06139, "precision_at_3": 0.04172, "precision_at_5": 0.03307, "precision_at_10": 0.02291, "precision_at_20": 0.01526, "precision_at_100": 0.00531, "precision_at_1000": 0.00107, "mrr_at_1": 0.06138790035587188, "mrr_at_3": 0.08311091340450771, "mrr_at_5": 0.09005782918149466, "mrr_at_10": 0.09632983580937318, "mrr_at_20": 0.10029345491156058, "mrr_at_100": 0.10418328055808178, "mrr_at_1000": 0.10551962334234978, "nauc_ndcg_at_1_max": 0.19046090779456812, "nauc_ndcg_at_1_std": 0.2005678293225855, "nauc_ndcg_at_1_diff1": 0.2732613145926889, "nauc_ndcg_at_3_max": 0.154650069438559, "nauc_ndcg_at_3_std": 0.21864693259137372, "nauc_ndcg_at_3_diff1": 0.19561940316458706, "nauc_ndcg_at_5_max": 0.14091485629423525, "nauc_ndcg_at_5_std": 0.23477553800788697, "nauc_ndcg_at_5_diff1": 0.17968447890707392, "nauc_ndcg_at_10_max": 0.12493779940679503, "nauc_ndcg_at_10_std": 0.25141145550716937, "nauc_ndcg_at_10_diff1": 0.16150290010603824, "nauc_ndcg_at_20_max": 0.11065180702665893, "nauc_ndcg_at_20_std": 0.27110351591711057, "nauc_ndcg_at_20_diff1": 0.15703674738718967, "nauc_ndcg_at_100_max": 0.08421783643347659, "nauc_ndcg_at_100_std": 0.2946243071156509, "nauc_ndcg_at_100_diff1": 0.1502281839569693, "nauc_ndcg_at_1000_max": 0.060271168700708054, "nauc_ndcg_at_1000_std": 0.30321027407391904, "nauc_ndcg_at_1000_diff1": 0.17005046587139194, "nauc_map_at_1_max": 0.16299753614775253, "nauc_map_at_1_std": 0.16346220475254442, "nauc_map_at_1_diff1": 0.2917363544554654, "nauc_map_at_3_max": 0.15598673546990813, "nauc_map_at_3_std": 0.19971704494790168, "nauc_map_at_3_diff1": 0.21645522028299932, "nauc_map_at_5_max": 0.15166101798657583, "nauc_map_at_5_std": 0.21573359427337446, "nauc_map_at_5_diff1": 0.20455508216082804, "nauc_map_at_10_max": 0.14475635349933652, "nauc_map_at_10_std": 0.22759247812792957, "nauc_map_at_10_diff1": 0.19358022834017713, "nauc_map_at_20_max": 0.13899000442931903, "nauc_map_at_20_std": 0.23628409823009514, "nauc_map_at_20_diff1": 0.1906825576599391, "nauc_map_at_100_max": 0.13372591272884576, "nauc_map_at_100_std": 0.2432631891975316, "nauc_map_at_100_diff1": 0.188127691740388, "nauc_map_at_1000_max": 0.13264497426057695, "nauc_map_at_1000_std": 0.2439228359663457, "nauc_map_at_1000_diff1": 0.18877656163071965, "nauc_recall_at_1_max": 0.16299753614775253, "nauc_recall_at_1_std": 0.16346220475254442, "nauc_recall_at_1_diff1": 0.2917363544554654, "nauc_recall_at_3_max": 0.1314709043759448, "nauc_recall_at_3_std": 0.22219290324299643, "nauc_recall_at_3_diff1": 0.15892511021348496, "nauc_recall_at_5_max": 0.11041311908878675, "nauc_recall_at_5_std": 0.2479865816737892, "nauc_recall_at_5_diff1": 0.12973531101440325, "nauc_recall_at_10_max": 0.08352674383767103, "nauc_recall_at_10_std": 0.27801950065943004, "nauc_recall_at_10_diff1": 0.10277047520590306, "nauc_recall_at_20_max": 0.055451740458610316, "nauc_recall_at_20_std": 0.30864764930100463, "nauc_recall_at_20_diff1": 0.10009990485099715, "nauc_recall_at_100_max": -0.010313300454429164, "nauc_recall_at_100_std": 0.3507341463365123, "nauc_recall_at_100_diff1": 0.09178275311751917, "nauc_recall_at_1000_max": -0.20168627852829185, "nauc_recall_at_1000_std": 0.42587214313005395, "nauc_recall_at_1000_diff1": 0.18865741402834005, "nauc_precision_at_1_max": 0.19046090779456812, "nauc_precision_at_1_std": 0.2005678293225855, "nauc_precision_at_1_diff1": 0.2732613145926889, "nauc_precision_at_3_max": 0.17065346765787537, "nauc_precision_at_3_std": 0.2580216755377953, "nauc_precision_at_3_diff1": 0.15208690889778514, "nauc_precision_at_5_max": 0.15966584831210248, "nauc_precision_at_5_std": 0.29340982416875505, "nauc_precision_at_5_diff1": 0.13560040963190081, "nauc_precision_at_10_max": 0.13047182792970813, "nauc_precision_at_10_std": 0.3126115268876884, "nauc_precision_at_10_diff1": 0.11276306288171808, "nauc_precision_at_20_max": 0.09951766493736443, "nauc_precision_at_20_std": 0.35593195934746613, "nauc_precision_at_20_diff1": 0.10597363396032576, "nauc_precision_at_100_max": 0.04010877959852337, "nauc_precision_at_100_std": 0.378379077826633, "nauc_precision_at_100_diff1": 0.0922392067579821, "nauc_precision_at_1000_max": -0.044271818009240606, "nauc_precision_at_1000_std": 0.29888109273282804, "nauc_precision_at_1000_diff1": 0.09766770636218437, "nauc_mrr_at_1_max": 0.19046090779456812, "nauc_mrr_at_1_std": 0.2005678293225855, "nauc_mrr_at_1_diff1": 0.2732613145926889, "nauc_mrr_at_3_max": 0.16211133977600667, "nauc_mrr_at_3_std": 0.22287755102083415, "nauc_mrr_at_3_diff1": 0.20847534169317564, "nauc_mrr_at_5_max": 0.15334494533630705, "nauc_mrr_at_5_std": 0.2342332780831409, "nauc_mrr_at_5_diff1": 0.19804894502376114, "nauc_mrr_at_10_max": 0.1440542234453121, "nauc_mrr_at_10_std": 0.240522209249698, "nauc_mrr_at_10_diff1": 0.1893144127381246, "nauc_mrr_at_20_max": 0.13965220841407827, "nauc_mrr_at_20_std": 0.246658048198982, "nauc_mrr_at_20_diff1": 0.18779493799431327, "nauc_mrr_at_100_max": 0.1352372575762705, "nauc_mrr_at_100_std": 0.24993578620339646, "nauc_mrr_at_100_diff1": 0.1864436719961944, "nauc_mrr_at_1000_max": 0.13462182002208006, "nauc_mrr_at_1000_std": 0.25008212756311865, "nauc_mrr_at_1000_diff1": 0.18711915110093033, "hit_rate_at_1": 0.06139, "hit_rate_at_3": 0.11254, "hit_rate_at_5": 0.14339, "hit_rate_at_10": 0.19024, "hit_rate_at_20": 0.24763, "hit_rate_at_100": 0.41429, "hit_rate_at_1000": 0.78455, "main_score": 0.10331, "hf_subset": "tr", "languages": [ "tur-Latn" ], "mteb_version": "2.18.16" } ] }, "evaluation_time": 25.021837311098352, "kg_co2_emissions": null, "date": "2026-08-11T08:43:34.083720Z", "evaluation_phases": [ { "name": "Data loading", "start": 4.032161086797714e-06, "end": 6.687524263979867, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 6.687531795119867, "end": 6.6875344440341, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 6.735912636155263, "end": 21.24591747415252, "split": "train", "subset": "tr" }, { "name": "Scoring", "start": 21.245933528989553, "end": 24.96005431120284, "split": "train", "subset": "tr" } ] }, { "dataset_revision": "979a211276faa22f671e69d096634193567cfd05", "task_name": "BelebeleRetrieval", "mteb_version": "2.18.16", "scores": { "test": [ { "ndcg_at_1": 0.83778, "ndcg_at_3": 0.8729, "ndcg_at_5": 0.8793, "ndcg_at_10": 0.88493, "ndcg_at_20": 0.89217, "ndcg_at_100": 0.89819, "ndcg_at_1000": 0.89954, "map_at_1": 0.83778, "map_at_3": 0.86463, "map_at_5": 0.86819, "map_at_10": 0.87044, "map_at_20": 0.87239, "map_at_100": 0.87331, "map_at_1000": 0.87337, "recall_at_1": 0.83778, "recall_at_3": 0.89667, "recall_at_5": 0.91222, "recall_at_10": 0.93, "recall_at_20": 0.95889, "recall_at_100": 0.99, "recall_at_1000": 1.0, "accuracy": 0.83778, "precision_at_1": 0.83778, "precision_at_3": 0.29889, "precision_at_5": 0.18244, "precision_at_10": 0.093, "precision_at_20": 0.04794, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.8377777777777777, "mrr_at_3": 0.8646296296296296, "mrr_at_5": 0.8676296296296296, "mrr_at_10": 0.8702103174603174, "mrr_at_20": 0.8720055372139995, "mrr_at_100": 0.8729395151413755, "mrr_at_1000": 0.8730008378211206, "nauc_ndcg_at_1_max": 0.8410244670966699, "nauc_ndcg_at_1_std": 0.06830117285862436, "nauc_ndcg_at_1_diff1": 0.920135195184682, "nauc_ndcg_at_3_max": 0.8482741017353032, "nauc_ndcg_at_3_std": 0.10547749274165154, "nauc_ndcg_at_3_diff1": 0.8954013102092868, "nauc_ndcg_at_5_max": 0.8494532972001592, "nauc_ndcg_at_5_std": 0.12742254228373348, "nauc_ndcg_at_5_diff1": 0.8968040492636123, "nauc_ndcg_at_10_max": 0.8533332736310034, "nauc_ndcg_at_10_std": 0.149236559008684, "nauc_ndcg_at_10_diff1": 0.8992038888020082, "nauc_ndcg_at_20_max": 0.851087799934009, "nauc_ndcg_at_20_std": 0.1380363533185576, "nauc_ndcg_at_20_diff1": 0.8999492972758715, "nauc_ndcg_at_100_max": 0.8513927339831183, "nauc_ndcg_at_100_std": 0.12804271152305366, "nauc_ndcg_at_100_diff1": 0.9025952980091414, "nauc_ndcg_at_1000_max": 0.8503838452506185, "nauc_ndcg_at_1000_std": 0.12168496863829963, "nauc_ndcg_at_1000_diff1": 0.9025192818485014, "nauc_map_at_1_max": 0.8410244670966699, "nauc_map_at_1_std": 0.06830117285862436, "nauc_map_at_1_diff1": 0.920135195184682, "nauc_map_at_3_max": 0.8472124322730213, "nauc_map_at_3_std": 0.09524294429360917, "nauc_map_at_3_diff1": 0.9024453786958063, "nauc_map_at_5_max": 0.8476935399614086, "nauc_map_at_5_std": 0.10596944315332903, "nauc_map_at_5_diff1": 0.9032210546789997, "nauc_map_at_10_max": 0.8491278258232463, "nauc_map_at_10_std": 0.1139024265659601, "nauc_map_at_10_diff1": 0.904189158426485, "nauc_map_at_20_max": 0.8486364332677006, "nauc_map_at_20_std": 0.11146918219618034, "nauc_map_at_20_diff1": 0.9044222430772109, "nauc_map_at_100_max": 0.8487478052591781, "nauc_map_at_100_std": 0.10976937220975148, "nauc_map_at_100_diff1": 0.9047515896507614, "nauc_map_at_1000_max": 0.8487193359313112, "nauc_map_at_1000_std": 0.10956032718263647, "nauc_map_at_1000_diff1": 0.9047484933605456, "nauc_recall_at_1_max": 0.8410244670966699, "nauc_recall_at_1_std": 0.06830117285862436, "nauc_recall_at_1_diff1": 0.920135195184682, "nauc_recall_at_3_max": 0.851642922678282, "nauc_recall_at_3_std": 0.14283193692467533, "nauc_recall_at_3_diff1": 0.8695643406632181, "nauc_recall_at_5_max": 0.8566109988299108, "nauc_recall_at_5_std": 0.22182037371910784, "nauc_recall_at_5_diff1": 0.8704629531137352, "nauc_recall_at_10_max": 0.8772175536881434, "nauc_recall_at_10_std": 0.3506513716597738, "nauc_recall_at_10_diff1": 0.8759133282942827, "nauc_recall_at_20_max": 0.8692305751129306, "nauc_recall_at_20_std": 0.36314886314886435, "nauc_recall_at_20_diff1": 0.867451485098547, "nauc_recall_at_100_max": 0.9286751737731979, "nauc_recall_at_100_std": 0.6017221703496162, "nauc_recall_at_100_diff1": 0.9074074074074023, "nauc_recall_at_1000_max": null, "nauc_recall_at_1000_std": null, "nauc_recall_at_1000_diff1": null, "nauc_precision_at_1_max": 0.8410244670966699, "nauc_precision_at_1_std": 0.06830117285862436, "nauc_precision_at_1_diff1": 0.920135195184682, "nauc_precision_at_3_max": 0.8516429226782833, "nauc_precision_at_3_std": 0.14283193692467333, "nauc_precision_at_3_diff1": 0.8695643406632189, "nauc_precision_at_5_max": 0.8566109988299109, "nauc_precision_at_5_std": 0.22182037371910543, "nauc_precision_at_5_diff1": 0.8704629531137342, "nauc_precision_at_10_max": 0.8772175536881401, "nauc_precision_at_10_std": 0.35065137165977484, "nauc_precision_at_10_diff1": 0.8759133282942804, "nauc_precision_at_20_max": 0.8692305751129216, "nauc_precision_at_20_std": 0.3631488631488554, "nauc_precision_at_20_diff1": 0.8674514850985429, "nauc_precision_at_100_max": 0.928675173773221, "nauc_precision_at_100_std": 0.6017221703496362, "nauc_precision_at_100_diff1": 0.9074074074074131, "nauc_precision_at_1000_max": null, "nauc_precision_at_1000_std": null, "nauc_precision_at_1000_diff1": null, "nauc_mrr_at_1_max": 0.8410244670966699, "nauc_mrr_at_1_std": 0.06830117285862436, "nauc_mrr_at_1_diff1": 0.920135195184682, "nauc_mrr_at_3_max": 0.8472124322730213, "nauc_mrr_at_3_std": 0.09524294429360917, "nauc_mrr_at_3_diff1": 0.9024453786958063, "nauc_mrr_at_5_max": 0.8483553382502018, "nauc_mrr_at_5_std": 0.10985416276732428, "nauc_mrr_at_5_diff1": 0.9036415761748603, "nauc_mrr_at_10_max": 0.8495478126860134, "nauc_mrr_at_10_std": 0.1158882768620596, "nauc_mrr_at_10_diff1": 0.9052043399121691, "nauc_mrr_at_20_max": 0.8494438987722414, "nauc_mrr_at_20_std": 0.11432323610147117, "nauc_mrr_at_20_diff1": 0.9048993393892497, "nauc_mrr_at_100_max": 0.8494035670115612, "nauc_mrr_at_100_std": 0.11245996264042155, "nauc_mrr_at_100_diff1": 0.9053150039850627, "nauc_mrr_at_1000_max": 0.8493755127679687, "nauc_mrr_at_1000_std": 0.11225290183205333, "nauc_mrr_at_1000_diff1": 0.9053121992092529, "hit_rate_at_1": 0.83778, "hit_rate_at_3": 0.89667, "hit_rate_at_5": 0.91222, "hit_rate_at_10": 0.93, "hit_rate_at_20": 0.95889, "hit_rate_at_100": 0.99, "hit_rate_at_1000": 1.0, "main_score": 0.88493, "hf_subset": "tur_Latn-tur_Latn", "languages": [ "tur-Latn", "tur-Latn" ], "mteb_version": "2.18.16" } ] }, "evaluation_time": 4.633956309873611, "kg_co2_emissions": null, "date": "2026-08-11T08:43:38.719847Z", "evaluation_phases": [ { "name": "Data loading", "start": 0.0, "end": 1.653581827878952, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 1.6664734289515764, "end": 4.349166231928393, "split": "test", "subset": "tur_Latn-tur_Latn" }, { "name": "Scoring", "start": 4.349181247875094, "end": 4.6296024329494685, "split": "test", "subset": "tur_Latn-tur_Latn" } ] } ], "exceptions": [], "experiment_name": null } }