diff --git "a/scores/LamRA-Ret.json" "b/scores/LamRA-Ret.json" --- "a/scores/LamRA-Ret.json" +++ "b/scores/LamRA-Ret.json" @@ -1,7 +1,7 @@ { "metadata": { "model_name": "LamRA-Ret", - "report_generated_date": "2025-06-28T08:54:18.459213", + "report_generated_date": "2025-12-29T09:27:50.385069", "model_size": 8.29, "url": "https://huggingface.co/code-kunkun/LamRA-Ret", "data_source": "TIGER-Lab" @@ -9,1010 +9,1010 @@ "metrics": { "image": { "ImageNet-1K": { - "hit@1": 0.723, + "hit@1": 0.726, "hit@5": 0.91, - "hit@10": 0.944, - "ndcg_linear@1": 0.723, - "ndcg_linear@5": 0.8268476572075378, - "ndcg_linear@10": 0.8378788241843189, - "ndcg_exponential@1": 0.723, - "ndcg_exponential@5": 0.8268476572075378, - "ndcg_exponential@10": 0.8378788241843189, - "precision@1": 0.723, + "hit@10": 0.943, + "ndcg_linear@1": 0.726, + "ndcg_linear@5": 0.8274672555530947, + "ndcg_linear@10": 0.8381890195352347, + "ndcg_exponential@1": 0.726, + "ndcg_exponential@5": 0.8274672555530947, + "ndcg_exponential@10": 0.8381890195352347, + "precision@1": 0.726, "precision@5": 0.18200000000000002, - "precision@10": 0.09440000000000001, - "recall@1": 0.723, + "precision@10": 0.09430000000000001, + "recall@1": 0.726, "recall@5": 0.91, - "recall@10": 0.944, - "f1@1": 0.723, + "recall@10": 0.943, + "f1@1": 0.726, "f1@5": 0.30333333333333334, - "f1@10": 0.17163636363636361, - "map@1": 0.723, - "map@5": 0.7987666666666666, - "map@10": 0.8033428571428571, - "mrr@1": 0.723, - "mrr@5": 0.7987666666666666, - "mrr@10": 0.8033428571428571, + "f1@10": 0.17145454545454544, + "map@1": 0.726, + "map@5": 0.7996500000000001, + "map@10": 0.8041055555555555, + "mrr@1": 0.726, + "mrr@5": 0.7996500000000001, + "mrr@10": 0.8041055555555555, "num_pred": 1000, "num_data": 1000 }, "N24News": { - "hit@1": 0.513, - "hit@5": 0.804, + "hit@1": 0.516, + "hit@5": 0.802, "hit@10": 0.929, - "ndcg_linear@1": 0.513, - "ndcg_linear@5": 0.6693842522947794, - "ndcg_linear@10": 0.7099458858472819, - "ndcg_exponential@1": 0.513, - "ndcg_exponential@5": 0.6693842522947794, - "ndcg_exponential@10": 0.7099458858472819, - "precision@1": 0.513, - "precision@5": 0.1608, + "ndcg_linear@1": 0.516, + "ndcg_linear@5": 0.6699102934848121, + "ndcg_linear@10": 0.7111290056251437, + "ndcg_exponential@1": 0.516, + "ndcg_exponential@5": 0.6699102934848121, + "ndcg_exponential@10": 0.7111290056251437, + "precision@1": 0.516, + "precision@5": 0.16040000000000001, "precision@10": 0.09290000000000001, - "recall@1": 0.513, - "recall@5": 0.804, + "recall@1": 0.516, + "recall@5": 0.802, "recall@10": 0.929, - "f1@1": 0.513, - "f1@5": 0.268, - "f1@10": 0.1689090909090909, - "map@1": 0.513, - "map@5": 0.6244166666666666, - "map@10": 0.641238492063492, - "mrr@1": 0.513, - "mrr@5": 0.6244166666666666, - "mrr@10": 0.641238492063492, + "f1@1": 0.516, + "f1@5": 0.26733333333333337, + "f1@10": 0.16890909090909087, + "map@1": 0.516, + "map@5": 0.6257666666666667, + "map@10": 0.6428686507936509, + "mrr@1": 0.516, + "mrr@5": 0.6257666666666667, + "mrr@10": 0.6428686507936509, "num_pred": 1000, "num_data": 1000 }, "HatefulMemes": { - "hit@1": 0.49, + "hit@1": 0.491, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.49, - "ndcg_linear@5": 0.8117741743214435, - "ndcg_linear@10": 0.8117741743214435, - "ndcg_exponential@1": 0.49, - "ndcg_exponential@5": 0.8117741743214435, - "ndcg_exponential@10": 0.8117741743214435, - "precision@1": 0.49, + "ndcg_linear@1": 0.491, + "ndcg_linear@5": 0.812143244567872, + "ndcg_linear@10": 0.812143244567872, + "ndcg_exponential@1": 0.491, + "ndcg_exponential@5": 0.812143244567872, + "ndcg_exponential@10": 0.812143244567872, + "precision@1": 0.491, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.49, + "recall@1": 0.491, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.49, + "f1@1": 0.491, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.49, - "map@5": 0.745, - "map@10": 0.745, - "mrr@1": 0.49, - "mrr@5": 0.745, - "mrr@10": 0.745, + "map@1": 0.491, + "map@5": 0.7455, + "map@10": 0.7455, + "mrr@1": 0.491, + "mrr@5": 0.7455, + "mrr@10": 0.7455, "num_pred": 1000, "num_data": 1000 }, "VOC2007": { - "hit@1": 0.801, - "hit@5": 0.976, - "hit@10": 0.992, - "ndcg_linear@1": 0.801, - "ndcg_linear@5": 0.9010597839851336, - "ndcg_linear@10": 0.9062744300991276, - "ndcg_exponential@1": 0.801, - "ndcg_exponential@5": 0.9010597839851336, - "ndcg_exponential@10": 0.9062744300991276, - "precision@1": 0.801, - "precision@5": 0.1952, - "precision@10": 0.09920000000000001, - "recall@1": 0.801, - "recall@5": 0.976, - "recall@10": 0.992, - "f1@1": 0.801, - "f1@5": 0.32533333333333336, - "f1@10": 0.18036363636363634, - "map@1": 0.801, - "map@5": 0.8754666666666667, - "map@10": 0.877640873015873, - "mrr@1": 0.801, - "mrr@5": 0.8754666666666667, - "mrr@10": 0.877640873015873, + "hit@1": 0.8, + "hit@5": 0.975, + "hit@10": 0.994, + "ndcg_linear@1": 0.8, + "ndcg_linear@5": 0.9006095441894524, + "ndcg_linear@10": 0.9067560574314143, + "ndcg_exponential@1": 0.8, + "ndcg_exponential@5": 0.9006095441894524, + "ndcg_exponential@10": 0.9067560574314143, + "precision@1": 0.8, + "precision@5": 0.195, + "precision@10": 0.09940000000000002, + "recall@1": 0.8, + "recall@5": 0.975, + "recall@10": 0.994, + "f1@1": 0.8, + "f1@5": 0.325, + "f1@10": 0.18072727272727268, + "map@1": 0.8, + "map@5": 0.87515, + "map@10": 0.8776880952380952, + "mrr@1": 0.8, + "mrr@5": 0.87515, + "mrr@10": 0.8776880952380952, "num_pred": 1000, "num_data": 1000 }, "SUN397": { - "hit@1": 0.685, - "hit@5": 0.928, + "hit@1": 0.687, + "hit@5": 0.926, "hit@10": 0.957, - "ndcg_linear@1": 0.685, - "ndcg_linear@5": 0.8224140927675592, - "ndcg_linear@10": 0.8318463032340889, - "ndcg_exponential@1": 0.685, - "ndcg_exponential@5": 0.8224140927675592, - "ndcg_exponential@10": 0.8318463032340889, - "precision@1": 0.685, - "precision@5": 0.18560000000000001, + "ndcg_linear@1": 0.687, + "ndcg_linear@5": 0.8220627789574411, + "ndcg_linear@10": 0.832235217075513, + "ndcg_exponential@1": 0.687, + "ndcg_exponential@5": 0.8220627789574411, + "ndcg_exponential@10": 0.832235217075513, + "precision@1": 0.687, + "precision@5": 0.1852, "precision@10": 0.09570000000000002, - "recall@1": 0.685, - "recall@5": 0.928, + "recall@1": 0.687, + "recall@5": 0.926, "recall@10": 0.957, - "f1@1": 0.685, - "f1@5": 0.30933333333333335, + "f1@1": 0.687, + "f1@5": 0.3086666666666667, "f1@10": 0.174, - "map@1": 0.685, - "map@5": 0.7865500000000001, - "map@10": 0.7904702380952381, - "mrr@1": 0.685, - "mrr@5": 0.7865500000000001, - "mrr@10": 0.7904702380952381, + "map@1": 0.687, + "map@5": 0.7867333333333334, + "map@10": 0.7910162698412698, + "mrr@1": 0.687, + "mrr@5": 0.7867333333333334, + "mrr@10": 0.7910162698412698, "num_pred": 1000, "num_data": 1000 }, "Place365": { - "hit@1": 0.406, - "hit@5": 0.684, - "hit@10": 0.787, - "ndcg_linear@1": 0.406, - "ndcg_linear@5": 0.5555467482483895, - "ndcg_linear@10": 0.5889145891921053, - "ndcg_exponential@1": 0.406, - "ndcg_exponential@5": 0.5555467482483895, - "ndcg_exponential@10": 0.5889145891921053, - "precision@1": 0.406, - "precision@5": 0.1368, - "precision@10": 0.0787, - "recall@1": 0.406, - "recall@5": 0.684, - "recall@10": 0.787, - "f1@1": 0.406, - "f1@5": 0.22800000000000006, - "f1@10": 0.1430909090909091, - "map@1": 0.406, - "map@5": 0.5126499999999999, - "map@10": 0.5264646825396826, - "mrr@1": 0.406, - "mrr@5": 0.5126499999999999, - "mrr@10": 0.5264646825396826, + "hit@1": 0.405, + "hit@5": 0.688, + "hit@10": 0.793, + "ndcg_linear@1": 0.405, + "ndcg_linear@5": 0.5560372236416723, + "ndcg_linear@10": 0.5897922531244657, + "ndcg_exponential@1": 0.405, + "ndcg_exponential@5": 0.5560372236416723, + "ndcg_exponential@10": 0.5897922531244657, + "precision@1": 0.405, + "precision@5": 0.1376, + "precision@10": 0.0793, + "recall@1": 0.405, + "recall@5": 0.688, + "recall@10": 0.793, + "f1@1": 0.405, + "f1@5": 0.22933333333333336, + "f1@10": 0.1441818181818182, + "map@1": 0.405, + "map@5": 0.5120833333333332, + "map@10": 0.5259043650793651, + "mrr@1": 0.405, + "mrr@5": 0.5120833333333332, + "mrr@10": 0.5259043650793651, "num_pred": 1000, "num_data": 1000 }, "ImageNet-A": { - "hit@1": 0.47, - "hit@5": 0.683, - "hit@10": 0.776, - "ndcg_linear@1": 0.47, - "ndcg_linear@5": 0.5846080932069478, - "ndcg_linear@10": 0.6143872769865809, - "ndcg_exponential@1": 0.47, - "ndcg_exponential@5": 0.5846080932069478, - "ndcg_exponential@10": 0.6143872769865809, - "precision@1": 0.47, - "precision@5": 0.13659999999999997, - "precision@10": 0.07759999999999999, - "recall@1": 0.47, - "recall@5": 0.683, - "recall@10": 0.776, - "f1@1": 0.47, - "f1@5": 0.2276666666666667, - "f1@10": 0.1410909090909091, - "map@1": 0.47, - "map@5": 0.5517333333333333, - "map@10": 0.5638619047619047, - "mrr@1": 0.47, - "mrr@5": 0.5517333333333333, - "mrr@10": 0.5638619047619047, + "hit@1": 0.472, + "hit@5": 0.688, + "hit@10": 0.781, + "ndcg_linear@1": 0.472, + "ndcg_linear@5": 0.587211174294051, + "ndcg_linear@10": 0.6169069190781477, + "ndcg_exponential@1": 0.472, + "ndcg_exponential@5": 0.587211174294051, + "ndcg_exponential@10": 0.6169069190781477, + "precision@1": 0.472, + "precision@5": 0.13759999999999997, + "precision@10": 0.07809999999999999, + "recall@1": 0.472, + "recall@5": 0.688, + "recall@10": 0.781, + "f1@1": 0.472, + "f1@5": 0.22933333333333336, + "f1@10": 0.142, + "map@1": 0.472, + "map@5": 0.55365, + "map@10": 0.5656960317460319, + "mrr@1": 0.472, + "mrr@5": 0.55365, + "mrr@10": 0.5656960317460319, "num_pred": 1000, "num_data": 1000 }, "ImageNet-R": { - "hit@1": 0.885, - "hit@5": 0.968, + "hit@1": 0.884, + "hit@5": 0.969, "hit@10": 0.981, - "ndcg_linear@1": 0.885, - "ndcg_linear@5": 0.9317946929268162, - "ndcg_linear@10": 0.9361395162551376, - "ndcg_exponential@1": 0.885, - "ndcg_exponential@5": 0.9317946929268162, - "ndcg_exponential@10": 0.9361395162551376, - "precision@1": 0.885, - "precision@5": 0.19360000000000002, + "ndcg_linear@1": 0.884, + "ndcg_linear@5": 0.9318124754876221, + "ndcg_linear@10": 0.9358060970260628, + "ndcg_exponential@1": 0.884, + "ndcg_exponential@5": 0.9318124754876221, + "ndcg_exponential@10": 0.9358060970260628, + "precision@1": 0.884, + "precision@5": 0.1938, "precision@10": 0.09810000000000002, - "recall@1": 0.885, - "recall@5": 0.968, + "recall@1": 0.884, + "recall@5": 0.969, "recall@10": 0.981, - "f1@1": 0.885, - "f1@5": 0.3226666666666666, + "f1@1": 0.884, + "f1@5": 0.323, "f1@10": 0.17836363636363634, - "map@1": 0.885, - "map@5": 0.9195166666666666, - "map@10": 0.9213904761904762, - "mrr@1": 0.885, - "mrr@5": 0.9195166666666666, - "mrr@10": 0.9213904761904762, + "map@1": 0.884, + "map@5": 0.9192166666666667, + "map@10": 0.920929761904762, + "mrr@1": 0.884, + "mrr@5": 0.9192166666666667, + "mrr@10": 0.920929761904762, "num_pred": 1000, "num_data": 1000 }, "ObjectNet": { - "hit@1": 0.664, + "hit@1": 0.663, "hit@5": 0.857, - "hit@10": 0.904, - "ndcg_linear@1": 0.664, - "ndcg_linear@5": 0.770759337476306, - "ndcg_linear@10": 0.7861961843019063, - "ndcg_exponential@1": 0.664, - "ndcg_exponential@5": 0.770759337476306, - "ndcg_exponential@10": 0.7861961843019063, - "precision@1": 0.664, + "hit@10": 0.908, + "ndcg_linear@1": 0.663, + "ndcg_linear@5": 0.7703286609182326, + "ndcg_linear@10": 0.7868437819775616, + "ndcg_exponential@1": 0.663, + "ndcg_exponential@5": 0.7703286609182326, + "ndcg_exponential@10": 0.7868437819775616, + "precision@1": 0.663, "precision@5": 0.17140000000000002, - "precision@10": 0.09040000000000001, - "recall@1": 0.664, + "precision@10": 0.0908, + "recall@1": 0.663, "recall@5": 0.857, - "recall@10": 0.904, - "f1@1": 0.664, + "recall@10": 0.908, + "f1@1": 0.663, "f1@5": 0.28566666666666674, - "f1@10": 0.16436363636363635, - "map@1": 0.664, - "map@5": 0.7417666666666667, - "map@10": 0.7482793650793651, - "mrr@1": 0.664, - "mrr@5": 0.7417666666666667, - "mrr@10": 0.7482793650793651, + "f1@10": 0.1650909090909091, + "map@1": 0.663, + "map@5": 0.7411833333333333, + "map@10": 0.7480170634920634, + "mrr@1": 0.663, + "mrr@5": 0.7411833333333333, + "mrr@10": 0.7480170634920634, "num_pred": 1000, "num_data": 1000 }, "Country211": { "hit@1": 0.283, "hit@5": 0.535, - "hit@10": 0.628, + "hit@10": 0.629, "ndcg_linear@1": 0.283, - "ndcg_linear@5": 0.4166269842150015, - "ndcg_linear@10": 0.44677345697388055, + "ndcg_linear@5": 0.4175838849427079, + "ndcg_linear@10": 0.4480794198740414, "ndcg_exponential@1": 0.283, - "ndcg_exponential@5": 0.4166269842150015, - "ndcg_exponential@10": 0.44677345697388055, + "ndcg_exponential@5": 0.4175838849427079, + "ndcg_exponential@10": 0.4480794198740414, "precision@1": 0.283, "precision@5": 0.107, - "precision@10": 0.0628, + "precision@10": 0.0629, "recall@1": 0.283, "recall@5": 0.535, - "recall@10": 0.628, + "recall@10": 0.629, "f1@1": 0.283, "f1@5": 0.17833333333333337, - "f1@10": 0.11418181818181819, + "f1@10": 0.11436363636363639, "map@1": 0.283, - "map@5": 0.37723333333333336, - "map@10": 0.3897083333333334, + "map@5": 0.37845, + "map@10": 0.3910920634920635, "mrr@1": 0.283, - "mrr@5": 0.37723333333333336, - "mrr@10": 0.3897083333333334, + "mrr@5": 0.37845, + "mrr@10": 0.3910920634920635, "num_pred": 1000, "num_data": 1000 }, "OK-VQA": { - "hit@1": 0.378, - "hit@5": 0.65, + "hit@1": 0.379, + "hit@5": 0.648, "hit@10": 0.748, - "ndcg_linear@1": 0.378, - "ndcg_linear@5": 0.523495748191076, - "ndcg_linear@10": 0.5551119641095289, - "ndcg_exponential@1": 0.378, - "ndcg_exponential@5": 0.523495748191076, - "ndcg_exponential@10": 0.5551119641095289, - "precision@1": 0.378, - "precision@5": 0.13, + "ndcg_linear@1": 0.379, + "ndcg_linear@5": 0.5233357312683175, + "ndcg_linear@10": 0.5556244245193712, + "ndcg_exponential@1": 0.379, + "ndcg_exponential@5": 0.5233357312683175, + "ndcg_exponential@10": 0.5556244245193712, + "precision@1": 0.379, + "precision@5": 0.1296, "precision@10": 0.07479999999999999, - "recall@1": 0.378, - "recall@5": 0.65, + "recall@1": 0.379, + "recall@5": 0.648, "recall@10": 0.748, - "f1@1": 0.378, - "f1@5": 0.2166666666666667, + "f1@1": 0.379, + "f1@5": 0.21600000000000005, "f1@10": 0.136, - "map@1": 0.378, - "map@5": 0.4814, - "map@10": 0.49440396825396826, - "mrr@1": 0.378, - "mrr@5": 0.4814, - "mrr@10": 0.49440396825396826, + "map@1": 0.379, + "map@5": 0.4817833333333333, + "map@10": 0.4950825396825397, + "mrr@1": 0.379, + "mrr@5": 0.4817833333333333, + "mrr@10": 0.4950825396825397, "num_pred": 1000, "num_data": 1000 }, "A-OKVQA": { - "hit@1": 0.27, - "hit@5": 0.566, - "hit@10": 0.693, - "ndcg_linear@1": 0.27, - "ndcg_linear@5": 0.425833923116744, - "ndcg_linear@10": 0.46711405957319874, - "ndcg_exponential@1": 0.27, - "ndcg_exponential@5": 0.425833923116744, - "ndcg_exponential@10": 0.46711405957319874, - "precision@1": 0.27, - "precision@5": 0.1132, - "precision@10": 0.0693, - "recall@1": 0.27, - "recall@5": 0.566, - "recall@10": 0.693, - "f1@1": 0.27, - "f1@5": 0.1886666666666667, - "f1@10": 0.126, - "map@1": 0.27, - "map@5": 0.3793666666666667, - "map@10": 0.39652301587301586, - "mrr@1": 0.27, - "mrr@5": 0.3793666666666667, - "mrr@10": 0.39652301587301586, + "hit@1": 0.269, + "hit@5": 0.571, + "hit@10": 0.694, + "ndcg_linear@1": 0.269, + "ndcg_linear@5": 0.42740683403676993, + "ndcg_linear@10": 0.46736896993098753, + "ndcg_exponential@1": 0.269, + "ndcg_exponential@5": 0.42740683403676993, + "ndcg_exponential@10": 0.46736896993098753, + "precision@1": 0.269, + "precision@5": 0.11419999999999998, + "precision@10": 0.0694, + "recall@1": 0.269, + "recall@5": 0.571, + "recall@10": 0.694, + "f1@1": 0.269, + "f1@5": 0.19033333333333338, + "f1@10": 0.1261818181818182, + "map@1": 0.269, + "map@5": 0.3798666666666667, + "map@10": 0.39646626984126987, + "mrr@1": 0.269, + "mrr@5": 0.3798666666666667, + "mrr@10": 0.39646626984126987, "num_pred": 1000, "num_data": 1000 }, "DocVQA": { - "hit@1": 0.223, - "hit@5": 0.382, - "hit@10": 0.448, - "ndcg_linear@1": 0.223, - "ndcg_linear@5": 0.3079314538526157, - "ndcg_linear@10": 0.32933154522617863, - "ndcg_exponential@1": 0.223, - "ndcg_exponential@5": 0.3079314538526157, - "ndcg_exponential@10": 0.32933154522617863, - "precision@1": 0.223, - "precision@5": 0.0764, - "precision@10": 0.0448, - "recall@1": 0.223, - "recall@5": 0.382, - "recall@10": 0.448, - "f1@1": 0.223, - "f1@5": 0.12733333333333338, - "f1@10": 0.08145454545454547, - "map@1": 0.223, - "map@5": 0.28323333333333334, - "map@10": 0.2921019841269841, - "mrr@1": 0.223, - "mrr@5": 0.28323333333333334, - "mrr@10": 0.2921019841269841, + "hit@1": 0.224, + "hit@5": 0.383, + "hit@10": 0.444, + "ndcg_linear@1": 0.224, + "ndcg_linear@5": 0.30862522909568885, + "ndcg_linear@10": 0.3283895045873121, + "ndcg_exponential@1": 0.224, + "ndcg_exponential@5": 0.30862522909568885, + "ndcg_exponential@10": 0.3283895045873121, + "precision@1": 0.224, + "precision@5": 0.07659999999999999, + "precision@10": 0.0444, + "recall@1": 0.224, + "recall@5": 0.383, + "recall@10": 0.444, + "f1@1": 0.224, + "f1@5": 0.1276666666666667, + "f1@10": 0.08072727272727274, + "map@1": 0.224, + "map@5": 0.2838666666666666, + "map@10": 0.2920448412698413, + "mrr@1": 0.224, + "mrr@5": 0.2838666666666666, + "mrr@10": 0.2920448412698413, "num_pred": 1000, "num_data": 1000 }, "InfographicsVQA": { "hit@1": 0.165, - "hit@5": 0.304, - "hit@10": 0.351, + "hit@5": 0.303, + "hit@10": 0.349, "ndcg_linear@1": 0.165, - "ndcg_linear@5": 0.24083608989579428, - "ndcg_linear@10": 0.25582539133402193, + "ndcg_linear@5": 0.24042084759828442, + "ndcg_linear@10": 0.2550576266410403, "ndcg_exponential@1": 0.165, - "ndcg_exponential@5": 0.24083608989579428, - "ndcg_exponential@10": 0.25582539133402193, + "ndcg_exponential@5": 0.24042084759828442, + "ndcg_exponential@10": 0.2550576266410403, "precision@1": 0.165, - "precision@5": 0.0608, - "precision@10": 0.03509999999999999, + "precision@5": 0.060599999999999994, + "precision@10": 0.034899999999999994, "recall@1": 0.165, - "recall@5": 0.304, - "recall@10": 0.351, + "recall@5": 0.303, + "recall@10": 0.349, "f1@1": 0.165, - "f1@5": 0.10133333333333334, - "f1@10": 0.06381818181818183, + "f1@5": 0.10100000000000002, + "f1@10": 0.06345454545454546, "map@1": 0.165, - "map@5": 0.21964999999999998, - "map@10": 0.22571349206349206, + "map@5": 0.2194, + "map@10": 0.22529563492063492, "mrr@1": 0.165, - "mrr@5": 0.21964999999999998, - "mrr@10": 0.22571349206349206, + "mrr@5": 0.2194, + "mrr@10": 0.22529563492063492, "num_pred": 1000, "num_data": 1000 }, "ChartQA": { - "hit@1": 0.117, - "hit@5": 0.162, + "hit@1": 0.116, + "hit@5": 0.158, "hit@10": 0.18, - "ndcg_linear@1": 0.117, - "ndcg_linear@5": 0.14132891302231343, - "ndcg_linear@10": 0.14726088628648307, - "ndcg_exponential@1": 0.117, - "ndcg_exponential@5": 0.14132891302231343, - "ndcg_exponential@10": 0.14726088628648307, - "precision@1": 0.117, - "precision@5": 0.032400000000000005, + "ndcg_linear@1": 0.116, + "ndcg_linear@5": 0.13915057203980383, + "ndcg_linear@10": 0.14655516875611413, + "ndcg_exponential@1": 0.116, + "ndcg_exponential@5": 0.13915057203980383, + "ndcg_exponential@10": 0.14655516875611413, + "precision@1": 0.116, + "precision@5": 0.0316, "precision@10": 0.018, - "recall@1": 0.117, - "recall@5": 0.162, + "recall@1": 0.116, + "recall@5": 0.158, "recall@10": 0.18, - "f1@1": 0.117, - "f1@5": 0.054, + "f1@1": 0.116, + "f1@5": 0.052666666666666674, "f1@10": 0.03272727272727273, - "map@1": 0.117, - "map@5": 0.13443333333333335, - "map@10": 0.13694960317460317, - "mrr@1": 0.117, - "mrr@5": 0.13443333333333335, - "mrr@10": 0.13694960317460317, + "map@1": 0.116, + "map@5": 0.1328, + "map@10": 0.13602698412698414, + "mrr@1": 0.116, + "mrr@5": 0.1328, + "mrr@10": 0.13602698412698414, "num_pred": 1000, "num_data": 1000 }, "Visual7W": { - "hit@1": 0.196, - "hit@5": 0.434, - "hit@10": 0.541, - "ndcg_linear@1": 0.196, - "ndcg_linear@5": 0.32114419847722886, - "ndcg_linear@10": 0.35574139683294936, - "ndcg_exponential@1": 0.196, - "ndcg_exponential@5": 0.32114419847722886, - "ndcg_exponential@10": 0.35574139683294936, - "precision@1": 0.196, - "precision@5": 0.0868, - "precision@10": 0.054099999999999995, - "recall@1": 0.196, - "recall@5": 0.434, - "recall@10": 0.541, - "f1@1": 0.196, - "f1@5": 0.1446666666666667, - "f1@10": 0.09836363636363639, - "map@1": 0.196, - "map@5": 0.28375, - "map@10": 0.29802619047619044, - "mrr@1": 0.196, - "mrr@5": 0.28375, - "mrr@10": 0.29802619047619044, + "hit@1": 0.198, + "hit@5": 0.432, + "hit@10": 0.536, + "ndcg_linear@1": 0.198, + "ndcg_linear@5": 0.32108259216558405, + "ndcg_linear@10": 0.355086593598355, + "ndcg_exponential@1": 0.198, + "ndcg_exponential@5": 0.32108259216558405, + "ndcg_exponential@10": 0.355086593598355, + "precision@1": 0.198, + "precision@5": 0.08639999999999999, + "precision@10": 0.053599999999999995, + "recall@1": 0.198, + "recall@5": 0.432, + "recall@10": 0.536, + "f1@1": 0.198, + "f1@5": 0.14400000000000002, + "f1@10": 0.09745454545454547, + "map@1": 0.198, + "map@5": 0.2843333333333333, + "map@10": 0.29858849206349203, + "mrr@1": 0.198, + "mrr@5": 0.2843333333333333, + "mrr@10": 0.29858849206349203, "num_pred": 1000, "num_data": 1000 }, "ScienceQA": { - "hit@1": 0.263, - "hit@5": 0.57, - "hit@10": 0.683, - "ndcg_linear@1": 0.263, - "ndcg_linear@5": 0.4232608546664754, - "ndcg_linear@10": 0.46061584821555157, - "ndcg_exponential@1": 0.263, - "ndcg_exponential@5": 0.4232608546664754, - "ndcg_exponential@10": 0.46061584821555157, - "precision@1": 0.263, - "precision@5": 0.114, - "precision@10": 0.0683, - "recall@1": 0.263, - "recall@5": 0.57, - "recall@10": 0.683, - "f1@1": 0.263, - "f1@5": 0.19000000000000003, - "f1@10": 0.12418181818181821, - "map@1": 0.263, - "map@5": 0.37475, - "map@10": 0.39065079365079364, - "mrr@1": 0.263, - "mrr@5": 0.37475, - "mrr@10": 0.39065079365079364, + "hit@1": 0.265, + "hit@5": 0.571, + "hit@10": 0.689, + "ndcg_linear@1": 0.265, + "ndcg_linear@5": 0.4250997547458267, + "ndcg_linear@10": 0.46395425910669463, + "ndcg_exponential@1": 0.265, + "ndcg_exponential@5": 0.4250997547458267, + "ndcg_exponential@10": 0.46395425910669463, + "precision@1": 0.265, + "precision@5": 0.11419999999999998, + "precision@10": 0.0689, + "recall@1": 0.265, + "recall@5": 0.571, + "recall@10": 0.689, + "f1@1": 0.265, + "f1@5": 0.19033333333333338, + "f1@10": 0.12527272727272729, + "map@1": 0.265, + "map@5": 0.37683333333333335, + "map@10": 0.39329166666666676, + "mrr@1": 0.265, + "mrr@5": 0.37683333333333335, + "mrr@10": 0.39329166666666676, "num_pred": 1000, "num_data": 1000 }, "VizWiz": { - "hit@1": 0.32, - "hit@5": 0.526, - "hit@10": 0.582, - "ndcg_linear@1": 0.32, - "ndcg_linear@5": 0.42951324327429996, - "ndcg_linear@10": 0.44772604037996566, - "ndcg_exponential@1": 0.32, - "ndcg_exponential@5": 0.42951324327429996, - "ndcg_exponential@10": 0.44772604037996566, - "precision@1": 0.32, - "precision@5": 0.10519999999999999, - "precision@10": 0.0582, - "recall@1": 0.32, - "recall@5": 0.526, - "recall@10": 0.582, - "f1@1": 0.32, - "f1@5": 0.17533333333333337, - "f1@10": 0.10581818181818184, - "map@1": 0.32, - "map@5": 0.39741666666666664, - "map@10": 0.4049861111111111, - "mrr@1": 0.32, - "mrr@5": 0.39741666666666664, - "mrr@10": 0.4049861111111111, + "hit@1": 0.319, + "hit@5": 0.524, + "hit@10": 0.585, + "ndcg_linear@1": 0.319, + "ndcg_linear@5": 0.4283011439714757, + "ndcg_linear@10": 0.4480825486246152, + "ndcg_exponential@1": 0.319, + "ndcg_exponential@5": 0.4283011439714757, + "ndcg_exponential@10": 0.4480825486246152, + "precision@1": 0.319, + "precision@5": 0.10479999999999998, + "precision@10": 0.0585, + "recall@1": 0.319, + "recall@5": 0.524, + "recall@10": 0.585, + "f1@1": 0.319, + "f1@5": 0.1746666666666667, + "f1@10": 0.10636363636363637, + "map@1": 0.319, + "map@5": 0.3964333333333333, + "map@10": 0.4046261904761905, + "mrr@1": 0.319, + "mrr@5": 0.3964333333333333, + "mrr@10": 0.4046261904761905, "num_pred": 1000, "num_data": 1000 }, "GQA": { - "hit@1": 0.385, - "hit@5": 0.709, - "hit@10": 0.832, - "ndcg_linear@1": 0.385, - "ndcg_linear@5": 0.5587878569139205, - "ndcg_linear@10": 0.5988209254862948, - "ndcg_exponential@1": 0.385, - "ndcg_exponential@5": 0.5587878569139205, - "ndcg_exponential@10": 0.5988209254862948, - "precision@1": 0.385, - "precision@5": 0.14179999999999998, - "precision@10": 0.08320000000000002, - "recall@1": 0.385, - "recall@5": 0.709, - "recall@10": 0.832, - "f1@1": 0.385, - "f1@5": 0.23633333333333337, - "f1@10": 0.15127272727272728, - "map@1": 0.385, - "map@5": 0.5087166666666667, - "map@10": 0.5253928571428572, - "mrr@1": 0.385, - "mrr@5": 0.5087166666666667, - "mrr@10": 0.5253928571428572, + "hit@1": 0.383, + "hit@5": 0.706, + "hit@10": 0.829, + "ndcg_linear@1": 0.383, + "ndcg_linear@5": 0.557227516579766, + "ndcg_linear@10": 0.5974087894179586, + "ndcg_exponential@1": 0.383, + "ndcg_exponential@5": 0.557227516579766, + "ndcg_exponential@10": 0.5974087894179586, + "precision@1": 0.383, + "precision@5": 0.1412, + "precision@10": 0.0829, + "recall@1": 0.383, + "recall@5": 0.706, + "recall@10": 0.829, + "f1@1": 0.383, + "f1@5": 0.23533333333333337, + "f1@10": 0.15072727272727274, + "map@1": 0.383, + "map@5": 0.50755, + "map@10": 0.5243698412698413, + "mrr@1": 0.383, + "mrr@5": 0.50755, + "mrr@10": 0.5243698412698413, "num_pred": 1000, "num_data": 1000 }, "TextVQA": { - "hit@1": 0.33, - "hit@5": 0.501, - "hit@10": 0.572, - "ndcg_linear@1": 0.33, - "ndcg_linear@5": 0.42213712812714405, - "ndcg_linear@10": 0.44510553711635736, - "ndcg_exponential@1": 0.33, - "ndcg_exponential@5": 0.42213712812714405, - "ndcg_exponential@10": 0.44510553711635736, - "precision@1": 0.33, - "precision@5": 0.10019999999999998, - "precision@10": 0.057199999999999994, - "recall@1": 0.33, - "recall@5": 0.501, - "recall@10": 0.572, - "f1@1": 0.33, - "f1@5": 0.16700000000000007, - "f1@10": 0.10400000000000001, - "map@1": 0.33, - "map@5": 0.39581666666666665, - "map@10": 0.40530873015873015, - "mrr@1": 0.33, - "mrr@5": 0.39581666666666665, - "mrr@10": 0.40530873015873015, + "hit@1": 0.331, + "hit@5": 0.506, + "hit@10": 0.574, + "ndcg_linear@1": 0.331, + "ndcg_linear@5": 0.42393452595626535, + "ndcg_linear@10": 0.4457358581969369, + "ndcg_exponential@1": 0.331, + "ndcg_exponential@5": 0.42393452595626535, + "ndcg_exponential@10": 0.4457358581969369, + "precision@1": 0.331, + "precision@5": 0.10119999999999998, + "precision@10": 0.05739999999999999, + "recall@1": 0.331, + "recall@5": 0.506, + "recall@10": 0.574, + "f1@1": 0.331, + "f1@5": 0.16866666666666671, + "f1@10": 0.1043636363636364, + "map@1": 0.331, + "map@5": 0.39668333333333333, + "map@10": 0.40557499999999996, + "mrr@1": 0.331, + "mrr@5": 0.39668333333333333, + "mrr@10": 0.40557499999999996, "num_pred": 1000, "num_data": 1000 }, "VisDial": { - "hit@1": 0.613, - "hit@5": 0.861, + "hit@1": 0.61, + "hit@5": 0.863, "hit@10": 0.923, - "ndcg_linear@1": 0.613, - "ndcg_linear@5": 0.7480330953397659, - "ndcg_linear@10": 0.7684474643796165, - "ndcg_exponential@1": 0.613, - "ndcg_exponential@5": 0.7480330953397659, - "ndcg_exponential@10": 0.7684474643796165, - "precision@1": 0.613, - "precision@5": 0.17220000000000002, + "ndcg_linear@1": 0.61, + "ndcg_linear@5": 0.7494181942698321, + "ndcg_linear@10": 0.7691660818855505, + "ndcg_exponential@1": 0.61, + "ndcg_exponential@5": 0.7494181942698321, + "ndcg_exponential@10": 0.7691660818855505, + "precision@1": 0.61, + "precision@5": 0.17260000000000003, "precision@10": 0.09230000000000001, - "recall@1": 0.613, - "recall@5": 0.861, + "recall@1": 0.61, + "recall@5": 0.863, "recall@10": 0.923, - "f1@1": 0.613, - "f1@5": 0.28700000000000003, + "f1@1": 0.61, + "f1@5": 0.2876666666666667, "f1@10": 0.1678181818181818, - "map@1": 0.613, - "map@5": 0.7101999999999999, - "map@10": 0.7188412698412698, - "mrr@1": 0.613, - "mrr@5": 0.7101999999999999, - "mrr@10": 0.7188412698412698, + "map@1": 0.61, + "map@5": 0.7112333333333334, + "map@10": 0.7195833333333332, + "mrr@1": 0.61, + "mrr@5": 0.7112333333333334, + "mrr@10": 0.7195833333333332, "num_pred": 1000, "num_data": 1000 }, "CIRR": { - "hit@1": 0.517, - "hit@5": 0.863, - "hit@10": 0.931, - "ndcg_linear@1": 0.517, - "ndcg_linear@5": 0.7053164568585676, - "ndcg_linear@10": 0.7275391721374201, - "ndcg_exponential@1": 0.517, - "ndcg_exponential@5": 0.7053164568585676, - "ndcg_exponential@10": 0.7275391721374201, - "precision@1": 0.517, - "precision@5": 0.17260000000000003, - "precision@10": 0.0931, - "recall@1": 0.517, - "recall@5": 0.863, - "recall@10": 0.931, - "f1@1": 0.517, - "f1@5": 0.28766666666666674, - "f1@10": 0.16927272727272724, - "map@1": 0.517, - "map@5": 0.6524333333333334, - "map@10": 0.6617317460317461, - "mrr@1": 0.517, - "mrr@5": 0.6524333333333334, - "mrr@10": 0.6617317460317461, + "hit@1": 0.521, + "hit@5": 0.861, + "hit@10": 0.934, + "ndcg_linear@1": 0.521, + "ndcg_linear@5": 0.7061831788047537, + "ndcg_linear@10": 0.7299819767400827, + "ndcg_exponential@1": 0.521, + "ndcg_exponential@5": 0.7061831788047537, + "ndcg_exponential@10": 0.7299819767400827, + "precision@1": 0.521, + "precision@5": 0.17220000000000002, + "precision@10": 0.09340000000000001, + "recall@1": 0.521, + "recall@5": 0.861, + "recall@10": 0.934, + "f1@1": 0.521, + "f1@5": 0.28700000000000003, + "f1@10": 0.1698181818181818, + "map@1": 0.521, + "map@5": 0.6542333333333333, + "map@10": 0.6641638888888889, + "mrr@1": 0.521, + "mrr@5": 0.6542333333333333, + "mrr@10": 0.6641638888888889, "num_pred": 1000, "num_data": 1000 }, "VisualNews_t2i": { - "hit@1": 0.704, - "hit@5": 0.875, + "hit@1": 0.709, + "hit@5": 0.877, "hit@10": 0.91, - "ndcg_linear@1": 0.704, - "ndcg_linear@5": 0.7989749726833766, - "ndcg_linear@10": 0.8104235859208543, - "ndcg_exponential@1": 0.704, - "ndcg_exponential@5": 0.7989749726833766, - "ndcg_exponential@10": 0.8104235859208543, - "precision@1": 0.704, - "precision@5": 0.17500000000000002, - "precision@10": 0.09100000000000001, - "recall@1": 0.704, - "recall@5": 0.875, + "ndcg_linear@1": 0.709, + "ndcg_linear@5": 0.8017332179127867, + "ndcg_linear@10": 0.8124801668655339, + "ndcg_exponential@1": 0.709, + "ndcg_exponential@5": 0.8017332179127867, + "ndcg_exponential@10": 0.8124801668655339, + "precision@1": 0.709, + "precision@5": 0.1754, + "precision@10": 0.091, + "recall@1": 0.709, + "recall@5": 0.877, "recall@10": 0.91, - "f1@1": 0.704, - "f1@5": 0.29166666666666663, + "f1@1": 0.709, + "f1@5": 0.2923333333333334, "f1@10": 0.16545454545454544, - "map@1": 0.704, - "map@5": 0.7733166666666667, - "map@10": 0.7781194444444445, - "mrr@1": 0.704, - "mrr@5": 0.7733166666666667, - "mrr@10": 0.7781194444444445, + "map@1": 0.709, + "map@5": 0.7763666666666666, + "map@10": 0.7808511904761904, + "mrr@1": 0.709, + "mrr@5": 0.7763666666666666, + "mrr@10": 0.7808511904761904, "num_pred": 1000, "num_data": 1000 }, "VisualNews_i2t": { - "hit@1": 0.839, + "hit@1": 0.841, "hit@5": 0.938, "hit@10": 0.953, - "ndcg_linear@1": 0.839, - "ndcg_linear@5": 0.8947751558137038, - "ndcg_linear@10": 0.8997533519597807, - "ndcg_exponential@1": 0.839, - "ndcg_exponential@5": 0.8947751558137038, - "ndcg_exponential@10": 0.8997533519597807, - "precision@1": 0.839, + "ndcg_linear@1": 0.841, + "ndcg_linear@5": 0.8956702672501651, + "ndcg_linear@10": 0.9006654339628153, + "ndcg_exponential@1": 0.841, + "ndcg_exponential@5": 0.8956702672501651, + "ndcg_exponential@10": 0.9006654339628153, + "precision@1": 0.841, "precision@5": 0.18760000000000002, "precision@10": 0.09530000000000001, - "recall@1": 0.839, + "recall@1": 0.841, "recall@5": 0.938, "recall@10": 0.953, - "f1@1": 0.839, + "f1@1": 0.841, "f1@5": 0.3126666666666667, "f1@10": 0.17327272727272724, - "map@1": 0.839, - "map@5": 0.8801333333333333, - "map@10": 0.8822630952380953, - "mrr@1": 0.839, - "mrr@5": 0.8801333333333333, - "mrr@10": 0.8822630952380953, + "map@1": 0.841, + "map@5": 0.8813166666666666, + "map@10": 0.8834634920634921, + "mrr@1": 0.841, + "mrr@5": 0.8813166666666666, + "mrr@10": 0.8834634920634921, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_t2i": { - "hit@1": 0.722, - "hit@5": 0.935, - "hit@10": 0.965, - "ndcg_linear@1": 0.722, - "ndcg_linear@5": 0.8400063048969685, - "ndcg_linear@10": 0.8496978391851898, - "ndcg_exponential@1": 0.722, - "ndcg_exponential@5": 0.8400063048969685, - "ndcg_exponential@10": 0.8496978391851898, - "precision@1": 0.722, - "precision@5": 0.187, - "precision@10": 0.09650000000000002, - "recall@1": 0.722, - "recall@5": 0.935, - "recall@10": 0.965, - "f1@1": 0.722, - "f1@5": 0.31166666666666665, - "f1@10": 0.17545454545454545, - "map@1": 0.722, - "map@5": 0.8079333333333333, - "map@10": 0.8119218253968254, - "mrr@1": 0.722, - "mrr@5": 0.8079333333333333, - "mrr@10": 0.8119218253968254, + "hit@1": 0.72, + "hit@5": 0.934, + "hit@10": 0.967, + "ndcg_linear@1": 0.72, + "ndcg_linear@5": 0.8388990941576828, + "ndcg_linear@10": 0.8495224955738723, + "ndcg_exponential@1": 0.72, + "ndcg_exponential@5": 0.8388990941576828, + "ndcg_exponential@10": 0.8495224955738723, + "precision@1": 0.72, + "precision@5": 0.18680000000000002, + "precision@10": 0.09670000000000002, + "recall@1": 0.72, + "recall@5": 0.934, + "recall@10": 0.967, + "f1@1": 0.72, + "f1@5": 0.3113333333333333, + "f1@10": 0.17581818181818182, + "map@1": 0.72, + "map@5": 0.8067666666666666, + "map@10": 0.8111190476190476, + "mrr@1": 0.72, + "mrr@5": 0.8067666666666666, + "mrr@10": 0.8111190476190476, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_i2t": { - "hit@1": 0.737, + "hit@1": 0.736, "hit@5": 0.957, "hit@10": 0.982, - "ndcg_linear@1": 0.737, - "ndcg_linear@5": 0.8594668192272695, - "ndcg_linear@10": 0.8676497166665621, - "ndcg_exponential@1": 0.737, - "ndcg_exponential@5": 0.8594668192272695, - "ndcg_exponential@10": 0.8676497166665621, - "precision@1": 0.737, + "ndcg_linear@1": 0.736, + "ndcg_linear@5": 0.8593163262360901, + "ndcg_linear@10": 0.8675170921319303, + "ndcg_exponential@1": 0.736, + "ndcg_exponential@5": 0.8593163262360901, + "ndcg_exponential@10": 0.8675170921319303, + "precision@1": 0.736, "precision@5": 0.19140000000000004, "precision@10": 0.09820000000000002, - "recall@1": 0.737, + "recall@1": 0.736, "recall@5": 0.957, "recall@10": 0.982, - "f1@1": 0.737, + "f1@1": 0.736, "f1@5": 0.319, "f1@10": 0.17854545454545453, - "map@1": 0.737, - "map@5": 0.8265833333333333, - "map@10": 0.8300186507936508, - "mrr@1": 0.737, - "mrr@5": 0.8265833333333333, - "mrr@10": 0.8300186507936508, + "map@1": 0.736, + "map@5": 0.8263499999999999, + "map@10": 0.8298031746031747, + "mrr@1": 0.736, + "mrr@5": 0.8263499999999999, + "mrr@10": 0.8298031746031747, "num_pred": 1000, "num_data": 1000 }, "NIGHTS": { - "hit@1": 0.656, + "hit@1": 0.657, "hit@5": 0.974, - "hit@10": 0.993, - "ndcg_linear@1": 0.656, - "ndcg_linear@5": 0.8392219252539634, - "ndcg_linear@10": 0.8454923298699608, - "ndcg_exponential@1": 0.656, - "ndcg_exponential@5": 0.8392219252539634, - "ndcg_exponential@10": 0.8454923298699608, - "precision@1": 0.656, + "hit@10": 0.994, + "ndcg_linear@1": 0.657, + "ndcg_linear@5": 0.8402634268290552, + "ndcg_linear@10": 0.8468119875869419, + "ndcg_exponential@1": 0.657, + "ndcg_exponential@5": 0.8402634268290552, + "ndcg_exponential@10": 0.8468119875869419, + "precision@1": 0.657, "precision@5": 0.1948, - "precision@10": 0.09930000000000001, - "recall@1": 0.656, + "precision@10": 0.09940000000000002, + "recall@1": 0.657, "recall@5": 0.974, - "recall@10": 0.993, - "f1@1": 0.656, + "recall@10": 0.994, + "f1@1": 0.657, "f1@5": 0.3246666666666666, - "f1@10": 0.1805454545454545, - "map@1": 0.656, - "map@5": 0.7931000000000001, - "map@10": 0.7957623015873017, - "mrr@1": 0.656, - "mrr@5": 0.7931000000000001, - "mrr@10": 0.7957623015873017, + "f1@10": 0.18072727272727268, + "map@1": 0.657, + "map@5": 0.7944666666666667, + "map@10": 0.7972162698412699, + "mrr@1": 0.657, + "mrr@5": 0.7944666666666667, + "mrr@10": 0.7972162698412699, "num_pred": 1000, "num_data": 1000 }, "WebQA": { - "hit@1": 0.81, + "hit@1": 0.812, "hit@5": 0.971, - "hit@10": 0.986, - "ndcg_linear@1": 0.81, - "ndcg_linear@5": 0.9037144327529909, - "ndcg_linear@10": 0.9087487966017537, - "ndcg_exponential@1": 0.81, - "ndcg_exponential@5": 0.9037144327529909, - "ndcg_exponential@10": 0.9087487966017537, - "precision@1": 0.81, + "hit@10": 0.987, + "ndcg_linear@1": 0.812, + "ndcg_linear@5": 0.904452573245848, + "ndcg_linear@10": 0.9097810073181558, + "ndcg_exponential@1": 0.812, + "ndcg_exponential@5": 0.904452573245848, + "ndcg_exponential@10": 0.9097810073181558, + "precision@1": 0.812, "precision@5": 0.1942, - "precision@10": 0.09860000000000002, - "recall@1": 0.81, + "precision@10": 0.09870000000000002, + "recall@1": 0.812, "recall@5": 0.971, - "recall@10": 0.986, - "f1@1": 0.81, + "recall@10": 0.987, + "f1@1": 0.812, "f1@5": 0.3236666666666666, - "f1@10": 0.17927272727272725, - "map@1": 0.81, - "map@5": 0.8806333333333335, - "map@10": 0.8828166666666666, - "mrr@1": 0.81, - "mrr@5": 0.8806333333333335, - "mrr@10": 0.8828166666666666, + "f1@10": 0.17945454545454545, + "map@1": 0.812, + "map@5": 0.8816333333333335, + "map@10": 0.883922619047619, + "mrr@1": 0.812, + "mrr@5": 0.8816333333333335, + "mrr@10": 0.883922619047619, "num_pred": 1000, "num_data": 1000 }, "FashionIQ": { - "hit@1": 0.42, - "hit@5": 0.689, + "hit@1": 0.417, + "hit@5": 0.684, "hit@10": 0.794, - "ndcg_linear@1": 0.42, - "ndcg_linear@5": 0.5636864513304993, - "ndcg_linear@10": 0.598000278473528, - "ndcg_exponential@1": 0.42, - "ndcg_exponential@5": 0.5636864513304993, - "ndcg_exponential@10": 0.598000278473528, - "precision@1": 0.42, - "precision@5": 0.13779999999999998, - "precision@10": 0.07940000000000001, - "recall@1": 0.42, - "recall@5": 0.689, + "ndcg_linear@1": 0.417, + "ndcg_linear@5": 0.5608008645007552, + "ndcg_linear@10": 0.5967823221741005, + "ndcg_exponential@1": 0.417, + "ndcg_exponential@5": 0.5608008645007552, + "ndcg_exponential@10": 0.5967823221741005, + "precision@1": 0.417, + "precision@5": 0.13679999999999998, + "precision@10": 0.0794, + "recall@1": 0.417, + "recall@5": 0.684, "recall@10": 0.794, - "f1@1": 0.42, - "f1@5": 0.2296666666666667, + "f1@1": 0.417, + "f1@5": 0.22800000000000006, "f1@10": 0.14436363636363636, - "map@1": 0.42, - "map@5": 0.5219166666666666, - "map@10": 0.5362988095238095, - "mrr@1": 0.42, - "mrr@5": 0.5219166666666666, - "mrr@10": 0.5362988095238095, + "map@1": 0.417, + "map@5": 0.5196333333333333, + "map@10": 0.5347309523809525, + "mrr@1": 0.417, + "mrr@5": 0.5196333333333333, + "mrr@10": 0.5347309523809525, "num_pred": 1000, "num_data": 1000 }, "Wiki-SS-NQ": { - "hit@1": 0.697, - "hit@5": 0.884, + "hit@1": 0.701, + "hit@5": 0.885, "hit@10": 0.921, - "ndcg_linear@1": 0.697, - "ndcg_linear@5": 0.801194685185507, - "ndcg_linear@10": 0.8131568214002816, - "ndcg_exponential@1": 0.697, - "ndcg_exponential@5": 0.801194685185507, - "ndcg_exponential@10": 0.8131568214002816, - "precision@1": 0.697, - "precision@5": 0.1768, - "precision@10": 0.09210000000000003, - "recall@1": 0.697, - "recall@5": 0.884, + "ndcg_linear@1": 0.701, + "ndcg_linear@5": 0.8032142484231147, + "ndcg_linear@10": 0.8149918635285255, + "ndcg_exponential@1": 0.701, + "ndcg_exponential@5": 0.8032142484231147, + "ndcg_exponential@10": 0.8149918635285255, + "precision@1": 0.701, + "precision@5": 0.17700000000000002, + "precision@10": 0.09210000000000002, + "recall@1": 0.701, + "recall@5": 0.885, "recall@10": 0.921, - "f1@1": 0.697, - "f1@5": 0.2946666666666667, + "f1@1": 0.701, + "f1@5": 0.295, "f1@10": 0.16745454545454544, - "map@1": 0.697, - "map@5": 0.77325, - "map@10": 0.7781892857142857, - "mrr@1": 0.697, - "mrr@5": 0.77325, - "mrr@10": 0.7781892857142857, + "map@1": 0.701, + "map@5": 0.77565, + "map@10": 0.7805908730158729, + "mrr@1": 0.701, + "mrr@5": 0.77565, + "mrr@10": 0.7805908730158729, "num_pred": 1000, "num_data": 1000 }, "OVEN": { - "hit@1": 0.82, - "hit@5": 0.974, + "hit@1": 0.822, + "hit@5": 0.973, "hit@10": 0.991, - "ndcg_linear@1": 0.82, - "ndcg_linear@5": 0.9071538558399693, - "ndcg_linear@10": 0.912778314510714, - "ndcg_exponential@1": 0.82, - "ndcg_exponential@5": 0.9071538558399693, - "ndcg_exponential@10": 0.912778314510714, - "precision@1": 0.82, - "precision@5": 0.19480000000000003, + "ndcg_linear@1": 0.822, + "ndcg_linear@5": 0.9068427776274527, + "ndcg_linear@10": 0.9128090086041838, + "ndcg_exponential@1": 0.822, + "ndcg_exponential@5": 0.9068427776274527, + "ndcg_exponential@10": 0.9128090086041838, + "precision@1": 0.822, + "precision@5": 0.19460000000000002, "precision@10": 0.09910000000000001, - "recall@1": 0.82, - "recall@5": 0.974, + "recall@1": 0.822, + "recall@5": 0.973, "recall@10": 0.991, - "f1@1": 0.82, - "f1@5": 0.3246666666666666, + "f1@1": 0.822, + "f1@5": 0.3243333333333333, "f1@10": 0.18018181818181817, - "map@1": 0.82, - "map@5": 0.88445, - "map@10": 0.8868444444444444, - "mrr@1": 0.82, - "mrr@5": 0.88445, - "mrr@10": 0.8868444444444444, + "map@1": 0.822, + "map@5": 0.8844166666666667, + "map@10": 0.8869638888888889, + "mrr@1": 0.822, + "mrr@5": 0.8844166666666667, + "mrr@10": 0.8869638888888889, "num_pred": 1000, "num_data": 1000 }, "EDIS": { - "hit@1": 0.859, - "hit@5": 0.987, - "hit@10": 0.997, - "ndcg_linear@1": 0.859, - "ndcg_linear@5": 0.9331780902287048, - "ndcg_linear@10": 0.9365900559178164, - "ndcg_exponential@1": 0.859, - "ndcg_exponential@5": 0.9331780902287048, - "ndcg_exponential@10": 0.9365900559178164, - "precision@1": 0.859, - "precision@5": 0.19740000000000005, - "precision@10": 0.09970000000000001, - "recall@1": 0.859, - "recall@5": 0.987, - "recall@10": 0.997, - "f1@1": 0.859, - "f1@5": 0.329, - "f1@10": 0.18127272727272725, - "map@1": 0.859, - "map@5": 0.91475, - "map@10": 0.9162619047619048, - "mrr@1": 0.859, - "mrr@5": 0.91475, - "mrr@10": 0.9162619047619048, + "hit@1": 0.861, + "hit@5": 0.989, + "hit@10": 0.998, + "ndcg_linear@1": 0.861, + "ndcg_linear@5": 0.9348801241010047, + "ndcg_linear@10": 0.9378916140959929, + "ndcg_exponential@1": 0.861, + "ndcg_exponential@5": 0.9348801241010047, + "ndcg_exponential@10": 0.9378916140959929, + "precision@1": 0.861, + "precision@5": 0.1978, + "precision@10": 0.09980000000000001, + "recall@1": 0.861, + "recall@5": 0.989, + "recall@10": 0.998, + "f1@1": 0.861, + "f1@5": 0.32966666666666666, + "f1@10": 0.18145454545454545, + "map@1": 0.861, + "map@5": 0.9163666666666667, + "map@10": 0.9176690476190477, + "mrr@1": 0.861, + "mrr@5": 0.9163666666666667, + "mrr@10": 0.9176690476190477, "num_pred": 1000, "num_data": 1000 }, "MSCOCO": { - "hit@1": 0.448, - "hit@5": 0.625, - "hit@10": 0.685, - "ndcg_linear@1": 0.448, - "ndcg_linear@5": 0.5408899777268813, - "ndcg_linear@10": 0.5602258062625833, - "ndcg_exponential@1": 0.448, - "ndcg_exponential@5": 0.5408899777268813, - "ndcg_exponential@10": 0.5602258062625833, - "precision@1": 0.448, - "precision@5": 0.12499999999999999, - "precision@10": 0.0685, - "recall@1": 0.448, - "recall@5": 0.625, - "recall@10": 0.685, - "f1@1": 0.448, - "f1@5": 0.20833333333333337, - "f1@10": 0.12454545454545456, - "map@1": 0.448, - "map@5": 0.5130333333333333, - "map@10": 0.5209734126984127, - "mrr@1": 0.448, - "mrr@5": 0.5130333333333333, - "mrr@10": 0.5209734126984127, + "hit@1": 0.447, + "hit@5": 0.623, + "hit@10": 0.683, + "ndcg_linear@1": 0.447, + "ndcg_linear@5": 0.5402969620703022, + "ndcg_linear@10": 0.559640107120064, + "ndcg_exponential@1": 0.447, + "ndcg_exponential@5": 0.5402969620703022, + "ndcg_exponential@10": 0.559640107120064, + "precision@1": 0.447, + "precision@5": 0.12459999999999997, + "precision@10": 0.0683, + "recall@1": 0.447, + "recall@5": 0.623, + "recall@10": 0.683, + "f1@1": 0.447, + "f1@5": 0.20766666666666672, + "f1@10": 0.12418181818181818, + "map@1": 0.447, + "map@5": 0.5128166666666667, + "map@10": 0.5207678571428571, + "mrr@1": 0.447, + "mrr@5": 0.5128166666666667, + "mrr@10": 0.5207678571428571, "num_pred": 1000, "num_data": 1000 }, "RefCOCO": { - "hit@1": 0.628, - "hit@5": 0.848, + "hit@1": 0.625, + "hit@5": 0.846, "hit@10": 0.897, - "ndcg_linear@1": 0.628, - "ndcg_linear@5": 0.7457621998007221, - "ndcg_linear@10": 0.7619311912107107, - "ndcg_exponential@1": 0.628, - "ndcg_exponential@5": 0.7457621998007221, - "ndcg_exponential@10": 0.7619311912107107, - "precision@1": 0.628, - "precision@5": 0.16960000000000003, - "precision@10": 0.0897, - "recall@1": 0.628, - "recall@5": 0.848, + "ndcg_linear@1": 0.625, + "ndcg_linear@5": 0.7443184379574657, + "ndcg_linear@10": 0.761331658751475, + "ndcg_exponential@1": 0.625, + "ndcg_exponential@5": 0.7443184379574657, + "ndcg_exponential@10": 0.761331658751475, + "precision@1": 0.625, + "precision@5": 0.1692, + "precision@10": 0.08970000000000002, + "recall@1": 0.625, + "recall@5": 0.846, "recall@10": 0.897, - "f1@1": 0.628, - "f1@5": 0.28266666666666673, + "f1@1": 0.625, + "f1@5": 0.2820000000000001, "f1@10": 0.1630909090909091, - "map@1": 0.628, - "map@5": 0.7117333333333333, - "map@10": 0.7185992063492063, - "mrr@1": 0.628, - "mrr@5": 0.7117333333333333, - "mrr@10": 0.7185992063492063, + "map@1": 0.625, + "map@5": 0.7103666666666667, + "map@10": 0.7176964285714285, + "mrr@1": 0.625, + "mrr@5": 0.7103666666666667, + "mrr@10": 0.7176964285714285, "num_pred": 1000, "num_data": 1000 }, "RefCOCO-Matching": { - "hit@1": 0.757, + "hit@1": 0.762, "hit@5": 0.999, "hit@10": 0.999, - "ndcg_linear@1": 0.757, - "ndcg_linear@5": 0.9102862746500986, - "ndcg_linear@10": 0.9102862746500986, - "ndcg_exponential@1": 0.757, - "ndcg_exponential@5": 0.9102862746500986, - "ndcg_exponential@10": 0.9102862746500986, - "precision@1": 0.757, + "ndcg_linear@1": 0.762, + "ndcg_linear@5": 0.9121932321938861, + "ndcg_linear@10": 0.9121932321938861, + "ndcg_exponential@1": 0.762, + "ndcg_exponential@5": 0.9121932321938861, + "ndcg_exponential@10": 0.9121932321938861, + "precision@1": 0.762, "precision@5": 0.19980000000000003, "precision@10": 0.09990000000000002, - "recall@1": 0.757, + "recall@1": 0.762, "recall@5": 0.999, "recall@10": 0.999, - "f1@1": 0.757, + "f1@1": 0.762, "f1@5": 0.333, "f1@10": 0.18163636363636362, - "map@1": 0.757, - "map@5": 0.8793666666666666, - "map@10": 0.8793666666666666, - "mrr@1": 0.757, - "mrr@5": 0.8763666666666666, - "mrr@10": 0.8763666666666666, + "map@1": 0.762, + "map@5": 0.88195, + "map@10": 0.88195, + "mrr@1": 0.762, + "mrr@5": 0.87895, + "mrr@10": 0.87895, "num_pred": 1000, "num_data": 1000 }, "Visual7W-Pointing": { - "hit@1": 0.673, - "hit@5": 0.863, - "hit@10": 0.919, - "ndcg_linear@1": 0.673, - "ndcg_linear@5": 0.7774031702105983, - "ndcg_linear@10": 0.7954662572023203, - "ndcg_exponential@1": 0.673, - "ndcg_exponential@5": 0.7774031702105983, - "ndcg_exponential@10": 0.7954662572023203, - "precision@1": 0.673, - "precision@5": 0.17260000000000003, - "precision@10": 0.09190000000000001, - "recall@1": 0.673, - "recall@5": 0.863, - "recall@10": 0.919, - "f1@1": 0.673, - "f1@5": 0.2876666666666667, - "f1@10": 0.16709090909090907, - "map@1": 0.673, - "map@5": 0.7486166666666667, - "map@10": 0.756040873015873, - "mrr@1": 0.673, - "mrr@5": 0.7486166666666667, - "mrr@10": 0.756040873015873, + "hit@1": 0.671, + "hit@5": 0.865, + "hit@10": 0.92, + "ndcg_linear@1": 0.671, + "ndcg_linear@5": 0.7771946583858733, + "ndcg_linear@10": 0.7947672268213155, + "ndcg_exponential@1": 0.671, + "ndcg_exponential@5": 0.7771946583858733, + "ndcg_exponential@10": 0.7947672268213155, + "precision@1": 0.671, + "precision@5": 0.173, + "precision@10": 0.092, + "recall@1": 0.671, + "recall@5": 0.865, + "recall@10": 0.92, + "f1@1": 0.671, + "f1@5": 0.2883333333333334, + "f1@10": 0.16727272727272724, + "map@1": 0.671, + "map@5": 0.7477166666666666, + "map@10": 0.7548440476190477, + "mrr@1": 0.671, + "mrr@5": 0.7477166666666666, + "mrr@10": 0.7548440476190477, "num_pred": 1000, "num_data": 1000 } @@ -1020,169 +1020,169 @@ "video": { "K700": { "hit@1": 0.423, - "hit@5": 0.653, - "hit@10": 0.731, + "hit@5": 0.649, + "hit@10": 0.732, "ndcg_linear@1": 0.423, - "ndcg_linear@5": 0.546371218016443, - "ndcg_linear@10": 0.5718997969605862, + "ndcg_linear@5": 0.5450341254135271, + "ndcg_linear@10": 0.572202681042046, "ndcg_exponential@1": 0.423, - "ndcg_exponential@5": 0.546371218016443, - "ndcg_exponential@10": 0.5718997969605862, + "ndcg_exponential@5": 0.5450341254135271, + "ndcg_exponential@10": 0.572202681042046, "precision@1": 0.423, - "precision@5": 0.13059999999999997, - "precision@10": 0.0731, + "precision@5": 0.1298, + "precision@10": 0.0732, "recall@1": 0.423, - "recall@5": 0.653, - "recall@10": 0.731, + "recall@5": 0.649, + "recall@10": 0.732, "f1@1": 0.423, - "f1@5": 0.21766666666666673, - "f1@10": 0.13290909090909092, + "f1@5": 0.2163333333333334, + "f1@10": 0.13309090909090912, "map@1": 0.423, - "map@5": 0.5107999999999999, - "map@10": 0.5215190476190475, + "map@5": 0.5102833333333333, + "map@10": 0.5216916666666667, "mrr@1": 0.423, - "mrr@5": 0.5107999999999999, - "mrr@10": 0.5215190476190475, + "mrr@5": 0.5102833333333333, + "mrr@10": 0.5216916666666667, "num_pred": 1000, "num_data": 1000 }, "SmthSmthV2": { - "hit@1": 0.363, - "hit@5": 0.602, - "hit@10": 0.698, - "ndcg_linear@1": 0.363, - "ndcg_linear@5": 0.4890606806277336, - "ndcg_linear@10": 0.5200630033328641, - "ndcg_exponential@1": 0.363, - "ndcg_exponential@5": 0.4890606806277336, - "ndcg_exponential@10": 0.5200630033328641, - "precision@1": 0.363, - "precision@5": 0.1204, - "precision@10": 0.06979999999999999, - "recall@1": 0.363, - "recall@5": 0.602, - "recall@10": 0.698, - "f1@1": 0.363, - "f1@5": 0.20066666666666674, - "f1@10": 0.12690909090909094, - "map@1": 0.363, - "map@5": 0.45156666666666667, - "map@10": 0.4643380952380952, - "mrr@1": 0.363, - "mrr@5": 0.45156666666666667, - "mrr@10": 0.4643380952380952, + "hit@1": 0.361, + "hit@5": 0.603, + "hit@10": 0.694, + "ndcg_linear@1": 0.361, + "ndcg_linear@5": 0.4884031681851842, + "ndcg_linear@10": 0.517829249638895, + "ndcg_exponential@1": 0.361, + "ndcg_exponential@5": 0.4884031681851842, + "ndcg_exponential@10": 0.517829249638895, + "precision@1": 0.361, + "precision@5": 0.1206, + "precision@10": 0.06939999999999999, + "recall@1": 0.361, + "recall@5": 0.603, + "recall@10": 0.694, + "f1@1": 0.361, + "f1@5": 0.20100000000000007, + "f1@10": 0.12618181818181823, + "map@1": 0.361, + "map@5": 0.45039999999999997, + "map@10": 0.46254166666666663, + "mrr@1": 0.361, + "mrr@5": 0.45039999999999997, + "mrr@10": 0.46254166666666663, "num_pred": 1000, "num_data": 1000 }, "HMDB51": { - "hit@1": 0.405, - "hit@5": 0.658, + "hit@1": 0.404, + "hit@5": 0.654, "hit@10": 0.783, - "ndcg_linear@1": 0.405, - "ndcg_linear@5": 0.5405643217050999, - "ndcg_linear@10": 0.5815107823708526, - "ndcg_exponential@1": 0.405, - "ndcg_exponential@5": 0.5405643217050999, - "ndcg_exponential@10": 0.5815107823708526, - "precision@1": 0.405, - "precision@5": 0.1316, + "ndcg_linear@1": 0.404, + "ndcg_linear@5": 0.537651401583337, + "ndcg_linear@10": 0.5798876010072345, + "ndcg_exponential@1": 0.404, + "ndcg_exponential@5": 0.537651401583337, + "ndcg_exponential@10": 0.5798876010072345, + "precision@1": 0.404, + "precision@5": 0.13079999999999997, "precision@10": 0.0783, - "recall@1": 0.405, - "recall@5": 0.658, + "recall@1": 0.404, + "recall@5": 0.654, "recall@10": 0.783, - "f1@1": 0.405, - "f1@5": 0.21933333333333338, + "f1@1": 0.404, + "f1@5": 0.21800000000000005, "f1@10": 0.1423636363636364, - "map@1": 0.405, - "map@5": 0.5013833333333333, - "map@10": 0.5185896825396826, - "mrr@1": 0.405, - "mrr@5": 0.5013833333333333, - "mrr@10": 0.5185896825396826, + "map@1": 0.404, + "map@5": 0.49881666666666674, + "map@10": 0.5165527777777779, + "mrr@1": 0.404, + "mrr@5": 0.49881666666666674, + "mrr@10": 0.5165527777777779, "num_pred": 1000, "num_data": 1000 }, "UCF101": { - "hit@1": 0.604, + "hit@1": 0.607, "hit@5": 0.852, "hit@10": 0.89, - "ndcg_linear@1": 0.604, - "ndcg_linear@5": 0.7417818963613594, - "ndcg_linear@10": 0.7541096692471368, - "ndcg_exponential@1": 0.604, - "ndcg_exponential@5": 0.7417818963613594, - "ndcg_exponential@10": 0.7541096692471368, - "precision@1": 0.604, + "ndcg_linear@1": 0.607, + "ndcg_linear@5": 0.7423416951965688, + "ndcg_linear@10": 0.7546406909413701, + "ndcg_exponential@1": 0.607, + "ndcg_exponential@5": 0.7423416951965688, + "ndcg_exponential@10": 0.7546406909413701, + "precision@1": 0.607, "precision@5": 0.17040000000000002, "precision@10": 0.08900000000000001, - "recall@1": 0.604, + "recall@1": 0.607, "recall@5": 0.852, "recall@10": 0.89, - "f1@1": 0.604, + "f1@1": 0.607, "f1@5": 0.284, "f1@10": 0.1618181818181818, - "map@1": 0.604, - "map@5": 0.7046166666666666, - "map@10": 0.7097305555555556, - "mrr@1": 0.604, - "mrr@5": 0.7046166666666666, - "mrr@10": 0.7097305555555556, + "map@1": 0.607, + "map@5": 0.7054666666666667, + "map@10": 0.7105499999999999, + "mrr@1": 0.607, + "mrr@5": 0.7054666666666667, + "mrr@10": 0.7105499999999999, "num_pred": 1000, "num_data": 1000 }, "Breakfast": { - "hit@1": 0.16859122401847576, - "hit@5": 0.6235565819861432, + "hit@1": 0.17551963048498845, + "hit@5": 0.625866050808314, "hit@10": 1.0, - "ndcg_linear@1": 0.16859122401847576, - "ndcg_linear@5": 0.400691767704696, - "ndcg_linear@10": 0.5235817090775436, - "ndcg_exponential@1": 0.16859122401847576, - "ndcg_exponential@5": 0.400691767704696, - "ndcg_exponential@10": 0.5235817090775436, - "precision@1": 0.16859122401847576, - "precision@5": 0.12471131639722864, + "ndcg_linear@1": 0.17551963048498845, + "ndcg_linear@5": 0.40363746352749275, + "ndcg_linear@10": 0.5256596448848111, + "ndcg_exponential@1": 0.17551963048498845, + "ndcg_exponential@5": 0.40363746352749275, + "ndcg_exponential@10": 0.5256596448848111, + "precision@1": 0.17551963048498845, + "precision@5": 0.1251732101616628, "precision@10": 0.1, - "recall@1": 0.16859122401847576, - "recall@5": 0.6235565819861432, + "recall@1": 0.17551963048498845, + "recall@5": 0.625866050808314, "recall@10": 1.0, - "f1@1": 0.16859122401847576, - "f1@5": 0.20785219399538113, + "f1@1": 0.17551963048498845, + "f1@5": 0.20862201693610477, "f1@10": 0.1818181818181818, - "map@1": 0.16859122401847576, - "map@5": 0.32763664357197847, - "map@10": 0.37902140840939913, - "mrr@1": 0.16859122401847576, - "mrr@5": 0.32763664357197847, - "mrr@10": 0.37902140840939913, + "map@1": 0.17551963048498845, + "map@5": 0.3309468822170901, + "map@10": 0.38190732798123095, + "mrr@1": 0.17551963048498845, + "mrr@5": 0.3309468822170901, + "mrr@10": 0.38190732798123095, "num_pred": 433, "num_data": 433 }, "MVBench": { - "hit@1": 0.372, + "hit@1": 0.37325, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.372, - "ndcg_linear@5": 0.7114371453270959, - "ndcg_linear@10": 0.7114371453270959, - "ndcg_exponential@1": 0.372, - "ndcg_exponential@5": 0.7114371453270959, - "ndcg_exponential@10": 0.7114371453270959, - "precision@1": 0.372, + "ndcg_linear@1": 0.37325, + "ndcg_linear@5": 0.7118542076839683, + "ndcg_linear@10": 0.7118542076839683, + "ndcg_exponential@1": 0.37325, + "ndcg_exponential@5": 0.7118542076839683, + "ndcg_exponential@10": 0.7118542076839683, + "precision@1": 0.37325, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.372, + "recall@1": 0.37325, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.372, + "f1@1": 0.37325, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.372, - "map@5": 0.6143833333333334, - "map@10": 0.6143833333333334, - "mrr@1": 0.372, - "mrr@5": 0.6143833333333334, - "mrr@10": 0.6143833333333334, + "map@1": 0.37325, + "map@5": 0.6149458333333333, + "map@10": 0.6149458333333333, + "mrr@1": 0.37325, + "mrr@5": 0.6149458333333333, + "mrr@10": 0.6149458333333333, "num_pred": 4000, "num_data": 4000 }, @@ -1191,11 +1191,11 @@ "hit@5": 1.0, "hit@10": 1.0, "ndcg_linear@1": 0.34074074074074073, - "ndcg_linear@5": 0.6905732385187221, - "ndcg_linear@10": 0.6905732385187221, + "ndcg_linear@5": 0.6905703803223214, + "ndcg_linear@10": 0.6905703803223214, "ndcg_exponential@1": 0.34074074074074073, - "ndcg_exponential@5": 0.6905732385187221, - "ndcg_exponential@10": 0.6905732385187221, + "ndcg_exponential@5": 0.6905703803223214, + "ndcg_exponential@10": 0.6905703803223214, "precision@1": 0.34074074074074073, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, @@ -1215,30 +1215,30 @@ "num_data": 2700 }, "NExTQA": { - "hit@1": 0.4371788883699206, + "hit@1": 0.43694535263895373, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.4371788883699206, - "ndcg_linear@5": 0.7269325524352686, - "ndcg_linear@10": 0.7269325524352686, - "ndcg_exponential@1": 0.4371788883699206, - "ndcg_exponential@5": 0.7269325524352686, - "ndcg_exponential@10": 0.7269325524352686, - "precision@1": 0.4371788883699206, + "ndcg_linear@1": 0.43694535263895373, + "ndcg_linear@5": 0.7268275682884282, + "ndcg_linear@10": 0.7268275682884282, + "ndcg_exponential@1": 0.43694535263895373, + "ndcg_exponential@5": 0.7268275682884282, + "ndcg_exponential@10": 0.7268275682884282, + "precision@1": 0.43694535263895373, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.4371788883699206, + "recall@1": 0.43694535263895373, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.4371788883699206, + "f1@1": 0.43694535263895373, "f1@5": 0.3333333333333333, "f1@10": 0.1818181818181818, - "map@1": 0.4371788883699206, - "map@5": 0.6369803829985987, - "map@10": 0.6369803829985987, - "mrr@1": 0.4371788883699206, - "mrr@5": 0.6367857698894598, - "mrr@10": 0.6367857698894598, + "map@1": 0.43694535263895373, + "map@5": 0.6368441538222015, + "map@10": 0.6368441538222015, + "mrr@1": 0.43694535263895373, + "mrr@5": 0.6366495407130625, + "mrr@10": 0.6366495407130625, "num_pred": 8564, "num_data": 8564 }, @@ -1247,11 +1247,11 @@ "hit@5": 1.0, "hit@10": 1.0, "ndcg_linear@1": 0.448, - "ndcg_linear@5": 0.7459721375093364, - "ndcg_linear@10": 0.7459721375093364, + "ndcg_linear@5": 0.7448582658380257, + "ndcg_linear@10": 0.7448582658380257, "ndcg_exponential@1": 0.448, - "ndcg_exponential@5": 0.7459721375093364, - "ndcg_exponential@10": 0.7459721375093364, + "ndcg_exponential@5": 0.7448582658380257, + "ndcg_exponential@10": 0.7448582658380257, "precision@1": 0.448, "precision@5": 0.2, "precision@10": 0.1, @@ -1262,207 +1262,207 @@ "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, "map@1": 0.448, - "map@5": 0.6608999999999999, - "map@10": 0.6608999999999999, + "map@5": 0.6595, + "map@10": 0.6595, "mrr@1": 0.448, - "mrr@5": 0.6608999999999999, - "mrr@10": 0.6608999999999999, + "mrr@5": 0.6595, + "mrr@10": 0.6595, "num_pred": 500, "num_data": 500 }, "ActivityNetQA": { - "hit@1": 0.532, + "hit@1": 0.538, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.532, - "ndcg_linear@5": 0.8272751246714422, - "ndcg_linear@10": 0.8272751246714422, - "ndcg_exponential@1": 0.532, - "ndcg_exponential@5": 0.8272751246714422, - "ndcg_exponential@10": 0.8272751246714422, - "precision@1": 0.532, + "ndcg_linear@1": 0.538, + "ndcg_linear@5": 0.8294895461500135, + "ndcg_linear@10": 0.8294895461500135, + "ndcg_exponential@1": 0.538, + "ndcg_exponential@5": 0.8294895461500135, + "ndcg_exponential@10": 0.8294895461500135, + "precision@1": 0.538, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.532, + "recall@1": 0.538, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.532, + "f1@1": 0.538, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.532, - "map@5": 0.766, - "map@10": 0.766, - "mrr@1": 0.532, - "mrr@5": 0.766, - "mrr@10": 0.766, + "map@1": 0.538, + "map@5": 0.769, + "map@10": 0.769, + "mrr@1": 0.538, + "mrr@5": 0.769, + "mrr@10": 0.769, "num_pred": 1000, "num_data": 1000 }, "DiDeMo": { - "hit@1": 0.24800796812749004, - "hit@5": 0.4711155378486056, - "hit@10": 0.5677290836653387, - "ndcg_linear@1": 0.24800796812749004, - "ndcg_linear@5": 0.36275185397452103, - "ndcg_linear@10": 0.3937078477068639, - "ndcg_exponential@1": 0.24800796812749004, - "ndcg_exponential@5": 0.36275185397452103, - "ndcg_exponential@10": 0.3937078477068639, - "precision@1": 0.24800796812749004, - "precision@5": 0.0942231075697211, - "precision@10": 0.05677290836653386, - "recall@1": 0.24800796812749004, - "recall@5": 0.4711155378486056, - "recall@10": 0.5677290836653387, - "f1@1": 0.24800796812749004, - "f1@5": 0.15703851261620191, - "f1@10": 0.10322346975733432, - "map@1": 0.24800796812749004, - "map@5": 0.3271082337317397, - "map@10": 0.33971653070258645, - "mrr@1": 0.24800796812749004, - "mrr@5": 0.3271082337317397, - "mrr@10": 0.33971653070258645, + "hit@1": 0.25, + "hit@5": 0.4681274900398406, + "hit@10": 0.5657370517928287, + "ndcg_linear@1": 0.25, + "ndcg_linear@5": 0.36244669367881044, + "ndcg_linear@10": 0.3939430128503134, + "ndcg_exponential@1": 0.25, + "ndcg_exponential@5": 0.36244669367881044, + "ndcg_exponential@10": 0.3939430128503134, + "precision@1": 0.25, + "precision@5": 0.09362549800796813, + "precision@10": 0.05657370517928287, + "recall@1": 0.25, + "recall@5": 0.4681274900398406, + "recall@10": 0.5657370517928287, + "f1@1": 0.25, + "f1@5": 0.15604249667994693, + "f1@10": 0.10286128214415068, + "map@1": 0.25, + "map@5": 0.32765604249667996, + "map@10": 0.34060938784544365, + "mrr@1": 0.25, + "mrr@5": 0.32765604249667996, + "mrr@10": 0.34060938784544365, "num_pred": 1004, "num_data": 1004 }, "MSR-VTT": { - "hit@1": 0.221, - "hit@5": 0.441, - "hit@10": 0.558, - "ndcg_linear@1": 0.221, - "ndcg_linear@5": 0.3378462330246669, - "ndcg_linear@10": 0.37615489678599645, - "ndcg_exponential@1": 0.221, - "ndcg_exponential@5": 0.3378462330246669, - "ndcg_exponential@10": 0.37615489678599645, - "precision@1": 0.221, - "precision@5": 0.08819999999999999, - "precision@10": 0.055799999999999995, - "recall@1": 0.221, - "recall@5": 0.441, - "recall@10": 0.558, - "f1@1": 0.221, - "f1@5": 0.14700000000000002, - "f1@10": 0.10145454545454546, - "map@1": 0.221, - "map@5": 0.30355, - "map@10": 0.31964126984126984, - "mrr@1": 0.221, - "mrr@5": 0.30355, - "mrr@10": 0.31964126984126984, + "hit@1": 0.226, + "hit@5": 0.446, + "hit@10": 0.55, + "ndcg_linear@1": 0.226, + "ndcg_linear@5": 0.34101683007726125, + "ndcg_linear@10": 0.3752643739670573, + "ndcg_exponential@1": 0.226, + "ndcg_exponential@5": 0.34101683007726125, + "ndcg_exponential@10": 0.3752643739670573, + "precision@1": 0.226, + "precision@5": 0.08919999999999999, + "precision@10": 0.055, + "recall@1": 0.226, + "recall@5": 0.446, + "recall@10": 0.55, + "f1@1": 0.226, + "f1@5": 0.1486666666666667, + "f1@10": 0.10000000000000003, + "map@1": 0.226, + "map@5": 0.3063166666666667, + "map@10": 0.3208087301587302, + "mrr@1": 0.226, + "mrr@5": 0.3063166666666667, + "mrr@10": 0.3208087301587302, "num_pred": 1000, "num_data": 1000 }, "MSVD": { - "hit@1": 0.46119402985074626, - "hit@5": 0.7, - "hit@10": 0.7850746268656716, - "ndcg_linear@1": 0.46119402985074626, - "ndcg_linear@5": 0.5909143508859, - "ndcg_linear@10": 0.6184940826509501, - "ndcg_exponential@1": 0.46119402985074626, - "ndcg_exponential@5": 0.5909143508859, - "ndcg_exponential@10": 0.6184940826509501, - "precision@1": 0.46119402985074626, - "precision@5": 0.13999999999999999, - "precision@10": 0.07850746268656715, - "recall@1": 0.46119402985074626, - "recall@5": 0.7, - "recall@10": 0.7850746268656716, - "f1@1": 0.46119402985074626, - "f1@5": 0.2333333333333334, - "f1@10": 0.14274084124830397, - "map@1": 0.46119402985074626, - "map@5": 0.5543781094527364, - "map@10": 0.5658019426676143, - "mrr@1": 0.46119402985074626, - "mrr@5": 0.5543781094527364, - "mrr@10": 0.5658019426676143, + "hit@1": 0.46417910447761196, + "hit@5": 0.7044776119402985, + "hit@10": 0.7880597014925373, + "ndcg_linear@1": 0.46417910447761196, + "ndcg_linear@5": 0.5945033588979008, + "ndcg_linear@10": 0.6213918548501596, + "ndcg_exponential@1": 0.46417910447761196, + "ndcg_exponential@5": 0.5945033588979008, + "ndcg_exponential@10": 0.6213918548501596, + "precision@1": 0.46417910447761196, + "precision@5": 0.1408955223880597, + "precision@10": 0.07880597014925372, + "recall@1": 0.46417910447761196, + "recall@5": 0.7044776119402985, + "recall@10": 0.7880597014925373, + "f1@1": 0.46417910447761196, + "f1@5": 0.23482587064676622, + "f1@10": 0.14328358208955225, + "map@1": 0.46417910447761196, + "map@5": 0.5577114427860697, + "map@10": 0.5687266050698887, + "mrr@1": 0.46417910447761196, + "mrr@5": 0.5577114427860697, + "mrr@10": 0.5687266050698887, "num_pred": 670, "num_data": 670 }, "VATEX": { - "hit@1": 0.19138008039303261, - "hit@5": 0.41692719964269764, - "hit@10": 0.5225547119249665, - "ndcg_linear@1": 0.19138008039303261, - "ndcg_linear@5": 0.31066559826092355, - "ndcg_linear@10": 0.3449304405187584, - "ndcg_exponential@1": 0.19138008039303261, - "ndcg_exponential@5": 0.31066559826092355, - "ndcg_exponential@10": 0.3449304405187584, - "precision@1": 0.19138008039303261, - "precision@5": 0.08338543992853951, - "precision@10": 0.05225547119249665, - "recall@1": 0.19138008039303261, - "recall@5": 0.41692719964269764, - "recall@10": 0.5225547119249665, - "f1@1": 0.19138008039303261, - "f1@5": 0.13897573321423257, - "f1@10": 0.09500994762272119, - "map@1": 0.19138008039303261, - "map@5": 0.27535730236712824, - "map@10": 0.28956069177708305, - "mrr@1": 0.19138008039303261, - "mrr@5": 0.27535730236712824, - "mrr@10": 0.28956069177708305, + "hit@1": 0.19093345243412238, + "hit@5": 0.4167038856632425, + "hit@10": 0.5214381420276909, + "ndcg_linear@1": 0.19093345243412238, + "ndcg_linear@5": 0.3100428592757246, + "ndcg_linear@10": 0.34396225202583136, + "ndcg_exponential@1": 0.19093345243412238, + "ndcg_exponential@5": 0.3100428592757246, + "ndcg_exponential@10": 0.34396225202583136, + "precision@1": 0.19093345243412238, + "precision@5": 0.08334077713264851, + "precision@10": 0.0521438142027691, + "recall@1": 0.19093345243412238, + "recall@5": 0.4167038856632425, + "recall@10": 0.5214381420276909, + "f1@1": 0.19093345243412238, + "f1@5": 0.13890129522108083, + "f1@10": 0.09480693491412563, + "map@1": 0.19093345243412238, + "map@5": 0.2746240881345839, + "map@10": 0.28865210517733964, + "mrr@1": 0.19093345243412238, + "mrr@5": 0.2746240881345839, + "mrr@10": 0.28865210517733964, "num_pred": 4478, "num_data": 4478 }, "YouCook2": { - "hit@1": 0.09153821956590123, - "hit@5": 0.21358917898710286, - "hit@10": 0.29285938974520287, - "ndcg_linear@1": 0.09153821956590123, - "ndcg_linear@5": 0.15446626761982413, - "ndcg_linear@10": 0.18024480954320224, - "ndcg_exponential@1": 0.09153821956590123, - "ndcg_exponential@5": 0.15446626761982413, - "ndcg_exponential@10": 0.18024480954320224, - "precision@1": 0.09153821956590123, - "precision@5": 0.04271783579742058, - "precision@10": 0.02928593897452029, - "recall@1": 0.09153821956590123, - "recall@5": 0.21358917898710286, - "recall@10": 0.29285938974520287, - "f1@1": 0.09153821956590123, - "f1@5": 0.07119639299570096, - "f1@10": 0.053247161771855074, - "map@1": 0.09153821956590123, - "map@5": 0.13498479605746042, - "map@10": 0.14571006655781743, - "mrr@1": 0.09153821956590123, - "mrr@5": 0.13498479605746042, - "mrr@10": 0.14571006655781743, + "hit@1": 0.09248191255111671, + "hit@5": 0.21516200062912866, + "hit@10": 0.29348851840201323, + "ndcg_linear@1": 0.09248191255111671, + "ndcg_linear@5": 0.15588090628774104, + "ndcg_linear@10": 0.18136804004616694, + "ndcg_exponential@1": 0.09248191255111671, + "ndcg_exponential@5": 0.15588090628774104, + "ndcg_exponential@10": 0.18136804004616694, + "precision@1": 0.09248191255111671, + "precision@5": 0.04303240012582574, + "precision@10": 0.029348851840201324, + "recall@1": 0.09248191255111671, + "recall@5": 0.21516200062912866, + "recall@10": 0.29348851840201323, + "f1@1": 0.09248191255111671, + "f1@5": 0.07172066687637622, + "f1@10": 0.05336154880036603, + "map@1": 0.09248191255111671, + "map@5": 0.13634790814721612, + "map@10": 0.1469588370107401, + "mrr@1": 0.09248191255111671, + "mrr@5": 0.13634790814721612, + "mrr@10": 0.1469588370107401, "num_pred": 3179, "num_data": 3179 }, "QVHighlight": { - "hit@1": 0.5383194829178208, + "hit@1": 0.5392428439519852, "hit@5": 0.863342566943675, "hit@10": 1.0, - "ndcg_linear@1": 0.5383194829178208, - "ndcg_linear@5": 0.7079527214519653, - "ndcg_linear@10": 0.7516086585604039, - "ndcg_exponential@1": 0.5383194829178208, - "ndcg_exponential@5": 0.7079527214519653, - "ndcg_exponential@10": 0.7516086585604039, - "precision@1": 0.5383194829178208, - "precision@5": 0.17266851338873498, + "ndcg_linear@1": 0.5392428439519852, + "ndcg_linear@5": 0.70888206397498, + "ndcg_linear@10": 0.7525081734401413, + "ndcg_exponential@1": 0.5392428439519852, + "ndcg_exponential@5": 0.70888206397498, + "ndcg_exponential@10": 0.7525081734401413, + "precision@1": 0.5392428439519852, + "precision@5": 0.17266851338873496, "precision@10": 0.09999999999999998, - "recall@1": 0.5383194829178208, + "recall@1": 0.5392428439519852, "recall@5": 0.863342566943675, "recall@10": 1.0, - "f1@1": 0.5383194829178208, + "f1@1": 0.5392428439519852, "f1@5": 0.28778085564789174, "f1@10": 0.18181818181818185, - "map@1": 0.5383194829178208, - "map@5": 0.6565712526931364, - "map@10": 0.6742862272640666, - "mrr@1": 0.5383194829178208, - "mrr@5": 0.6565712526931364, - "mrr@10": 0.6742862272640666, + "map@1": 0.5392428439519852, + "map@5": 0.6577562326869806, + "map@10": 0.6754418942092073, + "mrr@1": 0.5392428439519852, + "mrr@5": 0.6577562326869806, + "mrr@10": 0.6754418942092073, "num_pred": 1083, "num_data": 1083 }, @@ -1471,11 +1471,11 @@ "hit@5": 0.4786795048143054, "hit@10": 1.0, "ndcg_linear@1": 0.10866574965612105, - "ndcg_linear@5": 0.2926265568216908, - "ndcg_linear@10": 0.4575713199023012, + "ndcg_linear@5": 0.2920307085226254, + "ndcg_linear@10": 0.45711393209611073, "ndcg_exponential@1": 0.10866574965612105, - "ndcg_exponential@5": 0.2926265568216908, - "ndcg_exponential@10": 0.4575713199023012, + "ndcg_exponential@5": 0.2920307085226254, + "ndcg_exponential@10": 0.45711393209611073, "precision@1": 0.10866574965612105, "precision@5": 0.09573590096286107, "precision@10": 0.09999999999999998, @@ -1486,713 +1486,685 @@ "f1@5": 0.15955983493810183, "f1@10": 0.18181818181818185, "map@1": 0.10866574965612105, - "map@5": 0.23207244383310408, - "map@10": 0.2980508067509443, + "map@5": 0.23131591013296657, + "map@10": 0.29742418287810307, "mrr@1": 0.10866574965612105, - "mrr@5": 0.23207244383310408, - "mrr@10": 0.2980508067509443, + "mrr@5": 0.23131591013296657, + "mrr@10": 0.29742418287810307, "num_pred": 727, "num_data": 727 }, "MomentSeeker": { - "hit@1": 0.3383333333333333, - "hit@5": 0.8205555555555556, + "hit@1": 0.3595505617977528, + "hit@5": 0.8252184769038702, + "hit@10": 1.0, + "ndcg_linear@1": 0.3595505617977528, + "ndcg_linear@5": 0.5941517733286235, + "ndcg_linear@10": 0.6607268099543956, + "ndcg_exponential@1": 0.3595505617977528, + "ndcg_exponential@5": 0.5941517733286235, + "ndcg_exponential@10": 0.6607268099543956, + "precision@1": 0.3595505617977528, + "precision@5": 0.1818976279650437, + "precision@10": 0.11466916354556804, + "recall@1": 0.3341656263004577, + "recall@5": 0.8028506034124012, + "recall@10": 0.9987515605493134, + "f1@1": 0.3414402631631096, + "f1@5": 0.29094733120950356, + "f1@10": 0.20272001545409785, + "map@1": 0.3595505617977528, + "map@5": 0.5186889651824108, + "map@10": 0.5496653798327488, + "mrr@1": 0.3595505617977528, + "mrr@5": 0.5315126924677487, + "mrr@10": 0.5557661058597387, + "num_pred": 1602, + "num_data": 1602 + }, + "MomentSeeker_1k8": { + "hit@1": 0.3438888888888889, + "hit@5": 0.8194444444444444, "hit@10": 1.0, - "ndcg_linear@1": 0.3383333333333333, - "ndcg_linear@5": 0.5821798365943692, - "ndcg_linear@10": 0.6491829437988351, - "ndcg_exponential@1": 0.3383333333333333, - "ndcg_exponential@5": 0.5821798365943692, - "ndcg_exponential@10": 0.6491829437988351, - "precision@1": 0.3383333333333333, - "precision@5": 0.17922222222222223, + "ndcg_linear@1": 0.3438888888888889, + "ndcg_linear@5": 0.5837020570147833, + "ndcg_linear@10": 0.6511472349305348, + "ndcg_exponential@1": 0.3438888888888889, + "ndcg_exponential@5": 0.5837020570147833, + "ndcg_exponential@10": 0.6511472349305348, + "precision@1": 0.3438888888888889, + "precision@5": 0.17900000000000002, "precision@10": 0.11311111111111112, - "recall@1": 0.31546296296296295, - "recall@5": 0.8009259259259259, + "recall@1": 0.32129629629629625, + "recall@5": 0.799537037037037, "recall@10": 0.9988888888888889, - "f1@1": 0.3220299823633157, - "f1@5": 0.28780555863889207, + "f1@1": 0.3277707231040564, + "f1@5": 0.2874087332420666, "f1@10": 0.20050498883832216, - "map@1": 0.3383333333333333, - "map@5": 0.5041177469135804, - "map@10": 0.5349385196208113, - "mrr@1": 0.3383333333333333, - "mrr@5": 0.5154722222222222, - "mrr@10": 0.540335758377425, + "map@1": 0.3438888888888889, + "map@5": 0.5066424382716049, + "map@10": 0.5376596395502645, + "mrr@1": 0.3438888888888889, + "mrr@5": 0.5180092592592592, + "mrr@10": 0.5430429894179893, "num_pred": 1800, "num_data": 1800 } }, "visdoc": { "ViDoRe_arxivqa": { - "hit@1": 0.052, - "hit@5": 0.172, - "hit@10": 0.286, - "ndcg_linear@1": 0.052, - "ndcg_linear@5": 0.10801878370138777, - "ndcg_linear@10": 0.14373908906362196, - "ndcg_exponential@1": 0.052, - "ndcg_exponential@5": 0.10801878370138777, - "ndcg_exponential@10": 0.14373908906362196, - "precision@1": 0.052, - "precision@5": 0.03440000000000001, - "precision@10": 0.0286, - "recall@1": 0.052, - "recall@5": 0.172, - "recall@10": 0.286, - "f1@1": 0.052, - "f1@5": 0.05733333333333333, - "f1@10": 0.052, - "map@1": 0.052, - "map@5": 0.08753333333333334, - "map@10": 0.10160873015873015, - "mrr@1": 0.052, - "mrr@5": 0.08753333333333334, - "mrr@10": 0.10160873015873015, + "hit@1": 0.224, + "hit@5": 0.402, + "hit@10": 0.494, + "ndcg_linear@1": 0.224, + "ndcg_linear@5": 0.3153720115113372, + "ndcg_linear@10": 0.3448580633481198, + "ndcg_exponential@1": 0.224, + "ndcg_exponential@5": 0.3153720115113372, + "ndcg_exponential@10": 0.3448580633481198, + "precision@1": 0.224, + "precision@5": 0.0804, + "precision@10": 0.0494, + "recall@1": 0.224, + "recall@5": 0.402, + "recall@10": 0.494, + "f1@1": 0.224, + "f1@5": 0.13400000000000004, + "f1@10": 0.08981818181818182, + "map@1": 0.224, + "map@5": 0.28686666666666666, + "map@10": 0.29886984126984123, + "mrr@1": 0.224, + "mrr@5": 0.28686666666666666, + "mrr@10": 0.29886984126984123, "num_pred": 500, "num_data": 500 }, "ViDoRe_docvqa": { - "hit@1": 0.13747228381374724, - "hit@5": 0.2549889135254989, - "hit@10": 0.3148558758314856, - "ndcg_linear@1": 0.13747228381374724, - "ndcg_linear@5": 0.19129511635223329, - "ndcg_linear@10": 0.2105752120749416, - "ndcg_exponential@1": 0.13747228381374724, - "ndcg_exponential@5": 0.19129511635223329, - "ndcg_exponential@10": 0.2105752120749416, - "precision@1": 0.13747228381374724, - "precision@5": 0.050997782705099776, - "precision@10": 0.031929046563192905, - "recall@1": 0.13458980044345897, - "recall@5": 0.24811529933481155, - "recall@10": 0.30842572062084256, - "f1@1": 0.1352549889135255, - "f1@5": 0.08408826945412314, - "f1@10": 0.057562319424847144, - "map@1": 0.13747228381374724, - "map@5": 0.17147080561714712, - "map@10": 0.17921637982613592, - "mrr@1": 0.13747228381374724, - "mrr@5": 0.17597930524759794, - "mrr@10": 0.1834661950515609, + "hit@1": 0.11751662971175167, + "hit@5": 0.28159645232815966, + "hit@10": 0.36807095343680707, + "ndcg_linear@1": 0.11751662971175167, + "ndcg_linear@5": 0.19851344506911198, + "ndcg_linear@10": 0.2263303245060071, + "ndcg_exponential@1": 0.11751662971175167, + "ndcg_exponential@5": 0.19851344506911198, + "ndcg_exponential@10": 0.2263303245060071, + "precision@1": 0.11751662971175167, + "precision@5": 0.057649667405764965, + "precision@10": 0.037694013303769404, + "recall@1": 0.11574279379157429, + "recall@5": 0.2756836659275684, + "recall@10": 0.36049519586104956, + "f1@1": 0.11603843311160385, + "f1@5": 0.09420863689156372, + "f1@10": 0.06771425174972848, + "map@1": 0.11751662971175167, + "map@5": 0.17224685883222468, + "map@10": 0.1838724791468694, + "mrr@1": 0.11751662971175167, + "mrr@5": 0.17457501847745752, + "mrr@10": 0.18619733924611975, "num_pred": 451, "num_data": 451 }, "ViDoRe_infovqa": { - "hit@1": 0.3319838056680162, - "hit@5": 0.5748987854251012, - "hit@10": 0.6923076923076923, - "ndcg_linear@1": 0.3319838056680162, - "ndcg_linear@5": 0.46253535711806587, - "ndcg_linear@10": 0.5004886054716596, - "ndcg_exponential@1": 0.3319838056680162, - "ndcg_exponential@5": 0.46253535711806587, - "ndcg_exponential@10": 0.5004886054716596, - "precision@1": 0.3319838056680162, - "precision@5": 0.11497975708502024, - "precision@10": 0.06923076923076922, - "recall@1": 0.3319838056680162, - "recall@5": 0.5748987854251012, - "recall@10": 0.6912955465587044, - "f1@1": 0.3319838056680162, - "f1@5": 0.19163292847503377, - "f1@10": 0.12584345479082323, - "map@1": 0.3319838056680162, - "map@5": 0.4250337381916329, - "map@10": 0.4408127691022428, - "mrr@1": 0.3319838056680162, - "mrr@5": 0.4250337381916329, - "mrr@10": 0.4409252297410192, + "hit@1": 0.4959514170040486, + "hit@5": 0.7510121457489879, + "hit@10": 0.7955465587044535, + "ndcg_linear@1": 0.4959514170040486, + "ndcg_linear@5": 0.6367270116572423, + "ndcg_linear@10": 0.6511524672718955, + "ndcg_exponential@1": 0.4959514170040486, + "ndcg_exponential@5": 0.6367270116572423, + "ndcg_exponential@10": 0.6511524672718955, + "precision@1": 0.4959514170040486, + "precision@5": 0.1506072874493927, + "precision@10": 0.0797570850202429, + "recall@1": 0.4959514170040486, + "recall@5": 0.7510121457489879, + "recall@10": 0.7955465587044535, + "f1@1": 0.4959514170040486, + "f1@5": 0.25081935608251404, + "f1@10": 0.1449515396883818, + "map@1": 0.4959514170040486, + "map@5": 0.5982118758434548, + "map@10": 0.6041755028597133, + "mrr@1": 0.4959514170040486, + "mrr@5": 0.5980431848852901, + "mrr@10": 0.6040068119015487, "num_pred": 494, "num_data": 494 }, "ViDoRe_tabfquad": { - "hit@1": 0.275, - "hit@5": 0.5678571428571428, - "hit@10": 0.6928571428571428, - "ndcg_linear@1": 0.275, - "ndcg_linear@5": 0.42836855281007846, - "ndcg_linear@10": 0.4688038965463907, - "ndcg_exponential@1": 0.275, - "ndcg_exponential@5": 0.42836855281007846, - "ndcg_exponential@10": 0.4688038965463907, - "precision@1": 0.275, - "precision@5": 0.11357142857142856, - "precision@10": 0.06928571428571428, - "recall@1": 0.275, - "recall@5": 0.5678571428571428, - "recall@10": 0.6928571428571428, - "f1@1": 0.275, - "f1@5": 0.1892857142857143, - "f1@10": 0.125974025974026, - "map@1": 0.275, - "map@5": 0.3823214285714286, - "map@10": 0.3990192743764172, - "mrr@1": 0.275, - "mrr@5": 0.3823214285714286, - "mrr@10": 0.3990192743764172, + "hit@1": 0.36428571428571427, + "hit@5": 0.6821428571428572, + "hit@10": 0.7678571428571429, + "ndcg_linear@1": 0.36428571428571427, + "ndcg_linear@5": 0.534720551900067, + "ndcg_linear@10": 0.5622505955912354, + "ndcg_exponential@1": 0.36428571428571427, + "ndcg_exponential@5": 0.534720551900067, + "ndcg_exponential@10": 0.5622505955912354, + "precision@1": 0.36428571428571427, + "precision@5": 0.13642857142857143, + "precision@10": 0.07678571428571429, + "recall@1": 0.36428571428571427, + "recall@5": 0.6821428571428572, + "recall@10": 0.7678571428571429, + "f1@1": 0.36428571428571427, + "f1@5": 0.22738095238095243, + "f1@10": 0.13961038961038963, + "map@1": 0.36428571428571427, + "map@5": 0.4855952380952381, + "map@10": 0.4968494897959183, + "mrr@1": 0.36428571428571427, + "mrr@5": 0.4855952380952381, + "mrr@10": 0.4968494897959183, "num_pred": 280, "num_data": 280 }, "ViDoRe_tatdqa": { - "hit@1": 0.06804374240583232, - "hit@5": 0.1591737545565006, - "hit@10": 0.22053462940461727, - "ndcg_linear@1": 0.06804374240583232, - "ndcg_linear@5": 0.11430035232416713, - "ndcg_linear@10": 0.13412008753307786, - "ndcg_exponential@1": 0.06804374240583232, - "ndcg_exponential@5": 0.11430035232416713, - "ndcg_exponential@10": 0.13412008753307786, - "precision@1": 0.06804374240583232, - "precision@5": 0.03183475091130013, - "precision@10": 0.022053462940461725, - "recall@1": 0.06804374240583232, - "recall@5": 0.1586269744835966, - "recall@10": 0.21998784933171325, - "f1@1": 0.06804374240583232, - "f1@5": 0.052936411502632646, - "f1@10": 0.04004749806693913, - "map@1": 0.06804374240583232, - "map@5": 0.09965978128797083, - "map@10": 0.1078442689347914, - "mrr@1": 0.06804374240583232, - "mrr@5": 0.09975698663426488, - "mrr@10": 0.10795362494937222, + "hit@1": 0.02916160388821385, + "hit@5": 0.12636695018226002, + "hit@10": 0.1852976913730255, + "ndcg_linear@1": 0.02916160388821385, + "ndcg_linear@5": 0.07879193887194902, + "ndcg_linear@10": 0.09767103860648786, + "ndcg_exponential@1": 0.02916160388821385, + "ndcg_exponential@5": 0.07879193887194902, + "ndcg_exponential@10": 0.09767103860648786, + "precision@1": 0.02916160388821385, + "precision@5": 0.025273390036452004, + "precision@10": 0.018529769137302553, + "recall@1": 0.02916160388821385, + "recall@5": 0.12575941676792224, + "recall@10": 0.18469015795868773, + "f1@1": 0.02916160388821385, + "f1@5": 0.04206445640224498, + "f1@10": 0.03367207923708531, + "map@1": 0.02916160388821385, + "map@5": 0.06323916565411097, + "map@10": 0.07092181141391347, + "mrr@1": 0.02916160388821385, + "mrr@5": 0.0634163628999595, + "mrr@10": 0.071099008659762, "num_pred": 1646, "num_data": 1646 }, "ViDoRe_shiftproject": { - "hit@1": 0.07, - "hit@5": 0.17, - "hit@10": 0.27, - "ndcg_linear@1": 0.07, - "ndcg_linear@5": 0.12035565121611999, - "ndcg_linear@10": 0.1523429715395014, - "ndcg_exponential@1": 0.07, - "ndcg_exponential@5": 0.12035565121611999, - "ndcg_exponential@10": 0.1523429715395014, - "precision@1": 0.07, - "precision@5": 0.034, - "precision@10": 0.027000000000000003, - "recall@1": 0.07, - "recall@5": 0.17, - "recall@10": 0.27, - "f1@1": 0.07, - "f1@5": 0.05666666666666667, - "f1@10": 0.049090909090909095, - "map@1": 0.07, - "map@5": 0.10400000000000001, - "map@10": 0.116984126984127, - "mrr@1": 0.07, - "mrr@5": 0.10400000000000001, - "mrr@10": 0.116984126984127, + "hit@1": 0.09, + "hit@5": 0.22, + "hit@10": 0.34, + "ndcg_linear@1": 0.09, + "ndcg_linear@5": 0.15946678442077467, + "ndcg_linear@10": 0.1983380188867217, + "ndcg_exponential@1": 0.09, + "ndcg_exponential@5": 0.15946678442077467, + "ndcg_exponential@10": 0.1983380188867217, + "precision@1": 0.09, + "precision@5": 0.044000000000000004, + "precision@10": 0.034, + "recall@1": 0.09, + "recall@5": 0.22, + "recall@10": 0.34, + "f1@1": 0.09, + "f1@5": 0.07333333333333333, + "f1@10": 0.061818181818181814, + "map@1": 0.09, + "map@5": 0.13916666666666666, + "map@10": 0.15528968253968253, + "mrr@1": 0.09, + "mrr@5": 0.13916666666666666, + "mrr@10": 0.15528968253968253, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.05, - "hit@5": 0.15, - "hit@10": 0.22, - "ndcg_linear@1": 0.05, - "ndcg_linear@5": 0.1031574868850601, - "ndcg_linear@10": 0.12568086090486966, - "ndcg_exponential@1": 0.05, - "ndcg_exponential@5": 0.1031574868850601, - "ndcg_exponential@10": 0.12568086090486966, - "precision@1": 0.05, - "precision@5": 0.030000000000000006, - "precision@10": 0.022, - "recall@1": 0.05, - "recall@5": 0.15, - "recall@10": 0.22, - "f1@1": 0.05, - "f1@5": 0.05, - "f1@10": 0.04, - "map@1": 0.05, - "map@5": 0.0875, - "map@10": 0.09673412698412699, - "mrr@1": 0.05, - "mrr@5": 0.0875, - "mrr@10": 0.09673412698412699, + "hit@1": 0.21, + "hit@5": 0.38, + "hit@10": 0.49, + "ndcg_linear@1": 0.21, + "ndcg_linear@5": 0.2982527262008516, + "ndcg_linear@10": 0.33346429301328034, + "ndcg_exponential@1": 0.21, + "ndcg_exponential@5": 0.2982527262008516, + "ndcg_exponential@10": 0.33346429301328034, + "precision@1": 0.21, + "precision@5": 0.076, + "precision@10": 0.048999999999999995, + "recall@1": 0.21, + "recall@5": 0.38, + "recall@10": 0.49, + "f1@1": 0.21, + "f1@5": 0.12666666666666668, + "f1@10": 0.0890909090909091, + "map@1": 0.21, + "map@5": 0.2713333333333333, + "map@10": 0.2856190476190476, + "mrr@1": 0.21, + "mrr@5": 0.2713333333333333, + "mrr@10": 0.2856190476190476, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.18, - "hit@5": 0.31, - "hit@10": 0.41, - "ndcg_linear@1": 0.18, - "ndcg_linear@5": 0.24771924937667159, - "ndcg_linear@10": 0.2802583416144934, - "ndcg_exponential@1": 0.18, - "ndcg_exponential@5": 0.24771924937667159, - "ndcg_exponential@10": 0.2802583416144934, - "precision@1": 0.18, - "precision@5": 0.062, - "precision@10": 0.040999999999999995, - "recall@1": 0.18, - "recall@5": 0.31, - "recall@10": 0.41, - "f1@1": 0.18, - "f1@5": 0.10333333333333336, - "f1@10": 0.07454545454545455, - "map@1": 0.18, - "map@5": 0.22699999999999998, - "map@10": 0.24053968253968258, - "mrr@1": 0.18, - "mrr@5": 0.22699999999999998, - "mrr@10": 0.24053968253968258, + "hit@1": 0.24, + "hit@5": 0.48, + "hit@10": 0.59, + "ndcg_linear@1": 0.24, + "ndcg_linear@5": 0.3604743735343363, + "ndcg_linear@10": 0.3956497521671606, + "ndcg_exponential@1": 0.24, + "ndcg_exponential@5": 0.3604743735343363, + "ndcg_exponential@10": 0.3956497521671606, + "precision@1": 0.24, + "precision@5": 0.096, + "precision@10": 0.05899999999999998, + "recall@1": 0.24, + "recall@5": 0.48, + "recall@10": 0.59, + "f1@1": 0.24, + "f1@5": 0.16000000000000003, + "f1@10": 0.10727272727272728, + "map@1": 0.24, + "map@5": 0.3215, + "map@10": 0.33580158730158727, + "mrr@1": 0.24, + "mrr@5": 0.3215, + "mrr@10": 0.33580158730158727, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.09, - "hit@5": 0.23, - "hit@10": 0.27, - "ndcg_linear@1": 0.09, - "ndcg_linear@5": 0.16359030940399763, - "ndcg_linear@10": 0.17675975017719422, - "ndcg_exponential@1": 0.09, - "ndcg_exponential@5": 0.16359030940399763, - "ndcg_exponential@10": 0.17675975017719422, - "precision@1": 0.09, - "precision@5": 0.046000000000000006, - "precision@10": 0.027000000000000007, - "recall@1": 0.09, - "recall@5": 0.23, - "recall@10": 0.27, - "f1@1": 0.09, - "f1@5": 0.07666666666666667, - "f1@10": 0.04909090909090908, - "map@1": 0.09, - "map@5": 0.14150000000000001, - "map@10": 0.14708333333333334, - "mrr@1": 0.09, - "mrr@5": 0.14150000000000001, - "mrr@10": 0.14708333333333334, + "hit@1": 0.29, + "hit@5": 0.54, + "hit@10": 0.68, + "ndcg_linear@1": 0.29, + "ndcg_linear@5": 0.4118123078169721, + "ndcg_linear@10": 0.45602849143213303, + "ndcg_exponential@1": 0.29, + "ndcg_exponential@5": 0.4118123078169721, + "ndcg_exponential@10": 0.45602849143213303, + "precision@1": 0.29, + "precision@5": 0.10799999999999997, + "precision@10": 0.06799999999999998, + "recall@1": 0.29, + "recall@5": 0.54, + "recall@10": 0.68, + "f1@1": 0.29, + "f1@5": 0.18, + "f1@10": 0.12363636363636366, + "map@1": 0.29, + "map@5": 0.3698333333333334, + "map@10": 0.38744841269841274, + "mrr@1": 0.29, + "mrr@5": 0.3698333333333334, + "mrr@10": 0.38744841269841274, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.14, - "hit@5": 0.36, - "hit@10": 0.4, - "ndcg_linear@1": 0.14, - "ndcg_linear@5": 0.2593663913604869, - "ndcg_linear@10": 0.2717200597145647, - "ndcg_exponential@1": 0.14, - "ndcg_exponential@5": 0.2593663913604869, - "ndcg_exponential@10": 0.2717200597145647, - "precision@1": 0.14, - "precision@5": 0.07200000000000001, - "precision@10": 0.04, - "recall@1": 0.14, - "recall@5": 0.36, - "recall@10": 0.4, - "f1@1": 0.14, - "f1@5": 0.12000000000000002, - "f1@10": 0.07272727272727274, - "map@1": 0.14, - "map@5": 0.22566666666666665, - "map@10": 0.2304444444444444, - "mrr@1": 0.14, - "mrr@5": 0.22566666666666665, - "mrr@10": 0.2304444444444444, + "hit@1": 0.17, + "hit@5": 0.59, + "hit@10": 0.67, + "ndcg_linear@1": 0.17, + "ndcg_linear@5": 0.3882529874458062, + "ndcg_linear@10": 0.41469672868032115, + "ndcg_exponential@1": 0.17, + "ndcg_exponential@5": 0.3882529874458062, + "ndcg_exponential@10": 0.41469672868032115, + "precision@1": 0.17, + "precision@5": 0.118, + "precision@10": 0.06699999999999999, + "recall@1": 0.17, + "recall@5": 0.59, + "recall@10": 0.67, + "f1@1": 0.17, + "f1@5": 0.1966666666666667, + "f1@10": 0.12181818181818184, + "map@1": 0.17, + "map@5": 0.32150000000000006, + "map@10": 0.33273015873015876, + "mrr@1": 0.17, + "mrr@5": 0.32150000000000006, + "mrr@10": 0.33273015873015876, "num_pred": 100, "num_data": 100 }, "ViDoRe_esg_reports_human_labeled_v2": { "hit@1": 0.038461538461538464, - "hit@5": 0.17307692307692307, + "hit@5": 0.15384615384615385, "hit@10": 0.23076923076923078, "ndcg_linear@1": 0.038461538461538464, - "ndcg_linear@5": 0.07581945100217463, - "ndcg_linear@10": 0.08884346676199335, + "ndcg_linear@5": 0.06907796797594362, + "ndcg_linear@10": 0.09062983467663395, "ndcg_exponential@1": 0.038461538461538464, - "ndcg_exponential@5": 0.07527592503218486, - "ndcg_exponential@10": 0.0882008820280547, + "ndcg_exponential@5": 0.06797434842270504, + "ndcg_exponential@10": 0.08987952752166918, "precision@1": 0.038461538461538464, - "precision@5": 0.05000000000000001, + "precision@5": 0.04615384615384616, "precision@10": 0.038461538461538464, "recall@1": 0.020833333333333332, - "recall@5": 0.10961538461538461, + "recall@5": 0.09152930402930402, "recall@10": 0.14761904761904762, "f1@1": 0.022189349112426034, - "f1@5": 0.057642031171442946, + "f1@5": 0.05217445952740071, "f1@10": 0.054193845370315956, "map@1": 0.038461538461538464, - "map@5": 0.05387820512820513, - "map@10": 0.05896291208791209, + "map@5": 0.05282051282051282, + "map@10": 0.061855540293040294, "mrr@1": 0.038461538461538464, - "mrr@5": 0.07724358974358975, - "mrr@10": 0.08392094017094018, + "mrr@5": 0.07660256410256411, + "mrr@10": 0.08701923076923078, "num_pred": 52, "num_data": 52 }, "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.096875, - "hit@5": 0.296875, - "hit@10": 0.3796875, - "ndcg_linear@1": 0.096875, - "ndcg_linear@5": 0.1327817509891042, - "ndcg_linear@10": 0.15578005969658662, - "ndcg_exponential@1": 0.096875, - "ndcg_exponential@5": 0.1327817509891042, - "ndcg_exponential@10": 0.15578005969658662, - "precision@1": 0.096875, - "precision@5": 0.06625, - "precision@10": 0.05015625, - "recall@1": 0.05542025862068965, - "recall@5": 0.1645906222010748, - "recall@10": 0.23159770265864968, - "f1@1": 0.06360192325036075, - "f1@5": 0.08230852582874641, - "f1@10": 0.07338045655325068, - "map@1": 0.096875, - "map@5": 0.09772092013888889, - "map@10": 0.10609532519605062, - "mrr@1": 0.096875, - "mrr@5": 0.16531250000000003, - "mrr@10": 0.1761204117063492, + "hit@1": 0.103125, + "hit@5": 0.29375, + "hit@10": 0.3828125, + "ndcg_linear@1": 0.103125, + "ndcg_linear@5": 0.13424270653897868, + "ndcg_linear@10": 0.15803866047633233, + "ndcg_exponential@1": 0.103125, + "ndcg_exponential@5": 0.13424270653897868, + "ndcg_exponential@10": 0.15803866047633233, + "precision@1": 0.103125, + "precision@5": 0.0665625, + "precision@10": 0.0503125, + "recall@1": 0.05857130028735632, + "recall@5": 0.16345483001194208, + "recall@10": 0.2344351501376413, + "f1@1": 0.06733890241702742, + "f1@5": 0.08263266069102466, + "f1@10": 0.0736531918007102, + "map@1": 0.103125, + "map@5": 0.09984071180555557, + "map@10": 0.10825468572845805, + "mrr@1": 0.103125, + "mrr@5": 0.1678125, + "mrr@10": 0.17938864087301587, "num_pred": 640, "num_data": 640 }, "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.1939655172413793, - "hit@5": 0.46551724137931033, - "hit@10": 0.5948275862068966, - "ndcg_linear@1": 0.1939655172413793, - "ndcg_linear@5": 0.19075341400656756, - "ndcg_linear@10": 0.19262260915535673, - "ndcg_exponential@1": 0.1939655172413793, - "ndcg_exponential@5": 0.19075341400656748, - "ndcg_exponential@10": 0.19262260915535673, - "precision@1": 0.1939655172413793, - "precision@5": 0.18448275862068964, - "precision@10": 0.16724137931034483, - "recall@1": 0.016014854892863833, - "recall@5": 0.07300449328968175, - "recall@10": 0.14033065985008555, - "f1@1": 0.029036373118143485, - "f1@5": 0.0903316153795555, - "f1@10": 0.12402634113864117, - "map@1": 0.1939655172413793, - "map@5": 0.12855603448275862, - "map@10": 0.1083281829653956, - "mrr@1": 0.1939655172413793, - "mrr@5": 0.2959051724137931, - "mrr@10": 0.3119937055281883, + "hit@1": 0.1810344827586207, + "hit@5": 0.46120689655172414, + "hit@10": 0.5818965517241379, + "ndcg_linear@1": 0.1810344827586207, + "ndcg_linear@5": 0.19366068249110216, + "ndcg_linear@10": 0.18779289722293727, + "ndcg_exponential@1": 0.1810344827586207, + "ndcg_exponential@5": 0.19366068249110216, + "ndcg_exponential@10": 0.18779289722293727, + "precision@1": 0.1810344827586207, + "precision@5": 0.18965517241379312, + "precision@10": 0.16379310344827586, + "recall@1": 0.015114529092723818, + "recall@5": 0.07947393210360344, + "recall@10": 0.12966604156960443, + "f1@1": 0.027448351339559092, + "f1@5": 0.0949100771238182, + "f1@10": 0.11949919740879811, + "map@1": 0.1810344827586207, + "map@5": 0.13191091954022988, + "map@10": 0.10826013335570499, + "mrr@1": 0.1810344827586207, + "mrr@5": 0.2882183908045977, + "mrr@10": 0.30242884510125895, "num_pred": 232, "num_data": 232 }, "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.043859649122807015, - "hit@5": 0.16228070175438597, - "hit@10": 0.25877192982456143, - "ndcg_linear@1": 0.043859649122807015, - "ndcg_linear@5": 0.05924300567334283, - "ndcg_linear@10": 0.09666072127808396, - "ndcg_exponential@1": 0.043859649122807015, - "ndcg_exponential@5": 0.05924300567334283, - "ndcg_exponential@10": 0.09666072127808396, - "precision@1": 0.043859649122807015, - "precision@5": 0.04912280701754387, - "precision@10": 0.054385964912280704, - "recall@1": 0.008406432748538011, - "recall@5": 0.07328216374269006, - "recall@10": 0.17160087719298245, - "f1@1": 0.013420302893987103, - "f1@5": 0.05224007874936668, - "f1@10": 0.07439770268717638, - "map@1": 0.043859649122807015, - "map@5": 0.03577850877192982, - "map@10": 0.0544104619186856, - "mrr@1": 0.043859649122807015, - "mrr@5": 0.07339181286549706, - "mrr@10": 0.08703181565023671, + "hit@1": 0.04824561403508772, + "hit@5": 0.17543859649122806, + "hit@10": 0.2807017543859649, + "ndcg_linear@1": 0.04824561403508772, + "ndcg_linear@5": 0.06428169092377227, + "ndcg_linear@10": 0.10296018433382544, + "ndcg_exponential@1": 0.04824561403508772, + "ndcg_exponential@5": 0.06428169092377227, + "ndcg_exponential@10": 0.10296018433382544, + "precision@1": 0.04824561403508772, + "precision@5": 0.050877192982456146, + "precision@10": 0.056578947368421055, + "recall@1": 0.008954678362573099, + "recall@5": 0.07748538011695906, + "recall@10": 0.18318713450292398, + "f1@1": 0.014394961763382815, + "f1@5": 0.05543339386218644, + "f1@10": 0.07785995680732523, + "map@1": 0.04824561403508772, + "map@5": 0.03937499999999999, + "map@10": 0.05765989046690801, + "mrr@1": 0.04824561403508772, + "mrr@5": 0.08296783625730993, + "mrr@10": 0.09645815928710666, "num_pred": 228, "num_data": 228 }, "VisRAG_ArxivQA": { "hit@1": 0.012254901960784314, - "hit@5": 0.02696078431372549, - "hit@10": 0.03553921568627451, + "hit@5": 0.028186274509803922, + "hit@10": 0.03431372549019608, "ndcg_linear@1": 0.012254901960784314, - "ndcg_linear@5": 0.020155243188820424, - "ndcg_linear@10": 0.022825943867728897, + "ndcg_linear@5": 0.020436961158955515, + "ndcg_linear@10": 0.022337490218016186, "ndcg_exponential@1": 0.012254901960784314, - "ndcg_exponential@5": 0.020155243188820424, - "ndcg_exponential@10": 0.022825943867728897, + "ndcg_exponential@5": 0.020436961158955515, + "ndcg_exponential@10": 0.022337490218016186, "precision@1": 0.012254901960784314, - "precision@5": 0.005392156862745099, - "precision@10": 0.0035539215686274514, + "precision@5": 0.005637254901960785, + "precision@10": 0.0034313725490196082, "recall@1": 0.012254901960784314, - "recall@5": 0.02696078431372549, - "recall@10": 0.03553921568627451, + "recall@5": 0.028186274509803922, + "recall@10": 0.03431372549019608, "f1@1": 0.012254901960784314, - "f1@5": 0.008986928104575164, - "f1@10": 0.006461675579322638, + "f1@5": 0.009395424836601309, + "f1@10": 0.006238859180035651, "map@1": 0.012254901960784314, - "map@5": 0.01787173202614379, - "map@10": 0.018913398692810456, + "map@5": 0.0178921568627451, + "map@10": 0.018630855119825707, "mrr@1": 0.012254901960784314, - "mrr@5": 0.01787173202614379, - "mrr@10": 0.018913398692810456, + "mrr@5": 0.0178921568627451, + "mrr@10": 0.018630855119825707, "num_pred": 816, "num_data": 816 }, "VisRAG_ChartQA": { - "hit@1": 0.31746031746031744, - "hit@5": 0.49206349206349204, - "hit@10": 0.5396825396825397, - "ndcg_linear@1": 0.31746031746031744, - "ndcg_linear@5": 0.41335716785860044, - "ndcg_linear@10": 0.42995633781970166, - "ndcg_exponential@1": 0.31746031746031744, - "ndcg_exponential@5": 0.41335716785860044, - "ndcg_exponential@10": 0.42995633781970166, - "precision@1": 0.31746031746031744, - "precision@5": 0.09841269841269841, - "precision@10": 0.05396825396825397, - "recall@1": 0.31746031746031744, - "recall@5": 0.49206349206349204, - "recall@10": 0.5396825396825397, - "f1@1": 0.31746031746031744, - "f1@5": 0.1640211640211641, - "f1@10": 0.09812409812409813, - "map@1": 0.31746031746031744, - "map@5": 0.3867724867724867, - "map@10": 0.39433106575963717, - "mrr@1": 0.31746031746031744, - "mrr@5": 0.3867724867724867, - "mrr@10": 0.39433106575963717, + "hit@1": 0.3333333333333333, + "hit@5": 0.5079365079365079, + "hit@10": 0.5555555555555556, + "ndcg_linear@1": 0.3333333333333333, + "ndcg_linear@5": 0.427434196544863, + "ndcg_linear@10": 0.44403336650596426, + "ndcg_exponential@1": 0.3333333333333333, + "ndcg_exponential@5": 0.427434196544863, + "ndcg_exponential@10": 0.44403336650596426, + "precision@1": 0.3333333333333333, + "precision@5": 0.10158730158730159, + "precision@10": 0.05555555555555556, + "recall@1": 0.3333333333333333, + "recall@5": 0.5079365079365079, + "recall@10": 0.5555555555555556, + "f1@1": 0.3333333333333333, + "f1@5": 0.16931216931216939, + "f1@10": 0.10101010101010101, + "map@1": 0.3333333333333333, + "map@5": 0.4005291005291005, + "map@10": 0.4080876795162509, + "mrr@1": 0.3333333333333333, + "mrr@5": 0.4005291005291005, + "mrr@10": 0.4080876795162509, "num_pred": 63, "num_data": 63 }, "VisRAG_MP-DocVQA": { - "hit@1": 0.2436548223350254, - "hit@5": 0.41624365482233505, - "hit@10": 0.48223350253807107, - "ndcg_linear@1": 0.2436548223350254, - "ndcg_linear@5": 0.33378850914695807, - "ndcg_linear@10": 0.3551570419412752, - "ndcg_exponential@1": 0.2436548223350254, - "ndcg_exponential@5": 0.33378850914695807, - "ndcg_exponential@10": 0.3551570419412752, - "precision@1": 0.2436548223350254, - "precision@5": 0.08324873096446701, - "precision@10": 0.048223350253807105, - "recall@1": 0.2436548223350254, - "recall@5": 0.41624365482233505, - "recall@10": 0.48223350253807107, - "f1@1": 0.2436548223350254, - "f1@5": 0.13874788494077836, - "f1@10": 0.08767881864328564, - "map@1": 0.2436548223350254, - "map@5": 0.30654258319232935, - "map@10": 0.31538353073886066, - "mrr@1": 0.2436548223350254, - "mrr@5": 0.30654258319232935, - "mrr@10": 0.31538353073886066, + "hit@1": 0.24534686971235195, + "hit@5": 0.41455160744500846, + "hit@10": 0.4856175972927242, + "ndcg_linear@1": 0.24534686971235195, + "ndcg_linear@5": 0.3337283312762102, + "ndcg_linear@10": 0.3569109888725869, + "ndcg_exponential@1": 0.24534686971235195, + "ndcg_exponential@5": 0.3337283312762102, + "ndcg_exponential@10": 0.3569109888725869, + "precision@1": 0.24534686971235195, + "precision@5": 0.0829103214890017, + "precision@10": 0.04856175972927241, + "recall@1": 0.24534686971235195, + "recall@5": 0.41455160744500846, + "recall@10": 0.4856175972927242, + "f1@1": 0.24534686971235195, + "f1@5": 0.1381838691483362, + "f1@10": 0.08829410859867713, + "map@1": 0.24534686971235195, + "map@5": 0.30699379582628317, + "map@10": 0.3166827814035936, + "mrr@1": 0.24534686971235195, + "mrr@5": 0.30699379582628317, + "mrr@10": 0.3166827814035936, "num_pred": 591, "num_data": 591 }, "VisRAG_SlideVQA": { - "hit@1": 0.44964028776978415, - "hit@5": 0.7086330935251799, - "hit@10": 0.7859712230215827, - "ndcg_linear@1": 0.44964028776978415, - "ndcg_linear@5": 0.5648847466987875, - "ndcg_linear@10": 0.5955590007686733, - "ndcg_exponential@1": 0.44964028776978415, - "ndcg_exponential@5": 0.5648847466987875, - "ndcg_exponential@10": 0.5955590007686733, - "precision@1": 0.44964028776978415, - "precision@5": 0.1723021582733813, - "precision@10": 0.09766187050359712, - "recall@1": 0.36960431654676257, - "recall@5": 0.6735611510791367, - "recall@10": 0.7616906474820144, - "f1@1": 0.3962829736211031, - "f1@5": 0.2692702980472765, - "f1@10": 0.17105406583823854, - "map@1": 0.44964028776978415, - "map@5": 0.5169964028776978, - "map@10": 0.5313963115222108, - "mrr@1": 0.44964028776978415, - "mrr@5": 0.5481115107913669, - "mrr@10": 0.5588072399223478, + "hit@1": 0.4460431654676259, + "hit@5": 0.7050359712230215, + "hit@10": 0.7913669064748201, + "ndcg_linear@1": 0.4460431654676259, + "ndcg_linear@5": 0.5629991091942579, + "ndcg_linear@10": 0.5956221072567367, + "ndcg_exponential@1": 0.4460431654676259, + "ndcg_exponential@5": 0.5629991091942579, + "ndcg_exponential@10": 0.5956221072567367, + "precision@1": 0.4460431654676259, + "precision@5": 0.17194244604316547, + "precision@10": 0.09802158273381295, + "recall@1": 0.36510791366906475, + "recall@5": 0.670863309352518, + "recall@10": 0.7652877697841727, + "f1@1": 0.39208633093525175, + "f1@5": 0.26858513189448446, + "f1@10": 0.17170808807499457, + "map@1": 0.4460431654676259, + "map@5": 0.5149430455635492, + "map@10": 0.5299331963001028, + "mrr@1": 0.4460431654676259, + "mrr@5": 0.546732613908873, + "mrr@10": 0.5585467340413385, "num_pred": 556, "num_data": 556 }, "VisRAG_InfoVQA": { - "hit@1": 0.4415041782729805, - "hit@5": 0.6713091922005571, - "hit@10": 0.754874651810585, - "ndcg_linear@1": 0.4415041782729805, - "ndcg_linear@5": 0.5633673128694291, - "ndcg_linear@10": 0.5903761050947809, - "ndcg_exponential@1": 0.4415041782729805, - "ndcg_exponential@5": 0.5633673128694291, - "ndcg_exponential@10": 0.5903761050947809, - "precision@1": 0.4415041782729805, - "precision@5": 0.13426183844011141, - "precision@10": 0.07548746518105848, - "recall@1": 0.4415041782729805, - "recall@5": 0.6713091922005571, - "recall@10": 0.754874651810585, - "f1@1": 0.4415041782729805, - "f1@5": 0.2237697307335191, - "f1@10": 0.13724993669283364, - "map@1": 0.4415041782729805, - "map@5": 0.5274373259052925, - "map@10": 0.5385661228279612, - "mrr@1": 0.4415041782729805, - "mrr@5": 0.5274373259052925, - "mrr@10": 0.5385661228279612, + "hit@1": 0.45125348189415043, + "hit@5": 0.6727019498607242, + "hit@10": 0.7562674094707521, + "ndcg_linear@1": 0.45125348189415043, + "ndcg_linear@5": 0.5686346725403147, + "ndcg_linear@10": 0.5956607180873413, + "ndcg_exponential@1": 0.45125348189415043, + "ndcg_exponential@5": 0.5686346725403147, + "ndcg_exponential@10": 0.5956607180873413, + "precision@1": 0.45125348189415043, + "precision@5": 0.13454038997214485, + "precision@10": 0.07562674094707521, + "recall@1": 0.45125348189415043, + "recall@5": 0.6727019498607242, + "recall@10": 0.7562674094707521, + "f1@1": 0.45125348189415043, + "f1@5": 0.2242339832869081, + "f1@10": 0.13750316535831858, + "map@1": 0.45125348189415043, + "map@5": 0.5340064995357474, + "map@10": 0.5451596144493079, + "mrr@1": 0.45125348189415043, + "mrr@5": 0.5340064995357474, + "mrr@10": 0.5451596144493079, "num_pred": 718, "num_data": 718 }, "VisRAG_PlotQA": { - "hit@1": 0.2410196987253766, - "hit@5": 0.4380069524913094, - "hit@10": 0.5365005793742758, - "ndcg_linear@1": 0.2410196987253766, - "ndcg_linear@5": 0.3455373205552205, - "ndcg_linear@10": 0.3774838122200841, - "ndcg_exponential@1": 0.2410196987253766, - "ndcg_exponential@5": 0.3455373205552205, - "ndcg_exponential@10": 0.3774838122200841, - "precision@1": 0.2410196987253766, - "precision@5": 0.08760139049826188, - "precision@10": 0.053650057937427574, - "recall@1": 0.2410196987253766, - "recall@5": 0.4380069524913094, - "recall@10": 0.5365005793742758, - "f1@1": 0.2410196987253766, - "f1@5": 0.14600231749710316, - "f1@10": 0.09754555988623198, - "map@1": 0.2410196987253766, - "map@5": 0.3148319814600232, - "map@10": 0.32806792473652263, - "mrr@1": 0.2410196987253766, - "mrr@5": 0.3148319814600232, - "mrr@10": 0.32806792473652263, + "hit@1": 0.23870220162224798, + "hit@5": 0.4287369640787949, + "hit@10": 0.5434530706836617, + "ndcg_linear@1": 0.23870220162224798, + "ndcg_linear@5": 0.3406099336485858, + "ndcg_linear@10": 0.3775870792490953, + "ndcg_exponential@1": 0.23870220162224798, + "ndcg_exponential@5": 0.3406099336485858, + "ndcg_exponential@10": 0.3775870792490953, + "precision@1": 0.23870220162224798, + "precision@5": 0.08574739281575898, + "precision@10": 0.054345307068366155, + "recall@1": 0.23870220162224798, + "recall@5": 0.4287369640787949, + "recall@10": 0.5434530706836617, + "f1@1": 0.23870220162224798, + "f1@5": 0.1429123213595983, + "f1@10": 0.09880964921521122, + "map@1": 0.23870220162224798, + "map@5": 0.31123986095017386, + "map@10": 0.3264286633927422, + "mrr@1": 0.23870220162224798, + "mrr@5": 0.31123986095017386, + "mrr@10": 0.3264286633927422, "num_pred": 863, "num_data": 863 }, - "ViDoSeek-page": { - "hit@1": 0.040280210157618214, - "hit@5": 0.18563922942206654, - "hit@10": 0.287215411558669, - "ndcg_linear@1": 0.040280210157618214, - "ndcg_linear@5": 0.11294100510293231, - "ndcg_linear@10": 0.14594536433240723, - "ndcg_exponential@1": 0.040280210157618214, - "ndcg_exponential@5": 0.11294100510293234, - "ndcg_exponential@10": 0.14594536433240726, - "precision@1": 0.040280210157618214, - "precision@5": 0.03712784588441331, - "precision@10": 0.028721541155866902, - "recall@1": 0.040280210157618214, - "recall@5": 0.18563922942206654, - "recall@10": 0.287215411558669, - "f1@1": 0.040280210157618214, - "f1@5": 0.06187974314068886, - "f1@10": 0.052220983919758, - "map@1": 0.040280210157618214, - "map@5": 0.08917104495037945, - "map@10": 0.10288862480193478, - "mrr@1": 0.040280210157618214, - "mrr@5": 0.08917104495037945, - "mrr@10": 0.10288862480193478, - "num_pred": 1142, - "num_data": 1142 - }, - "ViDoSeek-doc": { - "hit@1": 0.4982486865148862, - "hit@5": 0.6567425569176882, - "hit@10": 0.7338003502626971, - "ndcg_linear@1": 0.3429655575014594, - "ndcg_linear@5": 0.3714170601904285, - "ndcg_linear@10": 0.35019901577656104, - "ndcg_exponential@1": 0.23204903677758318, - "ndcg_exponential@5": 0.3146728814563305, - "ndcg_exponential@10": 0.3162182310328309, - "precision@1": 0.4982486865148862, - "precision@5": 0.4001751313485114, - "precision@10": 0.3430823117338004, - "recall@1": 0.02844268284063671, - "recall@5": 0.11354378337205699, - "recall@10": 0.19116020046136178, - "f1@1": 0.05349597259677307, - "f1@5": 0.1736704243149188, - "f1@10": 0.24068949848035567, - "map@1": 0.4982486865148862, - "map@5": 0.3603765323992995, - "map@10": 0.2910387701490404, - "mrr@1": 0.4982486865148862, - "mrr@5": 0.5563776999416229, - "mrr@10": 0.5669967753592973, + "ViDoSeek-page-fixed": { + "hit@1": 0.1987740805604203, + "hit@5": 0.4754816112084063, + "hit@10": 0.5814360770577933, + "ndcg_linear@1": 0.1987740805604203, + "ndcg_linear@5": 0.34475184556996963, + "ndcg_linear@10": 0.37868926637669553, + "ndcg_exponential@1": 0.1987740805604203, + "ndcg_exponential@5": 0.3447518455699697, + "ndcg_exponential@10": 0.37868926637669553, + "precision@1": 0.1987740805604203, + "precision@5": 0.09509632224168126, + "precision@10": 0.05814360770577934, + "recall@1": 0.1987740805604203, + "recall@5": 0.4754816112084063, + "recall@10": 0.5814360770577933, + "f1@1": 0.1987740805604203, + "f1@5": 0.15849387040280216, + "f1@10": 0.10571565037414425, + "map@1": 0.1987740805604203, + "map@5": 0.3013572679509632, + "map@10": 0.3151742278931421, + "mrr@1": 0.1987740805604203, + "mrr@5": 0.3013572679509632, + "mrr@10": 0.3151742278931421, "num_pred": 1142, "num_data": 1142 }, - "MMLongBench-page": { - "hit@1": 0.059665871121718374, - "hit@5": 0.14797136038186157, - "hit@10": 0.2159904534606205, - "ndcg_linear@1": 0.059665871121718374, - "ndcg_linear@5": 0.07951086728623444, - "ndcg_linear@10": 0.09880430458699743, - "ndcg_exponential@1": 0.059665871121718374, - "ndcg_exponential@5": 0.07951086728623444, - "ndcg_exponential@10": 0.09880430458699746, - "precision@1": 0.059665871121718374, - "precision@5": 0.03293556085918855, - "precision@10": 0.0256563245823389, - "recall@1": 0.038929139674963066, - "recall@5": 0.10381908929044967, - "recall@10": 0.15813207940296245, - "f1@1": 0.04444323907331067, - "f1@5": 0.04737172798031987, - "f1@10": 0.042217267935243885, - "map@1": 0.059665871121718374, - "map@5": 0.06207769822328295, - "map@10": 0.06994553516182395, - "mrr@1": 0.059665871121718374, - "mrr@5": 0.08904136833731105, - "mrr@10": 0.09803860287153843, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.34725536992840095, - "hit@5": 0.5238663484486874, - "hit@10": 0.5883054892601431, - "ndcg_linear@1": 0.24701670644391405, - "ndcg_linear@5": 0.2758146276737757, - "ndcg_linear@10": 0.2683418556525889, - "ndcg_exponential@1": 0.17541766109785203, - "ndcg_exponential@5": 0.2302031604572782, - "ndcg_exponential@10": 0.23834477001816015, - "precision@1": 0.34725536992840095, - "precision@5": 0.31169451073985677, - "precision@10": 0.27792362768496426, - "recall@1": 0.011458219705852956, - "recall@5": 0.0473748134486053, - "recall@10": 0.08146579365576306, - "f1@1": 0.021977357536621576, - "f1@5": 0.0790594246656723, - "f1@10": 0.11819035378881726, - "map@1": 0.34725536992840095, - "map@5": 0.2678878281622912, - "map@10": 0.22411547713755348, - "mrr@1": 0.34725536992840095, - "mrr@5": 0.41585123309466987, - "mrr@10": 0.4247144561882032, + "MMLongBench-page-fixed": { + "hit@1": 0.13007159904534607, + "hit@5": 0.29236276849642007, + "hit@10": 0.360381861575179, + "ndcg_linear@1": 0.13007159904534607, + "ndcg_linear@5": 0.18293385514269955, + "ndcg_linear@10": 0.20739000825299356, + "ndcg_exponential@1": 0.13007159904534607, + "ndcg_exponential@5": 0.18293385514269958, + "ndcg_exponential@10": 0.20739000825299356, + "precision@1": 0.13007159904534607, + "precision@5": 0.06730310262529833, + "precision@10": 0.04486873508353222, + "recall@1": 0.09822044171686176, + "recall@5": 0.22969276811758912, + "recall@10": 0.2994459597681555, + "f1@1": 0.10726477182324436, + "f1@5": 0.09949609525981842, + "f1@10": 0.07540801510087859, + "map@1": 0.13007159904534607, + "map@5": 0.15353056218509678, + "map@10": 0.16460109103307194, + "mrr@1": 0.13007159904534607, + "mrr@5": 0.19015513126491643, + "mrr@10": 0.19920540212902982, "num_pred": 838, "num_data": 838 }