| { | |
| "metadata": { | |
| "timestamp": "2025-06-26T06:05:28.223089", | |
| "vidore_benchmark_version": "5.0.1.dev12+ge0eb903" | |
| }, | |
| "metrics": { | |
| "vidore/arxivqa_test_subsampled": { | |
| "ndcg_at_1": 0.816, | |
| "ndcg_at_3": 0.86359, | |
| "ndcg_at_5": 0.87608, | |
| "ndcg_at_10": 0.88392, | |
| "ndcg_at_20": 0.8879, | |
| "ndcg_at_50": 0.89151, | |
| "ndcg_at_100": 0.89314, | |
| "map_at_1": 0.816, | |
| "map_at_3": 0.85233, | |
| "map_at_5": 0.85933, | |
| "map_at_10": 0.86262, | |
| "map_at_20": 0.86367, | |
| "map_at_50": 0.86427, | |
| "map_at_100": 0.86442, | |
| "recall_at_1": 0.816, | |
| "recall_at_3": 0.896, | |
| "recall_at_5": 0.926, | |
| "recall_at_10": 0.95, | |
| "recall_at_20": 0.966, | |
| "recall_at_50": 0.984, | |
| "recall_at_100": 0.994, | |
| "precision_at_1": 0.816, | |
| "precision_at_3": 0.29867, | |
| "precision_at_5": 0.1852, | |
| "precision_at_10": 0.095, | |
| "precision_at_20": 0.0483, | |
| "precision_at_50": 0.01968, | |
| "precision_at_100": 0.00994, | |
| "mrr_at_1": 0.816, | |
| "mrr_at_3": 0.8523333333333332, | |
| "mrr_at_5": 0.8593333333333332, | |
| "mrr_at_10": 0.8626190476190474, | |
| "mrr_at_20": 0.8636749323775637, | |
| "mrr_at_50": 0.8642709526380625, | |
| "mrr_at_100": 0.8644171478708726, | |
| "naucs_at_1_max": 0.6293010404524454, | |
| "naucs_at_1_std": 0.4072658981997148, | |
| "naucs_at_1_diff1": 0.9408539481068943, | |
| "naucs_at_3_max": 0.7068956182776297, | |
| "naucs_at_3_std": 0.4853641810438182, | |
| "naucs_at_3_diff1": 0.919151086407275, | |
| "naucs_at_5_max": 0.6675372851843446, | |
| "naucs_at_5_std": 0.48681454563807564, | |
| "naucs_at_5_diff1": 0.9050521109344639, | |
| "naucs_at_10_max": 0.671241830065359, | |
| "naucs_at_10_std": 0.5544724556489257, | |
| "naucs_at_10_diff1": 0.9300653594771261, | |
| "naucs_at_20_max": 0.6462624265392473, | |
| "naucs_at_20_std": 0.5665952655572045, | |
| "naucs_at_20_diff1": 0.9288735101883935, | |
| "naucs_at_50_max": 0.6601890756302559, | |
| "naucs_at_50_std": 0.7135270774976739, | |
| "naucs_at_50_diff1": 0.9509803921568729, | |
| "naucs_at_100_max": 0.4788359788359729, | |
| "naucs_at_100_std": 0.7860255213196177, | |
| "naucs_at_100_diff1": 0.9128540305010608 | |
| }, | |
| "vidore/docvqa_test_subsampled": { | |
| "ndcg_at_1": 0.55432, | |
| "ndcg_at_3": 0.62271, | |
| "ndcg_at_5": 0.6418, | |
| "ndcg_at_10": 0.65981, | |
| "ndcg_at_20": 0.67245, | |
| "ndcg_at_50": 0.68717, | |
| "ndcg_at_100": 0.69371, | |
| "map_at_1": 0.55432, | |
| "map_at_3": 0.60643, | |
| "map_at_5": 0.61696, | |
| "map_at_10": 0.62444, | |
| "map_at_20": 0.62807, | |
| "map_at_50": 0.63053, | |
| "map_at_100": 0.63112, | |
| "recall_at_1": 0.55432, | |
| "recall_at_3": 0.66962, | |
| "recall_at_5": 0.71619, | |
| "recall_at_10": 0.77162, | |
| "recall_at_20": 0.8204, | |
| "recall_at_50": 0.89357, | |
| "recall_at_100": 0.93348, | |
| "precision_at_1": 0.55432, | |
| "precision_at_3": 0.22321, | |
| "precision_at_5": 0.14324, | |
| "precision_at_10": 0.07716, | |
| "precision_at_20": 0.04102, | |
| "precision_at_50": 0.01787, | |
| "precision_at_100": 0.00933, | |
| "mrr_at_1": 0.5543237250554324, | |
| "mrr_at_3": 0.6064301552106431, | |
| "mrr_at_5": 0.6169623059866962, | |
| "mrr_at_10": 0.6244403970013725, | |
| "mrr_at_20": 0.6280746690046116, | |
| "mrr_at_50": 0.6305269355853262, | |
| "mrr_at_100": 0.6311233105469878, | |
| "naucs_at_1_max": 0.26918057100652915, | |
| "naucs_at_1_std": 0.36012926047126925, | |
| "naucs_at_1_diff1": 0.8917667066190673, | |
| "naucs_at_3_max": 0.1874727550722851, | |
| "naucs_at_3_std": 0.3626474506990236, | |
| "naucs_at_3_diff1": 0.8344987396644553, | |
| "naucs_at_5_max": 0.15678371580205439, | |
| "naucs_at_5_std": 0.3744247784871604, | |
| "naucs_at_5_diff1": 0.8157913006713307, | |
| "naucs_at_10_max": 0.12657768477167816, | |
| "naucs_at_10_std": 0.39528373597721594, | |
| "naucs_at_10_diff1": 0.8084378962022776, | |
| "naucs_at_20_max": 0.07727578186474303, | |
| "naucs_at_20_std": 0.4426250569829898, | |
| "naucs_at_20_diff1": 0.7647475248422911, | |
| "naucs_at_50_max": 0.05408134294887712, | |
| "naucs_at_50_std": 0.6746820186486754, | |
| "naucs_at_50_diff1": 0.731714700634661, | |
| "naucs_at_100_max": 0.13603326841324073, | |
| "naucs_at_100_std": 0.8784558171048236, | |
| "naucs_at_100_diff1": 0.6908191020466624 | |
| }, | |
| "vidore/infovqa_test_subsampled": { | |
| "ndcg_at_1": 0.91093, | |
| "ndcg_at_3": 0.92978, | |
| "ndcg_at_5": 0.9364, | |
| "ndcg_at_10": 0.94263, | |
| "ndcg_at_20": 0.94514, | |
| "ndcg_at_50": 0.94634, | |
| "ndcg_at_100": 0.94739, | |
| "map_at_1": 0.91093, | |
| "map_at_3": 0.9251, | |
| "map_at_5": 0.92874, | |
| "map_at_10": 0.93152, | |
| "map_at_20": 0.93218, | |
| "map_at_50": 0.93237, | |
| "map_at_100": 0.93248, | |
| "recall_at_1": 0.91093, | |
| "recall_at_3": 0.94332, | |
| "recall_at_5": 0.95951, | |
| "recall_at_10": 0.97773, | |
| "recall_at_20": 0.98785, | |
| "recall_at_50": 0.99393, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.91093, | |
| "precision_at_3": 0.31444, | |
| "precision_at_5": 0.1919, | |
| "precision_at_10": 0.09777, | |
| "precision_at_20": 0.04939, | |
| "precision_at_50": 0.01988, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9109311740890689, | |
| "mrr_at_3": 0.9251012145748988, | |
| "mrr_at_5": 0.9287449392712549, | |
| "mrr_at_10": 0.9315162907268171, | |
| "mrr_at_20": 0.9321792672411868, | |
| "mrr_at_50": 0.9323702937480027, | |
| "mrr_at_100": 0.9324824899427555, | |
| "naucs_at_1_max": 0.5050488059362528, | |
| "naucs_at_1_std": 0.24635244116889507, | |
| "naucs_at_1_diff1": 0.9580692684878348, | |
| "naucs_at_3_max": 0.4711422227428897, | |
| "naucs_at_3_std": 0.23770030547234597, | |
| "naucs_at_3_diff1": 0.9626852988386382, | |
| "naucs_at_5_max": 0.6927616323055464, | |
| "naucs_at_5_std": 0.45649645323646826, | |
| "naucs_at_5_diff1": 0.9673496364838072, | |
| "naucs_at_10_max": 0.7590524361659158, | |
| "naucs_at_10_std": 0.529529634785233, | |
| "naucs_at_10_diff1": 0.9762542810791351, | |
| "naucs_at_20_max": 0.7830887900175995, | |
| "naucs_at_20_std": 0.7611980314414127, | |
| "naucs_at_20_diff1": 0.9782330909891938, | |
| "naucs_at_50_max": 0.9564661819784259, | |
| "naucs_at_50_std": 0.9074217540806789, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/tabfquad_test_subsampled": { | |
| "ndcg_at_1": 0.88929, | |
| "ndcg_at_3": 0.93699, | |
| "ndcg_at_5": 0.94298, | |
| "ndcg_at_10": 0.94649, | |
| "ndcg_at_20": 0.94746, | |
| "ndcg_at_50": 0.94746, | |
| "ndcg_at_100": 0.94808, | |
| "map_at_1": 0.88929, | |
| "map_at_3": 0.92619, | |
| "map_at_5": 0.92958, | |
| "map_at_10": 0.93105, | |
| "map_at_20": 0.93135, | |
| "map_at_50": 0.93135, | |
| "map_at_100": 0.93142, | |
| "recall_at_1": 0.88929, | |
| "recall_at_3": 0.96786, | |
| "recall_at_5": 0.98214, | |
| "recall_at_10": 0.99286, | |
| "recall_at_20": 0.99643, | |
| "recall_at_50": 0.99643, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.88929, | |
| "precision_at_3": 0.32262, | |
| "precision_at_5": 0.19643, | |
| "precision_at_10": 0.09929, | |
| "precision_at_20": 0.04982, | |
| "precision_at_50": 0.01993, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.8892857142857142, | |
| "mrr_at_3": 0.9261904761904763, | |
| "mrr_at_5": 0.9295833333333332, | |
| "mrr_at_10": 0.9310501700680273, | |
| "mrr_at_20": 0.9313477891156464, | |
| "mrr_at_50": 0.9313477891156464, | |
| "mrr_at_100": 0.9314164704343276, | |
| "naucs_at_1_max": 0.11535645648394, | |
| "naucs_at_1_std": 0.08977761508190756, | |
| "naucs_at_1_diff1": 0.9270919129983833, | |
| "naucs_at_3_max": 0.6139122315592889, | |
| "naucs_at_3_std": 0.5563336445689361, | |
| "naucs_at_3_diff1": 0.9564270152505452, | |
| "naucs_at_5_max": 0.5654528478057843, | |
| "naucs_at_5_std": 0.6578898225957153, | |
| "naucs_at_5_diff1": 0.9738562091503306, | |
| "naucs_at_10_max": 0.8611111111111035, | |
| "naucs_at_10_std": 0.9346405228758269, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": 1.0, | |
| "naucs_at_50_std": 1.0, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/tatdqa_test": { | |
| "ndcg_at_1": 0.69684, | |
| "ndcg_at_3": 0.78254, | |
| "ndcg_at_5": 0.79903, | |
| "ndcg_at_10": 0.81433, | |
| "ndcg_at_20": 0.82161, | |
| "ndcg_at_50": 0.82553, | |
| "ndcg_at_100": 0.82719, | |
| "map_at_1": 0.69684, | |
| "map_at_3": 0.76154, | |
| "map_at_5": 0.77069, | |
| "map_at_10": 0.77709, | |
| "map_at_20": 0.7792, | |
| "map_at_50": 0.77981, | |
| "map_at_100": 0.77995, | |
| "recall_at_1": 0.69684, | |
| "recall_at_3": 0.84326, | |
| "recall_at_5": 0.88335, | |
| "recall_at_10": 0.93013, | |
| "recall_at_20": 0.95808, | |
| "recall_at_50": 0.97813, | |
| "recall_at_100": 0.98846, | |
| "precision_at_1": 0.69684, | |
| "precision_at_3": 0.28109, | |
| "precision_at_5": 0.17667, | |
| "precision_at_10": 0.09301, | |
| "precision_at_20": 0.0479, | |
| "precision_at_50": 0.01956, | |
| "precision_at_100": 0.00988, | |
| "mrr_at_1": 0.695625759416768, | |
| "mrr_at_3": 0.7609356014580799, | |
| "mrr_at_5": 0.7699574726609966, | |
| "mrr_at_10": 0.7763932284132767, | |
| "mrr_at_20": 0.7785060539451517, | |
| "mrr_at_50": 0.7791113338667969, | |
| "mrr_at_100": 0.7792520714241775, | |
| "naucs_at_1_max": 0.23477193022892978, | |
| "naucs_at_1_std": 0.17073254873506194, | |
| "naucs_at_1_diff1": 0.845390421248477, | |
| "naucs_at_3_max": 0.21897649044161785, | |
| "naucs_at_3_std": 0.23263205468737608, | |
| "naucs_at_3_diff1": 0.7773076885537255, | |
| "naucs_at_5_max": 0.22908169584132324, | |
| "naucs_at_5_std": 0.25174255666871304, | |
| "naucs_at_5_diff1": 0.7439197717296311, | |
| "naucs_at_10_max": 0.23769949780387462, | |
| "naucs_at_10_std": 0.29069564332383074, | |
| "naucs_at_10_diff1": 0.7167658978463287, | |
| "naucs_at_20_max": 0.21326491789377966, | |
| "naucs_at_20_std": 0.26558383716997797, | |
| "naucs_at_20_diff1": 0.6754815504474063, | |
| "naucs_at_50_max": 0.12767303786085316, | |
| "naucs_at_50_std": 0.24485452538521443, | |
| "naucs_at_50_diff1": 0.6732815103835509, | |
| "naucs_at_100_max": 0.14158104162792087, | |
| "naucs_at_100_std": 0.324957585420732, | |
| "naucs_at_100_diff1": 0.6752814674223621 | |
| }, | |
| "vidore/shiftproject_test": { | |
| "ndcg_at_1": 0.84, | |
| "ndcg_at_3": 0.9194, | |
| "ndcg_at_5": 0.92327, | |
| "ndcg_at_10": 0.9266, | |
| "ndcg_at_20": 0.9266, | |
| "ndcg_at_50": 0.9266, | |
| "ndcg_at_100": 0.92821, | |
| "map_at_1": 0.84, | |
| "map_at_3": 0.90167, | |
| "map_at_5": 0.90367, | |
| "map_at_10": 0.9051, | |
| "map_at_20": 0.9051, | |
| "map_at_50": 0.9051, | |
| "map_at_100": 0.90523, | |
| "recall_at_1": 0.84, | |
| "recall_at_3": 0.97, | |
| "recall_at_5": 0.98, | |
| "recall_at_10": 0.99, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 0.99, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.84, | |
| "precision_at_3": 0.32333, | |
| "precision_at_5": 0.196, | |
| "precision_at_10": 0.099, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.0198, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.84, | |
| "mrr_at_3": 0.9016666666666666, | |
| "mrr_at_5": 0.9036666666666667, | |
| "mrr_at_10": 0.9050952380952382, | |
| "mrr_at_20": 0.9050952380952382, | |
| "mrr_at_50": 0.9050952380952382, | |
| "mrr_at_100": 0.905232224396608, | |
| "naucs_at_1_max": 0.19096710849288218, | |
| "naucs_at_1_std": -0.30166912125674944, | |
| "naucs_at_1_diff1": 0.8265218458517417, | |
| "naucs_at_3_max": -0.20401493930905265, | |
| "naucs_at_3_std": -0.7268907563025196, | |
| "naucs_at_3_diff1": 0.9564270152505466, | |
| "naucs_at_5_max": -0.3674136321195164, | |
| "naucs_at_5_std": -0.5144724556489195, | |
| "naucs_at_5_diff1": 0.9346405228758136, | |
| "naucs_at_10_max": -0.1713352007469681, | |
| "naucs_at_10_std": 0.12278244631185926, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": -0.1713352007469681, | |
| "naucs_at_20_std": 0.12278244631185926, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": -0.17133520074697067, | |
| "naucs_at_50_std": 0.12278244631185525, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_artificial_intelligence_test": { | |
| "ndcg_at_1": 1.0, | |
| "ndcg_at_3": 1.0, | |
| "ndcg_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "ndcg_at_20": 1.0, | |
| "ndcg_at_50": 1.0, | |
| "ndcg_at_100": 1.0, | |
| "map_at_1": 1.0, | |
| "map_at_3": 1.0, | |
| "map_at_5": 1.0, | |
| "map_at_10": 1.0, | |
| "map_at_20": 1.0, | |
| "map_at_50": 1.0, | |
| "map_at_100": 1.0, | |
| "recall_at_1": 1.0, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 1.0, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 1.0, | |
| "mrr_at_3": 1.0, | |
| "mrr_at_5": 1.0, | |
| "mrr_at_10": 1.0, | |
| "mrr_at_20": 1.0, | |
| "mrr_at_50": 1.0, | |
| "mrr_at_100": 1.0, | |
| "naucs_at_1_max": null, | |
| "naucs_at_1_std": null, | |
| "naucs_at_1_diff1": null, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_energy_test": { | |
| "ndcg_at_1": 0.96, | |
| "ndcg_at_3": 0.96631, | |
| "ndcg_at_5": 0.96631, | |
| "ndcg_at_10": 0.97235, | |
| "ndcg_at_20": 0.97235, | |
| "ndcg_at_50": 0.97451, | |
| "ndcg_at_100": 0.97451, | |
| "map_at_1": 0.96, | |
| "map_at_3": 0.965, | |
| "map_at_5": 0.965, | |
| "map_at_10": 0.96725, | |
| "map_at_20": 0.96725, | |
| "map_at_50": 0.96767, | |
| "map_at_100": 0.96767, | |
| "recall_at_1": 0.96, | |
| "recall_at_3": 0.97, | |
| "recall_at_5": 0.97, | |
| "recall_at_10": 0.99, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.96, | |
| "precision_at_3": 0.32333, | |
| "precision_at_5": 0.194, | |
| "precision_at_10": 0.099, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.96, | |
| "mrr_at_3": 0.965, | |
| "mrr_at_5": 0.965, | |
| "mrr_at_10": 0.9672499999999999, | |
| "mrr_at_20": 0.9672499999999999, | |
| "mrr_at_50": 0.9676666666666667, | |
| "mrr_at_100": 0.9676666666666667, | |
| "naucs_at_1_max": 0.5671101774042947, | |
| "naucs_at_1_std": -0.5088702147525661, | |
| "naucs_at_1_diff1": 1.0, | |
| "naucs_at_3_max": 0.7152194211017727, | |
| "naucs_at_3_std": -0.09850606909430029, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 0.7152194211017747, | |
| "naucs_at_5_std": -0.09850606909430323, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 0.8692810457516413, | |
| "naucs_at_10_std": 0.7222222222222276, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 0.8692810457516413, | |
| "naucs_at_20_std": 0.7222222222222276, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_government_reports_test": { | |
| "ndcg_at_1": 0.93, | |
| "ndcg_at_3": 0.96655, | |
| "ndcg_at_5": 0.96655, | |
| "ndcg_at_10": 0.97011, | |
| "ndcg_at_20": 0.97011, | |
| "ndcg_at_50": 0.97011, | |
| "ndcg_at_100": 0.97011, | |
| "map_at_1": 0.93, | |
| "map_at_3": 0.95833, | |
| "map_at_5": 0.95833, | |
| "map_at_10": 0.96, | |
| "map_at_20": 0.96, | |
| "map_at_50": 0.96, | |
| "map_at_100": 0.96, | |
| "recall_at_1": 0.93, | |
| "recall_at_3": 0.99, | |
| "recall_at_5": 0.99, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.93, | |
| "precision_at_3": 0.33, | |
| "precision_at_5": 0.198, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.93, | |
| "mrr_at_3": 0.9583333333333335, | |
| "mrr_at_5": 0.9583333333333335, | |
| "mrr_at_10": 0.96, | |
| "mrr_at_20": 0.96, | |
| "mrr_at_50": 0.96, | |
| "mrr_at_100": 0.96, | |
| "naucs_at_1_max": 0.771308523409364, | |
| "naucs_at_1_std": 0.25456849406429166, | |
| "naucs_at_1_diff1": 0.943977591036415, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_healthcare_industry_test": { | |
| "ndcg_at_1": 0.99, | |
| "ndcg_at_3": 0.99631, | |
| "ndcg_at_5": 0.99631, | |
| "ndcg_at_10": 0.99631, | |
| "ndcg_at_20": 0.99631, | |
| "ndcg_at_50": 0.99631, | |
| "ndcg_at_100": 0.99631, | |
| "map_at_1": 0.99, | |
| "map_at_3": 0.995, | |
| "map_at_5": 0.995, | |
| "map_at_10": 0.995, | |
| "map_at_20": 0.995, | |
| "map_at_50": 0.995, | |
| "map_at_100": 0.995, | |
| "recall_at_1": 0.99, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.99, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.99, | |
| "mrr_at_3": 0.995, | |
| "mrr_at_5": 0.995, | |
| "mrr_at_10": 0.995, | |
| "mrr_at_20": 0.995, | |
| "mrr_at_50": 0.995, | |
| "mrr_at_100": 0.995, | |
| "naucs_at_1_max": 0.7222222222222201, | |
| "naucs_at_1_std": 1.0, | |
| "naucs_at_1_diff1": 1.0, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.5, | |
| "ndcg_at_3": 0.51063, | |
| "ndcg_at_5": 0.56843, | |
| "ndcg_at_10": 0.62312, | |
| "ndcg_at_20": 0.65401, | |
| "ndcg_at_50": 0.67834, | |
| "ndcg_at_100": 0.6907, | |
| "map_at_1": 0.26391, | |
| "map_at_3": 0.38408, | |
| "map_at_5": 0.45528, | |
| "map_at_10": 0.5042, | |
| "map_at_20": 0.52273, | |
| "map_at_50": 0.53634, | |
| "map_at_100": 0.54254, | |
| "recall_at_1": 0.26391, | |
| "recall_at_3": 0.46643, | |
| "recall_at_5": 0.61986, | |
| "recall_at_10": 0.78311, | |
| "recall_at_20": 0.87897, | |
| "recall_at_50": 0.94043, | |
| "recall_at_100": 0.97277, | |
| "precision_at_1": 0.5, | |
| "precision_at_3": 0.35088, | |
| "precision_at_5": 0.30526, | |
| "precision_at_10": 0.20702, | |
| "precision_at_20": 0.12719, | |
| "precision_at_50": 0.06561, | |
| "precision_at_100": 0.03789, | |
| "mrr_at_1": 0.5, | |
| "mrr_at_3": 0.6089181286549706, | |
| "mrr_at_5": 0.6317251461988304, | |
| "mrr_at_10": 0.641208925090504, | |
| "mrr_at_20": 0.6448876756055548, | |
| "mrr_at_50": 0.6448876756055548, | |
| "mrr_at_100": 0.6450109692219045, | |
| "naucs_at_1_max": 0.06266633655005126, | |
| "naucs_at_1_std": 0.10328247420677374, | |
| "naucs_at_1_diff1": 0.32364592144068777, | |
| "naucs_at_3_max": -0.027204152815788592, | |
| "naucs_at_3_std": 0.10728940302719199, | |
| "naucs_at_3_diff1": 0.24109996679515233, | |
| "naucs_at_5_max": -0.08818416172256802, | |
| "naucs_at_5_std": 0.06933352726164131, | |
| "naucs_at_5_diff1": 0.1346725342215723, | |
| "naucs_at_10_max": -0.1634891544078435, | |
| "naucs_at_10_std": -0.07449668399775349, | |
| "naucs_at_10_diff1": 0.06840639837870822, | |
| "naucs_at_20_max": -0.23984204721411798, | |
| "naucs_at_20_std": -0.17420250142740507, | |
| "naucs_at_20_diff1": -0.03237996351004239, | |
| "naucs_at_50_max": -0.25657441933802366, | |
| "naucs_at_50_std": -0.21481884062099896, | |
| "naucs_at_50_diff1": -0.11144862322897976, | |
| "naucs_at_100_max": -0.26954674492082575, | |
| "naucs_at_100_std": -0.22275247242806823, | |
| "naucs_at_100_diff1": -0.12549423492642411 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { | |
| "ndcg_at_1": 0.61875, | |
| "ndcg_at_3": 0.60842, | |
| "ndcg_at_5": 0.62342, | |
| "ndcg_at_10": 0.65234, | |
| "ndcg_at_20": 0.67684, | |
| "ndcg_at_50": 0.70181, | |
| "ndcg_at_100": 0.71529, | |
| "map_at_1": 0.37744, | |
| "map_at_3": 0.49938, | |
| "map_at_5": 0.53387, | |
| "map_at_10": 0.56442, | |
| "map_at_20": 0.57848, | |
| "map_at_50": 0.58719, | |
| "map_at_100": 0.59025, | |
| "recall_at_1": 0.37744, | |
| "recall_at_3": 0.56678, | |
| "recall_at_5": 0.64475, | |
| "recall_at_10": 0.73598, | |
| "recall_at_20": 0.80296, | |
| "recall_at_50": 0.88095, | |
| "recall_at_100": 0.93199, | |
| "precision_at_1": 0.61875, | |
| "precision_at_3": 0.37188, | |
| "precision_at_5": 0.27312, | |
| "precision_at_10": 0.17187, | |
| "precision_at_20": 0.10141, | |
| "precision_at_50": 0.04884, | |
| "precision_at_100": 0.02719, | |
| "mrr_at_1": 0.61875, | |
| "mrr_at_3": 0.699739583333333, | |
| "mrr_at_5": 0.7123958333333327, | |
| "mrr_at_10": 0.7202827380952376, | |
| "mrr_at_20": 0.7221802283704852, | |
| "mrr_at_50": 0.7230994356972297, | |
| "mrr_at_100": 0.7232405617307813, | |
| "naucs_at_1_max": 0.165285331951999, | |
| "naucs_at_1_std": 0.034224006446228576, | |
| "naucs_at_1_diff1": 0.4806948881022956, | |
| "naucs_at_3_max": 0.024453257298345382, | |
| "naucs_at_3_std": -0.04382987653963637, | |
| "naucs_at_3_diff1": -0.031119586246158605, | |
| "naucs_at_5_max": 0.0044058378732677785, | |
| "naucs_at_5_std": -0.057741699699710346, | |
| "naucs_at_5_diff1": -0.10405546973817163, | |
| "naucs_at_10_max": -0.053355219862296764, | |
| "naucs_at_10_std": -0.0790732524974529, | |
| "naucs_at_10_diff1": -0.19731155024816296, | |
| "naucs_at_20_max": -0.08441971461943433, | |
| "naucs_at_20_std": -0.08182264243833959, | |
| "naucs_at_20_diff1": -0.2493971279113114, | |
| "naucs_at_50_max": -0.09688829418144233, | |
| "naucs_at_50_std": -0.0658329150011907, | |
| "naucs_at_50_diff1": -0.3027368345306483, | |
| "naucs_at_100_max": -0.10823541191893869, | |
| "naucs_at_100_std": -0.0945104052898891, | |
| "naucs_at_100_diff1": -0.3316920486637138 | |
| }, | |
| "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.62931, | |
| "ndcg_at_3": 0.59513, | |
| "ndcg_at_5": 0.56377, | |
| "ndcg_at_10": 0.56053, | |
| "ndcg_at_20": 0.58335, | |
| "ndcg_at_50": 0.65782, | |
| "ndcg_at_100": 0.69264, | |
| "map_at_1": 0.09011, | |
| "map_at_3": 0.19282, | |
| "map_at_5": 0.24036, | |
| "map_at_10": 0.31499, | |
| "map_at_20": 0.36938, | |
| "map_at_50": 0.43088, | |
| "map_at_100": 0.45631, | |
| "recall_at_1": 0.09011, | |
| "recall_at_3": 0.24556, | |
| "recall_at_5": 0.31701, | |
| "recall_at_10": 0.45534, | |
| "recall_at_20": 0.58907, | |
| "recall_at_50": 0.80132, | |
| "recall_at_100": 0.90565, | |
| "precision_at_1": 0.62931, | |
| "precision_at_3": 0.54598, | |
| "precision_at_5": 0.48276, | |
| "precision_at_10": 0.40216, | |
| "precision_at_20": 0.30345, | |
| "precision_at_50": 0.19431, | |
| "precision_at_100": 0.12522, | |
| "mrr_at_1": 0.6293103448275862, | |
| "mrr_at_3": 0.7471264367816095, | |
| "mrr_at_5": 0.7579022988505749, | |
| "mrr_at_10": 0.7618944991789821, | |
| "mrr_at_20": 0.7634015268231467, | |
| "mrr_at_50": 0.7639494630822898, | |
| "mrr_at_100": 0.7639494630822898, | |
| "naucs_at_1_max": 0.04993907976795083, | |
| "naucs_at_1_std": 0.18482267813483494, | |
| "naucs_at_1_diff1": 0.2858794410962534, | |
| "naucs_at_3_max": -0.08514210036309953, | |
| "naucs_at_3_std": 0.11808105150508759, | |
| "naucs_at_3_diff1": 0.027094035130762372, | |
| "naucs_at_5_max": -0.004367985895311936, | |
| "naucs_at_5_std": 0.15698404005850908, | |
| "naucs_at_5_diff1": -0.02650835347778693, | |
| "naucs_at_10_max": -0.02264677824492615, | |
| "naucs_at_10_std": 0.1395283970615093, | |
| "naucs_at_10_diff1": -0.0636982012078006, | |
| "naucs_at_20_max": -0.046111448743688874, | |
| "naucs_at_20_std": 0.08818519563045128, | |
| "naucs_at_20_diff1": -0.10045481147941789, | |
| "naucs_at_50_max": -0.031747430051497286, | |
| "naucs_at_50_std": 0.07071502869980983, | |
| "naucs_at_50_diff1": -0.14000365109174984, | |
| "naucs_at_100_max": -0.066374733897631, | |
| "naucs_at_100_std": -0.013446247932639044, | |
| "naucs_at_100_diff1": -0.16056962894086707 | |
| }, | |
| "vidore/restaurant_esg_reports_beir": { | |
| "ndcg_at_1": 0.72436, | |
| "ndcg_at_3": 0.75198, | |
| "ndcg_at_5": 0.76869, | |
| "ndcg_at_10": 0.79801, | |
| "ndcg_at_20": 0.81428, | |
| "ndcg_at_50": 0.82799, | |
| "ndcg_at_100": 0.8295, | |
| "map_at_1": 0.51909, | |
| "map_at_3": 0.66566, | |
| "map_at_5": 0.70632, | |
| "map_at_10": 0.73195, | |
| "map_at_20": 0.73946, | |
| "map_at_50": 0.74654, | |
| "map_at_100": 0.74697, | |
| "recall_at_1": 0.51909, | |
| "recall_at_3": 0.73545, | |
| "recall_at_5": 0.80596, | |
| "recall_at_10": 0.8816, | |
| "recall_at_20": 0.92428, | |
| "recall_at_50": 0.96885, | |
| "recall_at_100": 0.97445, | |
| "precision_at_1": 0.75, | |
| "precision_at_3": 0.41026, | |
| "precision_at_5": 0.3, | |
| "precision_at_10": 0.17308, | |
| "precision_at_20": 0.09712, | |
| "precision_at_50": 0.04462, | |
| "precision_at_100": 0.02269, | |
| "mrr_at_1": 0.75, | |
| "mrr_at_3": 0.8333333333333334, | |
| "mrr_at_5": 0.8410256410256411, | |
| "mrr_at_10": 0.8442307692307693, | |
| "mrr_at_20": 0.845979020979021, | |
| "mrr_at_50": 0.845979020979021, | |
| "mrr_at_100": 0.845979020979021, | |
| "naucs_at_1_max": 0.24468166513237982, | |
| "naucs_at_1_std": 0.19805637553820268, | |
| "naucs_at_1_diff1": 0.44490949154735066, | |
| "naucs_at_3_max": 0.04409608181340947, | |
| "naucs_at_3_std": 0.12191586031897299, | |
| "naucs_at_3_diff1": -0.20170920044109023, | |
| "naucs_at_5_max": -0.10046869346778849, | |
| "naucs_at_5_std": -0.021466277530853978, | |
| "naucs_at_5_diff1": -0.23631379826339902, | |
| "naucs_at_10_max": -0.11031194559055651, | |
| "naucs_at_10_std": -0.007349632141057487, | |
| "naucs_at_10_diff1": -0.1954112809364078, | |
| "naucs_at_20_max": -0.08664502573520873, | |
| "naucs_at_20_std": 0.0007084915425306685, | |
| "naucs_at_20_diff1": -0.3188566325096874, | |
| "naucs_at_50_max": -0.13648898619435715, | |
| "naucs_at_50_std": -0.04766895429454103, | |
| "naucs_at_50_diff1": -0.3833929317006285, | |
| "naucs_at_100_max": -0.15092442191928396, | |
| "naucs_at_100_std": -0.0640639845215107, | |
| "naucs_at_100_diff1": -0.3888345979263681 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0": { | |
| "ndcg_at_1": 0.49123, | |
| "ndcg_at_3": 0.51166, | |
| "ndcg_at_5": 0.57093, | |
| "ndcg_at_10": 0.62246, | |
| "ndcg_at_20": 0.64573, | |
| "ndcg_at_50": 0.67302, | |
| "ndcg_at_100": 0.68379, | |
| "map_at_1": 0.26696, | |
| "map_at_3": 0.38476, | |
| "map_at_5": 0.4576, | |
| "map_at_10": 0.50379, | |
| "map_at_20": 0.51794, | |
| "map_at_50": 0.5326, | |
| "map_at_100": 0.53858, | |
| "recall_at_1": 0.26696, | |
| "recall_at_3": 0.47153, | |
| "recall_at_5": 0.64025, | |
| "recall_at_10": 0.79041, | |
| "recall_at_20": 0.86345, | |
| "recall_at_50": 0.92982, | |
| "recall_at_100": 0.95175, | |
| "precision_at_1": 0.49123, | |
| "precision_at_3": 0.35673, | |
| "precision_at_5": 0.30526, | |
| "precision_at_10": 0.20702, | |
| "precision_at_20": 0.12193, | |
| "precision_at_50": 0.06456, | |
| "precision_at_100": 0.03737, | |
| "mrr_at_1": 0.49122807017543857, | |
| "mrr_at_3": 0.5964912280701753, | |
| "mrr_at_5": 0.6298245614035087, | |
| "mrr_at_10": 0.6337719298245614, | |
| "mrr_at_20": 0.6367131062951495, | |
| "mrr_at_50": 0.6367131062951495, | |
| "mrr_at_100": 0.6367131062951495, | |
| "naucs_at_1_max": 0.09118541685587986, | |
| "naucs_at_1_std": -0.008192518221940852, | |
| "naucs_at_1_diff1": 0.25538152453048424, | |
| "naucs_at_3_max": -0.07082338888627057, | |
| "naucs_at_3_std": 0.054108546626844174, | |
| "naucs_at_3_diff1": 0.2145148016267093, | |
| "naucs_at_5_max": -0.09990587583985085, | |
| "naucs_at_5_std": 0.15901900396869725, | |
| "naucs_at_5_diff1": 0.1903535434391372, | |
| "naucs_at_10_max": -0.2542485342376898, | |
| "naucs_at_10_std": -0.04845015739249387, | |
| "naucs_at_10_diff1": 0.0742124569628327, | |
| "naucs_at_20_max": -0.35959572979012006, | |
| "naucs_at_20_std": -0.16398049550481905, | |
| "naucs_at_20_diff1": -0.02970406796838026, | |
| "naucs_at_50_max": -0.39280580078248756, | |
| "naucs_at_50_std": -0.23497206052790093, | |
| "naucs_at_50_diff1": -0.13254644231801654, | |
| "naucs_at_100_max": -0.39460593340198014, | |
| "naucs_at_100_std": -0.24027067078494477, | |
| "naucs_at_100_diff1": -0.14366966729581043 | |
| }, | |
| "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { | |
| "ndcg_at_1": 0.74138, | |
| "ndcg_at_3": 0.69889, | |
| "ndcg_at_5": 0.64079, | |
| "ndcg_at_10": 0.6271, | |
| "ndcg_at_20": 0.63799, | |
| "ndcg_at_50": 0.70863, | |
| "ndcg_at_100": 0.74161, | |
| "map_at_1": 0.11186, | |
| "map_at_3": 0.23141, | |
| "map_at_5": 0.28257, | |
| "map_at_10": 0.3717, | |
| "map_at_20": 0.42631, | |
| "map_at_50": 0.49122, | |
| "map_at_100": 0.51792, | |
| "recall_at_1": 0.11186, | |
| "recall_at_3": 0.27369, | |
| "recall_at_5": 0.33748, | |
| "recall_at_10": 0.48707, | |
| "recall_at_20": 0.62067, | |
| "recall_at_50": 0.8278, | |
| "recall_at_100": 0.92727, | |
| "precision_at_1": 0.74138, | |
| "precision_at_3": 0.64943, | |
| "precision_at_5": 0.54483, | |
| "precision_at_10": 0.44655, | |
| "precision_at_20": 0.32241, | |
| "precision_at_50": 0.2031, | |
| "precision_at_100": 0.13, | |
| "mrr_at_1": 0.7413793103448276, | |
| "mrr_at_3": 0.8189655172413793, | |
| "mrr_at_5": 0.8224137931034482, | |
| "mrr_at_10": 0.8252873563218389, | |
| "mrr_at_20": 0.8293304396752672, | |
| "mrr_at_50": 0.8293304396752672, | |
| "mrr_at_100": 0.8293304396752672, | |
| "naucs_at_1_max": 0.35838166878631555, | |
| "naucs_at_1_std": 0.40425935610791647, | |
| "naucs_at_1_diff1": 0.31196864342648645, | |
| "naucs_at_3_max": -0.020321743603039347, | |
| "naucs_at_3_std": 0.16664971950540372, | |
| "naucs_at_3_diff1": -0.08724296354027215, | |
| "naucs_at_5_max": -0.032106566274432695, | |
| "naucs_at_5_std": 0.11470154885158665, | |
| "naucs_at_5_diff1": -0.10263779775029133, | |
| "naucs_at_10_max": -0.014430066686303686, | |
| "naucs_at_10_std": 0.10566168354354433, | |
| "naucs_at_10_diff1": -0.13830032198180964, | |
| "naucs_at_20_max": 0.048789989832314584, | |
| "naucs_at_20_std": 0.10940109965860004, | |
| "naucs_at_20_diff1": -0.05835957874375413, | |
| "naucs_at_50_max": -0.01769846400366676, | |
| "naucs_at_50_std": 0.03309578073622774, | |
| "naucs_at_50_diff1": -0.08590608452176329, | |
| "naucs_at_100_max": -0.10067244123611839, | |
| "naucs_at_100_std": -0.07858915244153995, | |
| "naucs_at_100_diff1": -0.10852488217820669 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { | |
| "ndcg_at_1": 0.6625, | |
| "ndcg_at_3": 0.64067, | |
| "ndcg_at_5": 0.64653, | |
| "ndcg_at_10": 0.68332, | |
| "ndcg_at_20": 0.70843, | |
| "ndcg_at_50": 0.72885, | |
| "ndcg_at_100": 0.74088, | |
| "map_at_1": 0.39814, | |
| "map_at_3": 0.52723, | |
| "map_at_5": 0.55866, | |
| "map_at_10": 0.59555, | |
| "map_at_20": 0.60975, | |
| "map_at_50": 0.61739, | |
| "map_at_100": 0.62024, | |
| "recall_at_1": 0.39814, | |
| "recall_at_3": 0.58754, | |
| "recall_at_5": 0.65552, | |
| "recall_at_10": 0.77079, | |
| "recall_at_20": 0.83799, | |
| "recall_at_50": 0.9003, | |
| "recall_at_100": 0.94656, | |
| "precision_at_1": 0.6625, | |
| "precision_at_3": 0.39583, | |
| "precision_at_5": 0.28375, | |
| "precision_at_10": 0.18188, | |
| "precision_at_20": 0.1075, | |
| "precision_at_50": 0.05025, | |
| "precision_at_100": 0.02756, | |
| "mrr_at_1": 0.6625, | |
| "mrr_at_3": 0.7291666666666665, | |
| "mrr_at_5": 0.7382291666666665, | |
| "mrr_at_10": 0.747906746031746, | |
| "mrr_at_20": 0.7496362257024021, | |
| "mrr_at_50": 0.7503963388309167, | |
| "mrr_at_100": 0.7506175950965809, | |
| "naucs_at_1_max": 0.41381934165815426, | |
| "naucs_at_1_std": 0.14434322166399152, | |
| "naucs_at_1_diff1": 0.47828806103942106, | |
| "naucs_at_3_max": -0.02337866180796834, | |
| "naucs_at_3_std": -0.092408589615909, | |
| "naucs_at_3_diff1": -0.04358601319008129, | |
| "naucs_at_5_max": -0.07407943615022797, | |
| "naucs_at_5_std": -0.12635065467026393, | |
| "naucs_at_5_diff1": -0.13429921326572394, | |
| "naucs_at_10_max": -0.07844324279953604, | |
| "naucs_at_10_std": -0.03584071250474527, | |
| "naucs_at_10_diff1": -0.21034229778676125, | |
| "naucs_at_20_max": -0.14896322731721384, | |
| "naucs_at_20_std": -0.06855841147784922, | |
| "naucs_at_20_diff1": -0.2730638665559868, | |
| "naucs_at_50_max": -0.19266770277649403, | |
| "naucs_at_50_std": -0.10702566472046023, | |
| "naucs_at_50_diff1": -0.3440668710883354, | |
| "naucs_at_100_max": -0.23435400494053177, | |
| "naucs_at_100_std": -0.15235408708713993, | |
| "naucs_at_100_diff1": -0.3875515616656985 | |
| } | |
| } | |
| } |