nips-2026-anon-artifacts / computed_values /data /postbugfix_convergence.json
warlockee's picture
Upload folder using huggingface_hub
bd3ae0e verified
{
"analysis": "post_bugfix_2026-02-27",
"bugfix_cutoff_date": "2026-02-27",
"n_experiments_total": 10252,
"n_experiments_postbugfix": 3003,
"metric_used": "val_ap (mAP_1000ms)",
"best_ap": 0.9972872876720983,
"power_law_fit": {
"a": 1.0309776612473043,
"b": 0.07660090971197056,
"c": 0.10562709364453839,
"r2": 0.9263119537339668
},
"bootstrap": {
"a": {
"mean": 1.0151743159991817,
"ci_lo": 0.994052817156907,
"ci_hi": 1.2752625110737068
},
"b": {
"mean": 0.2343487502975724,
"ci_lo": 0.023734076480255686,
"ci_hi": 0.9094941270919662
},
"c": {
"mean": 0.6877843846977298,
"ci_lo": 0.010000002818053202,
"ci_hi": 2.0447608771859778
},
"n_successful": 10000
},
"random_search": {
"c": 0.9933857531513521,
"r2": 0.9548032212359628,
"a": 0.9961533550294086,
"b": 0.22599530090921646
},
"tpe_search": {
"c": 1.1165134879545537,
"r2": 0.9695102537476521,
"a": 0.9963439634273502,
"b": 0.27383932783651854
},
"model_comparison": {
"power_law": {
"params": {
"a": 1.0309776612473043,
"b": 0.07660090971197056,
"c": 0.10562709364453839
},
"r2": 0.9263119537339668,
"aic": -10558.253134335755
},
"logarithmic": {
"params": {
"a": 0.963750631745776,
"b": 0.004347567696082376
},
"r2": 0.916676541840304,
"aic": -10437.240716268267
},
"exponential": {
"params": {
"a": 0.9977022599747527,
"b": 0.017007287605546248,
"c": 0.0016102846158961184
},
"r2": 0.8565181425141755,
"aic": -9891.218759341695
}
},
"best_aic_model": "power_law",
"ap_at_n": {
"llm_50": 0.9851625430854666,
"rand_50": 0.9876172902379546,
"tpe_50": 0.9873726456969698,
"llm_100": 0.9851625430854666,
"rand_100": 0.9897126421783415,
"tpe_100": 0.9909578992430562,
"llm_200": 0.9866374917320696,
"rand_200": 0.9918942890162451,
"tpe_200": 0.9928578507081098,
"llm_500": 0.9900740255959819,
"rand_500": 0.9944056260441696,
"tpe_500": 0.9948568130154374
},
"ken_test_postbugfix": {
"n_experiments": 603,
"best_test_ap": 0.9245057778280362,
"power_law": {
"a": 0.9330781493667619,
"b": 0.5763021851824278,
"c": 0.7629528207224087,
"r2": 0.8614031210842934
}
},
"stable_window": {
"window": "2026-02-27 to 2026-03-04",
"n_experiments": 1340,
"best_val_ap": 0.9957691426051499,
"power_law": {
"a": 1.0160590184039786,
"b": 0.0618827564638711,
"c": 0.14521913345183568,
"r2": 0.8882291869459537
}
},
"full_campaign": {
"n_experiments": 10252,
"best_val_ap": 0.9999992038926465,
"power_law": {
"a": 1.0007587191008416,
"b": 0.013028751330850845,
"c": 0.3415205999139974,
"r2": 0.4688207337442444
},
"random_search": {
"c": 0.8398420259260954,
"r2": 0.9993831778459062
}
},
"convergence_curve": {
"description": "Running max AP at sampled N values",
"N": [
1,
16,
31,
46,
61,
76,
91,
106,
121,
136,
151,
166,
181,
196,
211,
226,
241,
256,
271,
286,
301,
316,
331,
346,
361,
376,
391,
406,
421,
436,
451,
466,
481,
496,
511,
526,
541,
556,
571,
586,
601,
616,
631,
646,
661,
676,
691,
706,
721,
736,
751,
766,
781,
796,
811,
826,
841,
856,
871,
886,
901,
916,
931,
946,
961,
976,
991,
1006,
1021,
1036,
1051,
1066,
1081,
1096,
1111,
1126,
1141,
1156,
1171,
1186,
1201,
1216,
1231,
1246,
1261,
1276,
1291,
1306,
1321,
1336,
1351,
1366,
1381,
1396,
1411,
1426,
1441,
1456,
1471,
1486,
1501,
1516,
1531,
1546,
1561,
1576,
1591,
1606,
1621,
1636,
1651,
1666,
1681,
1696,
1711,
1726,
1741,
1756,
1771,
1786,
1801,
1816,
1831,
1846,
1861,
1876,
1891,
1906,
1921,
1936,
1951,
1966,
1981,
1996,
2011,
2026,
2041,
2056,
2071,
2086,
2101,
2116,
2131,
2146,
2161,
2176,
2191,
2206,
2221,
2236,
2251,
2266,
2281,
2296,
2311,
2326,
2341,
2356,
2371,
2386,
2401,
2416,
2431,
2446,
2461,
2476,
2491,
2506,
2521,
2536,
2551,
2566,
2581,
2596,
2611,
2626,
2641,
2656,
2671,
2686,
2701,
2716,
2731,
2746,
2761,
2776,
2791,
2806,
2821,
2836,
2851,
2866,
2881,
2896,
2911,
2926,
2941,
2956,
2971,
2986,
3001
],
"llm": [
0.940214964205142,
0.9733418823394715,
0.9838215474360069,
0.9851625430854666,
0.9851625430854666,
0.9851625430854666,
0.9851625430854666,
0.9851625430854666,
0.9852227386651115,
0.9852227386651115,
0.9862429823256287,
0.9862429823256287,
0.9862429823256287,
0.9866374917320696,
0.9866374917320696,
0.9866374917320696,
0.9866374917320696,
0.9866374917320696,
0.9866374917320696,
0.987301378033219,
0.987301378033219,
0.9887502127480645,
0.9887502127480645,
0.9887502127480645,
0.9891665525965191,
0.9891665525965191,
0.9891665525965191,
0.9891665525965191,
0.9891665525965191,
0.9891665525965191,
0.9891665525965191,
0.9891665525965191,
0.9900740255959819,
0.9900740255959819,
0.9900740255959819,
0.9900740255959819,
0.9900740255959819,
0.9900740255959819,
0.9911132045227066,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9919884196414142,
0.9935087343842928,
0.9935087343842928,
0.9935087343842928,
0.9935087343842928,
0.9935087343842928,
0.9935087343842928,
0.9935087343842928,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9957691426051499,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983,
0.9972872876720983
],
"rand_mean": [
0.7641192381220525,
0.9821905696585997,
0.9854825121133975,
0.9872677016258186,
0.9881660069314924,
0.9888754309420522,
0.9894104996458606,
0.9899089191745734,
0.9902914922141657,
0.9907474284275823,
0.991083207174302,
0.9913860952383124,
0.9916402355554694,
0.9918316903156567,
0.992035543210803,
0.9923106173951887,
0.9924650401627909,
0.9926411147427917,
0.9928207220606602,
0.9929696393088657,
0.9931369450689367,
0.9932696277710222,
0.9934172392551216,
0.9936004731672637,
0.9937228073238837,
0.9938193442399821,
0.9939092974031314,
0.9939683484430896,
0.9940372806558427,
0.9941416786888108,
0.9942348756540672,
0.9942905361162864,
0.9943429209596303,
0.9943982783043313,
0.9944583052501247,
0.9945306636129365,
0.994591432360381,
0.9946642192073715,
0.9947194018867423,
0.9947853228668996,
0.9948579613659693,
0.9949068905499888,
0.9949425642177339,
0.9950024743856584,
0.9950641607698634,
0.9951305850830622,
0.9951870799226641,
0.9952430245204394,
0.9952774435533205,
0.9953249128510142,
0.9953646484224329,
0.9953757509832035,
0.9954123247018488,
0.9954341597257605,
0.9954837719999129,
0.9955181322420918,
0.995551939903208,
0.9956083299140593,
0.9956315097692824,
0.9956521259275312,
0.9956822928288166,
0.9957082017651699,
0.9957534671744986,
0.9957794871544523,
0.9958063994170913,
0.9958311055126452,
0.9958590088941462,
0.995885501973166,
0.995907396373309,
0.9959314704075246,
0.9959684366822555,
0.9959865419068461,
0.9960048707534611,
0.9960192660966988,
0.9960407470799573,
0.9960628974458293,
0.9960913479541745,
0.9961113877759228,
0.9961328638344726,
0.9961504061666661,
0.9961613957927495,
0.9961867781849993,
0.9962042780539015,
0.9962191535114102,
0.9962327661873887,
0.9962652038704362,
0.9962755087083194,
0.9962886166120848,
0.996306739744561,
0.996316360324858,
0.9963373928910778,
0.9963567348103735,
0.9963664615096662,
0.9963820736082293,
0.9963988653062882,
0.9964108495682048,
0.9964203225930182,
0.9964280152796609,
0.9964429139149827,
0.9964539652735892,
0.9964652300050273,
0.9964831437126407,
0.9964881320752625,
0.9965024462452264,
0.9965075947056257,
0.9965224062378208,
0.9965328333548232,
0.9965395522966946,
0.9965443280873267,
0.9965589533323421,
0.9965722160041954,
0.9965804506505446,
0.9965909395000694,
0.9966047961971211,
0.9966129000114757,
0.9966247128137768,
0.9966367666410347,
0.9966531575759509,
0.9966660371809378,
0.9966723311166371,
0.9966839773012837,
0.9966882329970093,
0.9966952105166047,
0.9967145265881846,
0.9967234443435617,
0.9967329959248261,
0.996736346681224,
0.9967402879106857,
0.9967406111603911,
0.9967495319690334,
0.9967561403709962,
0.9967621443506619,
0.9967651032567518,
0.9967726939820865,
0.9967830310415214,
0.9967897995412617,
0.9967934210731524,
0.9968011224762028,
0.9968047033714735,
0.9968093684843903,
0.9968150492143502,
0.9968253430587686,
0.9968377409530454,
0.996842406065962,
0.9968485893239455,
0.99685541908134,
0.9968633330563802,
0.9968725526044976,
0.9968786251847654,
0.9968880573048727,
0.9968952561798997,
0.9969053806673449,
0.9969189604583725,
0.9969235148935732,
0.9969289371836163,
0.9969337129742486,
0.9969415250550148,
0.9969479296684296,
0.9969541129264131,
0.9969588799336038,
0.9969696176267882,
0.9969727645946377,
0.9969837236432536,
0.9969920566317423,
0.9970029050026423,
0.9970193181944795,
0.9970239833073961,
0.997028537742597,
0.997041006152838,
0.9970502257009555,
0.9970627028946383,
0.9970734405878225,
0.9970858071037897,
0.9970969699409538,
0.9971003294807931,
0.9971049945937096,
0.9971128066744758,
0.9971220262225932,
0.9971250625127271,
0.9971311350929949,
0.9971343839551187,
0.9971390490680352,
0.9971451216483032,
0.9971542305187048,
0.9971634500668222,
0.9971726696149397,
0.9971787421952074,
0.9971943575732982,
0.9972037896934055,
0.9972083441286064,
0.9972144167088743,
0.9972189711440752,
0.9972295981595438,
0.9972371888848786,
0.9972432614651465,
0.9972538884806151,
0.9972614792059499,
0.9972660336411506,
0.9972721062214185,
0.9972766606566194,
0.9972872876720881
],
"tpe_mean": [
0.7183762552679218,
0.9825082728357546,
0.9850026324344077,
0.9866569941763745,
0.9886785269590586,
0.9898765040952093,
0.9906653897525061,
0.991004014707496,
0.9911764772532742,
0.9918793533410196,
0.9923906683886939,
0.9924645844826219,
0.9927602909266886,
0.992855637153796,
0.9930927049610784,
0.9932115947416809,
0.9932420010365385,
0.9934047047368432,
0.9934429926177518,
0.993518563683508,
0.9936525291234416,
0.9937357037946231,
0.9938155122361023,
0.9938374386848795,
0.9938374386848795,
0.9939882069975381,
0.994275634497971,
0.994302668658671,
0.99444012082931,
0.9945041150145613,
0.9946538902203044,
0.9947001785175167,
0.9947950323845,
0.9948568130154374,
0.9948872193102949,
0.994930213920398,
0.9950185297904517,
0.995135167595045,
0.9951372054805273,
0.9951820447790871,
0.9952270772746726,
0.9955088981171349,
0.9956123916827625,
0.9957358733457502,
0.9958178953415434,
0.995845004719406,
0.9958934664072996,
0.9959280807484351,
0.9959323321882314,
0.9959669465293669,
0.9959669465293669,
0.9959995229850199,
0.9960037744248162,
0.9960277433747358,
0.9960277433747358,
0.9960655596714134,
0.9960655596714134,
0.9960655596714134,
0.9960655596714134,
0.9961115710708569,
0.9961115710708569,
0.9961600327587504,
0.9961600327587504,
0.9961600327587504,
0.9961600327587504,
0.9961600327587504,
0.9962294371098528,
0.9962294371098528,
0.9962294371098528,
0.9962294371098528,
0.9962620135655058,
0.9962662650053022,
0.9962662650053022,
0.9963104752533993,
0.9963104752533993,
0.9963104752533993,
0.9963125131388818,
0.9963631883810895,
0.9963631883810895,
0.9964239141837676,
0.9964420129703221,
0.9965027387730002,
0.9965027387730002,
0.9965027387730002,
0.9965027387730002,
0.9965027387730002,
0.9965047766584827,
0.9965047766584827,
0.9965047766584827,
0.9965047766584827,
0.9965047766584827,
0.9965532383463763,
0.9965910546430539,
0.9965910546430539,
0.9965910546430539,
0.9965932681973677,
0.9965932681973677,
0.9965932681973677,
0.9965932681973677,
0.9965932681973677,
0.9966278825385032,
0.9966278825385032,
0.9966278825385032,
0.9966604589941563,
0.9966604589941563,
0.9966604589941563,
0.9967276497909449,
0.9967580126922837,
0.9967580126922837,
0.9967580126922837,
0.9967580126922837,
0.9967580126922837,
0.9968187384949617,
0.9968187384949617,
0.9968491013963007,
0.9968794642976397,
0.9968794642976397,
0.9969120407532926,
0.9969446172089457,
0.9969792315500812,
0.9970095944514202,
0.9970095944514202,
0.9970095944514202,
0.9970095944514202,
0.9970399573527593,
0.9970399573527593,
0.9970399573527593,
0.9970399573527593,
0.9970399573527593,
0.9970399573527593,
0.9970399573527593,
0.9971310460567763,
0.9971636225124292,
0.9971658360667432,
0.9971658360667432,
0.9972265618694212,
0.9972265618694212,
0.9972265618694212,
0.9972265618694212,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972569247707601,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992,
0.9972872876720992
]
},
"reviewer_response": {
"original_fit": {
"c": 0.34,
"r2": 0.81,
"confound": "Fitted across bug-fix schedule (Mar 4-9), R^2 reflects infrastructure fixes not search dynamics"
},
"postbugfix_fit": {
"c": 0.10562709364453839,
"r2": 0.9263119537339668,
"note": "Restricted to 3003 experiments after bugfix cutoff (2026-02-27), eliminating infrastructure confound"
},
"improvement": {
"r2_delta": 0.11631195373396674,
"addresses_concern": true
}
}
}