{
  "analysis": "post_bugfix_2026-02-27",
  "bugfix_cutoff_date": "2026-02-27",
  "n_experiments_total": 10252,
  "n_experiments_postbugfix": 3003,
  "metric_used": "val_ap (mAP_1000ms)",
  "best_ap": 0.9972872876720983,
  "power_law_fit": {
    "a": 1.0309776612473043,
    "b": 0.07660090971197056,
    "c": 0.10562709364453839,
    "r2": 0.9263119537339668
  },
  "bootstrap": {
    "a": {
      "mean": 1.0151743159991817,
      "ci_lo": 0.994052817156907,
      "ci_hi": 1.2752625110737068
    },
    "b": {
      "mean": 0.2343487502975724,
      "ci_lo": 0.023734076480255686,
      "ci_hi": 0.9094941270919662
    },
    "c": {
      "mean": 0.6877843846977298,
      "ci_lo": 0.010000002818053202,
      "ci_hi": 2.0447608771859778
    },
    "n_successful": 10000
  },
  "random_search": {
    "c": 0.9933857531513521,
    "r2": 0.9548032212359628,
    "a": 0.9961533550294086,
    "b": 0.22599530090921646
  },
  "tpe_search": {
    "c": 1.1165134879545537,
    "r2": 0.9695102537476521,
    "a": 0.9963439634273502,
    "b": 0.27383932783651854
  },
  "model_comparison": {
    "power_law": {
      "params": {
        "a": 1.0309776612473043,
        "b": 0.07660090971197056,
        "c": 0.10562709364453839
      },
      "r2": 0.9263119537339668,
      "aic": -10558.253134335755
    },
    "logarithmic": {
      "params": {
        "a": 0.963750631745776,
        "b": 0.004347567696082376
      },
      "r2": 0.916676541840304,
      "aic": -10437.240716268267
    },
    "exponential": {
      "params": {
        "a": 0.9977022599747527,
        "b": 0.017007287605546248,
        "c": 0.0016102846158961184
      },
      "r2": 0.8565181425141755,
      "aic": -9891.218759341695
    }
  },
  "best_aic_model": "power_law",
  "ap_at_n": {
    "llm_50": 0.9851625430854666,
    "rand_50": 0.9876172902379546,
    "tpe_50": 0.9873726456969698,
    "llm_100": 0.9851625430854666,
    "rand_100": 0.9897126421783415,
    "tpe_100": 0.9909578992430562,
    "llm_200": 0.9866374917320696,
    "rand_200": 0.9918942890162451,
    "tpe_200": 0.9928578507081098,
    "llm_500": 0.9900740255959819,
    "rand_500": 0.9944056260441696,
    "tpe_500": 0.9948568130154374
  },
  "ken_test_postbugfix": {
    "n_experiments": 603,
    "best_test_ap": 0.9245057778280362,
    "power_law": {
      "a": 0.9330781493667619,
      "b": 0.5763021851824278,
      "c": 0.7629528207224087,
      "r2": 0.8614031210842934
    }
  },
  "stable_window": {
    "window": "2026-02-27 to 2026-03-04",
    "n_experiments": 1340,
    "best_val_ap": 0.9957691426051499,
    "power_law": {
      "a": 1.0160590184039786,
      "b": 0.0618827564638711,
      "c": 0.14521913345183568,
      "r2": 0.8882291869459537
    }
  },
  "full_campaign": {
    "n_experiments": 10252,
    "best_val_ap": 0.9999992038926465,
    "power_law": {
      "a": 1.0007587191008416,
      "b": 0.013028751330850845,
      "c": 0.3415205999139974,
      "r2": 0.4688207337442444
    },
    "random_search": {
      "c": 0.8398420259260954,
      "r2": 0.9993831778459062
    }
  },
  "convergence_curve": {
    "description": "Running max AP at sampled N values",
    "N": [
      1,
      16,
      31,
      46,
      61,
      76,
      91,
      106,
      121,
      136,
      151,
      166,
      181,
      196,
      211,
      226,
      241,
      256,
      271,
      286,
      301,
      316,
      331,
      346,
      361,
      376,
      391,
      406,
      421,
      436,
      451,
      466,
      481,
      496,
      511,
      526,
      541,
      556,
      571,
      586,
      601,
      616,
      631,
      646,
      661,
      676,
      691,
      706,
      721,
      736,
      751,
      766,
      781,
      796,
      811,
      826,
      841,
      856,
      871,
      886,
      901,
      916,
      931,
      946,
      961,
      976,
      991,
      1006,
      1021,
      1036,
      1051,
      1066,
      1081,
      1096,
      1111,
      1126,
      1141,
      1156,
      1171,
      1186,
      1201,
      1216,
      1231,
      1246,
      1261,
      1276,
      1291,
      1306,
      1321,
      1336,
      1351,
      1366,
      1381,
      1396,
      1411,
      1426,
      1441,
      1456,
      1471,
      1486,
      1501,
      1516,
      1531,
      1546,
      1561,
      1576,
      1591,
      1606,
      1621,
      1636,
      1651,
      1666,
      1681,
      1696,
      1711,
      1726,
      1741,
      1756,
      1771,
      1786,
      1801,
      1816,
      1831,
      1846,
      1861,
      1876,
      1891,
      1906,
      1921,
      1936,
      1951,
      1966,
      1981,
      1996,
      2011,
      2026,
      2041,
      2056,
      2071,
      2086,
      2101,
      2116,
      2131,
      2146,
      2161,
      2176,
      2191,
      2206,
      2221,
      2236,
      2251,
      2266,
      2281,
      2296,
      2311,
      2326,
      2341,
      2356,
      2371,
      2386,
      2401,
      2416,
      2431,
      2446,
      2461,
      2476,
      2491,
      2506,
      2521,
      2536,
      2551,
      2566,
      2581,
      2596,
      2611,
      2626,
      2641,
      2656,
      2671,
      2686,
      2701,
      2716,
      2731,
      2746,
      2761,
      2776,
      2791,
      2806,
      2821,
      2836,
      2851,
      2866,
      2881,
      2896,
      2911,
      2926,
      2941,
      2956,
      2971,
      2986,
      3001
    ],
    "llm": [
      0.940214964205142,
      0.9733418823394715,
      0.9838215474360069,
      0.9851625430854666,
      0.9851625430854666,
      0.9851625430854666,
      0.9851625430854666,
      0.9851625430854666,
      0.9852227386651115,
      0.9852227386651115,
      0.9862429823256287,
      0.9862429823256287,
      0.9862429823256287,
      0.9866374917320696,
      0.9866374917320696,
      0.9866374917320696,
      0.9866374917320696,
      0.9866374917320696,
      0.9866374917320696,
      0.987301378033219,
      0.987301378033219,
      0.9887502127480645,
      0.9887502127480645,
      0.9887502127480645,
      0.9891665525965191,
      0.9891665525965191,
      0.9891665525965191,
      0.9891665525965191,
      0.9891665525965191,
      0.9891665525965191,
      0.9891665525965191,
      0.9891665525965191,
      0.9900740255959819,
      0.9900740255959819,
      0.9900740255959819,
      0.9900740255959819,
      0.9900740255959819,
      0.9900740255959819,
      0.9911132045227066,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9919884196414142,
      0.9935087343842928,
      0.9935087343842928,
      0.9935087343842928,
      0.9935087343842928,
      0.9935087343842928,
      0.9935087343842928,
      0.9935087343842928,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9957691426051499,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983,
      0.9972872876720983
    ],
    "rand_mean": [
      0.7641192381220525,
      0.9821905696585997,
      0.9854825121133975,
      0.9872677016258186,
      0.9881660069314924,
      0.9888754309420522,
      0.9894104996458606,
      0.9899089191745734,
      0.9902914922141657,
      0.9907474284275823,
      0.991083207174302,
      0.9913860952383124,
      0.9916402355554694,
      0.9918316903156567,
      0.992035543210803,
      0.9923106173951887,
      0.9924650401627909,
      0.9926411147427917,
      0.9928207220606602,
      0.9929696393088657,
      0.9931369450689367,
      0.9932696277710222,
      0.9934172392551216,
      0.9936004731672637,
      0.9937228073238837,
      0.9938193442399821,
      0.9939092974031314,
      0.9939683484430896,
      0.9940372806558427,
      0.9941416786888108,
      0.9942348756540672,
      0.9942905361162864,
      0.9943429209596303,
      0.9943982783043313,
      0.9944583052501247,
      0.9945306636129365,
      0.994591432360381,
      0.9946642192073715,
      0.9947194018867423,
      0.9947853228668996,
      0.9948579613659693,
      0.9949068905499888,
      0.9949425642177339,
      0.9950024743856584,
      0.9950641607698634,
      0.9951305850830622,
      0.9951870799226641,
      0.9952430245204394,
      0.9952774435533205,
      0.9953249128510142,
      0.9953646484224329,
      0.9953757509832035,
      0.9954123247018488,
      0.9954341597257605,
      0.9954837719999129,
      0.9955181322420918,
      0.995551939903208,
      0.9956083299140593,
      0.9956315097692824,
      0.9956521259275312,
      0.9956822928288166,
      0.9957082017651699,
      0.9957534671744986,
      0.9957794871544523,
      0.9958063994170913,
      0.9958311055126452,
      0.9958590088941462,
      0.995885501973166,
      0.995907396373309,
      0.9959314704075246,
      0.9959684366822555,
      0.9959865419068461,
      0.9960048707534611,
      0.9960192660966988,
      0.9960407470799573,
      0.9960628974458293,
      0.9960913479541745,
      0.9961113877759228,
      0.9961328638344726,
      0.9961504061666661,
      0.9961613957927495,
      0.9961867781849993,
      0.9962042780539015,
      0.9962191535114102,
      0.9962327661873887,
      0.9962652038704362,
      0.9962755087083194,
      0.9962886166120848,
      0.996306739744561,
      0.996316360324858,
      0.9963373928910778,
      0.9963567348103735,
      0.9963664615096662,
      0.9963820736082293,
      0.9963988653062882,
      0.9964108495682048,
      0.9964203225930182,
      0.9964280152796609,
      0.9964429139149827,
      0.9964539652735892,
      0.9964652300050273,
      0.9964831437126407,
      0.9964881320752625,
      0.9965024462452264,
      0.9965075947056257,
      0.9965224062378208,
      0.9965328333548232,
      0.9965395522966946,
      0.9965443280873267,
      0.9965589533323421,
      0.9965722160041954,
      0.9965804506505446,
      0.9965909395000694,
      0.9966047961971211,
      0.9966129000114757,
      0.9966247128137768,
      0.9966367666410347,
      0.9966531575759509,
      0.9966660371809378,
      0.9966723311166371,
      0.9966839773012837,
      0.9966882329970093,
      0.9966952105166047,
      0.9967145265881846,
      0.9967234443435617,
      0.9967329959248261,
      0.996736346681224,
      0.9967402879106857,
      0.9967406111603911,
      0.9967495319690334,
      0.9967561403709962,
      0.9967621443506619,
      0.9967651032567518,
      0.9967726939820865,
      0.9967830310415214,
      0.9967897995412617,
      0.9967934210731524,
      0.9968011224762028,
      0.9968047033714735,
      0.9968093684843903,
      0.9968150492143502,
      0.9968253430587686,
      0.9968377409530454,
      0.996842406065962,
      0.9968485893239455,
      0.99685541908134,
      0.9968633330563802,
      0.9968725526044976,
      0.9968786251847654,
      0.9968880573048727,
      0.9968952561798997,
      0.9969053806673449,
      0.9969189604583725,
      0.9969235148935732,
      0.9969289371836163,
      0.9969337129742486,
      0.9969415250550148,
      0.9969479296684296,
      0.9969541129264131,
      0.9969588799336038,
      0.9969696176267882,
      0.9969727645946377,
      0.9969837236432536,
      0.9969920566317423,
      0.9970029050026423,
      0.9970193181944795,
      0.9970239833073961,
      0.997028537742597,
      0.997041006152838,
      0.9970502257009555,
      0.9970627028946383,
      0.9970734405878225,
      0.9970858071037897,
      0.9970969699409538,
      0.9971003294807931,
      0.9971049945937096,
      0.9971128066744758,
      0.9971220262225932,
      0.9971250625127271,
      0.9971311350929949,
      0.9971343839551187,
      0.9971390490680352,
      0.9971451216483032,
      0.9971542305187048,
      0.9971634500668222,
      0.9971726696149397,
      0.9971787421952074,
      0.9971943575732982,
      0.9972037896934055,
      0.9972083441286064,
      0.9972144167088743,
      0.9972189711440752,
      0.9972295981595438,
      0.9972371888848786,
      0.9972432614651465,
      0.9972538884806151,
      0.9972614792059499,
      0.9972660336411506,
      0.9972721062214185,
      0.9972766606566194,
      0.9972872876720881
    ],
    "tpe_mean": [
      0.7183762552679218,
      0.9825082728357546,
      0.9850026324344077,
      0.9866569941763745,
      0.9886785269590586,
      0.9898765040952093,
      0.9906653897525061,
      0.991004014707496,
      0.9911764772532742,
      0.9918793533410196,
      0.9923906683886939,
      0.9924645844826219,
      0.9927602909266886,
      0.992855637153796,
      0.9930927049610784,
      0.9932115947416809,
      0.9932420010365385,
      0.9934047047368432,
      0.9934429926177518,
      0.993518563683508,
      0.9936525291234416,
      0.9937357037946231,
      0.9938155122361023,
      0.9938374386848795,
      0.9938374386848795,
      0.9939882069975381,
      0.994275634497971,
      0.994302668658671,
      0.99444012082931,
      0.9945041150145613,
      0.9946538902203044,
      0.9947001785175167,
      0.9947950323845,
      0.9948568130154374,
      0.9948872193102949,
      0.994930213920398,
      0.9950185297904517,
      0.995135167595045,
      0.9951372054805273,
      0.9951820447790871,
      0.9952270772746726,
      0.9955088981171349,
      0.9956123916827625,
      0.9957358733457502,
      0.9958178953415434,
      0.995845004719406,
      0.9958934664072996,
      0.9959280807484351,
      0.9959323321882314,
      0.9959669465293669,
      0.9959669465293669,
      0.9959995229850199,
      0.9960037744248162,
      0.9960277433747358,
      0.9960277433747358,
      0.9960655596714134,
      0.9960655596714134,
      0.9960655596714134,
      0.9960655596714134,
      0.9961115710708569,
      0.9961115710708569,
      0.9961600327587504,
      0.9961600327587504,
      0.9961600327587504,
      0.9961600327587504,
      0.9961600327587504,
      0.9962294371098528,
      0.9962294371098528,
      0.9962294371098528,
      0.9962294371098528,
      0.9962620135655058,
      0.9962662650053022,
      0.9962662650053022,
      0.9963104752533993,
      0.9963104752533993,
      0.9963104752533993,
      0.9963125131388818,
      0.9963631883810895,
      0.9963631883810895,
      0.9964239141837676,
      0.9964420129703221,
      0.9965027387730002,
      0.9965027387730002,
      0.9965027387730002,
      0.9965027387730002,
      0.9965027387730002,
      0.9965047766584827,
      0.9965047766584827,
      0.9965047766584827,
      0.9965047766584827,
      0.9965047766584827,
      0.9965532383463763,
      0.9965910546430539,
      0.9965910546430539,
      0.9965910546430539,
      0.9965932681973677,
      0.9965932681973677,
      0.9965932681973677,
      0.9965932681973677,
      0.9965932681973677,
      0.9966278825385032,
      0.9966278825385032,
      0.9966278825385032,
      0.9966604589941563,
      0.9966604589941563,
      0.9966604589941563,
      0.9967276497909449,
      0.9967580126922837,
      0.9967580126922837,
      0.9967580126922837,
      0.9967580126922837,
      0.9967580126922837,
      0.9968187384949617,
      0.9968187384949617,
      0.9968491013963007,
      0.9968794642976397,
      0.9968794642976397,
      0.9969120407532926,
      0.9969446172089457,
      0.9969792315500812,
      0.9970095944514202,
      0.9970095944514202,
      0.9970095944514202,
      0.9970095944514202,
      0.9970399573527593,
      0.9970399573527593,
      0.9970399573527593,
      0.9970399573527593,
      0.9970399573527593,
      0.9970399573527593,
      0.9970399573527593,
      0.9971310460567763,
      0.9971636225124292,
      0.9971658360667432,
      0.9971658360667432,
      0.9972265618694212,
      0.9972265618694212,
      0.9972265618694212,
      0.9972265618694212,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972569247707601,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992,
      0.9972872876720992
    ]
  },
  "reviewer_response": {
    "original_fit": {
      "c": 0.34,
      "r2": 0.81,
      "confound": "Fitted across bug-fix schedule (Mar 4-9), R^2 reflects infrastructure fixes not search dynamics"
    },
    "postbugfix_fit": {
      "c": 0.10562709364453839,
      "r2": 0.9263119537339668,
      "note": "Restricted to 3003 experiments after bugfix cutoff (2026-02-27), eliminating infrastructure confound"
    },
    "improvement": {
      "r2_delta": 0.11631195373396674,
      "addresses_concern": true
    }
  }
}