{
  "n_experiments": 1337,
  "best_ap": 0.9245057778280362,
  "power_law_fit": {
    "a": 0.9797451854364101,
    "b": 0.7041564427568653,
    "c": 0.3408828354213971,
    "r2": 0.8131605516787284
  },
  "bootstrap": {
    "a": {
      "mean": 0.9827887830845106,
      "ci_lo": 0.912302980949136,
      "ci_hi": 1.4999999999999767
    },
    "b": {
      "mean": 0.4371570277279218,
      "ci_lo": 0.08032745979173711,
      "ci_hi": 0.9041926705501468
    },
    "c": {
      "mean": 0.7311506858658928,
      "ci_lo": 0.014452905651612627,
      "ci_hi": 2.4073841308636217
    },
    "n_successful": 10000
  },
  "random_search": {
    "c": 0.6612503819238731,
    "r2": 0.956669727864147
  },
  "tpe_search": {
    "c": 0.6877971382555464,
    "r2": 0.9891756864378389
  },
  "model_comparison": {
    "power_law": {
      "params": {
        "a": 0.9797451854364101,
        "b": 0.7041564427568653,
        "c": 0.3408828354213971
      },
      "r2": 0.8131605516787284,
      "aic": -6339.723049489921,
      "bic": -6324.128498758616
    },
    "logarithmic": {
      "params": {
        "a": 0.6054871110593983,
        "b": 0.04561921689225508
      },
      "r2": 0.7463509035589249,
      "aic": -5932.999316614736,
      "bic": -5922.602949460532
    },
    "exponential": {
      "params": {
        "a": 0.9230100607548066,
        "b": 0.1680534844691711,
        "c": 0.0035953246989852954
      },
      "r2": 0.6352810459224858,
      "aic": -5445.433985463425,
      "bic": -5429.83943473212
    }
  },
  "best_aic_model": "power_law",
  "sample_efficiency": {
    "N_llm_95": 334,
    "N_llm_99": 771,
    "N_rand_95": 15,
    "N_rand_99": 275,
    "ratio_95": 0.04491017964071856,
    "ratio_99": 0.35667963683527887,
    "crossover_n": 771,
    "post_crossover_ratio": 1.3333333333333333,
    "note": "Random search from the LLM-curated experiment pool reaches targets faster because it uniformly samples good configs the LLM created. The meaningful metric is that LLM-guided search DISCOVERS the best architectures; random search cannot create novel configurations outside the observed pool."
  },
  "innovation_rate": {
    "fit": {
      "alpha": 0.34390287993523255,
      "k": 0.5826839121515532
    },
    "bin_centers": [
      6,
      19,
      32,
      45,
      58,
      71,
      84,
      97,
      110,
      123,
      136,
      149,
      162,
      175,
      188,
      201,
      214,
      227,
      240,
      253
    ],
    "rates": [
      0.3076923076923077,
      0.0,
      0.0,
      0.23076923076923078,
      0.0,
      0.0,
      0.07692307692307693,
      0.0,
      0.0,
      0.07692307692307693,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0
    ]
  },
  "ap_at_n": {
    "llm_100": 0.8140566199161836,
    "rand_100": 0.9049494607160214,
    "tpe_100": 0.911042121889541,
    "llm_500": 0.8851484305948076,
    "rand_500": 0.9198625661758042,
    "tpe_500": 0.9227963023842508,
    "llm_1000": 0.9245057778280362,
    "rand_1000": 0.9233831111858435,
    "tpe_1000": 0.9245057778280364,
    "llm_1337": 0.9245057778280362,
    "rand_1337": 0.9245057778280233,
    "tpe_1337": 0.9245057778280364
  },
  "predictive_test": {
    "predicted_at_n": 0.8868135813688272,
    "actual_at_n": 0.9245057778280362,
    "prediction_error": 0.03769219645920907
  },
  "convergence_curve": {
    "description": "Running max AP at sampled N values",
    "N": [
      1,
      7,
      13,
      19,
      25,
      31,
      37,
      43,
      49,
      55,
      61,
      67,
      73,
      79,
      85,
      91,
      97,
      103,
      109,
      115,
      121,
      127,
      133,
      139,
      145,
      151,
      157,
      163,
      169,
      175,
      181,
      187,
      193,
      199,
      205,
      211,
      217,
      223,
      229,
      235,
      241,
      247,
      253,
      259,
      265,
      271,
      277,
      283,
      289,
      295,
      301,
      307,
      313,
      319,
      325,
      331,
      337,
      343,
      349,
      355,
      361,
      367,
      373,
      379,
      385,
      391,
      397,
      403,
      409,
      415,
      421,
      427,
      433,
      439,
      445,
      451,
      457,
      463,
      469,
      475,
      481,
      487,
      493,
      499,
      505,
      511,
      517,
      523,
      529,
      535,
      541,
      547,
      553,
      559,
      565,
      571,
      577,
      583,
      589,
      595,
      601,
      607,
      613,
      619,
      625,
      631,
      637,
      643,
      649,
      655,
      661,
      667,
      673,
      679,
      685,
      691,
      697,
      703,
      709,
      715,
      721,
      727,
      733,
      739,
      745,
      751,
      757,
      763,
      769,
      775,
      781,
      787,
      793,
      799,
      805,
      811,
      817,
      823,
      829,
      835,
      841,
      847,
      853,
      859,
      865,
      871,
      877,
      883,
      889,
      895,
      901,
      907,
      913,
      919,
      925,
      931,
      937,
      943,
      949,
      955,
      961,
      967,
      973,
      979,
      985,
      991,
      997,
      1003,
      1009,
      1015,
      1021,
      1027,
      1033,
      1039,
      1045,
      1051,
      1057,
      1063,
      1069,
      1075,
      1081,
      1087,
      1093,
      1099,
      1105,
      1111,
      1117,
      1123,
      1129,
      1135,
      1141,
      1147,
      1153,
      1159,
      1165,
      1171,
      1177,
      1183,
      1189,
      1195,
      1201,
      1207,
      1213,
      1219,
      1225,
      1231,
      1237,
      1243,
      1249,
      1255,
      1261,
      1267,
      1273,
      1279,
      1285,
      1291,
      1297,
      1303,
      1309,
      1315,
      1321,
      1327,
      1333
    ],
    "llm": [
      0.15148902405674883,
      0.7937975097761555,
      0.7937975097761555,
      0.7937975097761555,
      0.7937975097761555,
      0.7937975097761555,
      0.7937975097761555,
      0.7967345288723343,
      0.8080247440516302,
      0.8080247440516302,
      0.8080247440516302,
      0.8080247440516302,
      0.8080247440516302,
      0.8080247440516302,
      0.8140566199161836,
      0.8140566199161836,
      0.8140566199161836,
      0.8140566199161836,
      0.8140566199161836,
      0.8140566199161836,
      0.8140566199161836,
      0.8140566199161836,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8478092069786651,
      0.8653678579286025,
      0.8653678579286025,
      0.8653678579286025,
      0.8653678579286025,
      0.8653678579286025,
      0.8653678579286025,
      0.8653678579286025,
      0.8653678579286025,
      0.8653678579286025,
      0.8759324263880269,
      0.8759324263880269,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8851484305948076,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.8962403030508167,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362,
      0.9245057778280362
    ],
    "rand_mean": [
      0.5539919620591378,
      0.8586148580927718,
      0.876315322495589,
      0.883287370103437,
      0.8878750136620434,
      0.8915583791179627,
      0.8940736549301979,
      0.8956787931255444,
      0.8969885029320731,
      0.898601141395943,
      0.8997513400383906,
      0.9007156984383211,
      0.9015536565265153,
      0.902519805861566,
      0.9032930530573099,
      0.9039880787121927,
      0.9045773309117836,
      0.9052926543131274,
      0.9060126521860954,
      0.9067262072709764,
      0.9071970085930197,
      0.9077913725004447,
      0.9081232782736762,
      0.9085760217750625,
      0.9090213861221228,
      0.909541424849955,
      0.909905461371397,
      0.9102836264706451,
      0.9106457643304203,
      0.9109442496509794,
      0.9112882798507187,
      0.9116601415476588,
      0.9119926131150268,
      0.9122379833851116,
      0.9124605120814768,
      0.9127843567444064,
      0.9131765838300125,
      0.9133230817733851,
      0.9136011606800336,
      0.9138549628429847,
      0.9140601861588387,
      0.9142147146186077,
      0.9144582718558307,
      0.9148072333264828,
      0.9149726871649635,
      0.9151359004009482,
      0.9153730759229252,
      0.9154954722376893,
      0.9156948459499265,
      0.9158361052846274,
      0.9159694516258681,
      0.9161648706718712,
      0.9162129954066505,
      0.9164166152218198,
      0.9165421910032218,
      0.9167025880851393,
      0.9168103719312732,
      0.9170519908138618,
      0.9172484387591234,
      0.9173372287813165,
      0.9174540167579135,
      0.9175799957165547,
      0.9178481331815044,
      0.917947386308874,
      0.9180762785293521,
      0.9181739015378276,
      0.9182478416960321,
      0.9183219759038872,
      0.9183813734877466,
      0.9184883050178207,
      0.9185854311746318,
      0.9187539444450935,
      0.9188562326879866,
      0.9189247034773074,
      0.9190074690618844,
      0.9191757398095576,
      0.9193020729586202,
      0.9194202282424166,
      0.9194680469570539,
      0.9195309983107063,
      0.9195801615011558,
      0.9196790703952726,
      0.9197446413771064,
      0.9198408687750251,
      0.9199247521705163,
      0.9199784356508849,
      0.9200507084607743,
      0.920101371334301,
      0.9202031405601224,
      0.9202936752299448,
      0.9203422066388265,
      0.9204711086704153,
      0.920511272677044,
      0.9205667935295574,
      0.9206338814784774,
      0.9206881407062355,
      0.9207326101163311,
      0.9208407068425861,
      0.9209527424688814,
      0.9210776026367988,
      0.9211387896680517,
      0.921265125733149,
      0.9212854790136747,
      0.921335139475762,
      0.9214025727186796,
      0.9214304737127009,
      0.9214620881563054,
      0.9214998477547909,
      0.9215570851623022,
      0.9216073027495086,
      0.9216468334017336,
      0.9216948016248729,
      0.9217206239623292,
      0.9217457022516953,
      0.92178233037658,
      0.9218388412764298,
      0.9218776974143904,
      0.9219261023027553,
      0.9219658796238922,
      0.9220359321729906,
      0.9220873704931913,
      0.922111426040804,
      0.9221699201259789,
      0.9221903646700947,
      0.9222070957646278,
      0.9222402200211914,
      0.9222850699910528,
      0.9223115524357048,
      0.9223446151037924,
      0.9223657974163956,
      0.9223907547670579,
      0.9224137868255495,
      0.9224407378487897,
      0.9224725726297565,
      0.9224998912606126,
      0.9225312990837355,
      0.9225674816557451,
      0.922599566916186,
      0.9226355090859012,
      0.9226558942797894,
      0.9226837727528238,
      0.9227242667965744,
      0.92275673994737,
      0.9227890433069248,
      0.9228252618206656,
      0.9228670096160885,
      0.9229114888525765,
      0.9229269864781756,
      0.9229388074834326,
      0.9229593683653812,
      0.9229997117953139,
      0.9230124211828757,
      0.9230416968672733,
      0.9230626940994408,
      0.9231087254659656,
      0.9231407810512927,
      0.9231450175138133,
      0.9231781395321286,
      0.9232135690622227,
      0.923236927574584,
      0.9232800201750652,
      0.923292729562627,
      0.9233124114546458,
      0.9233206080356613,
      0.9233498018488442,
      0.9233664416797869,
      0.9233791807424625,
      0.9233997510167864,
      0.9234154727903104,
      0.9234437188709607,
      0.9234855627214775,
      0.9235016201892549,
      0.9235339817937215,
      0.9235421486996231,
      0.9235621069356676,
      0.9235866670036001,
      0.9236184462449016,
      0.9236426706185809,
      0.9236586984112444,
      0.9236920990118339,
      0.923708432823637,
      0.9237244902914146,
      0.9237368936598368,
      0.9237568518958814,
      0.9237811675331506,
      0.9238060039451089,
      0.923835047144474,
      0.9238507689179979,
      0.9238628662672805,
      0.9238749636165632,
      0.9238831601975785,
      0.9239028124144835,
      0.9239191462262868,
      0.9239387984431918,
      0.923962993141757,
      0.9239750904910397,
      0.9240032453080997,
      0.9240189670816238,
      0.9240353008934271,
      0.9240473982427098,
      0.9240634260353734,
      0.9240797598471765,
      0.9241112033942246,
      0.9241347860545107,
      0.9241583687147967,
      0.9241898122618447,
      0.9242016035919878,
      0.9242212558088927,
      0.9242330471390358,
      0.9242487689125598,
      0.9242726575919854,
      0.9242889914037886,
      0.9243089496398332,
      0.9243246714133573,
      0.9243403931868812,
      0.9243521845170243,
      0.9243639758471673,
      0.9243796976206915,
      0.9243875585074534,
      0.9244150716111205,
      0.9244386542714066,
      0.9244586125074512,
      0.9244821951677372
    ],
    "tpe_mean": [
      0.5912913686350796,
      0.85161265335282,
      0.8735164828388184,
      0.8831368565985873,
      0.8917299193133655,
      0.8954685656571313,
      0.8997251851035619,
      0.9005363613119652,
      0.9021125706867079,
      0.9032347411494064,
      0.9042820656690665,
      0.9055377104329506,
      0.9071154135907004,
      0.9088460423784389,
      0.9091735421288114,
      0.9106747828986519,
      0.9107874531332466,
      0.9120269952129328,
      0.913308039924551,
      0.9137765880345954,
      0.9138378796758597,
      0.9139460304794171,
      0.9142738450860755,
      0.9149995024016598,
      0.915842453156816,
      0.9164865673834638,
      0.91664143321591,
      0.9170844470193547,
      0.9177401248666308,
      0.9177401248666308,
      0.9180248144976662,
      0.9181887461179751,
      0.9181887461179751,
      0.9181887461179751,
      0.918269180257398,
      0.9184288167667174,
      0.9184288167667174,
      0.9184288167667174,
      0.9186798964115175,
      0.9186798964115175,
      0.9192563718888969,
      0.9197234168037659,
      0.9197234168037659,
      0.9197234168037659,
      0.9201866958785448,
      0.9202738439030361,
      0.9203524527706564,
      0.9203524527706564,
      0.9206586818273198,
      0.9206586818273198,
      0.9206586818273198,
      0.9206586818273198,
      0.9206586818273198,
      0.9206586818273198,
      0.9206586818273198,
      0.9211270390790988,
      0.9211270390790988,
      0.9211270390790988,
      0.9211270390790988,
      0.9211270390790988,
      0.9211270390790988,
      0.9217556044958645,
      0.9217556044958645,
      0.9217556044958645,
      0.9218427525203556,
      0.9218427525203556,
      0.9218427525203556,
      0.9218427525203556,
      0.9218427525203556,
      0.9222228416143752,
      0.9223075708647874,
      0.9224672073741068,
      0.9224672073741068,
      0.9224672073741068,
      0.9227176935166306,
      0.9227176935166306,
      0.9227176935166306,
      0.9227176935166306,
      0.9227963023842508,
      0.9227963023842508,
      0.9227963023842508,
      0.9227963023842508,
      0.9227963023842508,
      0.9227963023842508,
      0.9227963023842508,
      0.9227963023842508,
      0.9227963023842508,
      0.9229602340045598,
      0.9229602340045598,
      0.9229602340045598,
      0.9229663543873518,
      0.9229663543873518,
      0.923053502411843,
      0.923053502411843,
      0.9231382316622553,
      0.9231382316622553,
      0.9231443520450473,
      0.923301569780288,
      0.923301569780288,
      0.923301569780288,
      0.923301569780288,
      0.923301569780288,
      0.9233801786479081,
      0.9234593810178049,
      0.9235379898854251,
      0.9235379898854251,
      0.9235379898854251,
      0.9237799368710781,
      0.9237799368710781,
      0.9237799368710781,
      0.9237799368710781,
      0.9237799368710781,
      0.9237799368710781,
      0.9237799368710781,
      0.9239493953719027,
      0.9239493953719027,
      0.9240280042395228,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241066131071431,
      0.9241852219747634,
      0.9242638308423836,
      0.9242638308423836,
      0.924342439710004,
      0.924342439710004,
      0.924342439710004,
      0.924342439710004,
      0.924342439710004,
      0.924342439710004,
      0.9244271689604162,
      0.9244271689604162,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364,
      0.9245057778280364
    ]
  }
}