warlockee's picture
Upload folder using huggingface_hub
bd3ae0e verified
{
"n_experiments": 1337,
"best_ap": 0.9245057778280362,
"power_law_fit": {
"a": 0.9797451854364101,
"b": 0.7041564427568653,
"c": 0.3408828354213971,
"r2": 0.8131605516787284
},
"bootstrap": {
"a": {
"mean": 0.9827887830845106,
"ci_lo": 0.912302980949136,
"ci_hi": 1.4999999999999767
},
"b": {
"mean": 0.4371570277279218,
"ci_lo": 0.08032745979173711,
"ci_hi": 0.9041926705501468
},
"c": {
"mean": 0.7311506858658928,
"ci_lo": 0.014452905651612627,
"ci_hi": 2.4073841308636217
},
"n_successful": 10000
},
"random_search": {
"c": 0.6612503819238731,
"r2": 0.956669727864147
},
"tpe_search": {
"c": 0.6877971382555464,
"r2": 0.9891756864378389
},
"model_comparison": {
"power_law": {
"params": {
"a": 0.9797451854364101,
"b": 0.7041564427568653,
"c": 0.3408828354213971
},
"r2": 0.8131605516787284,
"aic": -6339.723049489921,
"bic": -6324.128498758616
},
"logarithmic": {
"params": {
"a": 0.6054871110593983,
"b": 0.04561921689225508
},
"r2": 0.7463509035589249,
"aic": -5932.999316614736,
"bic": -5922.602949460532
},
"exponential": {
"params": {
"a": 0.9230100607548066,
"b": 0.1680534844691711,
"c": 0.0035953246989852954
},
"r2": 0.6352810459224858,
"aic": -5445.433985463425,
"bic": -5429.83943473212
}
},
"best_aic_model": "power_law",
"sample_efficiency": {
"N_llm_95": 334,
"N_llm_99": 771,
"N_rand_95": 15,
"N_rand_99": 275,
"ratio_95": 0.04491017964071856,
"ratio_99": 0.35667963683527887,
"crossover_n": 771,
"post_crossover_ratio": 1.3333333333333333,
"note": "Random search from the LLM-curated experiment pool reaches targets faster because it uniformly samples good configs the LLM created. The meaningful metric is that LLM-guided search DISCOVERS the best architectures; random search cannot create novel configurations outside the observed pool."
},
"innovation_rate": {
"fit": {
"alpha": 0.34390287993523255,
"k": 0.5826839121515532
},
"bin_centers": [
6,
19,
32,
45,
58,
71,
84,
97,
110,
123,
136,
149,
162,
175,
188,
201,
214,
227,
240,
253
],
"rates": [
0.3076923076923077,
0.0,
0.0,
0.23076923076923078,
0.0,
0.0,
0.07692307692307693,
0.0,
0.0,
0.07692307692307693,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"ap_at_n": {
"llm_100": 0.8140566199161836,
"rand_100": 0.9049494607160214,
"tpe_100": 0.911042121889541,
"llm_500": 0.8851484305948076,
"rand_500": 0.9198625661758042,
"tpe_500": 0.9227963023842508,
"llm_1000": 0.9245057778280362,
"rand_1000": 0.9233831111858435,
"tpe_1000": 0.9245057778280364,
"llm_1337": 0.9245057778280362,
"rand_1337": 0.9245057778280233,
"tpe_1337": 0.9245057778280364
},
"predictive_test": {
"predicted_at_n": 0.8868135813688272,
"actual_at_n": 0.9245057778280362,
"prediction_error": 0.03769219645920907
},
"convergence_curve": {
"description": "Running max AP at sampled N values",
"N": [
1,
7,
13,
19,
25,
31,
37,
43,
49,
55,
61,
67,
73,
79,
85,
91,
97,
103,
109,
115,
121,
127,
133,
139,
145,
151,
157,
163,
169,
175,
181,
187,
193,
199,
205,
211,
217,
223,
229,
235,
241,
247,
253,
259,
265,
271,
277,
283,
289,
295,
301,
307,
313,
319,
325,
331,
337,
343,
349,
355,
361,
367,
373,
379,
385,
391,
397,
403,
409,
415,
421,
427,
433,
439,
445,
451,
457,
463,
469,
475,
481,
487,
493,
499,
505,
511,
517,
523,
529,
535,
541,
547,
553,
559,
565,
571,
577,
583,
589,
595,
601,
607,
613,
619,
625,
631,
637,
643,
649,
655,
661,
667,
673,
679,
685,
691,
697,
703,
709,
715,
721,
727,
733,
739,
745,
751,
757,
763,
769,
775,
781,
787,
793,
799,
805,
811,
817,
823,
829,
835,
841,
847,
853,
859,
865,
871,
877,
883,
889,
895,
901,
907,
913,
919,
925,
931,
937,
943,
949,
955,
961,
967,
973,
979,
985,
991,
997,
1003,
1009,
1015,
1021,
1027,
1033,
1039,
1045,
1051,
1057,
1063,
1069,
1075,
1081,
1087,
1093,
1099,
1105,
1111,
1117,
1123,
1129,
1135,
1141,
1147,
1153,
1159,
1165,
1171,
1177,
1183,
1189,
1195,
1201,
1207,
1213,
1219,
1225,
1231,
1237,
1243,
1249,
1255,
1261,
1267,
1273,
1279,
1285,
1291,
1297,
1303,
1309,
1315,
1321,
1327,
1333
],
"llm": [
0.15148902405674883,
0.7937975097761555,
0.7937975097761555,
0.7937975097761555,
0.7937975097761555,
0.7937975097761555,
0.7937975097761555,
0.7967345288723343,
0.8080247440516302,
0.8080247440516302,
0.8080247440516302,
0.8080247440516302,
0.8080247440516302,
0.8080247440516302,
0.8140566199161836,
0.8140566199161836,
0.8140566199161836,
0.8140566199161836,
0.8140566199161836,
0.8140566199161836,
0.8140566199161836,
0.8140566199161836,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8478092069786651,
0.8653678579286025,
0.8653678579286025,
0.8653678579286025,
0.8653678579286025,
0.8653678579286025,
0.8653678579286025,
0.8653678579286025,
0.8653678579286025,
0.8653678579286025,
0.8759324263880269,
0.8759324263880269,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8851484305948076,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.8962403030508167,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362,
0.9245057778280362
],
"rand_mean": [
0.5539919620591378,
0.8586148580927718,
0.876315322495589,
0.883287370103437,
0.8878750136620434,
0.8915583791179627,
0.8940736549301979,
0.8956787931255444,
0.8969885029320731,
0.898601141395943,
0.8997513400383906,
0.9007156984383211,
0.9015536565265153,
0.902519805861566,
0.9032930530573099,
0.9039880787121927,
0.9045773309117836,
0.9052926543131274,
0.9060126521860954,
0.9067262072709764,
0.9071970085930197,
0.9077913725004447,
0.9081232782736762,
0.9085760217750625,
0.9090213861221228,
0.909541424849955,
0.909905461371397,
0.9102836264706451,
0.9106457643304203,
0.9109442496509794,
0.9112882798507187,
0.9116601415476588,
0.9119926131150268,
0.9122379833851116,
0.9124605120814768,
0.9127843567444064,
0.9131765838300125,
0.9133230817733851,
0.9136011606800336,
0.9138549628429847,
0.9140601861588387,
0.9142147146186077,
0.9144582718558307,
0.9148072333264828,
0.9149726871649635,
0.9151359004009482,
0.9153730759229252,
0.9154954722376893,
0.9156948459499265,
0.9158361052846274,
0.9159694516258681,
0.9161648706718712,
0.9162129954066505,
0.9164166152218198,
0.9165421910032218,
0.9167025880851393,
0.9168103719312732,
0.9170519908138618,
0.9172484387591234,
0.9173372287813165,
0.9174540167579135,
0.9175799957165547,
0.9178481331815044,
0.917947386308874,
0.9180762785293521,
0.9181739015378276,
0.9182478416960321,
0.9183219759038872,
0.9183813734877466,
0.9184883050178207,
0.9185854311746318,
0.9187539444450935,
0.9188562326879866,
0.9189247034773074,
0.9190074690618844,
0.9191757398095576,
0.9193020729586202,
0.9194202282424166,
0.9194680469570539,
0.9195309983107063,
0.9195801615011558,
0.9196790703952726,
0.9197446413771064,
0.9198408687750251,
0.9199247521705163,
0.9199784356508849,
0.9200507084607743,
0.920101371334301,
0.9202031405601224,
0.9202936752299448,
0.9203422066388265,
0.9204711086704153,
0.920511272677044,
0.9205667935295574,
0.9206338814784774,
0.9206881407062355,
0.9207326101163311,
0.9208407068425861,
0.9209527424688814,
0.9210776026367988,
0.9211387896680517,
0.921265125733149,
0.9212854790136747,
0.921335139475762,
0.9214025727186796,
0.9214304737127009,
0.9214620881563054,
0.9214998477547909,
0.9215570851623022,
0.9216073027495086,
0.9216468334017336,
0.9216948016248729,
0.9217206239623292,
0.9217457022516953,
0.92178233037658,
0.9218388412764298,
0.9218776974143904,
0.9219261023027553,
0.9219658796238922,
0.9220359321729906,
0.9220873704931913,
0.922111426040804,
0.9221699201259789,
0.9221903646700947,
0.9222070957646278,
0.9222402200211914,
0.9222850699910528,
0.9223115524357048,
0.9223446151037924,
0.9223657974163956,
0.9223907547670579,
0.9224137868255495,
0.9224407378487897,
0.9224725726297565,
0.9224998912606126,
0.9225312990837355,
0.9225674816557451,
0.922599566916186,
0.9226355090859012,
0.9226558942797894,
0.9226837727528238,
0.9227242667965744,
0.92275673994737,
0.9227890433069248,
0.9228252618206656,
0.9228670096160885,
0.9229114888525765,
0.9229269864781756,
0.9229388074834326,
0.9229593683653812,
0.9229997117953139,
0.9230124211828757,
0.9230416968672733,
0.9230626940994408,
0.9231087254659656,
0.9231407810512927,
0.9231450175138133,
0.9231781395321286,
0.9232135690622227,
0.923236927574584,
0.9232800201750652,
0.923292729562627,
0.9233124114546458,
0.9233206080356613,
0.9233498018488442,
0.9233664416797869,
0.9233791807424625,
0.9233997510167864,
0.9234154727903104,
0.9234437188709607,
0.9234855627214775,
0.9235016201892549,
0.9235339817937215,
0.9235421486996231,
0.9235621069356676,
0.9235866670036001,
0.9236184462449016,
0.9236426706185809,
0.9236586984112444,
0.9236920990118339,
0.923708432823637,
0.9237244902914146,
0.9237368936598368,
0.9237568518958814,
0.9237811675331506,
0.9238060039451089,
0.923835047144474,
0.9238507689179979,
0.9238628662672805,
0.9238749636165632,
0.9238831601975785,
0.9239028124144835,
0.9239191462262868,
0.9239387984431918,
0.923962993141757,
0.9239750904910397,
0.9240032453080997,
0.9240189670816238,
0.9240353008934271,
0.9240473982427098,
0.9240634260353734,
0.9240797598471765,
0.9241112033942246,
0.9241347860545107,
0.9241583687147967,
0.9241898122618447,
0.9242016035919878,
0.9242212558088927,
0.9242330471390358,
0.9242487689125598,
0.9242726575919854,
0.9242889914037886,
0.9243089496398332,
0.9243246714133573,
0.9243403931868812,
0.9243521845170243,
0.9243639758471673,
0.9243796976206915,
0.9243875585074534,
0.9244150716111205,
0.9244386542714066,
0.9244586125074512,
0.9244821951677372
],
"tpe_mean": [
0.5912913686350796,
0.85161265335282,
0.8735164828388184,
0.8831368565985873,
0.8917299193133655,
0.8954685656571313,
0.8997251851035619,
0.9005363613119652,
0.9021125706867079,
0.9032347411494064,
0.9042820656690665,
0.9055377104329506,
0.9071154135907004,
0.9088460423784389,
0.9091735421288114,
0.9106747828986519,
0.9107874531332466,
0.9120269952129328,
0.913308039924551,
0.9137765880345954,
0.9138378796758597,
0.9139460304794171,
0.9142738450860755,
0.9149995024016598,
0.915842453156816,
0.9164865673834638,
0.91664143321591,
0.9170844470193547,
0.9177401248666308,
0.9177401248666308,
0.9180248144976662,
0.9181887461179751,
0.9181887461179751,
0.9181887461179751,
0.918269180257398,
0.9184288167667174,
0.9184288167667174,
0.9184288167667174,
0.9186798964115175,
0.9186798964115175,
0.9192563718888969,
0.9197234168037659,
0.9197234168037659,
0.9197234168037659,
0.9201866958785448,
0.9202738439030361,
0.9203524527706564,
0.9203524527706564,
0.9206586818273198,
0.9206586818273198,
0.9206586818273198,
0.9206586818273198,
0.9206586818273198,
0.9206586818273198,
0.9206586818273198,
0.9211270390790988,
0.9211270390790988,
0.9211270390790988,
0.9211270390790988,
0.9211270390790988,
0.9211270390790988,
0.9217556044958645,
0.9217556044958645,
0.9217556044958645,
0.9218427525203556,
0.9218427525203556,
0.9218427525203556,
0.9218427525203556,
0.9218427525203556,
0.9222228416143752,
0.9223075708647874,
0.9224672073741068,
0.9224672073741068,
0.9224672073741068,
0.9227176935166306,
0.9227176935166306,
0.9227176935166306,
0.9227176935166306,
0.9227963023842508,
0.9227963023842508,
0.9227963023842508,
0.9227963023842508,
0.9227963023842508,
0.9227963023842508,
0.9227963023842508,
0.9227963023842508,
0.9227963023842508,
0.9229602340045598,
0.9229602340045598,
0.9229602340045598,
0.9229663543873518,
0.9229663543873518,
0.923053502411843,
0.923053502411843,
0.9231382316622553,
0.9231382316622553,
0.9231443520450473,
0.923301569780288,
0.923301569780288,
0.923301569780288,
0.923301569780288,
0.923301569780288,
0.9233801786479081,
0.9234593810178049,
0.9235379898854251,
0.9235379898854251,
0.9235379898854251,
0.9237799368710781,
0.9237799368710781,
0.9237799368710781,
0.9237799368710781,
0.9237799368710781,
0.9237799368710781,
0.9237799368710781,
0.9239493953719027,
0.9239493953719027,
0.9240280042395228,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241066131071431,
0.9241852219747634,
0.9242638308423836,
0.9242638308423836,
0.924342439710004,
0.924342439710004,
0.924342439710004,
0.924342439710004,
0.924342439710004,
0.924342439710004,
0.9244271689604162,
0.9244271689604162,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364,
0.9245057778280364
]
}
}