| { |
| "generated_at": "2026-06-09T22:14:43.622408+00:00", |
| "reference_core": { |
| "openai-community/gpt2": 0.113891, |
| "openai-community/gpt2-medium": 0.184903, |
| "openai-community/gpt2-large": 0.214641, |
| "openai-community/gpt2-xl": 0.256525 |
| }, |
| "lobotomy_floor": 0.15, |
| "runs": { |
| "gpt2_124m": { |
| "model": "openai-community/gpt2", |
| "params_M": 124.439808, |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "max_per_task": -1, |
| "wall_clock_sec": 891.2480754852295, |
| "family": "huggingface", |
| "results": { |
| "hellaswag_zeroshot": 0.30920135431189005, |
| "jeopardy": 0.001889466225791214, |
| "bigbench_qa_wikidata": 0.28340140741105263, |
| "arc_easy": 0.414983164983165, |
| "arc_challenge": 0.21843003412969283, |
| "copa": 0.63, |
| "commonsense_qa": 0.2325962325962326, |
| "piqa": 0.6289445048966268, |
| "openbook_qa": 0.264, |
| "lambada_openai": 0.32078400931496215, |
| "hellaswag": 0.30770762796255724, |
| "winograd": 0.5897435897435898, |
| "winogrande": 0.5114443567482242, |
| "bigbench_dyck_languages": 0.158, |
| "agi_eval_lsat_ar": 0.22608695652173913, |
| "bigbench_cs_algorithms": 0.42803030303030304, |
| "bigbench_operators": 0.09523809523809523, |
| "bigbench_repeat_copy_logic": 0.03125, |
| "squad": 0.05742667928098392, |
| "coqa": 0.13616434924213955, |
| "boolq": 0.5541284403669725, |
| "bigbench_language_identification": 0.2546 |
| }, |
| "centered_results": { |
| "hellaswag_zeroshot": 0.07893513908252008, |
| "jeopardy": 0.001889466225791214, |
| "bigbench_qa_wikidata": 0.28340140741105263, |
| "arc_easy": 0.21997755331088667, |
| "arc_challenge": -0.04209328782707622, |
| "copa": 0.26, |
| "commonsense_qa": 0.04074529074529074, |
| "piqa": 0.2578890097932536, |
| "openbook_qa": 0.018666666666666682, |
| "lambada_openai": 0.32078400931496215, |
| "hellaswag": 0.07694350395007632, |
| "winograd": 0.17948717948717952, |
| "winogrande": 0.022888713496448387, |
| "bigbench_dyck_languages": 0.158, |
| "agi_eval_lsat_ar": 0.0326086956521739, |
| "bigbench_cs_algorithms": 0.42803030303030304, |
| "bigbench_operators": 0.09523809523809523, |
| "bigbench_repeat_copy_logic": 0.03125, |
| "squad": 0.05742667928098392, |
| "coqa": 0.13616434924213955, |
| "boolq": -0.17334620956059865, |
| "bigbench_language_identification": 0.17997799779977997 |
| }, |
| "task_timings_sec": { |
| "hellaswag_zeroshot": 63.30279040336609, |
| "jeopardy": 14.193135976791382, |
| "bigbench_qa_wikidata": 124.85103392601013, |
| "arc_easy": 22.142359495162964, |
| "arc_challenge": 10.91347050666809, |
| "copa": 0.6135663986206055, |
| "commonsense_qa": 13.434561729431152, |
| "piqa": 14.689457654953003, |
| "openbook_qa": 2.6003074645996094, |
| "lambada_openai": 27.92130184173584, |
| "hellaswag": 142.64234709739685, |
| "winograd": 1.3682467937469482, |
| "winogrande": 6.364197015762329, |
| "bigbench_dyck_languages": 8.250524520874023, |
| "agi_eval_lsat_ar": 3.1633238792419434, |
| "bigbench_cs_algorithms": 9.292541980743408, |
| "bigbench_operators": 1.407222032546997, |
| "bigbench_repeat_copy_logic": 0.2225968837738037, |
| "squad": 139.18381094932556, |
| "coqa": 59.376181840896606, |
| "boolq": 41.74283003807068, |
| "bigbench_language_identification": 183.55174899101257 |
| }, |
| "core_metric": 0.12113020737908765, |
| "n_tasks_evaluated": 22 |
| }, |
| "gpt2_medium_355m": { |
| "model": "openai-community/gpt2-medium", |
| "params_M": 354.823168, |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "max_per_task": -1, |
| "wall_clock_sec": 1346.1762642860413, |
| "family": "huggingface", |
| "results": { |
| "hellaswag_zeroshot": 0.3937462656841267, |
| "jeopardy": 0.038734057628719884, |
| "bigbench_qa_wikidata": 0.4208946410117612, |
| "arc_easy": 0.4852693602693603, |
| "arc_challenge": 0.2696245733788396, |
| "copa": 0.64, |
| "commonsense_qa": 0.21375921375921375, |
| "piqa": 0.6594124047878128, |
| "openbook_qa": 0.306, |
| "lambada_openai": 0.42111391422472344, |
| "hellaswag": 0.3981278629755029, |
| "winograd": 0.6153846153846154, |
| "winogrande": 0.5130228887134964, |
| "bigbench_dyck_languages": 0.18, |
| "agi_eval_lsat_ar": 0.22608695652173913, |
| "bigbench_cs_algorithms": 0.42954545454545456, |
| "bigbench_operators": 0.11428571428571428, |
| "bigbench_repeat_copy_logic": 0.0625, |
| "squad": 0.15496688741721854, |
| "coqa": 0.2233496179381185, |
| "boolq": 0.6024464831804281, |
| "bigbench_language_identification": 0.259 |
| }, |
| "centered_results": { |
| "hellaswag_zeroshot": 0.1916616875788356, |
| "jeopardy": 0.038734057628719884, |
| "bigbench_qa_wikidata": 0.4208946410117612, |
| "arc_easy": 0.31369248035914704, |
| "arc_challenge": 0.026166097838452806, |
| "copa": 0.28, |
| "commonsense_qa": 0.017199017199017175, |
| "piqa": 0.31882480957562565, |
| "openbook_qa": 0.07466666666666666, |
| "lambada_openai": 0.42111391422472344, |
| "hellaswag": 0.19750381730067054, |
| "winograd": 0.23076923076923084, |
| "winogrande": 0.026045777426992878, |
| "bigbench_dyck_languages": 0.18, |
| "agi_eval_lsat_ar": 0.0326086956521739, |
| "bigbench_cs_algorithms": 0.42954545454545456, |
| "bigbench_operators": 0.11428571428571428, |
| "bigbench_repeat_copy_logic": 0.0625, |
| "squad": 0.15496688741721854, |
| "coqa": 0.2233496179381185, |
| "boolq": -0.04619346531466275, |
| "bigbench_language_identification": 0.18481848184818483 |
| }, |
| "task_timings_sec": { |
| "hellaswag_zeroshot": 91.82639169692993, |
| "jeopardy": 21.153838396072388, |
| "bigbench_qa_wikidata": 186.42186880111694, |
| "arc_easy": 31.19749355316162, |
| "arc_challenge": 16.568403244018555, |
| "copa": 1.0189661979675293, |
| "commonsense_qa": 16.761160135269165, |
| "piqa": 21.31744146347046, |
| "openbook_qa": 4.463853359222412, |
| "lambada_openai": 49.6056182384491, |
| "hellaswag": 239.63831663131714, |
| "winograd": 2.319700002670288, |
| "winogrande": 10.733928442001343, |
| "bigbench_dyck_languages": 11.990600109100342, |
| "agi_eval_lsat_ar": 5.203017950057983, |
| "bigbench_cs_algorithms": 20.828216552734375, |
| "bigbench_operators": 3.4815895557403564, |
| "bigbench_repeat_copy_logic": 0.5363271236419678, |
| "squad": 192.21085810661316, |
| "coqa": 93.88718795776367, |
| "boolq": 55.57720470428467, |
| "bigbench_language_identification": 269.4145817756653 |
| }, |
| "core_metric": 0.1769615265432748, |
| "n_tasks_evaluated": 22 |
| }, |
| "pythia_410m": { |
| "model": "EleutherAI/pythia-410m", |
| "params_M": 405.334016, |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "max_per_task": -1, |
| "wall_clock_sec": 1300.899761915207, |
| "family": "huggingface", |
| "results": { |
| "hellaswag_zeroshot": 0.39772953594901417, |
| "jeopardy": 0.03920642418516769, |
| "bigbench_qa_wikidata": 0.5093253284779292, |
| "arc_easy": 0.5273569023569024, |
| "arc_challenge": 0.24829351535836178, |
| "copa": 0.64, |
| "commonsense_qa": 0.24078624078624078, |
| "piqa": 0.6855277475516867, |
| "openbook_qa": 0.306, |
| "lambada_openai": 0.4758393169027751, |
| "hellaswag": 0.39772953594901417, |
| "winograd": 0.6996336996336996, |
| "winogrande": 0.5295974743488555, |
| "bigbench_dyck_languages": 0.272, |
| "agi_eval_lsat_ar": 0.2608695652173913, |
| "bigbench_cs_algorithms": 0.4643939393939394, |
| "bigbench_operators": 0.13333333333333333, |
| "bigbench_repeat_copy_logic": 0.0, |
| "squad": 0.2173131504257332, |
| "coqa": 0.20130276838281347, |
| "boolq": 0.47003058103975537, |
| "bigbench_language_identification": 0.2537 |
| }, |
| "centered_results": { |
| "hellaswag_zeroshot": 0.19697271459868557, |
| "jeopardy": 0.03920642418516769, |
| "bigbench_qa_wikidata": 0.5093253284779292, |
| "arc_easy": 0.36980920314253646, |
| "arc_challenge": -0.0022753128555176305, |
| "copa": 0.28, |
| "commonsense_qa": 0.05098280098280096, |
| "piqa": 0.3710554951033733, |
| "openbook_qa": 0.07466666666666666, |
| "lambada_openai": 0.4758393169027751, |
| "hellaswag": 0.19697271459868557, |
| "winograd": 0.3992673992673992, |
| "winogrande": 0.059194948697711025, |
| "bigbench_dyck_languages": 0.272, |
| "agi_eval_lsat_ar": 0.07608695652173911, |
| "bigbench_cs_algorithms": 0.4643939393939394, |
| "bigbench_operators": 0.13333333333333333, |
| "bigbench_repeat_copy_logic": 0.0, |
| "squad": 0.2173131504257332, |
| "coqa": 0.20130276838281347, |
| "boolq": -0.3946563656848543, |
| "bigbench_language_identification": 0.17898789878987897 |
| }, |
| "task_timings_sec": { |
| "hellaswag_zeroshot": 100.39278149604797, |
| "jeopardy": 21.593817472457886, |
| "bigbench_qa_wikidata": 198.73203802108765, |
| "arc_easy": 34.205162048339844, |
| "arc_challenge": 15.374195575714111, |
| "copa": 0.9315333366394043, |
| "commonsense_qa": 14.663639783859253, |
| "piqa": 19.43335795402527, |
| "openbook_qa": 4.472408056259155, |
| "lambada_openai": 47.07043409347534, |
| "hellaswag": 193.5711019039154, |
| "winograd": 2.3898563385009766, |
| "winogrande": 11.632225036621094, |
| "bigbench_dyck_languages": 10.497180938720703, |
| "agi_eval_lsat_ar": 4.13561487197876, |
| "bigbench_cs_algorithms": 13.70287275314331, |
| "bigbench_operators": 2.1963233947753906, |
| "bigbench_repeat_copy_logic": 0.3281850814819336, |
| "squad": 178.40499138832092, |
| "coqa": 85.92737078666687, |
| "boolq": 60.55928707122803, |
| "bigbench_language_identification": 280.67118740081787 |
| }, |
| "core_metric": 0.18953542640594528, |
| "n_tasks_evaluated": 22 |
| }, |
| "rrt_355m": { |
| "model": "RRT-355M", |
| "params_M": 353.55136, |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "max_per_task": -1, |
| "wall_clock_sec": 1566.2221701145172, |
| "family": "rrt", |
| "ckpt_iter": 21000, |
| "ckpt_best_val_loss": 2.8001298904418945, |
| "snapshot_dir": "/workspace/eval_355m/snapshot", |
| "bound_override": null, |
| "results": { |
| "hellaswag_zeroshot": 0.3690499900418243, |
| "jeopardy": 0.013226263580538498, |
| "bigbench_qa_wikidata": 0.3665173957974509, |
| "arc_easy": 0.5732323232323232, |
| "arc_challenge": 0.2883959044368601, |
| "copa": 0.64, |
| "commonsense_qa": 0.20147420147420148, |
| "piqa": 0.6550598476605005, |
| "openbook_qa": 0.36, |
| "lambada_openai": 0.2633417426741704, |
| "hellaswag": 0.3625771758613822, |
| "winograd": 0.5860805860805861, |
| "winogrande": 0.5098658247829518, |
| "bigbench_dyck_languages": 0.129, |
| "agi_eval_lsat_ar": 0.3, |
| "bigbench_cs_algorithms": 0.4431818181818182, |
| "bigbench_operators": 0.07142857142857142, |
| "bigbench_repeat_copy_logic": 0.0, |
| "squad": 0.08580889309366131, |
| "coqa": 0.09106852060628837, |
| "boolq": 0.5963302752293578, |
| "bigbench_language_identification": 0.2549 |
| }, |
| "centered_results": { |
| "hellaswag_zeroshot": 0.15873332005576576, |
| "jeopardy": 0.013226263580538498, |
| "bigbench_qa_wikidata": 0.3665173957974509, |
| "arc_easy": 0.43097643097643096, |
| "arc_challenge": 0.05119453924914678, |
| "copa": 0.28, |
| "commonsense_qa": 0.0018427518427518302, |
| "piqa": 0.31011969532100103, |
| "openbook_qa": 0.14666666666666664, |
| "lambada_openai": 0.2633417426741704, |
| "hellaswag": 0.15010290114850963, |
| "winograd": 0.17216117216117222, |
| "winogrande": 0.019731649565903675, |
| "bigbench_dyck_languages": 0.129, |
| "agi_eval_lsat_ar": 0.12499999999999997, |
| "bigbench_cs_algorithms": 0.4431818181818182, |
| "bigbench_operators": 0.07142857142857142, |
| "bigbench_repeat_copy_logic": 0.0, |
| "squad": 0.08580889309366131, |
| "coqa": 0.09106852060628837, |
| "boolq": -0.062288749396426896, |
| "bigbench_language_identification": 0.18030803080308033 |
| }, |
| "task_timings_sec": { |
| "hellaswag_zeroshot": 118.51018595695496, |
| "jeopardy": 24.857576847076416, |
| "bigbench_qa_wikidata": 225.71323251724243, |
| "arc_easy": 33.35843825340271, |
| "arc_challenge": 15.301321268081665, |
| "copa": 1.1727240085601807, |
| "commonsense_qa": 16.000224351882935, |
| "piqa": 21.52322506904602, |
| "openbook_qa": 5.0123610496521, |
| "lambada_openai": 51.571826219558716, |
| "hellaswag": 212.81852674484253, |
| "winograd": 2.6677236557006836, |
| "winogrande": 12.38275671005249, |
| "bigbench_dyck_languages": 12.239135503768921, |
| "agi_eval_lsat_ar": 4.667552709579468, |
| "bigbench_cs_algorithms": 14.96208667755127, |
| "bigbench_operators": 2.4181301593780518, |
| "bigbench_repeat_copy_logic": 0.39531755447387695, |
| "squad": 226.37038016319275, |
| "coqa": 93.64458155632019, |
| "boolq": 78.85809707641602, |
| "bigbench_language_identification": 391.75933027267456 |
| }, |
| "core_metric": 0.1558237097162046, |
| "n_tasks_evaluated": 22 |
| } |
| }, |
| "headline_core": { |
| "gpt2_124m": 0.12113020737908765, |
| "gpt2_medium_355m": 0.1769615265432748, |
| "pythia_410m": 0.18953542640594528, |
| "rrt_355m": 0.1558237097162046 |
| } |
| } |
