{
  "generated_at": "2026-06-09T22:14:43.622408+00:00",
  "reference_core": {
    "openai-community/gpt2": 0.113891,
    "openai-community/gpt2-medium": 0.184903,
    "openai-community/gpt2-large": 0.214641,
    "openai-community/gpt2-xl": 0.256525
  },
  "lobotomy_floor": 0.15,
  "runs": {
    "gpt2_124m": {
      "model": "openai-community/gpt2",
      "params_M": 124.439808,
      "device": "cuda",
      "dtype": "bfloat16",
      "max_per_task": -1,
      "wall_clock_sec": 891.2480754852295,
      "family": "huggingface",
      "results": {
        "hellaswag_zeroshot": 0.30920135431189005,
        "jeopardy": 0.001889466225791214,
        "bigbench_qa_wikidata": 0.28340140741105263,
        "arc_easy": 0.414983164983165,
        "arc_challenge": 0.21843003412969283,
        "copa": 0.63,
        "commonsense_qa": 0.2325962325962326,
        "piqa": 0.6289445048966268,
        "openbook_qa": 0.264,
        "lambada_openai": 0.32078400931496215,
        "hellaswag": 0.30770762796255724,
        "winograd": 0.5897435897435898,
        "winogrande": 0.5114443567482242,
        "bigbench_dyck_languages": 0.158,
        "agi_eval_lsat_ar": 0.22608695652173913,
        "bigbench_cs_algorithms": 0.42803030303030304,
        "bigbench_operators": 0.09523809523809523,
        "bigbench_repeat_copy_logic": 0.03125,
        "squad": 0.05742667928098392,
        "coqa": 0.13616434924213955,
        "boolq": 0.5541284403669725,
        "bigbench_language_identification": 0.2546
      },
      "centered_results": {
        "hellaswag_zeroshot": 0.07893513908252008,
        "jeopardy": 0.001889466225791214,
        "bigbench_qa_wikidata": 0.28340140741105263,
        "arc_easy": 0.21997755331088667,
        "arc_challenge": -0.04209328782707622,
        "copa": 0.26,
        "commonsense_qa": 0.04074529074529074,
        "piqa": 0.2578890097932536,
        "openbook_qa": 0.018666666666666682,
        "lambada_openai": 0.32078400931496215,
        "hellaswag": 0.07694350395007632,
        "winograd": 0.17948717948717952,
        "winogrande": 0.022888713496448387,
        "bigbench_dyck_languages": 0.158,
        "agi_eval_lsat_ar": 0.0326086956521739,
        "bigbench_cs_algorithms": 0.42803030303030304,
        "bigbench_operators": 0.09523809523809523,
        "bigbench_repeat_copy_logic": 0.03125,
        "squad": 0.05742667928098392,
        "coqa": 0.13616434924213955,
        "boolq": -0.17334620956059865,
        "bigbench_language_identification": 0.17997799779977997
      },
      "task_timings_sec": {
        "hellaswag_zeroshot": 63.30279040336609,
        "jeopardy": 14.193135976791382,
        "bigbench_qa_wikidata": 124.85103392601013,
        "arc_easy": 22.142359495162964,
        "arc_challenge": 10.91347050666809,
        "copa": 0.6135663986206055,
        "commonsense_qa": 13.434561729431152,
        "piqa": 14.689457654953003,
        "openbook_qa": 2.6003074645996094,
        "lambada_openai": 27.92130184173584,
        "hellaswag": 142.64234709739685,
        "winograd": 1.3682467937469482,
        "winogrande": 6.364197015762329,
        "bigbench_dyck_languages": 8.250524520874023,
        "agi_eval_lsat_ar": 3.1633238792419434,
        "bigbench_cs_algorithms": 9.292541980743408,
        "bigbench_operators": 1.407222032546997,
        "bigbench_repeat_copy_logic": 0.2225968837738037,
        "squad": 139.18381094932556,
        "coqa": 59.376181840896606,
        "boolq": 41.74283003807068,
        "bigbench_language_identification": 183.55174899101257
      },
      "core_metric": 0.12113020737908765,
      "n_tasks_evaluated": 22
    },
    "gpt2_medium_355m": {
      "model": "openai-community/gpt2-medium",
      "params_M": 354.823168,
      "device": "cuda",
      "dtype": "bfloat16",
      "max_per_task": -1,
      "wall_clock_sec": 1346.1762642860413,
      "family": "huggingface",
      "results": {
        "hellaswag_zeroshot": 0.3937462656841267,
        "jeopardy": 0.038734057628719884,
        "bigbench_qa_wikidata": 0.4208946410117612,
        "arc_easy": 0.4852693602693603,
        "arc_challenge": 0.2696245733788396,
        "copa": 0.64,
        "commonsense_qa": 0.21375921375921375,
        "piqa": 0.6594124047878128,
        "openbook_qa": 0.306,
        "lambada_openai": 0.42111391422472344,
        "hellaswag": 0.3981278629755029,
        "winograd": 0.6153846153846154,
        "winogrande": 0.5130228887134964,
        "bigbench_dyck_languages": 0.18,
        "agi_eval_lsat_ar": 0.22608695652173913,
        "bigbench_cs_algorithms": 0.42954545454545456,
        "bigbench_operators": 0.11428571428571428,
        "bigbench_repeat_copy_logic": 0.0625,
        "squad": 0.15496688741721854,
        "coqa": 0.2233496179381185,
        "boolq": 0.6024464831804281,
        "bigbench_language_identification": 0.259
      },
      "centered_results": {
        "hellaswag_zeroshot": 0.1916616875788356,
        "jeopardy": 0.038734057628719884,
        "bigbench_qa_wikidata": 0.4208946410117612,
        "arc_easy": 0.31369248035914704,
        "arc_challenge": 0.026166097838452806,
        "copa": 0.28,
        "commonsense_qa": 0.017199017199017175,
        "piqa": 0.31882480957562565,
        "openbook_qa": 0.07466666666666666,
        "lambada_openai": 0.42111391422472344,
        "hellaswag": 0.19750381730067054,
        "winograd": 0.23076923076923084,
        "winogrande": 0.026045777426992878,
        "bigbench_dyck_languages": 0.18,
        "agi_eval_lsat_ar": 0.0326086956521739,
        "bigbench_cs_algorithms": 0.42954545454545456,
        "bigbench_operators": 0.11428571428571428,
        "bigbench_repeat_copy_logic": 0.0625,
        "squad": 0.15496688741721854,
        "coqa": 0.2233496179381185,
        "boolq": -0.04619346531466275,
        "bigbench_language_identification": 0.18481848184818483
      },
      "task_timings_sec": {
        "hellaswag_zeroshot": 91.82639169692993,
        "jeopardy": 21.153838396072388,
        "bigbench_qa_wikidata": 186.42186880111694,
        "arc_easy": 31.19749355316162,
        "arc_challenge": 16.568403244018555,
        "copa": 1.0189661979675293,
        "commonsense_qa": 16.761160135269165,
        "piqa": 21.31744146347046,
        "openbook_qa": 4.463853359222412,
        "lambada_openai": 49.6056182384491,
        "hellaswag": 239.63831663131714,
        "winograd": 2.319700002670288,
        "winogrande": 10.733928442001343,
        "bigbench_dyck_languages": 11.990600109100342,
        "agi_eval_lsat_ar": 5.203017950057983,
        "bigbench_cs_algorithms": 20.828216552734375,
        "bigbench_operators": 3.4815895557403564,
        "bigbench_repeat_copy_logic": 0.5363271236419678,
        "squad": 192.21085810661316,
        "coqa": 93.88718795776367,
        "boolq": 55.57720470428467,
        "bigbench_language_identification": 269.4145817756653
      },
      "core_metric": 0.1769615265432748,
      "n_tasks_evaluated": 22
    },
    "pythia_410m": {
      "model": "EleutherAI/pythia-410m",
      "params_M": 405.334016,
      "device": "cuda",
      "dtype": "bfloat16",
      "max_per_task": -1,
      "wall_clock_sec": 1300.899761915207,
      "family": "huggingface",
      "results": {
        "hellaswag_zeroshot": 0.39772953594901417,
        "jeopardy": 0.03920642418516769,
        "bigbench_qa_wikidata": 0.5093253284779292,
        "arc_easy": 0.5273569023569024,
        "arc_challenge": 0.24829351535836178,
        "copa": 0.64,
        "commonsense_qa": 0.24078624078624078,
        "piqa": 0.6855277475516867,
        "openbook_qa": 0.306,
        "lambada_openai": 0.4758393169027751,
        "hellaswag": 0.39772953594901417,
        "winograd": 0.6996336996336996,
        "winogrande": 0.5295974743488555,
        "bigbench_dyck_languages": 0.272,
        "agi_eval_lsat_ar": 0.2608695652173913,
        "bigbench_cs_algorithms": 0.4643939393939394,
        "bigbench_operators": 0.13333333333333333,
        "bigbench_repeat_copy_logic": 0.0,
        "squad": 0.2173131504257332,
        "coqa": 0.20130276838281347,
        "boolq": 0.47003058103975537,
        "bigbench_language_identification": 0.2537
      },
      "centered_results": {
        "hellaswag_zeroshot": 0.19697271459868557,
        "jeopardy": 0.03920642418516769,
        "bigbench_qa_wikidata": 0.5093253284779292,
        "arc_easy": 0.36980920314253646,
        "arc_challenge": -0.0022753128555176305,
        "copa": 0.28,
        "commonsense_qa": 0.05098280098280096,
        "piqa": 0.3710554951033733,
        "openbook_qa": 0.07466666666666666,
        "lambada_openai": 0.4758393169027751,
        "hellaswag": 0.19697271459868557,
        "winograd": 0.3992673992673992,
        "winogrande": 0.059194948697711025,
        "bigbench_dyck_languages": 0.272,
        "agi_eval_lsat_ar": 0.07608695652173911,
        "bigbench_cs_algorithms": 0.4643939393939394,
        "bigbench_operators": 0.13333333333333333,
        "bigbench_repeat_copy_logic": 0.0,
        "squad": 0.2173131504257332,
        "coqa": 0.20130276838281347,
        "boolq": -0.3946563656848543,
        "bigbench_language_identification": 0.17898789878987897
      },
      "task_timings_sec": {
        "hellaswag_zeroshot": 100.39278149604797,
        "jeopardy": 21.593817472457886,
        "bigbench_qa_wikidata": 198.73203802108765,
        "arc_easy": 34.205162048339844,
        "arc_challenge": 15.374195575714111,
        "copa": 0.9315333366394043,
        "commonsense_qa": 14.663639783859253,
        "piqa": 19.43335795402527,
        "openbook_qa": 4.472408056259155,
        "lambada_openai": 47.07043409347534,
        "hellaswag": 193.5711019039154,
        "winograd": 2.3898563385009766,
        "winogrande": 11.632225036621094,
        "bigbench_dyck_languages": 10.497180938720703,
        "agi_eval_lsat_ar": 4.13561487197876,
        "bigbench_cs_algorithms": 13.70287275314331,
        "bigbench_operators": 2.1963233947753906,
        "bigbench_repeat_copy_logic": 0.3281850814819336,
        "squad": 178.40499138832092,
        "coqa": 85.92737078666687,
        "boolq": 60.55928707122803,
        "bigbench_language_identification": 280.67118740081787
      },
      "core_metric": 0.18953542640594528,
      "n_tasks_evaluated": 22
    },
    "rrt_355m": {
      "model": "RRT-355M",
      "params_M": 353.55136,
      "device": "cuda",
      "dtype": "bfloat16",
      "max_per_task": -1,
      "wall_clock_sec": 1566.2221701145172,
      "family": "rrt",
      "ckpt_iter": 21000,
      "ckpt_best_val_loss": 2.8001298904418945,
      "snapshot_dir": "/workspace/eval_355m/snapshot",
      "bound_override": null,
      "results": {
        "hellaswag_zeroshot": 0.3690499900418243,
        "jeopardy": 0.013226263580538498,
        "bigbench_qa_wikidata": 0.3665173957974509,
        "arc_easy": 0.5732323232323232,
        "arc_challenge": 0.2883959044368601,
        "copa": 0.64,
        "commonsense_qa": 0.20147420147420148,
        "piqa": 0.6550598476605005,
        "openbook_qa": 0.36,
        "lambada_openai": 0.2633417426741704,
        "hellaswag": 0.3625771758613822,
        "winograd": 0.5860805860805861,
        "winogrande": 0.5098658247829518,
        "bigbench_dyck_languages": 0.129,
        "agi_eval_lsat_ar": 0.3,
        "bigbench_cs_algorithms": 0.4431818181818182,
        "bigbench_operators": 0.07142857142857142,
        "bigbench_repeat_copy_logic": 0.0,
        "squad": 0.08580889309366131,
        "coqa": 0.09106852060628837,
        "boolq": 0.5963302752293578,
        "bigbench_language_identification": 0.2549
      },
      "centered_results": {
        "hellaswag_zeroshot": 0.15873332005576576,
        "jeopardy": 0.013226263580538498,
        "bigbench_qa_wikidata": 0.3665173957974509,
        "arc_easy": 0.43097643097643096,
        "arc_challenge": 0.05119453924914678,
        "copa": 0.28,
        "commonsense_qa": 0.0018427518427518302,
        "piqa": 0.31011969532100103,
        "openbook_qa": 0.14666666666666664,
        "lambada_openai": 0.2633417426741704,
        "hellaswag": 0.15010290114850963,
        "winograd": 0.17216117216117222,
        "winogrande": 0.019731649565903675,
        "bigbench_dyck_languages": 0.129,
        "agi_eval_lsat_ar": 0.12499999999999997,
        "bigbench_cs_algorithms": 0.4431818181818182,
        "bigbench_operators": 0.07142857142857142,
        "bigbench_repeat_copy_logic": 0.0,
        "squad": 0.08580889309366131,
        "coqa": 0.09106852060628837,
        "boolq": -0.062288749396426896,
        "bigbench_language_identification": 0.18030803080308033
      },
      "task_timings_sec": {
        "hellaswag_zeroshot": 118.51018595695496,
        "jeopardy": 24.857576847076416,
        "bigbench_qa_wikidata": 225.71323251724243,
        "arc_easy": 33.35843825340271,
        "arc_challenge": 15.301321268081665,
        "copa": 1.1727240085601807,
        "commonsense_qa": 16.000224351882935,
        "piqa": 21.52322506904602,
        "openbook_qa": 5.0123610496521,
        "lambada_openai": 51.571826219558716,
        "hellaswag": 212.81852674484253,
        "winograd": 2.6677236557006836,
        "winogrande": 12.38275671005249,
        "bigbench_dyck_languages": 12.239135503768921,
        "agi_eval_lsat_ar": 4.667552709579468,
        "bigbench_cs_algorithms": 14.96208667755127,
        "bigbench_operators": 2.4181301593780518,
        "bigbench_repeat_copy_logic": 0.39531755447387695,
        "squad": 226.37038016319275,
        "coqa": 93.64458155632019,
        "boolq": 78.85809707641602,
        "bigbench_language_identification": 391.75933027267456
      },
      "core_metric": 0.1558237097162046,
      "n_tasks_evaluated": 22
    }
  },
  "headline_core": {
    "gpt2_124m": 0.12113020737908765,
    "gpt2_medium_355m": 0.1769615265432748,
    "pythia_410m": 0.18953542640594528,
    "rrt_355m": 0.1558237097162046
  }
}