ablations/downstream_evals.csv

budget,checkpoint,model,task,n_shot,metric,value,stderr
1e17,dense_s2_1e17,dense,arc_challenge,10,acc,0.1681,0.0109
1e17,dense_s2_1e17,dense,arc_challenge,10,acc_norm,0.2244,0.0122
1e17,v16k_sweep_s2_1e17,moe,arc_challenge,10,acc,0.1706,0.011
1e17,v16k_sweep_s2_1e17,moe,arc_challenge,10,acc_norm,0.2125,0.012
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,arc_challenge,10,acc,0.1613,0.0107
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,arc_challenge,10,acc_norm,0.215,0.012
1e17,dense_s2_1e17,dense,arc_easy,10,acc,0.3371,0.0097
1e17,dense_s2_1e17,dense,arc_easy,10,acc_norm,0.3262,0.0096
1e17,v16k_sweep_s2_1e17,moe,arc_easy,10,acc,0.3455,0.0098
1e17,v16k_sweep_s2_1e17,moe,arc_easy,10,acc_norm,0.3413,0.0097
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,arc_easy,10,acc,0.3422,0.0097
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,arc_easy,10,acc_norm,0.3409,0.0097
1e17,dense_s2_1e17,dense,hellaswag,10,acc,0.2638,0.0044
1e17,dense_s2_1e17,dense,hellaswag,10,acc_norm,0.2651,0.0044
1e17,v16k_sweep_s2_1e17,moe,hellaswag,10,acc,0.2657,0.0044
1e17,v16k_sweep_s2_1e17,moe,hellaswag,10,acc_norm,0.2625,0.0044
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,hellaswag,10,acc,0.2678,0.0044
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,hellaswag,10,acc_norm,0.2664,0.0044
1e17,dense_s2_1e17,dense,openbookqa,0,acc,0.132,0.0152
1e17,dense_s2_1e17,dense,openbookqa,0,acc_norm,0.236,0.019
1e17,v16k_sweep_s2_1e17,moe,openbookqa,0,acc,0.118,0.0144
1e17,v16k_sweep_s2_1e17,moe,openbookqa,0,acc_norm,0.244,0.0192
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,openbookqa,0,acc,0.136,0.0153
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,openbookqa,0,acc_norm,0.234,0.019
1e17,dense_s2_1e17,dense,piqa,10,acc,0.5724,0.0115
1e17,dense_s2_1e17,dense,piqa,10,acc_norm,0.5647,0.0116
1e17,v16k_sweep_s2_1e17,moe,piqa,10,acc,0.5794,0.0115
1e17,v16k_sweep_s2_1e17,moe,piqa,10,acc_norm,0.5686,0.0116
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,piqa,10,acc,0.5789,0.0115
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,piqa,10,acc_norm,0.5843,0.0115
1e17,dense_s2_1e17,dense,winogrande,0,acc,0.5193,0.014
1e17,v16k_sweep_s2_1e17,moe,winogrande,0,acc,0.5051,0.0141
1e17,tmoe_minlogit_sh1_s2_1e17,temporal,winogrande,0,acc,0.517,0.014
1e18,flame38m_dense,dense,arc_challenge,10,acc,0.1886,0.0114
1e18,flame38m_dense,dense,arc_challenge,10,acc_norm,0.215,0.012
1e18,flame38m_temporal_minlogit,temporal,arc_challenge,10,acc,0.1886,0.0114
1e18,flame38m_temporal_minlogit,temporal,arc_challenge,10,acc_norm,0.2218,0.0121
1e18,flame38m_dense,dense,arc_easy,10,acc,0.3434,0.0097
1e18,flame38m_dense,dense,arc_easy,10,acc_norm,0.322,0.0096
1e18,flame38m_temporal_minlogit,temporal,arc_easy,10,acc,0.3607,0.0099
1e18,flame38m_temporal_minlogit,temporal,arc_easy,10,acc_norm,0.3401,0.0097
1e18,flame38m_dense,dense,hellaswag,10,acc,0.2582,0.0044
1e18,flame38m_dense,dense,hellaswag,10,acc_norm,0.2526,0.0043
1e18,flame38m_temporal_minlogit,temporal,hellaswag,10,acc,0.268,0.0044
1e18,flame38m_temporal_minlogit,temporal,hellaswag,10,acc_norm,0.259,0.0044
1e18,flame38m_dense,dense,openbookqa,0,acc,0.134,0.0152
1e18,flame38m_dense,dense,openbookqa,0,acc_norm,0.236,0.019
1e18,flame38m_temporal_minlogit,temporal,openbookqa,0,acc,0.136,0.0153
1e18,flame38m_temporal_minlogit,temporal,openbookqa,0,acc_norm,0.246,0.0193
1e18,flame38m_dense,dense,piqa,10,acc,0.5789,0.0115
1e18,flame38m_dense,dense,piqa,10,acc_norm,0.5544,0.0116
1e18,flame38m_temporal_minlogit,temporal,piqa,10,acc,0.5767,0.0115
1e18,flame38m_temporal_minlogit,temporal,piqa,10,acc_norm,0.5696,0.0116
1e18,flame38m_dense,dense,winogrande,0,acc,0.5075,0.0141
1e18,flame38m_temporal_minlogit,temporal,winogrande,0,acc,0.4878,0.014