model,task,acc,acc_norm,stderr
flame38m_dense_local,arc_challenge,0.178328,0.221843,0.011186
flame38m_dense_local,arc_easy,0.340488,0.326178,0.009724
flame38m_dense_local,boolq,0.39633,,0.008555
flame38m_dense_local,copa,0.53,,0.050161
flame38m_dense_local,hellaswag,0.259311,0.256025,0.004374
flame38m_dense_local,lambada_openai,0.027751,,0.002288
flame38m_dense_local,openbookqa,0.128,0.254,0.014956
flame38m_dense_local,piqa,0.565288,0.552231,0.011566
flame38m_dense_local,sciq,0.455,0.437,0.015755
flame38m_dense_local,winogrande,0.507498,,0.014051
flame38m_g1_moe,arc_challenge,0.183447,0.217577,0.01131
flame38m_g1_moe,arc_easy,0.363636,0.347643,0.009871
flame38m_g1_moe,boolq,0.377064,,0.008477
flame38m_g1_moe,copa,0.56,,0.049889
flame38m_g1_moe,hellaswag,0.267078,0.257917,0.004415
flame38m_g1_moe,lambada_openai,0.082282,,0.003828
flame38m_g1_moe,openbookqa,0.144,0.242,0.015717
flame38m_g1_moe,piqa,0.570185,0.566921,0.01155
flame38m_g1_moe,sciq,0.492,0.434,0.015817
flame38m_g1_moe,winogrande,0.487766,,0.014048
flame38m_g1_moe_s2,arc_challenge,0.183447,0.21843,0.01131
flame38m_g1_moe_s2,arc_easy,0.353956,0.328283,0.009812
flame38m_g1_moe_s2,boolq,0.493884,,0.008744
flame38m_g1_moe_s2,copa,0.57,,0.049757
flame38m_g1_moe_s2,hellaswag,0.260406,0.258614,0.00438
flame38m_g1_moe_s2,lambada_openai,0.049486,,0.003022
flame38m_g1_moe_s2,openbookqa,0.126,0.224,0.014856
flame38m_g1_moe_s2,piqa,0.579434,0.566921,0.011518
flame38m_g1_moe_s2,sciq,0.459,0.431,0.015766
flame38m_g1_moe_s2,winogrande,0.486977,,0.014048
flame38m_g1_moe_s3,arc_challenge,0.183447,0.215017,0.01131
flame38m_g1_moe_s3,arc_easy,0.342593,0.337121,0.009738
flame38m_g1_moe_s3,boolq,0.589297,,0.008604
flame38m_g1_moe_s3,copa,0.6,,0.049237
flame38m_g1_moe_s3,hellaswag,0.261502,0.26419,0.004386
flame38m_g1_moe_s3,lambada_openai,0.067533,,0.003496
flame38m_g1_moe_s3,openbookqa,0.13,0.256,0.015055
flame38m_g1_moe_s3,piqa,0.575082,0.570185,0.011534
flame38m_g1_moe_s3,sciq,0.479,0.443,0.015805
flame38m_g1_moe_s3,winogrande,0.492502,,0.014051
flame38m_g1_temporal,arc_challenge,0.183447,0.236348,0.01131
flame38m_g1_temporal,arc_easy,0.37037,0.357323,0.009909
flame38m_g1_temporal,boolq,0.442813,,0.008688
flame38m_g1_temporal,copa,0.56,,0.049889
flame38m_g1_temporal,hellaswag,0.265186,0.263294,0.004405
flame38m_g1_temporal,lambada_openai,0.072967,,0.003623
flame38m_g1_temporal,openbookqa,0.14,0.26,0.015533
flame38m_g1_temporal,piqa,0.576714,0.57617,0.011528
flame38m_g1_temporal,sciq,0.499,0.436,0.015819
flame38m_g1_temporal,winogrande,0.479084,,0.01404
flame38m_g1_temporal_s2,arc_challenge,0.171502,0.203072,0.011015
flame38m_g1_temporal_s2,arc_easy,0.366582,0.344276,0.009888
flame38m_g1_temporal_s2,boolq,0.409174,,0.0086
flame38m_g1_temporal_s2,copa,0.53,,0.050161
flame38m_g1_temporal_s2,hellaswag,0.263892,0.258514,0.004398
flame38m_g1_temporal_s2,lambada_openai,0.072773,,0.003619
flame38m_g1_temporal_s2,openbookqa,0.134,0.248,0.01525
flame38m_g1_temporal_s2,piqa,0.58161,0.573449,0.011509
flame38m_g1_temporal_s2,sciq,0.482,0.442,0.015809
flame38m_g1_temporal_s2,winogrande,0.479874,,0.014041
flame38m_g1_temporal_s3,arc_challenge,0.177474,0.209898,0.011165
flame38m_g1_temporal_s3,arc_easy,0.351431,0.339226,0.009796
flame38m_g1_temporal_s3,boolq,0.447401,,0.008697
flame38m_g1_temporal_s3,copa,0.61,,0.049021
flame38m_g1_temporal_s3,hellaswag,0.26419,0.2618,0.0044
flame38m_g1_temporal_s3,lambada_openai,0.057248,,0.003237
flame38m_g1_temporal_s3,openbookqa,0.128,0.268,0.014956
flame38m_g1_temporal_s3,piqa,0.577258,0.572361,0.011526
flame38m_g1_temporal_s3,sciq,0.486,0.434,0.015813
flame38m_g1_temporal_s3,winogrande,0.488556,,0.014049
flame38m_g3_moe,arc_challenge,0.182594,0.209898,0.01129
flame38m_g3_moe,arc_easy,0.367424,0.3367,0.009893
flame38m_g3_moe,boolq,0.395107,,0.00855
flame38m_g3_moe,copa,0.54,,0.050091
flame38m_g3_moe,hellaswag,0.263394,0.258713,0.004396
flame38m_g3_moe,lambada_openai,0.043858,,0.002853
flame38m_g3_moe,openbookqa,0.144,0.254,0.015717
flame38m_g3_moe,piqa,0.569097,0.5642,0.011554
flame38m_g3_moe,sciq,0.47,0.454,0.015791
flame38m_g3_moe,winogrande,0.507498,,0.014051
flame38m_g3_moe_s2,arc_challenge,0.188567,0.223549,0.011431
flame38m_g3_moe_s2,arc_easy,0.341751,0.322391,0.009732
flame38m_g3_moe_s2,boolq,0.413456,,0.008613
flame38m_g3_moe_s2,copa,0.6,,0.049237
flame38m_g3_moe_s2,hellaswag,0.259311,0.258215,0.004374
flame38m_g3_moe_s2,lambada_openai,0.038036,,0.002665
flame38m_g3_moe_s2,openbookqa,0.118,0.232,0.014442
flame38m_g3_moe_s2,piqa,0.577802,0.56148,0.011524
flame38m_g3_moe_s2,sciq,0.47,0.437,0.015791
flame38m_g3_moe_s2,winogrande,0.520916,,0.01404
flame38m_g3_moe_s3,arc_challenge,0.188567,0.221843,0.011431
flame38m_g3_moe_s3,arc_easy,0.337542,0.328283,0.009703
flame38m_g3_moe_s3,boolq,0.484404,,0.008741
flame38m_g3_moe_s3,copa,0.6,,0.049237
flame38m_g3_moe_s3,hellaswag,0.262199,0.256622,0.004389
flame38m_g3_moe_s3,lambada_openai,0.047351,,0.002959
flame38m_g3_moe_s3,openbookqa,0.13,0.236,0.015055
flame38m_g3_moe_s3,piqa,0.577802,0.57617,0.011524
flame38m_g3_moe_s3,sciq,0.448,0.434,0.015734
flame38m_g3_moe_s3,winogrande,0.487766,,0.014048
flame38m_g3_temporal,arc_challenge,0.178328,0.214164,0.011186
flame38m_g3_temporal,arc_easy,0.355219,0.348064,0.00982
flame38m_g3_temporal,boolq,0.412844,,0.008611
flame38m_g3_temporal,copa,0.6,,0.049237
flame38m_g3_temporal,hellaswag,0.262099,0.257021,0.004389
flame38m_g3_temporal,lambada_openai,0.048127,,0.002982
flame38m_g3_temporal,openbookqa,0.122,0.254,0.014651
flame38m_g3_temporal,piqa,0.581066,0.5642,0.011511
flame38m_g3_temporal,sciq,0.477,0.445,0.015803
flame38m_g3_temporal,winogrande,0.518548,,0.014043
flame38m_g3_temporal_s2,arc_challenge,0.179181,0.21587,0.011207
flame38m_g3_temporal_s2,arc_easy,0.364478,0.335438,0.009876
flame38m_g3_temporal_s2,boolq,0.540061,,0.008717
flame38m_g3_temporal_s2,copa,0.55,,0.05
flame38m_g3_temporal_s2,hellaswag,0.263095,0.260605,0.004394
flame38m_g3_temporal_s2,lambada_openai,0.050844,,0.003061
flame38m_g3_temporal_s2,openbookqa,0.126,0.242,0.014856
flame38m_g3_temporal_s2,piqa,0.568553,0.562568,0.011556
flame38m_g3_temporal_s2,sciq,0.473,0.431,0.015796
flame38m_g3_temporal_s2,winogrande,0.500395,,0.014052
flame38m_g3_temporal_s3,arc_challenge,0.187713,0.209044,0.011411
flame38m_g3_temporal_s3,arc_easy,0.364899,0.332071,0.009878
flame38m_g3_temporal_s3,boolq,0.604893,,0.00855
flame38m_g3_temporal_s3,copa,0.58,,0.049604
flame38m_g3_temporal_s3,hellaswag,0.265186,0.258215,0.004405
flame38m_g3_temporal_s3,lambada_openai,0.055502,,0.00319
flame38m_g3_temporal_s3,openbookqa,0.132,0.26,0.015153
flame38m_g3_temporal_s3,piqa,0.582699,0.577802,0.011505
flame38m_g3_temporal_s3,sciq,0.488,0.449,0.015815
flame38m_g3_temporal_s3,winogrande,0.490134,,0.01405
flame192_g3_moe,arc_challenge,0.176621,0.209044,0.011144
flame192_g3_moe,arc_easy,0.348485,0.332912,0.009777
flame192_g3_moe,boolq,0.402446,,0.008577
flame192_g3_moe,copa,0.56,,0.049889
flame192_g3_moe,hellaswag,0.262995,0.258913,0.004394
flame192_g3_moe,lambada_openai,0.046575,,0.002936
flame192_g3_moe,openbookqa,0.15,0.234,0.015985
flame192_g3_moe,piqa,0.572905,0.560392,0.011541
flame192_g3_moe,sciq,0.479,0.429,0.015805
flame192_g3_moe,winogrande,0.501973,,0.014052
flame192_g3_temporal,arc_challenge,0.190273,0.224403,0.01147
flame192_g3_temporal,arc_easy,0.361953,0.335438,0.009861
flame192_g3_temporal,boolq,0.38318,,0.008503
flame192_g3_temporal,copa,0.53,,0.050161
flame192_g3_temporal,hellaswag,0.262298,0.259809,0.00439
flame192_g3_temporal,lambada_openai,0.059771,,0.003303
flame192_g3_temporal,openbookqa,0.14,0.248,0.015533
flame192_g3_temporal,piqa,0.568009,0.562024,0.011557
flame192_g3_temporal,sciq,0.493,0.455,0.015818
flame192_g3_temporal,winogrande,0.475138,,0.014035
flame512_dense,arc_challenge,0.178328,0.221843,0.011186
flame512_dense,arc_easy,0.339646,0.320707,0.009718
flame512_dense,boolq,0.525076,,0.008734
flame512_dense,copa,0.54,,0.050091
flame512_dense,hellaswag,0.258813,0.257419,0.004371
flame512_dense,lambada_openai,0.011256,,0.00147
flame512_dense,openbookqa,0.112,0.24,0.014118
flame512_dense,piqa,0.57617,0.562024,0.01153
flame512_dense,sciq,0.377,0.367,0.015333
flame512_dense,winogrande,0.501184,,0.014052
flame512_g1_moe,arc_challenge,0.198805,0.236348,0.011663
flame512_g1_moe,arc_easy,0.340488,0.319444,0.009724
flame512_g1_moe,boolq,0.484098,,0.008741
flame512_g1_moe,copa,0.54,,0.050091
flame512_g1_moe,hellaswag,0.263394,0.256821,0.004396
flame512_g1_moe,lambada_openai,0.019988,,0.00195
flame512_g1_moe,openbookqa,0.124,0.248,0.014754
flame512_g1_moe,piqa,0.568553,0.557127,0.011556
flame512_g1_moe,sciq,0.4,0.388,0.0155
flame512_g1_moe,winogrande,0.513812,,0.014047
flame512_g1_temporal,arc_challenge,0.178328,0.213311,0.011186
flame512_g1_temporal,arc_easy,0.353956,0.32702,0.009812
flame512_g1_temporal,boolq,0.378593,,0.008483
flame512_g1_temporal,copa,0.56,,0.049889
flame512_g1_temporal,hellaswag,0.262896,0.257718,0.004393
flame512_g1_temporal,lambada_openai,0.01863,,0.001884
flame512_g1_temporal,openbookqa,0.132,0.24,0.015153
flame512_g1_temporal,piqa,0.567465,0.563656,0.011559
flame512_g1_temporal,sciq,0.407,0.387,0.015543
flame512_g1_temporal,winogrande,0.524073,,0.014036
flame512_g3_moe,arc_challenge,0.185154,0.219283,0.011351
flame512_g3_moe,arc_easy,0.339226,0.318182,0.009715
flame512_g3_moe,boolq,0.591131,,0.008599
flame512_g3_moe,copa,0.49,,0.050242
flame512_g3_moe,hellaswag,0.257518,0.256224,0.004364
flame512_g3_moe,lambada_openai,0.012808,,0.001567
flame512_g3_moe,openbookqa,0.112,0.234,0.014118
flame512_g3_moe,piqa,0.567465,0.553319,0.011559
flame512_g3_moe,sciq,0.383,0.358,0.01538
flame512_g3_moe,winogrande,0.500395,,0.014052
flame512_g3_temporal,arc_challenge,0.171502,0.214164,0.011015
flame512_g3_temporal,arc_easy,0.341751,0.32702,0.009732
flame512_g3_temporal,boolq,0.449541,,0.0087
flame512_g3_temporal,copa,0.57,,0.049757
flame512_g3_temporal,hellaswag,0.259211,0.259709,0.004373
flame512_g3_temporal,lambada_openai,0.012032,,0.001519
flame512_g3_temporal,openbookqa,0.118,0.226,0.014442
flame512_g3_temporal,piqa,0.569097,0.551687,0.011554
flame512_g3_temporal,sciq,0.4,0.373,0.0155
flame512_g3_temporal,winogrande,0.528808,,0.014029
flame38m_g5_temporal,arc_challenge,0.18686,0.217577,0.011391
flame38m_g5_temporal,arc_easy,0.343013,0.326599,0.009741
flame38m_g5_temporal,boolq,0.389297,,0.008528
flame38m_g5_temporal,copa,0.57,,0.049757
flame38m_g5_temporal,hellaswag,0.261203,0.259709,0.004384
flame38m_g5_temporal,lambada_openai,0.041917,,0.002792
flame38m_g5_temporal,openbookqa,0.142,0.244,0.015626
flame38m_g5_temporal,piqa,0.575082,0.565288,0.011534
flame38m_g5_temporal,sciq,0.465,0.443,0.01578
flame38m_g5_temporal,winogrande,0.504341,,0.014052
flame38m_g5_moe,arc_challenge,0.182594,0.21587,0.01129
flame38m_g5_moe,arc_easy,0.339226,0.324495,0.009715
flame38m_g5_moe,boolq,0.515291,,0.008741
flame38m_g5_moe,copa,0.59,,0.049431
flame38m_g5_moe,hellaswag,0.262995,0.260207,0.004394
flame38m_g5_moe,lambada_openai,0.040753,,0.002755
flame38m_g5_moe,openbookqa,0.128,0.25,0.014956
flame38m_g5_moe,piqa,0.577802,0.562024,0.011524
flame38m_g5_moe,sciq,0.471,0.438,0.015793
flame38m_g5_moe,winogrande,0.503552,,0.014052