| model,task,acc,acc_norm,stderr | |
| flame38m_dense_local,arc_challenge,0.178328,0.221843,0.011186 | |
| flame38m_dense_local,arc_easy,0.340488,0.326178,0.009724 | |
| flame38m_dense_local,boolq,0.39633,,0.008555 | |
| flame38m_dense_local,copa,0.53,,0.050161 | |
| flame38m_dense_local,hellaswag,0.259311,0.256025,0.004374 | |
| flame38m_dense_local,lambada_openai,0.027751,,0.002288 | |
| flame38m_dense_local,openbookqa,0.128,0.254,0.014956 | |
| flame38m_dense_local,piqa,0.565288,0.552231,0.011566 | |
| flame38m_dense_local,sciq,0.455,0.437,0.015755 | |
| flame38m_dense_local,winogrande,0.507498,,0.014051 | |
| flame38m_g1_moe,arc_challenge,0.183447,0.217577,0.01131 | |
| flame38m_g1_moe,arc_easy,0.363636,0.347643,0.009871 | |
| flame38m_g1_moe,boolq,0.377064,,0.008477 | |
| flame38m_g1_moe,copa,0.56,,0.049889 | |
| flame38m_g1_moe,hellaswag,0.267078,0.257917,0.004415 | |
| flame38m_g1_moe,lambada_openai,0.082282,,0.003828 | |
| flame38m_g1_moe,openbookqa,0.144,0.242,0.015717 | |
| flame38m_g1_moe,piqa,0.570185,0.566921,0.01155 | |
| flame38m_g1_moe,sciq,0.492,0.434,0.015817 | |
| flame38m_g1_moe,winogrande,0.487766,,0.014048 | |
| flame38m_g1_moe_s2,arc_challenge,0.183447,0.21843,0.01131 | |
| flame38m_g1_moe_s2,arc_easy,0.353956,0.328283,0.009812 | |
| flame38m_g1_moe_s2,boolq,0.493884,,0.008744 | |
| flame38m_g1_moe_s2,copa,0.57,,0.049757 | |
| flame38m_g1_moe_s2,hellaswag,0.260406,0.258614,0.00438 | |
| flame38m_g1_moe_s2,lambada_openai,0.049486,,0.003022 | |
| flame38m_g1_moe_s2,openbookqa,0.126,0.224,0.014856 | |
| flame38m_g1_moe_s2,piqa,0.579434,0.566921,0.011518 | |
| flame38m_g1_moe_s2,sciq,0.459,0.431,0.015766 | |
| flame38m_g1_moe_s2,winogrande,0.486977,,0.014048 | |
| flame38m_g1_moe_s3,arc_challenge,0.183447,0.215017,0.01131 | |
| flame38m_g1_moe_s3,arc_easy,0.342593,0.337121,0.009738 | |
| flame38m_g1_moe_s3,boolq,0.589297,,0.008604 | |
| flame38m_g1_moe_s3,copa,0.6,,0.049237 | |
| flame38m_g1_moe_s3,hellaswag,0.261502,0.26419,0.004386 | |
| flame38m_g1_moe_s3,lambada_openai,0.067533,,0.003496 | |
| flame38m_g1_moe_s3,openbookqa,0.13,0.256,0.015055 | |
| flame38m_g1_moe_s3,piqa,0.575082,0.570185,0.011534 | |
| flame38m_g1_moe_s3,sciq,0.479,0.443,0.015805 | |
| flame38m_g1_moe_s3,winogrande,0.492502,,0.014051 | |
| flame38m_g1_temporal,arc_challenge,0.183447,0.236348,0.01131 | |
| flame38m_g1_temporal,arc_easy,0.37037,0.357323,0.009909 | |
| flame38m_g1_temporal,boolq,0.442813,,0.008688 | |
| flame38m_g1_temporal,copa,0.56,,0.049889 | |
| flame38m_g1_temporal,hellaswag,0.265186,0.263294,0.004405 | |
| flame38m_g1_temporal,lambada_openai,0.072967,,0.003623 | |
| flame38m_g1_temporal,openbookqa,0.14,0.26,0.015533 | |
| flame38m_g1_temporal,piqa,0.576714,0.57617,0.011528 | |
| flame38m_g1_temporal,sciq,0.499,0.436,0.015819 | |
| flame38m_g1_temporal,winogrande,0.479084,,0.01404 | |
| flame38m_g1_temporal_s2,arc_challenge,0.171502,0.203072,0.011015 | |
| flame38m_g1_temporal_s2,arc_easy,0.366582,0.344276,0.009888 | |
| flame38m_g1_temporal_s2,boolq,0.409174,,0.0086 | |
| flame38m_g1_temporal_s2,copa,0.53,,0.050161 | |
| flame38m_g1_temporal_s2,hellaswag,0.263892,0.258514,0.004398 | |
| flame38m_g1_temporal_s2,lambada_openai,0.072773,,0.003619 | |
| flame38m_g1_temporal_s2,openbookqa,0.134,0.248,0.01525 | |
| flame38m_g1_temporal_s2,piqa,0.58161,0.573449,0.011509 | |
| flame38m_g1_temporal_s2,sciq,0.482,0.442,0.015809 | |
| flame38m_g1_temporal_s2,winogrande,0.479874,,0.014041 | |
| flame38m_g1_temporal_s3,arc_challenge,0.177474,0.209898,0.011165 | |
| flame38m_g1_temporal_s3,arc_easy,0.351431,0.339226,0.009796 | |
| flame38m_g1_temporal_s3,boolq,0.447401,,0.008697 | |
| flame38m_g1_temporal_s3,copa,0.61,,0.049021 | |
| flame38m_g1_temporal_s3,hellaswag,0.26419,0.2618,0.0044 | |
| flame38m_g1_temporal_s3,lambada_openai,0.057248,,0.003237 | |
| flame38m_g1_temporal_s3,openbookqa,0.128,0.268,0.014956 | |
| flame38m_g1_temporal_s3,piqa,0.577258,0.572361,0.011526 | |
| flame38m_g1_temporal_s3,sciq,0.486,0.434,0.015813 | |
| flame38m_g1_temporal_s3,winogrande,0.488556,,0.014049 | |
| flame38m_g3_moe,arc_challenge,0.182594,0.209898,0.01129 | |
| flame38m_g3_moe,arc_easy,0.367424,0.3367,0.009893 | |
| flame38m_g3_moe,boolq,0.395107,,0.00855 | |
| flame38m_g3_moe,copa,0.54,,0.050091 | |
| flame38m_g3_moe,hellaswag,0.263394,0.258713,0.004396 | |
| flame38m_g3_moe,lambada_openai,0.043858,,0.002853 | |
| flame38m_g3_moe,openbookqa,0.144,0.254,0.015717 | |
| flame38m_g3_moe,piqa,0.569097,0.5642,0.011554 | |
| flame38m_g3_moe,sciq,0.47,0.454,0.015791 | |
| flame38m_g3_moe,winogrande,0.507498,,0.014051 | |
| flame38m_g3_moe_s2,arc_challenge,0.188567,0.223549,0.011431 | |
| flame38m_g3_moe_s2,arc_easy,0.341751,0.322391,0.009732 | |
| flame38m_g3_moe_s2,boolq,0.413456,,0.008613 | |
| flame38m_g3_moe_s2,copa,0.6,,0.049237 | |
| flame38m_g3_moe_s2,hellaswag,0.259311,0.258215,0.004374 | |
| flame38m_g3_moe_s2,lambada_openai,0.038036,,0.002665 | |
| flame38m_g3_moe_s2,openbookqa,0.118,0.232,0.014442 | |
| flame38m_g3_moe_s2,piqa,0.577802,0.56148,0.011524 | |
| flame38m_g3_moe_s2,sciq,0.47,0.437,0.015791 | |
| flame38m_g3_moe_s2,winogrande,0.520916,,0.01404 | |
| flame38m_g3_moe_s3,arc_challenge,0.188567,0.221843,0.011431 | |
| flame38m_g3_moe_s3,arc_easy,0.337542,0.328283,0.009703 | |
| flame38m_g3_moe_s3,boolq,0.484404,,0.008741 | |
| flame38m_g3_moe_s3,copa,0.6,,0.049237 | |
| flame38m_g3_moe_s3,hellaswag,0.262199,0.256622,0.004389 | |
| flame38m_g3_moe_s3,lambada_openai,0.047351,,0.002959 | |
| flame38m_g3_moe_s3,openbookqa,0.13,0.236,0.015055 | |
| flame38m_g3_moe_s3,piqa,0.577802,0.57617,0.011524 | |
| flame38m_g3_moe_s3,sciq,0.448,0.434,0.015734 | |
| flame38m_g3_moe_s3,winogrande,0.487766,,0.014048 | |
| flame38m_g3_temporal,arc_challenge,0.178328,0.214164,0.011186 | |
| flame38m_g3_temporal,arc_easy,0.355219,0.348064,0.00982 | |
| flame38m_g3_temporal,boolq,0.412844,,0.008611 | |
| flame38m_g3_temporal,copa,0.6,,0.049237 | |
| flame38m_g3_temporal,hellaswag,0.262099,0.257021,0.004389 | |
| flame38m_g3_temporal,lambada_openai,0.048127,,0.002982 | |
| flame38m_g3_temporal,openbookqa,0.122,0.254,0.014651 | |
| flame38m_g3_temporal,piqa,0.581066,0.5642,0.011511 | |
| flame38m_g3_temporal,sciq,0.477,0.445,0.015803 | |
| flame38m_g3_temporal,winogrande,0.518548,,0.014043 | |
| flame38m_g3_temporal_s2,arc_challenge,0.179181,0.21587,0.011207 | |
| flame38m_g3_temporal_s2,arc_easy,0.364478,0.335438,0.009876 | |
| flame38m_g3_temporal_s2,boolq,0.540061,,0.008717 | |
| flame38m_g3_temporal_s2,copa,0.55,,0.05 | |
| flame38m_g3_temporal_s2,hellaswag,0.263095,0.260605,0.004394 | |
| flame38m_g3_temporal_s2,lambada_openai,0.050844,,0.003061 | |
| flame38m_g3_temporal_s2,openbookqa,0.126,0.242,0.014856 | |
| flame38m_g3_temporal_s2,piqa,0.568553,0.562568,0.011556 | |
| flame38m_g3_temporal_s2,sciq,0.473,0.431,0.015796 | |
| flame38m_g3_temporal_s2,winogrande,0.500395,,0.014052 | |
| flame38m_g3_temporal_s3,arc_challenge,0.187713,0.209044,0.011411 | |
| flame38m_g3_temporal_s3,arc_easy,0.364899,0.332071,0.009878 | |
| flame38m_g3_temporal_s3,boolq,0.604893,,0.00855 | |
| flame38m_g3_temporal_s3,copa,0.58,,0.049604 | |
| flame38m_g3_temporal_s3,hellaswag,0.265186,0.258215,0.004405 | |
| flame38m_g3_temporal_s3,lambada_openai,0.055502,,0.00319 | |
| flame38m_g3_temporal_s3,openbookqa,0.132,0.26,0.015153 | |
| flame38m_g3_temporal_s3,piqa,0.582699,0.577802,0.011505 | |
| flame38m_g3_temporal_s3,sciq,0.488,0.449,0.015815 | |
| flame38m_g3_temporal_s3,winogrande,0.490134,,0.01405 | |
| flame192_g3_moe,arc_challenge,0.176621,0.209044,0.011144 | |
| flame192_g3_moe,arc_easy,0.348485,0.332912,0.009777 | |
| flame192_g3_moe,boolq,0.402446,,0.008577 | |
| flame192_g3_moe,copa,0.56,,0.049889 | |
| flame192_g3_moe,hellaswag,0.262995,0.258913,0.004394 | |
| flame192_g3_moe,lambada_openai,0.046575,,0.002936 | |
| flame192_g3_moe,openbookqa,0.15,0.234,0.015985 | |
| flame192_g3_moe,piqa,0.572905,0.560392,0.011541 | |
| flame192_g3_moe,sciq,0.479,0.429,0.015805 | |
| flame192_g3_moe,winogrande,0.501973,,0.014052 | |
| flame192_g3_temporal,arc_challenge,0.190273,0.224403,0.01147 | |
| flame192_g3_temporal,arc_easy,0.361953,0.335438,0.009861 | |
| flame192_g3_temporal,boolq,0.38318,,0.008503 | |
| flame192_g3_temporal,copa,0.53,,0.050161 | |
| flame192_g3_temporal,hellaswag,0.262298,0.259809,0.00439 | |
| flame192_g3_temporal,lambada_openai,0.059771,,0.003303 | |
| flame192_g3_temporal,openbookqa,0.14,0.248,0.015533 | |
| flame192_g3_temporal,piqa,0.568009,0.562024,0.011557 | |
| flame192_g3_temporal,sciq,0.493,0.455,0.015818 | |
| flame192_g3_temporal,winogrande,0.475138,,0.014035 | |
| flame512_dense,arc_challenge,0.178328,0.221843,0.011186 | |
| flame512_dense,arc_easy,0.339646,0.320707,0.009718 | |
| flame512_dense,boolq,0.525076,,0.008734 | |
| flame512_dense,copa,0.54,,0.050091 | |
| flame512_dense,hellaswag,0.258813,0.257419,0.004371 | |
| flame512_dense,lambada_openai,0.011256,,0.00147 | |
| flame512_dense,openbookqa,0.112,0.24,0.014118 | |
| flame512_dense,piqa,0.57617,0.562024,0.01153 | |
| flame512_dense,sciq,0.377,0.367,0.015333 | |
| flame512_dense,winogrande,0.501184,,0.014052 | |
| flame512_g1_moe,arc_challenge,0.198805,0.236348,0.011663 | |
| flame512_g1_moe,arc_easy,0.340488,0.319444,0.009724 | |
| flame512_g1_moe,boolq,0.484098,,0.008741 | |
| flame512_g1_moe,copa,0.54,,0.050091 | |
| flame512_g1_moe,hellaswag,0.263394,0.256821,0.004396 | |
| flame512_g1_moe,lambada_openai,0.019988,,0.00195 | |
| flame512_g1_moe,openbookqa,0.124,0.248,0.014754 | |
| flame512_g1_moe,piqa,0.568553,0.557127,0.011556 | |
| flame512_g1_moe,sciq,0.4,0.388,0.0155 | |
| flame512_g1_moe,winogrande,0.513812,,0.014047 | |
| flame512_g1_temporal,arc_challenge,0.178328,0.213311,0.011186 | |
| flame512_g1_temporal,arc_easy,0.353956,0.32702,0.009812 | |
| flame512_g1_temporal,boolq,0.378593,,0.008483 | |
| flame512_g1_temporal,copa,0.56,,0.049889 | |
| flame512_g1_temporal,hellaswag,0.262896,0.257718,0.004393 | |
| flame512_g1_temporal,lambada_openai,0.01863,,0.001884 | |
| flame512_g1_temporal,openbookqa,0.132,0.24,0.015153 | |
| flame512_g1_temporal,piqa,0.567465,0.563656,0.011559 | |
| flame512_g1_temporal,sciq,0.407,0.387,0.015543 | |
| flame512_g1_temporal,winogrande,0.524073,,0.014036 | |
| flame512_g3_moe,arc_challenge,0.185154,0.219283,0.011351 | |
| flame512_g3_moe,arc_easy,0.339226,0.318182,0.009715 | |
| flame512_g3_moe,boolq,0.591131,,0.008599 | |
| flame512_g3_moe,copa,0.49,,0.050242 | |
| flame512_g3_moe,hellaswag,0.257518,0.256224,0.004364 | |
| flame512_g3_moe,lambada_openai,0.012808,,0.001567 | |
| flame512_g3_moe,openbookqa,0.112,0.234,0.014118 | |
| flame512_g3_moe,piqa,0.567465,0.553319,0.011559 | |
| flame512_g3_moe,sciq,0.383,0.358,0.01538 | |
| flame512_g3_moe,winogrande,0.500395,,0.014052 | |
| flame512_g3_temporal,arc_challenge,0.171502,0.214164,0.011015 | |
| flame512_g3_temporal,arc_easy,0.341751,0.32702,0.009732 | |
| flame512_g3_temporal,boolq,0.449541,,0.0087 | |
| flame512_g3_temporal,copa,0.57,,0.049757 | |
| flame512_g3_temporal,hellaswag,0.259211,0.259709,0.004373 | |
| flame512_g3_temporal,lambada_openai,0.012032,,0.001519 | |
| flame512_g3_temporal,openbookqa,0.118,0.226,0.014442 | |
| flame512_g3_temporal,piqa,0.569097,0.551687,0.011554 | |
| flame512_g3_temporal,sciq,0.4,0.373,0.0155 | |
| flame512_g3_temporal,winogrande,0.528808,,0.014029 | |
| flame38m_g5_temporal,arc_challenge,0.18686,0.217577,0.011391 | |
| flame38m_g5_temporal,arc_easy,0.343013,0.326599,0.009741 | |
| flame38m_g5_temporal,boolq,0.389297,,0.008528 | |
| flame38m_g5_temporal,copa,0.57,,0.049757 | |
| flame38m_g5_temporal,hellaswag,0.261203,0.259709,0.004384 | |
| flame38m_g5_temporal,lambada_openai,0.041917,,0.002792 | |
| flame38m_g5_temporal,openbookqa,0.142,0.244,0.015626 | |
| flame38m_g5_temporal,piqa,0.575082,0.565288,0.011534 | |
| flame38m_g5_temporal,sciq,0.465,0.443,0.01578 | |
| flame38m_g5_temporal,winogrande,0.504341,,0.014052 | |
| flame38m_g5_moe,arc_challenge,0.182594,0.21587,0.01129 | |
| flame38m_g5_moe,arc_easy,0.339226,0.324495,0.009715 | |
| flame38m_g5_moe,boolq,0.515291,,0.008741 | |
| flame38m_g5_moe,copa,0.59,,0.049431 | |
| flame38m_g5_moe,hellaswag,0.262995,0.260207,0.004394 | |
| flame38m_g5_moe,lambada_openai,0.040753,,0.002755 | |
| flame38m_g5_moe,openbookqa,0.128,0.25,0.014956 | |
| flame38m_g5_moe,piqa,0.577802,0.562024,0.011524 | |
| flame38m_g5_moe,sciq,0.471,0.438,0.015793 | |
| flame38m_g5_moe,winogrande,0.503552,,0.014052 | |