{ "config": { "compressed_weights": "outputs/microsoft/Phi-mini-MoE-instruct/mobe_weights", "start_layer": 3, "end_layer": 32, "matrix_types": [ "w1" ], "tasks": [ "mmlu", "arc_easy", "arc_challenge" ], "num_fewshot": 0 }, "comparison_summary": { "arc_challenge": { "acc,none": { "original": 0.5750853242320819, "compressed": 0.5247440273037542, "delta": -0.05034129692832767 }, "acc_stderr,none": { "original": 0.014445698968520772, "compressed": 0.014593487694937738, "delta": 0.0001477887264169659 }, "acc_norm,none": { "original": 0.5878839590443686, "compressed": 0.5793515358361775, "delta": -0.008532423208191142 }, "acc_norm_stderr,none": { "original": 0.014383915302225402, "compressed": 0.014426211252508404, "delta": 4.2295950283002814e-05 } }, "arc_easy": { "acc,none": { "original": 0.8261784511784511, "compressed": 0.8131313131313131, "delta": -0.013047138047137974 }, "acc_stderr,none": { "original": 0.007776012182869763, "compressed": 0.007998652011081697, "delta": 0.00022263982821193323 }, "acc_norm,none": { "original": 0.811026936026936, "compressed": 0.7954545454545454, "delta": -0.015572390572390571 }, "acc_norm_stderr,none": { "original": 0.008033148299801935, "compressed": 0.008276958800002999, "delta": 0.00024381050020106342 } }, "mmlu": { "acc,none": { "original": 0.6791055405212932, "compressed": 0.6536105967810854, "delta": -0.025494943740207887 }, "acc_stderr,none": { "original": 0.003724961489256017, "compressed": 0.0038123134918508976, "delta": 8.735200259488065e-05 } }, "mmlu_humanities": { "acc,none": { "original": 0.620403825717322, "compressed": 0.5957492029755579, "delta": -0.02465462274176411 }, "acc_stderr,none": { "original": 0.0066572161384182, "compressed": 0.006759316550944608, "delta": 0.00010210041252640793 } }, "mmlu_formal_logic": { "acc,none": { "original": 0.5396825396825397, "compressed": 0.5158730158730159, "delta": -0.023809523809523725 }, "acc_stderr,none": { "original": 0.04458029125470973, "compressed": 0.044698818540726076, "delta": 0.00011852728601634971 } }, "mmlu_high_school_european_history": { "acc,none": { "original": 0.8181818181818182, "compressed": 0.7696969696969697, "delta": -0.048484848484848575 }, "acc_stderr,none": { "original": 0.030117688929503582, "compressed": 0.03287666758603489, "delta": 0.0027589786565313107 } }, "mmlu_high_school_us_history": { "acc,none": { "original": 0.8725490196078431, "compressed": 0.8480392156862745, "delta": -0.02450980392156865 }, "acc_stderr,none": { "original": 0.023405530480846315, "compressed": 0.025195658428931792, "delta": 0.0017901279480854772 } }, "mmlu_high_school_world_history": { "acc,none": { "original": 0.7932489451476793, "compressed": 0.7932489451476793, "delta": 0.0 }, "acc_stderr,none": { "original": 0.026361651668389094, "compressed": 0.026361651668389094, "delta": 0.0 } }, "mmlu_international_law": { "acc,none": { "original": 0.8760330578512396, "compressed": 0.8016528925619835, "delta": -0.07438016528925617 }, "acc_stderr,none": { "original": 0.030083098716035216, "compressed": 0.03640118271990947, "delta": 0.006318084003874253 } }, "mmlu_jurisprudence": { "acc,none": { "original": 0.8240740740740741, "compressed": 0.7962962962962963, "delta": -0.02777777777777779 }, "acc_stderr,none": { "original": 0.036809181416738807, "compressed": 0.03893542518824847, "delta": 0.0021262437715096616 } }, "mmlu_logical_fallacies": { "acc,none": { "original": 0.8159509202453987, "compressed": 0.8098159509202454, "delta": -0.006134969325153339 }, "acc_stderr,none": { "original": 0.030446777687971726, "compressed": 0.03083349114628123, "delta": 0.00038671345830950526 } }, "mmlu_moral_disputes": { "acc,none": { "original": 0.7572254335260116, "compressed": 0.7167630057803468, "delta": -0.040462427745664775 }, "acc_stderr,none": { "original": 0.023083658586984204, "compressed": 0.02425790170532337, "delta": 0.0011742431183391668 } }, "mmlu_moral_scenarios": { "acc,none": { "original": 0.4033519553072626, "compressed": 0.376536312849162, "delta": -0.026815642458100586 }, "acc_stderr,none": { "original": 0.01640712303219525, "compressed": 0.016204672385106596, "delta": -0.00020245064708865335 } }, "mmlu_philosophy": { "acc,none": { "original": 0.7427652733118971, "compressed": 0.707395498392283, "delta": -0.03536977491961413 }, "acc_stderr,none": { "original": 0.024826171289250888, "compressed": 0.02583989833487798, "delta": 0.0010137270456270914 } }, "mmlu_prehistory": { "acc,none": { "original": 0.7469135802469136, "compressed": 0.7191358024691358, "delta": -0.02777777777777779 }, "acc_stderr,none": { "original": 0.024191808600713002, "compressed": 0.025006469755799208, "delta": 0.0008146611550862055 } }, "mmlu_professional_law": { "acc,none": { "original": 0.5117340286831812, "compressed": 0.4954367666232073, "delta": -0.016297262059973894 }, "acc_stderr,none": { "original": 0.01276671901968672, "compressed": 0.012769704263117531, "delta": 2.985243430810719e-06 } }, "mmlu_world_religions": { "acc,none": { "original": 0.8245614035087719, "compressed": 0.8011695906432749, "delta": -0.023391812865497075 }, "acc_stderr,none": { "original": 0.029170885500727654, "compressed": 0.03061111655743253, "delta": 0.001440231056704877 } }, "mmlu_other": { "acc,none": { "original": 0.7306083038300611, "compressed": 0.7071129707112971, "delta": -0.02349533311876406 }, "acc_stderr,none": { "original": 0.007703905997463136, "compressed": 0.00792634679462611, "delta": 0.00022244079716297448 } }, "mmlu_business_ethics": { "acc,none": { "original": 0.72, "compressed": 0.71, "delta": -0.010000000000000009 }, "acc_stderr,none": { "original": 0.045126085985421276, "compressed": 0.045604802157206845, "delta": 0.00047871617178556897 } }, "mmlu_clinical_knowledge": { "acc,none": { "original": 0.7584905660377359, "compressed": 0.7283018867924528, "delta": -0.030188679245283123 }, "acc_stderr,none": { "original": 0.02634148037111836, "compressed": 0.027377706624670713, "delta": 0.001036226253552354 } }, "mmlu_college_medicine": { "acc,none": { "original": 0.6820809248554913, "compressed": 0.630057803468208, "delta": -0.052023121387283267 }, "acc_stderr,none": { "original": 0.0355068398916558, "compressed": 0.0368122963339432, "delta": 0.0013054564422873977 } }, "mmlu_global_facts": { "acc,none": { "original": 0.39, "compressed": 0.46, "delta": 0.07 }, "acc_stderr,none": { "original": 0.04902071300001974, "compressed": 0.05009082659620333, "delta": 0.0010701135961835861 } }, "mmlu_human_aging": { "acc,none": { "original": 0.7085201793721974, "compressed": 0.6681614349775785, "delta": -0.04035874439461884 }, "acc_stderr,none": { "original": 0.030500283176545843, "compressed": 0.03160295143776679, "delta": 0.0011026682612209487 } }, "mmlu_management": { "acc,none": { "original": 0.7961165048543689, "compressed": 0.7475728155339806, "delta": -0.04854368932038833 }, "acc_stderr,none": { "original": 0.0398913985953177, "compressed": 0.04301250399690879, "delta": 0.0031211054015910927 } }, "mmlu_marketing": { "acc,none": { "original": 0.8931623931623932, "compressed": 0.8974358974358975, "delta": 0.004273504273504258 }, "acc_stderr,none": { "original": 0.020237149008990922, "compressed": 0.019875655027867457, "delta": -0.00036149398112346465 } }, "mmlu_medical_genetics": { "acc,none": { "original": 0.76, "compressed": 0.74, "delta": -0.020000000000000018 }, "acc_stderr,none": { "original": 0.04292346959909283, "compressed": 0.04408440022768078, "delta": 0.00116093062858795 } }, "mmlu_miscellaneous": { "acc,none": { "original": 0.8173690932311622, "compressed": 0.7956577266922095, "delta": -0.02171136653895278 }, "acc_stderr,none": { "original": 0.013816335389973124, "compressed": 0.0144191239809319, "delta": 0.0006027885909587766 } }, "mmlu_nutrition": { "acc,none": { "original": 0.7516339869281046, "compressed": 0.7287581699346405, "delta": -0.022875816993464082 }, "acc_stderr,none": { "original": 0.02473998135511359, "compressed": 0.025457756696667864, "delta": 0.0007177753415542748 } }, "mmlu_professional_accounting": { "acc,none": { "original": 0.5425531914893617, "compressed": 0.5035460992907801, "delta": -0.03900709219858156 }, "acc_stderr,none": { "original": 0.02971928127223684, "compressed": 0.02982674915328092, "delta": 0.00010746788104407795 } }, "mmlu_professional_medicine": { "acc,none": { "original": 0.7426470588235294, "compressed": 0.7058823529411765, "delta": -0.03676470588235292 }, "acc_stderr,none": { "original": 0.026556519470041517, "compressed": 0.0276784686421447, "delta": 0.001121949172103183 } }, "mmlu_virology": { "acc,none": { "original": 0.5421686746987951, "compressed": 0.5301204819277109, "delta": -0.012048192771084265 }, "acc_stderr,none": { "original": 0.038786267710023595, "compressed": 0.03885425420866766, "delta": 6.798649864406259e-05 } }, "mmlu_social_sciences": { "acc,none": { "original": 0.7923301917452064, "compressed": 0.7630809229769255, "delta": -0.029249268768280823 }, "acc_stderr,none": { "original": 0.007164065545433766, "compressed": 0.007521922102925205, "delta": 0.00035785655749143944 } }, "mmlu_econometrics": { "acc,none": { "original": 0.5087719298245614, "compressed": 0.5087719298245614, "delta": 0.0 }, "acc_stderr,none": { "original": 0.04702880432049615, "compressed": 0.04702880432049615, "delta": 0.0 } }, "mmlu_high_school_geography": { "acc,none": { "original": 0.8434343434343434, "compressed": 0.8181818181818182, "delta": -0.025252525252525193 }, "acc_stderr,none": { "original": 0.025890520358141454, "compressed": 0.0274796030105388, "delta": 0.0015890826523973466 } }, "mmlu_high_school_government_and_politics": { "acc,none": { "original": 0.9067357512953368, "compressed": 0.8860103626943006, "delta": -0.020725388601036232 }, "acc_stderr,none": { "original": 0.02098685459328971, "compressed": 0.022935144053919436, "delta": 0.0019482894606297242 } }, "mmlu_high_school_macroeconomics": { "acc,none": { "original": 0.735897435897436, "compressed": 0.6846153846153846, "delta": -0.05128205128205132 }, "acc_stderr,none": { "original": 0.022352193737453268, "compressed": 0.02355964698318994, "delta": 0.0012074532457366713 } }, "mmlu_high_school_microeconomics": { "acc,none": { "original": 0.865546218487395, "compressed": 0.8277310924369747, "delta": -0.037815126050420256 }, "acc_stderr,none": { "original": 0.022159373072744442, "compressed": 0.024528664971305427, "delta": 0.002369291898560985 } }, "mmlu_high_school_psychology": { "acc,none": { "original": 0.8642201834862385, "compressed": 0.8477064220183487, "delta": -0.016513761467889854 }, "acc_stderr,none": { "original": 0.014686907556340017, "compressed": 0.015405084393157069, "delta": 0.0007181768368170523 } }, "mmlu_human_sexuality": { "acc,none": { "original": 0.7786259541984732, "compressed": 0.7557251908396947, "delta": -0.022900763358778553 }, "acc_stderr,none": { "original": 0.036412970813137296, "compressed": 0.037683359597287434, "delta": 0.001270388784150138 } }, "mmlu_professional_psychology": { "acc,none": { "original": 0.7205882352941176, "compressed": 0.6879084967320261, "delta": -0.0326797385620915 }, "acc_stderr,none": { "original": 0.018152871051538812, "compressed": 0.01874501120127766, "delta": 0.0005921401497388477 } }, "mmlu_public_relations": { "acc,none": { "original": 0.7181818181818181, "compressed": 0.7090909090909091, "delta": -0.009090909090909038 }, "acc_stderr,none": { "original": 0.043091187099464585, "compressed": 0.04350271442923243, "delta": 0.0004115273297678468 } }, "mmlu_security_studies": { "acc,none": { "original": 0.8040816326530612, "compressed": 0.7714285714285715, "delta": -0.03265306122448974 }, "acc_stderr,none": { "original": 0.02540930195322568, "compressed": 0.026882144922307744, "delta": 0.001472842969082063 } }, "mmlu_sociology": { "acc,none": { "original": 0.845771144278607, "compressed": 0.8059701492537313, "delta": -0.03980099502487566 }, "acc_stderr,none": { "original": 0.025538433368578334, "compressed": 0.027962677604768896, "delta": 0.0024242442361905626 } }, "mmlu_us_foreign_policy": { "acc,none": { "original": 0.85, "compressed": 0.82, "delta": -0.030000000000000027 }, "acc_stderr,none": { "original": 0.0358870281282637, "compressed": 0.03861229196653694, "delta": 0.002725263838273241 } }, "mmlu_stem": { "acc,none": { "original": 0.6054551221059309, "compressed": 0.5803996194100857, "delta": -0.025055502695845222 }, "acc_stderr,none": { "original": 0.008367541993189256, "compressed": 0.008464141586351462, "delta": 9.659959316220615e-05 } }, "mmlu_abstract_algebra": { "acc,none": { "original": 0.5, "compressed": 0.42, "delta": -0.08000000000000002 }, "acc_stderr,none": { "original": 0.050251890762960605, "compressed": 0.049604496374885836, "delta": -0.0006473943880747693 } }, "mmlu_anatomy": { "acc,none": { "original": 0.6518518518518519, "compressed": 0.6148148148148148, "delta": -0.03703703703703709 }, "acc_stderr,none": { "original": 0.041153246103369526, "compressed": 0.042039210401562783, "delta": 0.0008859642981932578 } }, "mmlu_astronomy": { "acc,none": { "original": 0.7697368421052632, "compressed": 0.743421052631579, "delta": -0.02631578947368418 }, "acc_stderr,none": { "original": 0.03426059424403165, "compressed": 0.035541803680256896, "delta": 0.0012812094362252488 } }, "mmlu_college_biology": { "acc,none": { "original": 0.8263888888888888, "compressed": 0.7986111111111112, "delta": -0.02777777777777768 }, "acc_stderr,none": { "original": 0.03167473383795718, "compressed": 0.03353647469713839, "delta": 0.001861740859181213 } }, "mmlu_college_chemistry": { "acc,none": { "original": 0.45, "compressed": 0.47, "delta": 0.019999999999999962 }, "acc_stderr,none": { "original": 0.049999999999999996, "compressed": 0.05016135580465919, "delta": 0.00016135580465919536 } }, "mmlu_college_computer_science": { "acc,none": { "original": 0.6, "compressed": 0.53, "delta": -0.06999999999999995 }, "acc_stderr,none": { "original": 0.049236596391733084, "compressed": 0.050161355804659205, "delta": 0.0009247594129261211 } }, "mmlu_college_mathematics": { "acc,none": { "original": 0.43, "compressed": 0.4, "delta": -0.02999999999999997 }, "acc_stderr,none": { "original": 0.04975698519562427, "compressed": 0.04923659639173309, "delta": -0.0005203888038911794 } }, "mmlu_college_physics": { "acc,none": { "original": 0.39215686274509803, "compressed": 0.37254901960784315, "delta": -0.019607843137254888 }, "acc_stderr,none": { "original": 0.048580835742663434, "compressed": 0.04810840148082634, "delta": -0.0004724342618370944 } }, "mmlu_computer_security": { "acc,none": { "original": 0.77, "compressed": 0.73, "delta": -0.040000000000000036 }, "acc_stderr,none": { "original": 0.04229525846816505, "compressed": 0.044619604333847394, "delta": 0.0023243458656823437 } }, "mmlu_conceptual_physics": { "acc,none": { "original": 0.6553191489361702, "compressed": 0.6382978723404256, "delta": -0.017021276595744594 }, "acc_stderr,none": { "original": 0.031068985963122145, "compressed": 0.03141082197596241, "delta": 0.0003418360128402617 } }, "mmlu_electrical_engineering": { "acc,none": { "original": 0.6137931034482759, "compressed": 0.593103448275862, "delta": -0.020689655172413834 }, "acc_stderr,none": { "original": 0.04057324734419036, "compressed": 0.04093793981266236, "delta": 0.00036469246847200404 } }, "mmlu_elementary_mathematics": { "acc,none": { "original": 0.5582010582010583, "compressed": 0.5661375661375662, "delta": 0.007936507936507908 }, "acc_stderr,none": { "original": 0.025576257061253833, "compressed": 0.02552503438247489, "delta": -5.122267877894285e-05 } }, "mmlu_high_school_biology": { "acc,none": { "original": 0.8290322580645161, "compressed": 0.7935483870967742, "delta": -0.03548387096774186 }, "acc_stderr,none": { "original": 0.021417242936321544, "compressed": 0.023025899617188744, "delta": 0.0016086566808671994 } }, "mmlu_high_school_chemistry": { "acc,none": { "original": 0.6059113300492611, "compressed": 0.5566502463054187, "delta": -0.049261083743842415 }, "acc_stderr,none": { "original": 0.03438157967036543, "compressed": 0.03495334582162934, "delta": 0.0005717661512639055 } }, "mmlu_high_school_computer_science": { "acc,none": { "original": 0.77, "compressed": 0.75, "delta": -0.020000000000000018 }, "acc_stderr,none": { "original": 0.04229525846816506, "compressed": 0.04351941398892446, "delta": 0.0012241555207594054 } }, "mmlu_high_school_mathematics": { "acc,none": { "original": 0.3925925925925926, "compressed": 0.36666666666666664, "delta": -0.025925925925925963 }, "acc_stderr,none": { "original": 0.02977384701253297, "compressed": 0.029381620726465073, "delta": -0.00039222628606789753 } }, "mmlu_high_school_physics": { "acc,none": { "original": 0.4768211920529801, "compressed": 0.46357615894039733, "delta": -0.013245033112582794 }, "acc_stderr,none": { "original": 0.04078093859163083, "compressed": 0.04071636065944216, "delta": -6.45779321886697e-05 } }, "mmlu_high_school_statistics": { "acc,none": { "original": 0.5833333333333334, "compressed": 0.5787037037037037, "delta": -0.00462962962962965 }, "acc_stderr,none": { "original": 0.03362277436608043, "compressed": 0.033674621388960775, "delta": 5.184702288034415e-05 } }, "mmlu_machine_learning": { "acc,none": { "original": 0.49107142857142855, "compressed": 0.42857142857142855, "delta": -0.0625 }, "acc_stderr,none": { "original": 0.04745033255489123, "compressed": 0.04697113923010212, "delta": -0.0004791933247891081 } } }, "original_model_results": { "results": { "arc_challenge": { "alias": "arc_challenge", "acc,none": 0.5750853242320819, "acc_stderr,none": 0.014445698968520772, "acc_norm,none": 0.5878839590443686, "acc_norm_stderr,none": 0.014383915302225402 }, "arc_easy": { "alias": "arc_easy", "acc,none": 0.8261784511784511, "acc_stderr,none": 0.007776012182869763, "acc_norm,none": 0.811026936026936, "acc_norm_stderr,none": 0.008033148299801935 }, "mmlu": { "acc,none": 0.6791055405212932, "acc_stderr,none": 0.003724961489256017, "alias": "mmlu" }, "mmlu_humanities": { "acc,none": 0.620403825717322, "acc_stderr,none": 0.0066572161384182, "alias": " - humanities" }, "mmlu_formal_logic": { "alias": " - formal_logic", "acc,none": 0.5396825396825397, "acc_stderr,none": 0.04458029125470973 }, "mmlu_high_school_european_history": { "alias": " - high_school_european_history", "acc,none": 0.8181818181818182, "acc_stderr,none": 0.030117688929503582 }, "mmlu_high_school_us_history": { "alias": " - high_school_us_history", "acc,none": 0.8725490196078431, "acc_stderr,none": 0.023405530480846315 }, "mmlu_high_school_world_history": { "alias": " - high_school_world_history", "acc,none": 0.7932489451476793, "acc_stderr,none": 0.026361651668389094 }, "mmlu_international_law": { "alias": " - international_law", "acc,none": 0.8760330578512396, "acc_stderr,none": 0.030083098716035216 }, "mmlu_jurisprudence": { "alias": " - jurisprudence", "acc,none": 0.8240740740740741, "acc_stderr,none": 0.036809181416738807 }, "mmlu_logical_fallacies": { "alias": " - logical_fallacies", "acc,none": 0.8159509202453987, "acc_stderr,none": 0.030446777687971726 }, "mmlu_moral_disputes": { "alias": " - moral_disputes", "acc,none": 0.7572254335260116, "acc_stderr,none": 0.023083658586984204 }, "mmlu_moral_scenarios": { "alias": " - moral_scenarios", "acc,none": 0.4033519553072626, "acc_stderr,none": 0.01640712303219525 }, "mmlu_philosophy": { "alias": " - philosophy", "acc,none": 0.7427652733118971, "acc_stderr,none": 0.024826171289250888 }, "mmlu_prehistory": { "alias": " - prehistory", "acc,none": 0.7469135802469136, "acc_stderr,none": 0.024191808600713002 }, "mmlu_professional_law": { "alias": " - professional_law", "acc,none": 0.5117340286831812, "acc_stderr,none": 0.01276671901968672 }, "mmlu_world_religions": { "alias": " - world_religions", "acc,none": 0.8245614035087719, "acc_stderr,none": 0.029170885500727654 }, "mmlu_other": { "acc,none": 0.7306083038300611, "acc_stderr,none": 0.007703905997463136, "alias": " - other" }, "mmlu_business_ethics": { "alias": " - business_ethics", "acc,none": 0.72, "acc_stderr,none": 0.045126085985421276 }, "mmlu_clinical_knowledge": { "alias": " - clinical_knowledge", "acc,none": 0.7584905660377359, "acc_stderr,none": 0.02634148037111836 }, "mmlu_college_medicine": { "alias": " - college_medicine", "acc,none": 0.6820809248554913, "acc_stderr,none": 0.0355068398916558 }, "mmlu_global_facts": { "alias": " - global_facts", "acc,none": 0.39, "acc_stderr,none": 0.04902071300001974 }, "mmlu_human_aging": { "alias": " - human_aging", "acc,none": 0.7085201793721974, "acc_stderr,none": 0.030500283176545843 }, "mmlu_management": { "alias": " - management", "acc,none": 0.7961165048543689, "acc_stderr,none": 0.0398913985953177 }, "mmlu_marketing": { "alias": " - marketing", "acc,none": 0.8931623931623932, "acc_stderr,none": 0.020237149008990922 }, "mmlu_medical_genetics": { "alias": " - medical_genetics", "acc,none": 0.76, "acc_stderr,none": 0.04292346959909283 }, "mmlu_miscellaneous": { "alias": " - miscellaneous", "acc,none": 0.8173690932311622, "acc_stderr,none": 0.013816335389973124 }, "mmlu_nutrition": { "alias": " - nutrition", "acc,none": 0.7516339869281046, "acc_stderr,none": 0.02473998135511359 }, "mmlu_professional_accounting": { "alias": " - professional_accounting", "acc,none": 0.5425531914893617, "acc_stderr,none": 0.02971928127223684 }, "mmlu_professional_medicine": { "alias": " - professional_medicine", "acc,none": 0.7426470588235294, "acc_stderr,none": 0.026556519470041517 }, "mmlu_virology": { "alias": " - virology", "acc,none": 0.5421686746987951, "acc_stderr,none": 0.038786267710023595 }, "mmlu_social_sciences": { "acc,none": 0.7923301917452064, "acc_stderr,none": 0.007164065545433766, "alias": " - social sciences" }, "mmlu_econometrics": { "alias": " - econometrics", "acc,none": 0.5087719298245614, "acc_stderr,none": 0.04702880432049615 }, "mmlu_high_school_geography": { "alias": " - high_school_geography", "acc,none": 0.8434343434343434, "acc_stderr,none": 0.025890520358141454 }, "mmlu_high_school_government_and_politics": { "alias": " - high_school_government_and_politics", "acc,none": 0.9067357512953368, "acc_stderr,none": 0.02098685459328971 }, "mmlu_high_school_macroeconomics": { "alias": " - high_school_macroeconomics", "acc,none": 0.735897435897436, "acc_stderr,none": 0.022352193737453268 }, "mmlu_high_school_microeconomics": { "alias": " - high_school_microeconomics", "acc,none": 0.865546218487395, "acc_stderr,none": 0.022159373072744442 }, "mmlu_high_school_psychology": { "alias": " - high_school_psychology", "acc,none": 0.8642201834862385, "acc_stderr,none": 0.014686907556340017 }, "mmlu_human_sexuality": { "alias": " - human_sexuality", "acc,none": 0.7786259541984732, "acc_stderr,none": 0.036412970813137296 }, "mmlu_professional_psychology": { "alias": " - professional_psychology", "acc,none": 0.7205882352941176, "acc_stderr,none": 0.018152871051538812 }, "mmlu_public_relations": { "alias": " - public_relations", "acc,none": 0.7181818181818181, "acc_stderr,none": 0.043091187099464585 }, "mmlu_security_studies": { "alias": " - security_studies", "acc,none": 0.8040816326530612, "acc_stderr,none": 0.02540930195322568 }, "mmlu_sociology": { "alias": " - sociology", "acc,none": 0.845771144278607, "acc_stderr,none": 0.025538433368578334 }, "mmlu_us_foreign_policy": { "alias": " - us_foreign_policy", "acc,none": 0.85, "acc_stderr,none": 0.0358870281282637 }, "mmlu_stem": { "acc,none": 0.6054551221059309, "acc_stderr,none": 0.008367541993189256, "alias": " - stem" }, "mmlu_abstract_algebra": { "alias": " - abstract_algebra", "acc,none": 0.5, "acc_stderr,none": 0.050251890762960605 }, "mmlu_anatomy": { "alias": " - anatomy", "acc,none": 0.6518518518518519, "acc_stderr,none": 0.041153246103369526 }, "mmlu_astronomy": { "alias": " - astronomy", "acc,none": 0.7697368421052632, "acc_stderr,none": 0.03426059424403165 }, "mmlu_college_biology": { "alias": " - college_biology", "acc,none": 0.8263888888888888, "acc_stderr,none": 0.03167473383795718 }, "mmlu_college_chemistry": { "alias": " - college_chemistry", "acc,none": 0.45, "acc_stderr,none": 0.049999999999999996 }, "mmlu_college_computer_science": { "alias": " - college_computer_science", "acc,none": 0.6, "acc_stderr,none": 0.049236596391733084 }, "mmlu_college_mathematics": { "alias": " - college_mathematics", "acc,none": 0.43, "acc_stderr,none": 0.04975698519562427 }, "mmlu_college_physics": { "alias": " - college_physics", "acc,none": 0.39215686274509803, "acc_stderr,none": 0.048580835742663434 }, "mmlu_computer_security": { "alias": " - computer_security", "acc,none": 0.77, "acc_stderr,none": 0.04229525846816505 }, "mmlu_conceptual_physics": { "alias": " - conceptual_physics", "acc,none": 0.6553191489361702, "acc_stderr,none": 0.031068985963122145 }, "mmlu_electrical_engineering": { "alias": " - electrical_engineering", "acc,none": 0.6137931034482759, "acc_stderr,none": 0.04057324734419036 }, "mmlu_elementary_mathematics": { "alias": " - elementary_mathematics", "acc,none": 0.5582010582010583, "acc_stderr,none": 0.025576257061253833 }, "mmlu_high_school_biology": { "alias": " - high_school_biology", "acc,none": 0.8290322580645161, "acc_stderr,none": 0.021417242936321544 }, "mmlu_high_school_chemistry": { "alias": " - high_school_chemistry", "acc,none": 0.6059113300492611, "acc_stderr,none": 0.03438157967036543 }, "mmlu_high_school_computer_science": { "alias": " - high_school_computer_science", "acc,none": 0.77, "acc_stderr,none": 0.04229525846816506 }, "mmlu_high_school_mathematics": { "alias": " - high_school_mathematics", "acc,none": 0.3925925925925926, "acc_stderr,none": 0.02977384701253297 }, "mmlu_high_school_physics": { "alias": " - high_school_physics", "acc,none": 0.4768211920529801, "acc_stderr,none": 0.04078093859163083 }, "mmlu_high_school_statistics": { "alias": " - high_school_statistics", "acc,none": 0.5833333333333334, "acc_stderr,none": 0.03362277436608043 }, "mmlu_machine_learning": { "alias": " - machine_learning", "acc,none": 0.49107142857142855, "acc_stderr,none": 0.04745033255489123 } }, "groups": { "mmlu": { "acc,none": 0.6791055405212932, "acc_stderr,none": 0.003724961489256017, "alias": "mmlu" }, "mmlu_humanities": { "acc,none": 0.620403825717322, "acc_stderr,none": 0.0066572161384182, "alias": " - humanities" }, "mmlu_other": { "acc,none": 0.7306083038300611, "acc_stderr,none": 0.007703905997463136, "alias": " - other" }, "mmlu_social_sciences": { "acc,none": 0.7923301917452064, "acc_stderr,none": 0.007164065545433766, "alias": " - social sciences" }, "mmlu_stem": { "acc,none": 0.6054551221059309, "acc_stderr,none": 0.008367541993189256, "alias": " - stem" } }, "group_subtasks": { "mmlu_humanities": [ "mmlu_formal_logic", "mmlu_high_school_european_history", "mmlu_high_school_us_history", "mmlu_high_school_world_history", "mmlu_international_law", "mmlu_jurisprudence", "mmlu_logical_fallacies", "mmlu_moral_disputes", "mmlu_moral_scenarios", "mmlu_philosophy", "mmlu_prehistory", "mmlu_professional_law", "mmlu_world_religions" ], "mmlu_social_sciences": [ "mmlu_econometrics", "mmlu_high_school_geography", "mmlu_high_school_government_and_politics", "mmlu_high_school_macroeconomics", "mmlu_high_school_microeconomics", "mmlu_high_school_psychology", "mmlu_human_sexuality", "mmlu_professional_psychology", "mmlu_public_relations", "mmlu_security_studies", "mmlu_sociology", "mmlu_us_foreign_policy" ], "mmlu_other": [ "mmlu_business_ethics", "mmlu_clinical_knowledge", "mmlu_college_medicine", "mmlu_global_facts", "mmlu_human_aging", "mmlu_management", "mmlu_marketing", "mmlu_medical_genetics", "mmlu_miscellaneous", "mmlu_nutrition", "mmlu_professional_accounting", "mmlu_professional_medicine", "mmlu_virology" ], "mmlu_stem": [ "mmlu_abstract_algebra", "mmlu_anatomy", "mmlu_astronomy", "mmlu_college_biology", "mmlu_college_chemistry", "mmlu_college_computer_science", "mmlu_college_mathematics", "mmlu_college_physics", "mmlu_computer_security", "mmlu_conceptual_physics", "mmlu_electrical_engineering", "mmlu_elementary_mathematics", "mmlu_high_school_biology", "mmlu_high_school_chemistry", "mmlu_high_school_computer_science", "mmlu_high_school_mathematics", "mmlu_high_school_physics", "mmlu_high_school_statistics", "mmlu_machine_learning" ], "mmlu": [ "mmlu_stem", "mmlu_other", "mmlu_social_sciences", "mmlu_humanities" ], "arc_easy": [], "arc_challenge": [] }, "configs": { "arc_challenge": { "task": "arc_challenge", "tag": [ "ai2_arc" ], "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Challenge", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0 } }, "arc_easy": { "task": "arc_easy", "tag": [ "ai2_arc" ], "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Easy", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0 } }, "mmlu_abstract_algebra": { "task": "mmlu_abstract_algebra", "task_alias": "abstract_algebra", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_anatomy": { "task": "mmlu_anatomy", "task_alias": "anatomy", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "anatomy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_astronomy": { "task": "mmlu_astronomy", "task_alias": "astronomy", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "astronomy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_business_ethics": { "task": "mmlu_business_ethics", "task_alias": "business_ethics", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "business_ethics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_clinical_knowledge": { "task": "mmlu_clinical_knowledge", "task_alias": "clinical_knowledge", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "clinical_knowledge", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_biology": { "task": "mmlu_college_biology", "task_alias": "college_biology", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_biology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college biology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_chemistry": { "task": "mmlu_college_chemistry", "task_alias": "college_chemistry", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_chemistry", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_computer_science": { "task": "mmlu_college_computer_science", "task_alias": "college_computer_science", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_computer_science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_mathematics": { "task": "mmlu_college_mathematics", "task_alias": "college_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_medicine": { "task": "mmlu_college_medicine", "task_alias": "college_medicine", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_medicine", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_physics": { "task": "mmlu_college_physics", "task_alias": "college_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_computer_security": { "task": "mmlu_computer_security", "task_alias": "computer_security", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "computer_security", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about computer security.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_conceptual_physics": { "task": "mmlu_conceptual_physics", "task_alias": "conceptual_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "conceptual_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_econometrics": { "task": "mmlu_econometrics", "task_alias": "econometrics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "econometrics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_electrical_engineering": { "task": "mmlu_electrical_engineering", "task_alias": "electrical_engineering", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "electrical_engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_elementary_mathematics": { "task": "mmlu_elementary_mathematics", "task_alias": "elementary_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "elementary_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_formal_logic": { "task": "mmlu_formal_logic", "task_alias": "formal_logic", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "formal_logic", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_global_facts": { "task": "mmlu_global_facts", "task_alias": "global_facts", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "global_facts", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about global facts.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_biology": { "task": "mmlu_high_school_biology", "task_alias": "high_school_biology", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_biology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_chemistry": { "task": "mmlu_high_school_chemistry", "task_alias": "high_school_chemistry", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_chemistry", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_computer_science": { "task": "mmlu_high_school_computer_science", "task_alias": "high_school_computer_science", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_computer_science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_european_history": { "task": "mmlu_high_school_european_history", "task_alias": "high_school_european_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_european_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_geography": { "task": "mmlu_high_school_geography", "task_alias": "high_school_geography", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_geography", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_government_and_politics": { "task": "mmlu_high_school_government_and_politics", "task_alias": "high_school_government_and_politics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_government_and_politics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_macroeconomics": { "task": "mmlu_high_school_macroeconomics", "task_alias": "high_school_macroeconomics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_macroeconomics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_mathematics": { "task": "mmlu_high_school_mathematics", "task_alias": "high_school_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_microeconomics": { "task": "mmlu_high_school_microeconomics", "task_alias": "high_school_microeconomics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_microeconomics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_physics": { "task": "mmlu_high_school_physics", "task_alias": "high_school_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_psychology": { "task": "mmlu_high_school_psychology", "task_alias": "high_school_psychology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_psychology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_statistics": { "task": "mmlu_high_school_statistics", "task_alias": "high_school_statistics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_statistics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_us_history": { "task": "mmlu_high_school_us_history", "task_alias": "high_school_us_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_us_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_world_history": { "task": "mmlu_high_school_world_history", "task_alias": "high_school_world_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_world_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_human_aging": { "task": "mmlu_human_aging", "task_alias": "human_aging", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "human_aging", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about human aging.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_human_sexuality": { "task": "mmlu_human_sexuality", "task_alias": "human_sexuality", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "human_sexuality", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_international_law": { "task": "mmlu_international_law", "task_alias": "international_law", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "international_law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about international law.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_jurisprudence": { "task": "mmlu_jurisprudence", "task_alias": "jurisprudence", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "jurisprudence", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_logical_fallacies": { "task": "mmlu_logical_fallacies", "task_alias": "logical_fallacies", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "logical_fallacies", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_machine_learning": { "task": "mmlu_machine_learning", "task_alias": "machine_learning", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "machine_learning", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_management": { "task": "mmlu_management", "task_alias": "management", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "management", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about management.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_marketing": { "task": "mmlu_marketing", "task_alias": "marketing", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "marketing", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about marketing.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_medical_genetics": { "task": "mmlu_medical_genetics", "task_alias": "medical_genetics", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "medical_genetics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_miscellaneous": { "task": "mmlu_miscellaneous", "task_alias": "miscellaneous", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "miscellaneous", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_moral_disputes": { "task": "mmlu_moral_disputes", "task_alias": "moral_disputes", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "moral_disputes", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_moral_scenarios": { "task": "mmlu_moral_scenarios", "task_alias": "moral_scenarios", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "moral_scenarios", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_nutrition": { "task": "mmlu_nutrition", "task_alias": "nutrition", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "nutrition", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_philosophy": { "task": "mmlu_philosophy", "task_alias": "philosophy", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "philosophy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_prehistory": { "task": "mmlu_prehistory", "task_alias": "prehistory", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "prehistory", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_accounting": { "task": "mmlu_professional_accounting", "task_alias": "professional_accounting", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_accounting", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_law": { "task": "mmlu_professional_law", "task_alias": "professional_law", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional law.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_medicine": { "task": "mmlu_professional_medicine", "task_alias": "professional_medicine", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_medicine", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_psychology": { "task": "mmlu_professional_psychology", "task_alias": "professional_psychology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_psychology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_public_relations": { "task": "mmlu_public_relations", "task_alias": "public_relations", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "public_relations", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about public relations.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_security_studies": { "task": "mmlu_security_studies", "task_alias": "security_studies", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "security_studies", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about security studies.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_sociology": { "task": "mmlu_sociology", "task_alias": "sociology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "sociology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about sociology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_us_foreign_policy": { "task": "mmlu_us_foreign_policy", "task_alias": "us_foreign_policy", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "us_foreign_policy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_virology": { "task": "mmlu_virology", "task_alias": "virology", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "virology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about virology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_world_religions": { "task": "mmlu_world_religions", "task_alias": "world_religions", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "world_religions", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about world religions.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } } }, "versions": { "arc_challenge": 1.0, "arc_easy": 1.0, "mmlu": 2, "mmlu_abstract_algebra": 1.0, "mmlu_anatomy": 1.0, "mmlu_astronomy": 1.0, "mmlu_business_ethics": 1.0, "mmlu_clinical_knowledge": 1.0, "mmlu_college_biology": 1.0, "mmlu_college_chemistry": 1.0, "mmlu_college_computer_science": 1.0, "mmlu_college_mathematics": 1.0, "mmlu_college_medicine": 1.0, "mmlu_college_physics": 1.0, "mmlu_computer_security": 1.0, "mmlu_conceptual_physics": 1.0, "mmlu_econometrics": 1.0, "mmlu_electrical_engineering": 1.0, "mmlu_elementary_mathematics": 1.0, "mmlu_formal_logic": 1.0, "mmlu_global_facts": 1.0, "mmlu_high_school_biology": 1.0, "mmlu_high_school_chemistry": 1.0, "mmlu_high_school_computer_science": 1.0, "mmlu_high_school_european_history": 1.0, "mmlu_high_school_geography": 1.0, "mmlu_high_school_government_and_politics": 1.0, "mmlu_high_school_macroeconomics": 1.0, "mmlu_high_school_mathematics": 1.0, "mmlu_high_school_microeconomics": 1.0, "mmlu_high_school_physics": 1.0, "mmlu_high_school_psychology": 1.0, "mmlu_high_school_statistics": 1.0, "mmlu_high_school_us_history": 1.0, "mmlu_high_school_world_history": 1.0, "mmlu_human_aging": 1.0, "mmlu_human_sexuality": 1.0, "mmlu_humanities": 2, "mmlu_international_law": 1.0, "mmlu_jurisprudence": 1.0, "mmlu_logical_fallacies": 1.0, "mmlu_machine_learning": 1.0, "mmlu_management": 1.0, "mmlu_marketing": 1.0, "mmlu_medical_genetics": 1.0, "mmlu_miscellaneous": 1.0, "mmlu_moral_disputes": 1.0, "mmlu_moral_scenarios": 1.0, "mmlu_nutrition": 1.0, "mmlu_other": 2, "mmlu_philosophy": 1.0, "mmlu_prehistory": 1.0, "mmlu_professional_accounting": 1.0, "mmlu_professional_law": 1.0, "mmlu_professional_medicine": 1.0, "mmlu_professional_psychology": 1.0, "mmlu_public_relations": 1.0, "mmlu_security_studies": 1.0, "mmlu_social_sciences": 2, "mmlu_sociology": 1.0, "mmlu_stem": 2, "mmlu_us_foreign_policy": 1.0, "mmlu_virology": 1.0, "mmlu_world_religions": 1.0 }, "n-shot": { "arc_challenge": 0, "arc_easy": 0, "mmlu_abstract_algebra": 0, "mmlu_anatomy": 0, "mmlu_astronomy": 0, "mmlu_business_ethics": 0, "mmlu_clinical_knowledge": 0, "mmlu_college_biology": 0, "mmlu_college_chemistry": 0, "mmlu_college_computer_science": 0, "mmlu_college_mathematics": 0, "mmlu_college_medicine": 0, "mmlu_college_physics": 0, "mmlu_computer_security": 0, "mmlu_conceptual_physics": 0, "mmlu_econometrics": 0, "mmlu_electrical_engineering": 0, "mmlu_elementary_mathematics": 0, "mmlu_formal_logic": 0, "mmlu_global_facts": 0, "mmlu_high_school_biology": 0, "mmlu_high_school_chemistry": 0, "mmlu_high_school_computer_science": 0, "mmlu_high_school_european_history": 0, "mmlu_high_school_geography": 0, "mmlu_high_school_government_and_politics": 0, "mmlu_high_school_macroeconomics": 0, "mmlu_high_school_mathematics": 0, "mmlu_high_school_microeconomics": 0, "mmlu_high_school_physics": 0, "mmlu_high_school_psychology": 0, "mmlu_high_school_statistics": 0, "mmlu_high_school_us_history": 0, "mmlu_high_school_world_history": 0, "mmlu_human_aging": 0, "mmlu_human_sexuality": 0, "mmlu_international_law": 0, "mmlu_jurisprudence": 0, "mmlu_logical_fallacies": 0, "mmlu_machine_learning": 0, "mmlu_management": 0, "mmlu_marketing": 0, "mmlu_medical_genetics": 0, "mmlu_miscellaneous": 0, "mmlu_moral_disputes": 0, "mmlu_moral_scenarios": 0, "mmlu_nutrition": 0, "mmlu_philosophy": 0, "mmlu_prehistory": 0, "mmlu_professional_accounting": 0, "mmlu_professional_law": 0, "mmlu_professional_medicine": 0, "mmlu_professional_psychology": 0, "mmlu_public_relations": 0, "mmlu_security_studies": 0, "mmlu_sociology": 0, "mmlu_us_foreign_policy": 0, "mmlu_virology": 0, "mmlu_world_religions": 0 }, "higher_is_better": { "arc_challenge": { "acc": true, "acc_norm": true }, "arc_easy": { "acc": true, "acc_norm": true }, "mmlu": { "acc": true }, "mmlu_abstract_algebra": { "acc": true }, "mmlu_anatomy": { "acc": true }, "mmlu_astronomy": { "acc": true }, "mmlu_business_ethics": { "acc": true }, "mmlu_clinical_knowledge": { "acc": true }, "mmlu_college_biology": { "acc": true }, "mmlu_college_chemistry": { "acc": true }, "mmlu_college_computer_science": { "acc": true }, "mmlu_college_mathematics": { "acc": true }, "mmlu_college_medicine": { "acc": true }, "mmlu_college_physics": { "acc": true }, "mmlu_computer_security": { "acc": true }, "mmlu_conceptual_physics": { "acc": true }, "mmlu_econometrics": { "acc": true }, "mmlu_electrical_engineering": { "acc": true }, "mmlu_elementary_mathematics": { "acc": true }, "mmlu_formal_logic": { "acc": true }, "mmlu_global_facts": { "acc": true }, "mmlu_high_school_biology": { "acc": true }, "mmlu_high_school_chemistry": { "acc": true }, "mmlu_high_school_computer_science": { "acc": true }, "mmlu_high_school_european_history": { "acc": true }, "mmlu_high_school_geography": { "acc": true }, "mmlu_high_school_government_and_politics": { "acc": true }, "mmlu_high_school_macroeconomics": { "acc": true }, "mmlu_high_school_mathematics": { "acc": true }, "mmlu_high_school_microeconomics": { "acc": true }, "mmlu_high_school_physics": { "acc": true }, "mmlu_high_school_psychology": { "acc": true }, "mmlu_high_school_statistics": { "acc": true }, "mmlu_high_school_us_history": { "acc": true }, "mmlu_high_school_world_history": { "acc": true }, "mmlu_human_aging": { "acc": true }, "mmlu_human_sexuality": { "acc": true }, "mmlu_humanities": { "acc": true }, "mmlu_international_law": { "acc": true }, "mmlu_jurisprudence": { "acc": true }, "mmlu_logical_fallacies": { "acc": true }, "mmlu_machine_learning": { "acc": true }, "mmlu_management": { "acc": true }, "mmlu_marketing": { "acc": true }, "mmlu_medical_genetics": { "acc": true }, "mmlu_miscellaneous": { "acc": true }, "mmlu_moral_disputes": { "acc": true }, "mmlu_moral_scenarios": { "acc": true }, "mmlu_nutrition": { "acc": true }, "mmlu_other": { "acc": true }, "mmlu_philosophy": { "acc": true }, "mmlu_prehistory": { "acc": true }, "mmlu_professional_accounting": { "acc": true }, "mmlu_professional_law": { "acc": true }, "mmlu_professional_medicine": { "acc": true }, "mmlu_professional_psychology": { "acc": true }, "mmlu_public_relations": { "acc": true }, "mmlu_security_studies": { "acc": true }, "mmlu_social_sciences": { "acc": true }, "mmlu_sociology": { "acc": true }, "mmlu_stem": { "acc": true }, "mmlu_us_foreign_policy": { "acc": true }, "mmlu_virology": { "acc": true }, "mmlu_world_religions": { "acc": true } }, "n-samples": { "arc_challenge": { "original": 1172, "effective": 1172 }, "arc_easy": { "original": 2376, "effective": 2376 }, "mmlu_abstract_algebra": { "original": 100, "effective": 100 }, "mmlu_anatomy": { "original": 135, "effective": 135 }, "mmlu_astronomy": { "original": 152, "effective": 152 }, "mmlu_college_biology": { "original": 144, "effective": 144 }, "mmlu_college_chemistry": { "original": 100, "effective": 100 }, "mmlu_college_computer_science": { "original": 100, "effective": 100 }, "mmlu_college_mathematics": { "original": 100, "effective": 100 }, "mmlu_college_physics": { "original": 102, "effective": 102 }, "mmlu_computer_security": { "original": 100, "effective": 100 }, "mmlu_conceptual_physics": { "original": 235, "effective": 235 }, "mmlu_electrical_engineering": { "original": 145, "effective": 145 }, "mmlu_elementary_mathematics": { "original": 378, "effective": 378 }, "mmlu_high_school_biology": { "original": 310, "effective": 310 }, "mmlu_high_school_chemistry": { "original": 203, "effective": 203 }, "mmlu_high_school_computer_science": { "original": 100, "effective": 100 }, "mmlu_high_school_mathematics": { "original": 270, "effective": 270 }, "mmlu_high_school_physics": { "original": 151, "effective": 151 }, "mmlu_high_school_statistics": { "original": 216, "effective": 216 }, "mmlu_machine_learning": { "original": 112, "effective": 112 }, "mmlu_business_ethics": { "original": 100, "effective": 100 }, "mmlu_clinical_knowledge": { "original": 265, "effective": 265 }, "mmlu_college_medicine": { "original": 173, "effective": 173 }, "mmlu_global_facts": { "original": 100, "effective": 100 }, "mmlu_human_aging": { "original": 223, "effective": 223 }, "mmlu_management": { "original": 103, "effective": 103 }, "mmlu_marketing": { "original": 234, "effective": 234 }, "mmlu_medical_genetics": { "original": 100, "effective": 100 }, "mmlu_miscellaneous": { "original": 783, "effective": 783 }, "mmlu_nutrition": { "original": 306, "effective": 306 }, "mmlu_professional_accounting": { "original": 282, "effective": 282 }, "mmlu_professional_medicine": { "original": 272, "effective": 272 }, "mmlu_virology": { "original": 166, "effective": 166 }, "mmlu_econometrics": { "original": 114, "effective": 114 }, "mmlu_high_school_geography": { "original": 198, "effective": 198 }, "mmlu_high_school_government_and_politics": { "original": 193, "effective": 193 }, "mmlu_high_school_macroeconomics": { "original": 390, "effective": 390 }, "mmlu_high_school_microeconomics": { "original": 238, "effective": 238 }, "mmlu_high_school_psychology": { "original": 545, "effective": 545 }, "mmlu_human_sexuality": { "original": 131, "effective": 131 }, "mmlu_professional_psychology": { "original": 612, "effective": 612 }, "mmlu_public_relations": { "original": 110, "effective": 110 }, "mmlu_security_studies": { "original": 245, "effective": 245 }, "mmlu_sociology": { "original": 201, "effective": 201 }, "mmlu_us_foreign_policy": { "original": 100, "effective": 100 }, "mmlu_formal_logic": { "original": 126, "effective": 126 }, "mmlu_high_school_european_history": { "original": 165, "effective": 165 }, "mmlu_high_school_us_history": { "original": 204, "effective": 204 }, "mmlu_high_school_world_history": { "original": 237, "effective": 237 }, "mmlu_international_law": { "original": 121, "effective": 121 }, "mmlu_jurisprudence": { "original": 108, "effective": 108 }, "mmlu_logical_fallacies": { "original": 163, "effective": 163 }, "mmlu_moral_disputes": { "original": 346, "effective": 346 }, "mmlu_moral_scenarios": { "original": 895, "effective": 895 }, "mmlu_philosophy": { "original": 311, "effective": 311 }, "mmlu_prehistory": { "original": 324, "effective": 324 }, "mmlu_professional_law": { "original": 1534, "effective": 1534 }, "mmlu_world_religions": { "original": 171, "effective": 171 } }, "config": { "model": "/home/jovyan/.cache/huggingface/models--microsoft--Phi-mini-MoE-instruct/snapshots/62d7eed72cc0420fc924c800721b1401d628395c", "model_args": null, "model_num_parameters": 7647632704, "model_dtype": "torch.bfloat16", "model_revision": "main", "model_sha": "", "batch_size": "auto", "batch_sizes": [ 64 ], "device": null, "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "0cffc7f", "date": 1763176280.2870958, "pretty_env_info": "PyTorch version: 2.7.1+cu118\nIs debug build: False\nCUDA used to build PyTorch: 11.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.9\nLibc version: glibc-2.35\n\nPython version: 3.10.13 | packaged by conda-forge | (main, Oct 26 2023, 18:07:37) [GCC 12.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-138-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 11.8.89\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA A100 80GB PCIe\nNvidia driver version: 555.42.06\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7443 24-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 1\nFrequency boost: enabled\nCPU max MHz: 4035.6440\nCPU min MHz: 1500.0000\nBogoMIPS: 5699.48\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload vgif v_spec_ctrl umip pku ospke vaes vpclmulqdq rdpid overflow_recov succor smca fsrm\nVirtualization: AMD-V\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 256 MiB (8 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.2.6\n[pip3] nvidia-cublas-cu11==11.11.3.6\n[pip3] nvidia-cuda-cupti-cu11==11.8.87\n[pip3] nvidia-cuda-nvrtc-cu11==11.8.89\n[pip3] nvidia-cuda-runtime-cu11==11.8.89\n[pip3] nvidia-cudnn-cu11==9.1.0.70\n[pip3] nvidia-cufft-cu11==10.9.0.58\n[pip3] nvidia-curand-cu11==10.3.0.86\n[pip3] nvidia-cusolver-cu11==11.4.1.48\n[pip3] nvidia-cusparse-cu11==11.7.5.86\n[pip3] nvidia-nccl-cu11==2.21.5\n[pip3] nvidia-nvtx-cu11==11.8.86\n[pip3] torch==2.7.1+cu118\n[pip3] triton==3.3.1\n[conda] nomkl 1.0 h5ca1d4c_0 conda-forge\n[conda] numpy 1.26.2 py310hb13e2d6_0 conda-forge\n[conda] nvidia-cublas-cu11 11.11.3.6 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu11 11.8.87 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu11 11.8.89 pypi_0 pypi\n[conda] nvidia-cudnn-cu11 8.7.0.84 pypi_0 pypi\n[conda] nvidia-cufft-cu11 10.9.0.58 pypi_0 pypi\n[conda] nvidia-curand-cu11 10.3.0.86 pypi_0 pypi\n[conda] nvidia-cusolver-cu11 11.4.1.48 pypi_0 pypi\n[conda] nvidia-cusparse-cu11 11.7.5.86 pypi_0 pypi\n[conda] nvidia-nccl-cu11 2.16.5 pypi_0 pypi\n[conda] pytorch-lightning 2.1.2 pypi_0 pypi\n[conda] torch 2.1.1+cu118 pypi_0 pypi\n[conda] torchaudio 2.1.1+cu118 pypi_0 pypi\n[conda] torchdata 0.7.1+cpu pypi_0 pypi\n[conda] torchmetrics 1.2.0 pypi_0 pypi\n[conda] torchtext 0.16.1+cpu pypi_0 pypi\n[conda] torchvision 0.16.1+cu118 pypi_0 pypi\n[conda] triton 2.1.0 pypi_0 pypi", "transformers_version": "4.55.0", "lm_eval_version": "0.4.9.1", "upper_git_hash": null, "tokenizer_pad_token": [ "<|endoftext|>", "32000" ], "tokenizer_eos_token": [ "<|endoftext|>", "32000" ], "tokenizer_bos_token": [ "", "1" ], "eot_token_id": 32000, "max_length": 4096 }, "compressed_model_results": { "results": { "arc_challenge": { "alias": "arc_challenge", "acc,none": 0.5247440273037542, "acc_stderr,none": 0.014593487694937738, "acc_norm,none": 0.5793515358361775, "acc_norm_stderr,none": 0.014426211252508404 }, "arc_easy": { "alias": "arc_easy", "acc,none": 0.8131313131313131, "acc_stderr,none": 0.007998652011081697, "acc_norm,none": 0.7954545454545454, "acc_norm_stderr,none": 0.008276958800002999 }, "mmlu": { "acc,none": 0.6536105967810854, "acc_stderr,none": 0.0038123134918508976, "alias": "mmlu" }, "mmlu_humanities": { "acc,none": 0.5957492029755579, "acc_stderr,none": 0.006759316550944608, "alias": " - humanities" }, "mmlu_formal_logic": { "alias": " - formal_logic", "acc,none": 0.5158730158730159, "acc_stderr,none": 0.044698818540726076 }, "mmlu_high_school_european_history": { "alias": " - high_school_european_history", "acc,none": 0.7696969696969697, "acc_stderr,none": 0.03287666758603489 }, "mmlu_high_school_us_history": { "alias": " - high_school_us_history", "acc,none": 0.8480392156862745, "acc_stderr,none": 0.025195658428931792 }, "mmlu_high_school_world_history": { "alias": " - high_school_world_history", "acc,none": 0.7932489451476793, "acc_stderr,none": 0.026361651668389094 }, "mmlu_international_law": { "alias": " - international_law", "acc,none": 0.8016528925619835, "acc_stderr,none": 0.03640118271990947 }, "mmlu_jurisprudence": { "alias": " - jurisprudence", "acc,none": 0.7962962962962963, "acc_stderr,none": 0.03893542518824847 }, "mmlu_logical_fallacies": { "alias": " - logical_fallacies", "acc,none": 0.8098159509202454, "acc_stderr,none": 0.03083349114628123 }, "mmlu_moral_disputes": { "alias": " - moral_disputes", "acc,none": 0.7167630057803468, "acc_stderr,none": 0.02425790170532337 }, "mmlu_moral_scenarios": { "alias": " - moral_scenarios", "acc,none": 0.376536312849162, "acc_stderr,none": 0.016204672385106596 }, "mmlu_philosophy": { "alias": " - philosophy", "acc,none": 0.707395498392283, "acc_stderr,none": 0.02583989833487798 }, "mmlu_prehistory": { "alias": " - prehistory", "acc,none": 0.7191358024691358, "acc_stderr,none": 0.025006469755799208 }, "mmlu_professional_law": { "alias": " - professional_law", "acc,none": 0.4954367666232073, "acc_stderr,none": 0.012769704263117531 }, "mmlu_world_religions": { "alias": " - world_religions", "acc,none": 0.8011695906432749, "acc_stderr,none": 0.03061111655743253 }, "mmlu_other": { "acc,none": 0.7071129707112971, "acc_stderr,none": 0.00792634679462611, "alias": " - other" }, "mmlu_business_ethics": { "alias": " - business_ethics", "acc,none": 0.71, "acc_stderr,none": 0.045604802157206845 }, "mmlu_clinical_knowledge": { "alias": " - clinical_knowledge", "acc,none": 0.7283018867924528, "acc_stderr,none": 0.027377706624670713 }, "mmlu_college_medicine": { "alias": " - college_medicine", "acc,none": 0.630057803468208, "acc_stderr,none": 0.0368122963339432 }, "mmlu_global_facts": { "alias": " - global_facts", "acc,none": 0.46, "acc_stderr,none": 0.05009082659620333 }, "mmlu_human_aging": { "alias": " - human_aging", "acc,none": 0.6681614349775785, "acc_stderr,none": 0.03160295143776679 }, "mmlu_management": { "alias": " - management", "acc,none": 0.7475728155339806, "acc_stderr,none": 0.04301250399690879 }, "mmlu_marketing": { "alias": " - marketing", "acc,none": 0.8974358974358975, "acc_stderr,none": 0.019875655027867457 }, "mmlu_medical_genetics": { "alias": " - medical_genetics", "acc,none": 0.74, "acc_stderr,none": 0.04408440022768078 }, "mmlu_miscellaneous": { "alias": " - miscellaneous", "acc,none": 0.7956577266922095, "acc_stderr,none": 0.0144191239809319 }, "mmlu_nutrition": { "alias": " - nutrition", "acc,none": 0.7287581699346405, "acc_stderr,none": 0.025457756696667864 }, "mmlu_professional_accounting": { "alias": " - professional_accounting", "acc,none": 0.5035460992907801, "acc_stderr,none": 0.02982674915328092 }, "mmlu_professional_medicine": { "alias": " - professional_medicine", "acc,none": 0.7058823529411765, "acc_stderr,none": 0.0276784686421447 }, "mmlu_virology": { "alias": " - virology", "acc,none": 0.5301204819277109, "acc_stderr,none": 0.03885425420866766 }, "mmlu_social_sciences": { "acc,none": 0.7630809229769255, "acc_stderr,none": 0.007521922102925205, "alias": " - social sciences" }, "mmlu_econometrics": { "alias": " - econometrics", "acc,none": 0.5087719298245614, "acc_stderr,none": 0.04702880432049615 }, "mmlu_high_school_geography": { "alias": " - high_school_geography", "acc,none": 0.8181818181818182, "acc_stderr,none": 0.0274796030105388 }, "mmlu_high_school_government_and_politics": { "alias": " - high_school_government_and_politics", "acc,none": 0.8860103626943006, "acc_stderr,none": 0.022935144053919436 }, "mmlu_high_school_macroeconomics": { "alias": " - high_school_macroeconomics", "acc,none": 0.6846153846153846, "acc_stderr,none": 0.02355964698318994 }, "mmlu_high_school_microeconomics": { "alias": " - high_school_microeconomics", "acc,none": 0.8277310924369747, "acc_stderr,none": 0.024528664971305427 }, "mmlu_high_school_psychology": { "alias": " - high_school_psychology", "acc,none": 0.8477064220183487, "acc_stderr,none": 0.015405084393157069 }, "mmlu_human_sexuality": { "alias": " - human_sexuality", "acc,none": 0.7557251908396947, "acc_stderr,none": 0.037683359597287434 }, "mmlu_professional_psychology": { "alias": " - professional_psychology", "acc,none": 0.6879084967320261, "acc_stderr,none": 0.01874501120127766 }, "mmlu_public_relations": { "alias": " - public_relations", "acc,none": 0.7090909090909091, "acc_stderr,none": 0.04350271442923243 }, "mmlu_security_studies": { "alias": " - security_studies", "acc,none": 0.7714285714285715, "acc_stderr,none": 0.026882144922307744 }, "mmlu_sociology": { "alias": " - sociology", "acc,none": 0.8059701492537313, "acc_stderr,none": 0.027962677604768896 }, "mmlu_us_foreign_policy": { "alias": " - us_foreign_policy", "acc,none": 0.82, "acc_stderr,none": 0.03861229196653694 }, "mmlu_stem": { "acc,none": 0.5803996194100857, "acc_stderr,none": 0.008464141586351462, "alias": " - stem" }, "mmlu_abstract_algebra": { "alias": " - abstract_algebra", "acc,none": 0.42, "acc_stderr,none": 0.049604496374885836 }, "mmlu_anatomy": { "alias": " - anatomy", "acc,none": 0.6148148148148148, "acc_stderr,none": 0.042039210401562783 }, "mmlu_astronomy": { "alias": " - astronomy", "acc,none": 0.743421052631579, "acc_stderr,none": 0.035541803680256896 }, "mmlu_college_biology": { "alias": " - college_biology", "acc,none": 0.7986111111111112, "acc_stderr,none": 0.03353647469713839 }, "mmlu_college_chemistry": { "alias": " - college_chemistry", "acc,none": 0.47, "acc_stderr,none": 0.05016135580465919 }, "mmlu_college_computer_science": { "alias": " - college_computer_science", "acc,none": 0.53, "acc_stderr,none": 0.050161355804659205 }, "mmlu_college_mathematics": { "alias": " - college_mathematics", "acc,none": 0.4, "acc_stderr,none": 0.04923659639173309 }, "mmlu_college_physics": { "alias": " - college_physics", "acc,none": 0.37254901960784315, "acc_stderr,none": 0.04810840148082634 }, "mmlu_computer_security": { "alias": " - computer_security", "acc,none": 0.73, "acc_stderr,none": 0.044619604333847394 }, "mmlu_conceptual_physics": { "alias": " - conceptual_physics", "acc,none": 0.6382978723404256, "acc_stderr,none": 0.03141082197596241 }, "mmlu_electrical_engineering": { "alias": " - electrical_engineering", "acc,none": 0.593103448275862, "acc_stderr,none": 0.04093793981266236 }, "mmlu_elementary_mathematics": { "alias": " - elementary_mathematics", "acc,none": 0.5661375661375662, "acc_stderr,none": 0.02552503438247489 }, "mmlu_high_school_biology": { "alias": " - high_school_biology", "acc,none": 0.7935483870967742, "acc_stderr,none": 0.023025899617188744 }, "mmlu_high_school_chemistry": { "alias": " - high_school_chemistry", "acc,none": 0.5566502463054187, "acc_stderr,none": 0.03495334582162934 }, "mmlu_high_school_computer_science": { "alias": " - high_school_computer_science", "acc,none": 0.75, "acc_stderr,none": 0.04351941398892446 }, "mmlu_high_school_mathematics": { "alias": " - high_school_mathematics", "acc,none": 0.36666666666666664, "acc_stderr,none": 0.029381620726465073 }, "mmlu_high_school_physics": { "alias": " - high_school_physics", "acc,none": 0.46357615894039733, "acc_stderr,none": 0.04071636065944216 }, "mmlu_high_school_statistics": { "alias": " - high_school_statistics", "acc,none": 0.5787037037037037, "acc_stderr,none": 0.033674621388960775 }, "mmlu_machine_learning": { "alias": " - machine_learning", "acc,none": 0.42857142857142855, "acc_stderr,none": 0.04697113923010212 } }, "groups": { "mmlu": { "acc,none": 0.6536105967810854, "acc_stderr,none": 0.0038123134918508976, "alias": "mmlu" }, "mmlu_humanities": { "acc,none": 0.5957492029755579, "acc_stderr,none": 0.006759316550944608, "alias": " - humanities" }, "mmlu_other": { "acc,none": 0.7071129707112971, "acc_stderr,none": 0.00792634679462611, "alias": " - other" }, "mmlu_social_sciences": { "acc,none": 0.7630809229769255, "acc_stderr,none": 0.007521922102925205, "alias": " - social sciences" }, "mmlu_stem": { "acc,none": 0.5803996194100857, "acc_stderr,none": 0.008464141586351462, "alias": " - stem" } }, "group_subtasks": { "mmlu_humanities": [ "mmlu_formal_logic", "mmlu_high_school_european_history", "mmlu_high_school_us_history", "mmlu_high_school_world_history", "mmlu_international_law", "mmlu_jurisprudence", "mmlu_logical_fallacies", "mmlu_moral_disputes", "mmlu_moral_scenarios", "mmlu_philosophy", "mmlu_prehistory", "mmlu_professional_law", "mmlu_world_religions" ], "mmlu_social_sciences": [ "mmlu_econometrics", "mmlu_high_school_geography", "mmlu_high_school_government_and_politics", "mmlu_high_school_macroeconomics", "mmlu_high_school_microeconomics", "mmlu_high_school_psychology", "mmlu_human_sexuality", "mmlu_professional_psychology", "mmlu_public_relations", "mmlu_security_studies", "mmlu_sociology", "mmlu_us_foreign_policy" ], "mmlu_other": [ "mmlu_business_ethics", "mmlu_clinical_knowledge", "mmlu_college_medicine", "mmlu_global_facts", "mmlu_human_aging", "mmlu_management", "mmlu_marketing", "mmlu_medical_genetics", "mmlu_miscellaneous", "mmlu_nutrition", "mmlu_professional_accounting", "mmlu_professional_medicine", "mmlu_virology" ], "mmlu_stem": [ "mmlu_abstract_algebra", "mmlu_anatomy", "mmlu_astronomy", "mmlu_college_biology", "mmlu_college_chemistry", "mmlu_college_computer_science", "mmlu_college_mathematics", "mmlu_college_physics", "mmlu_computer_security", "mmlu_conceptual_physics", "mmlu_electrical_engineering", "mmlu_elementary_mathematics", "mmlu_high_school_biology", "mmlu_high_school_chemistry", "mmlu_high_school_computer_science", "mmlu_high_school_mathematics", "mmlu_high_school_physics", "mmlu_high_school_statistics", "mmlu_machine_learning" ], "mmlu": [ "mmlu_stem", "mmlu_other", "mmlu_social_sciences", "mmlu_humanities" ], "arc_easy": [], "arc_challenge": [] }, "configs": { "arc_challenge": { "task": "arc_challenge", "tag": [ "ai2_arc" ], "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Challenge", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0 } }, "arc_easy": { "task": "arc_easy", "tag": [ "ai2_arc" ], "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Easy", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0 } }, "mmlu_abstract_algebra": { "task": "mmlu_abstract_algebra", "task_alias": "abstract_algebra", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_anatomy": { "task": "mmlu_anatomy", "task_alias": "anatomy", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "anatomy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_astronomy": { "task": "mmlu_astronomy", "task_alias": "astronomy", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "astronomy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_business_ethics": { "task": "mmlu_business_ethics", "task_alias": "business_ethics", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "business_ethics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_clinical_knowledge": { "task": "mmlu_clinical_knowledge", "task_alias": "clinical_knowledge", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "clinical_knowledge", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_biology": { "task": "mmlu_college_biology", "task_alias": "college_biology", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_biology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college biology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_chemistry": { "task": "mmlu_college_chemistry", "task_alias": "college_chemistry", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_chemistry", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_computer_science": { "task": "mmlu_college_computer_science", "task_alias": "college_computer_science", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_computer_science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_mathematics": { "task": "mmlu_college_mathematics", "task_alias": "college_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_medicine": { "task": "mmlu_college_medicine", "task_alias": "college_medicine", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_medicine", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_college_physics": { "task": "mmlu_college_physics", "task_alias": "college_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_computer_security": { "task": "mmlu_computer_security", "task_alias": "computer_security", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "computer_security", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about computer security.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_conceptual_physics": { "task": "mmlu_conceptual_physics", "task_alias": "conceptual_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "conceptual_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_econometrics": { "task": "mmlu_econometrics", "task_alias": "econometrics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "econometrics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_electrical_engineering": { "task": "mmlu_electrical_engineering", "task_alias": "electrical_engineering", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "electrical_engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_elementary_mathematics": { "task": "mmlu_elementary_mathematics", "task_alias": "elementary_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "elementary_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_formal_logic": { "task": "mmlu_formal_logic", "task_alias": "formal_logic", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "formal_logic", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_global_facts": { "task": "mmlu_global_facts", "task_alias": "global_facts", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "global_facts", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about global facts.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_biology": { "task": "mmlu_high_school_biology", "task_alias": "high_school_biology", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_biology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_chemistry": { "task": "mmlu_high_school_chemistry", "task_alias": "high_school_chemistry", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_chemistry", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_computer_science": { "task": "mmlu_high_school_computer_science", "task_alias": "high_school_computer_science", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_computer_science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_european_history": { "task": "mmlu_high_school_european_history", "task_alias": "high_school_european_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_european_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_geography": { "task": "mmlu_high_school_geography", "task_alias": "high_school_geography", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_geography", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_government_and_politics": { "task": "mmlu_high_school_government_and_politics", "task_alias": "high_school_government_and_politics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_government_and_politics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_macroeconomics": { "task": "mmlu_high_school_macroeconomics", "task_alias": "high_school_macroeconomics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_macroeconomics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_mathematics": { "task": "mmlu_high_school_mathematics", "task_alias": "high_school_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_microeconomics": { "task": "mmlu_high_school_microeconomics", "task_alias": "high_school_microeconomics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_microeconomics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_physics": { "task": "mmlu_high_school_physics", "task_alias": "high_school_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_psychology": { "task": "mmlu_high_school_psychology", "task_alias": "high_school_psychology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_psychology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_statistics": { "task": "mmlu_high_school_statistics", "task_alias": "high_school_statistics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_statistics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_us_history": { "task": "mmlu_high_school_us_history", "task_alias": "high_school_us_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_us_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_high_school_world_history": { "task": "mmlu_high_school_world_history", "task_alias": "high_school_world_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_world_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_human_aging": { "task": "mmlu_human_aging", "task_alias": "human_aging", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "human_aging", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about human aging.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_human_sexuality": { "task": "mmlu_human_sexuality", "task_alias": "human_sexuality", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "human_sexuality", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_international_law": { "task": "mmlu_international_law", "task_alias": "international_law", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "international_law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about international law.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_jurisprudence": { "task": "mmlu_jurisprudence", "task_alias": "jurisprudence", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "jurisprudence", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_logical_fallacies": { "task": "mmlu_logical_fallacies", "task_alias": "logical_fallacies", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "logical_fallacies", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_machine_learning": { "task": "mmlu_machine_learning", "task_alias": "machine_learning", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "machine_learning", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_management": { "task": "mmlu_management", "task_alias": "management", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "management", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about management.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_marketing": { "task": "mmlu_marketing", "task_alias": "marketing", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "marketing", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about marketing.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_medical_genetics": { "task": "mmlu_medical_genetics", "task_alias": "medical_genetics", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "medical_genetics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_miscellaneous": { "task": "mmlu_miscellaneous", "task_alias": "miscellaneous", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "miscellaneous", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_moral_disputes": { "task": "mmlu_moral_disputes", "task_alias": "moral_disputes", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "moral_disputes", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_moral_scenarios": { "task": "mmlu_moral_scenarios", "task_alias": "moral_scenarios", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "moral_scenarios", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_nutrition": { "task": "mmlu_nutrition", "task_alias": "nutrition", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "nutrition", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_philosophy": { "task": "mmlu_philosophy", "task_alias": "philosophy", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "philosophy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_prehistory": { "task": "mmlu_prehistory", "task_alias": "prehistory", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "prehistory", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_accounting": { "task": "mmlu_professional_accounting", "task_alias": "professional_accounting", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_accounting", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_law": { "task": "mmlu_professional_law", "task_alias": "professional_law", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional law.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_medicine": { "task": "mmlu_professional_medicine", "task_alias": "professional_medicine", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_medicine", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_professional_psychology": { "task": "mmlu_professional_psychology", "task_alias": "professional_psychology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_psychology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_public_relations": { "task": "mmlu_public_relations", "task_alias": "public_relations", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "public_relations", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about public relations.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_security_studies": { "task": "mmlu_security_studies", "task_alias": "security_studies", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "security_studies", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about security studies.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_sociology": { "task": "mmlu_sociology", "task_alias": "sociology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "sociology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about sociology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_us_foreign_policy": { "task": "mmlu_us_foreign_policy", "task_alias": "us_foreign_policy", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "us_foreign_policy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_virology": { "task": "mmlu_virology", "task_alias": "virology", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "virology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about virology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "mmlu_world_religions": { "task": "mmlu_world_religions", "task_alias": "world_religions", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "world_religions", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about world religions.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } } }, "versions": { "arc_challenge": 1.0, "arc_easy": 1.0, "mmlu": 2, "mmlu_abstract_algebra": 1.0, "mmlu_anatomy": 1.0, "mmlu_astronomy": 1.0, "mmlu_business_ethics": 1.0, "mmlu_clinical_knowledge": 1.0, "mmlu_college_biology": 1.0, "mmlu_college_chemistry": 1.0, "mmlu_college_computer_science": 1.0, "mmlu_college_mathematics": 1.0, "mmlu_college_medicine": 1.0, "mmlu_college_physics": 1.0, "mmlu_computer_security": 1.0, "mmlu_conceptual_physics": 1.0, "mmlu_econometrics": 1.0, "mmlu_electrical_engineering": 1.0, "mmlu_elementary_mathematics": 1.0, "mmlu_formal_logic": 1.0, "mmlu_global_facts": 1.0, "mmlu_high_school_biology": 1.0, "mmlu_high_school_chemistry": 1.0, "mmlu_high_school_computer_science": 1.0, "mmlu_high_school_european_history": 1.0, "mmlu_high_school_geography": 1.0, "mmlu_high_school_government_and_politics": 1.0, "mmlu_high_school_macroeconomics": 1.0, "mmlu_high_school_mathematics": 1.0, "mmlu_high_school_microeconomics": 1.0, "mmlu_high_school_physics": 1.0, "mmlu_high_school_psychology": 1.0, "mmlu_high_school_statistics": 1.0, "mmlu_high_school_us_history": 1.0, "mmlu_high_school_world_history": 1.0, "mmlu_human_aging": 1.0, "mmlu_human_sexuality": 1.0, "mmlu_humanities": 2, "mmlu_international_law": 1.0, "mmlu_jurisprudence": 1.0, "mmlu_logical_fallacies": 1.0, "mmlu_machine_learning": 1.0, "mmlu_management": 1.0, "mmlu_marketing": 1.0, "mmlu_medical_genetics": 1.0, "mmlu_miscellaneous": 1.0, "mmlu_moral_disputes": 1.0, "mmlu_moral_scenarios": 1.0, "mmlu_nutrition": 1.0, "mmlu_other": 2, "mmlu_philosophy": 1.0, "mmlu_prehistory": 1.0, "mmlu_professional_accounting": 1.0, "mmlu_professional_law": 1.0, "mmlu_professional_medicine": 1.0, "mmlu_professional_psychology": 1.0, "mmlu_public_relations": 1.0, "mmlu_security_studies": 1.0, "mmlu_social_sciences": 2, "mmlu_sociology": 1.0, "mmlu_stem": 2, "mmlu_us_foreign_policy": 1.0, "mmlu_virology": 1.0, "mmlu_world_religions": 1.0 }, "n-shot": { "arc_challenge": 0, "arc_easy": 0, "mmlu_abstract_algebra": 0, "mmlu_anatomy": 0, "mmlu_astronomy": 0, "mmlu_business_ethics": 0, "mmlu_clinical_knowledge": 0, "mmlu_college_biology": 0, "mmlu_college_chemistry": 0, "mmlu_college_computer_science": 0, "mmlu_college_mathematics": 0, "mmlu_college_medicine": 0, "mmlu_college_physics": 0, "mmlu_computer_security": 0, "mmlu_conceptual_physics": 0, "mmlu_econometrics": 0, "mmlu_electrical_engineering": 0, "mmlu_elementary_mathematics": 0, "mmlu_formal_logic": 0, "mmlu_global_facts": 0, "mmlu_high_school_biology": 0, "mmlu_high_school_chemistry": 0, "mmlu_high_school_computer_science": 0, "mmlu_high_school_european_history": 0, "mmlu_high_school_geography": 0, "mmlu_high_school_government_and_politics": 0, "mmlu_high_school_macroeconomics": 0, "mmlu_high_school_mathematics": 0, "mmlu_high_school_microeconomics": 0, "mmlu_high_school_physics": 0, "mmlu_high_school_psychology": 0, "mmlu_high_school_statistics": 0, "mmlu_high_school_us_history": 0, "mmlu_high_school_world_history": 0, "mmlu_human_aging": 0, "mmlu_human_sexuality": 0, "mmlu_international_law": 0, "mmlu_jurisprudence": 0, "mmlu_logical_fallacies": 0, "mmlu_machine_learning": 0, "mmlu_management": 0, "mmlu_marketing": 0, "mmlu_medical_genetics": 0, "mmlu_miscellaneous": 0, "mmlu_moral_disputes": 0, "mmlu_moral_scenarios": 0, "mmlu_nutrition": 0, "mmlu_philosophy": 0, "mmlu_prehistory": 0, "mmlu_professional_accounting": 0, "mmlu_professional_law": 0, "mmlu_professional_medicine": 0, "mmlu_professional_psychology": 0, "mmlu_public_relations": 0, "mmlu_security_studies": 0, "mmlu_sociology": 0, "mmlu_us_foreign_policy": 0, "mmlu_virology": 0, "mmlu_world_religions": 0 }, "higher_is_better": { "arc_challenge": { "acc": true, "acc_norm": true }, "arc_easy": { "acc": true, "acc_norm": true }, "mmlu": { "acc": true }, "mmlu_abstract_algebra": { "acc": true }, "mmlu_anatomy": { "acc": true }, "mmlu_astronomy": { "acc": true }, "mmlu_business_ethics": { "acc": true }, "mmlu_clinical_knowledge": { "acc": true }, "mmlu_college_biology": { "acc": true }, "mmlu_college_chemistry": { "acc": true }, "mmlu_college_computer_science": { "acc": true }, "mmlu_college_mathematics": { "acc": true }, "mmlu_college_medicine": { "acc": true }, "mmlu_college_physics": { "acc": true }, "mmlu_computer_security": { "acc": true }, "mmlu_conceptual_physics": { "acc": true }, "mmlu_econometrics": { "acc": true }, "mmlu_electrical_engineering": { "acc": true }, "mmlu_elementary_mathematics": { "acc": true }, "mmlu_formal_logic": { "acc": true }, "mmlu_global_facts": { "acc": true }, "mmlu_high_school_biology": { "acc": true }, "mmlu_high_school_chemistry": { "acc": true }, "mmlu_high_school_computer_science": { "acc": true }, "mmlu_high_school_european_history": { "acc": true }, "mmlu_high_school_geography": { "acc": true }, "mmlu_high_school_government_and_politics": { "acc": true }, "mmlu_high_school_macroeconomics": { "acc": true }, "mmlu_high_school_mathematics": { "acc": true }, "mmlu_high_school_microeconomics": { "acc": true }, "mmlu_high_school_physics": { "acc": true }, "mmlu_high_school_psychology": { "acc": true }, "mmlu_high_school_statistics": { "acc": true }, "mmlu_high_school_us_history": { "acc": true }, "mmlu_high_school_world_history": { "acc": true }, "mmlu_human_aging": { "acc": true }, "mmlu_human_sexuality": { "acc": true }, "mmlu_humanities": { "acc": true }, "mmlu_international_law": { "acc": true }, "mmlu_jurisprudence": { "acc": true }, "mmlu_logical_fallacies": { "acc": true }, "mmlu_machine_learning": { "acc": true }, "mmlu_management": { "acc": true }, "mmlu_marketing": { "acc": true }, "mmlu_medical_genetics": { "acc": true }, "mmlu_miscellaneous": { "acc": true }, "mmlu_moral_disputes": { "acc": true }, "mmlu_moral_scenarios": { "acc": true }, "mmlu_nutrition": { "acc": true }, "mmlu_other": { "acc": true }, "mmlu_philosophy": { "acc": true }, "mmlu_prehistory": { "acc": true }, "mmlu_professional_accounting": { "acc": true }, "mmlu_professional_law": { "acc": true }, "mmlu_professional_medicine": { "acc": true }, "mmlu_professional_psychology": { "acc": true }, "mmlu_public_relations": { "acc": true }, "mmlu_security_studies": { "acc": true }, "mmlu_social_sciences": { "acc": true }, "mmlu_sociology": { "acc": true }, "mmlu_stem": { "acc": true }, "mmlu_us_foreign_policy": { "acc": true }, "mmlu_virology": { "acc": true }, "mmlu_world_religions": { "acc": true } }, "n-samples": { "arc_challenge": { "original": 1172, "effective": 1172 }, "arc_easy": { "original": 2376, "effective": 2376 }, "mmlu_abstract_algebra": { "original": 100, "effective": 100 }, "mmlu_anatomy": { "original": 135, "effective": 135 }, "mmlu_astronomy": { "original": 152, "effective": 152 }, "mmlu_college_biology": { "original": 144, "effective": 144 }, "mmlu_college_chemistry": { "original": 100, "effective": 100 }, "mmlu_college_computer_science": { "original": 100, "effective": 100 }, "mmlu_college_mathematics": { "original": 100, "effective": 100 }, "mmlu_college_physics": { "original": 102, "effective": 102 }, "mmlu_computer_security": { "original": 100, "effective": 100 }, "mmlu_conceptual_physics": { "original": 235, "effective": 235 }, "mmlu_electrical_engineering": { "original": 145, "effective": 145 }, "mmlu_elementary_mathematics": { "original": 378, "effective": 378 }, "mmlu_high_school_biology": { "original": 310, "effective": 310 }, "mmlu_high_school_chemistry": { "original": 203, "effective": 203 }, "mmlu_high_school_computer_science": { "original": 100, "effective": 100 }, "mmlu_high_school_mathematics": { "original": 270, "effective": 270 }, "mmlu_high_school_physics": { "original": 151, "effective": 151 }, "mmlu_high_school_statistics": { "original": 216, "effective": 216 }, "mmlu_machine_learning": { "original": 112, "effective": 112 }, "mmlu_business_ethics": { "original": 100, "effective": 100 }, "mmlu_clinical_knowledge": { "original": 265, "effective": 265 }, "mmlu_college_medicine": { "original": 173, "effective": 173 }, "mmlu_global_facts": { "original": 100, "effective": 100 }, "mmlu_human_aging": { "original": 223, "effective": 223 }, "mmlu_management": { "original": 103, "effective": 103 }, "mmlu_marketing": { "original": 234, "effective": 234 }, "mmlu_medical_genetics": { "original": 100, "effective": 100 }, "mmlu_miscellaneous": { "original": 783, "effective": 783 }, "mmlu_nutrition": { "original": 306, "effective": 306 }, "mmlu_professional_accounting": { "original": 282, "effective": 282 }, "mmlu_professional_medicine": { "original": 272, "effective": 272 }, "mmlu_virology": { "original": 166, "effective": 166 }, "mmlu_econometrics": { "original": 114, "effective": 114 }, "mmlu_high_school_geography": { "original": 198, "effective": 198 }, "mmlu_high_school_government_and_politics": { "original": 193, "effective": 193 }, "mmlu_high_school_macroeconomics": { "original": 390, "effective": 390 }, "mmlu_high_school_microeconomics": { "original": 238, "effective": 238 }, "mmlu_high_school_psychology": { "original": 545, "effective": 545 }, "mmlu_human_sexuality": { "original": 131, "effective": 131 }, "mmlu_professional_psychology": { "original": 612, "effective": 612 }, "mmlu_public_relations": { "original": 110, "effective": 110 }, "mmlu_security_studies": { "original": 245, "effective": 245 }, "mmlu_sociology": { "original": 201, "effective": 201 }, "mmlu_us_foreign_policy": { "original": 100, "effective": 100 }, "mmlu_formal_logic": { "original": 126, "effective": 126 }, "mmlu_high_school_european_history": { "original": 165, "effective": 165 }, "mmlu_high_school_us_history": { "original": 204, "effective": 204 }, "mmlu_high_school_world_history": { "original": 237, "effective": 237 }, "mmlu_international_law": { "original": 121, "effective": 121 }, "mmlu_jurisprudence": { "original": 108, "effective": 108 }, "mmlu_logical_fallacies": { "original": 163, "effective": 163 }, "mmlu_moral_disputes": { "original": 346, "effective": 346 }, "mmlu_moral_scenarios": { "original": 895, "effective": 895 }, "mmlu_philosophy": { "original": 311, "effective": 311 }, "mmlu_prehistory": { "original": 324, "effective": 324 }, "mmlu_professional_law": { "original": 1534, "effective": 1534 }, "mmlu_world_religions": { "original": 171, "effective": 171 } }, "config": { "model": "/home/jovyan/.cache/huggingface/models--microsoft--Phi-mini-MoE-instruct/snapshots/62d7eed72cc0420fc924c800721b1401d628395c", "model_args": null, "model_num_parameters": 7647632704, "model_dtype": "torch.bfloat16", "model_revision": "main", "model_sha": "", "batch_size": "auto", "batch_sizes": [ 64 ], "device": null, "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "0cffc7f", "date": 1763176707.4563708, "pretty_env_info": "PyTorch version: 2.7.1+cu118\nIs debug build: False\nCUDA used to build PyTorch: 11.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.9\nLibc version: glibc-2.35\n\nPython version: 3.10.13 | packaged by conda-forge | (main, Oct 26 2023, 18:07:37) [GCC 12.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-138-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 11.8.89\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA A100 80GB PCIe\nNvidia driver version: 555.42.06\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7443 24-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 1\nFrequency boost: enabled\nCPU max MHz: 4035.6440\nCPU min MHz: 1500.0000\nBogoMIPS: 5699.48\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload vgif v_spec_ctrl umip pku ospke vaes vpclmulqdq rdpid overflow_recov succor smca fsrm\nVirtualization: AMD-V\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 256 MiB (8 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.2.6\n[pip3] nvidia-cublas-cu11==11.11.3.6\n[pip3] nvidia-cuda-cupti-cu11==11.8.87\n[pip3] nvidia-cuda-nvrtc-cu11==11.8.89\n[pip3] nvidia-cuda-runtime-cu11==11.8.89\n[pip3] nvidia-cudnn-cu11==9.1.0.70\n[pip3] nvidia-cufft-cu11==10.9.0.58\n[pip3] nvidia-curand-cu11==10.3.0.86\n[pip3] nvidia-cusolver-cu11==11.4.1.48\n[pip3] nvidia-cusparse-cu11==11.7.5.86\n[pip3] nvidia-nccl-cu11==2.21.5\n[pip3] nvidia-nvtx-cu11==11.8.86\n[pip3] torch==2.7.1+cu118\n[pip3] triton==3.3.1\n[conda] nomkl 1.0 h5ca1d4c_0 conda-forge\n[conda] numpy 1.26.2 py310hb13e2d6_0 conda-forge\n[conda] nvidia-cublas-cu11 11.11.3.6 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu11 11.8.87 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu11 11.8.89 pypi_0 pypi\n[conda] nvidia-cudnn-cu11 8.7.0.84 pypi_0 pypi\n[conda] nvidia-cufft-cu11 10.9.0.58 pypi_0 pypi\n[conda] nvidia-curand-cu11 10.3.0.86 pypi_0 pypi\n[conda] nvidia-cusolver-cu11 11.4.1.48 pypi_0 pypi\n[conda] nvidia-cusparse-cu11 11.7.5.86 pypi_0 pypi\n[conda] nvidia-nccl-cu11 2.16.5 pypi_0 pypi\n[conda] pytorch-lightning 2.1.2 pypi_0 pypi\n[conda] torch 2.1.1+cu118 pypi_0 pypi\n[conda] torchaudio 2.1.1+cu118 pypi_0 pypi\n[conda] torchdata 0.7.1+cpu pypi_0 pypi\n[conda] torchmetrics 1.2.0 pypi_0 pypi\n[conda] torchtext 0.16.1+cpu pypi_0 pypi\n[conda] torchvision 0.16.1+cu118 pypi_0 pypi\n[conda] triton 2.1.0 pypi_0 pypi", "transformers_version": "4.55.0", "lm_eval_version": "0.4.9.1", "upper_git_hash": null, "tokenizer_pad_token": [ "<|endoftext|>", "32000" ], "tokenizer_eos_token": [ "<|endoftext|>", "32000" ], "tokenizer_bos_token": [ "", "1" ], "eot_token_id": 32000, "max_length": 4096 } }