| { |
| "config": { |
| "compressed_weights": "outputs/microsoft/Phi-mini-MoE-instruct/mobe_weights", |
| "start_layer": 3, |
| "end_layer": 32, |
| "matrix_types": [ |
| "w1" |
| ], |
| "tasks": [ |
| "mmlu", |
| "arc_easy", |
| "arc_challenge" |
| ], |
| "num_fewshot": 0 |
| }, |
| "comparison_summary": { |
| "arc_challenge": { |
| "acc,none": { |
| "original": 0.5750853242320819, |
| "compressed": 0.5247440273037542, |
| "delta": -0.05034129692832767 |
| }, |
| "acc_stderr,none": { |
| "original": 0.014445698968520772, |
| "compressed": 0.014593487694937738, |
| "delta": 0.0001477887264169659 |
| }, |
| "acc_norm,none": { |
| "original": 0.5878839590443686, |
| "compressed": 0.5793515358361775, |
| "delta": -0.008532423208191142 |
| }, |
| "acc_norm_stderr,none": { |
| "original": 0.014383915302225402, |
| "compressed": 0.014426211252508404, |
| "delta": 4.2295950283002814e-05 |
| } |
| }, |
| "arc_easy": { |
| "acc,none": { |
| "original": 0.8261784511784511, |
| "compressed": 0.8131313131313131, |
| "delta": -0.013047138047137974 |
| }, |
| "acc_stderr,none": { |
| "original": 0.007776012182869763, |
| "compressed": 0.007998652011081697, |
| "delta": 0.00022263982821193323 |
| }, |
| "acc_norm,none": { |
| "original": 0.811026936026936, |
| "compressed": 0.7954545454545454, |
| "delta": -0.015572390572390571 |
| }, |
| "acc_norm_stderr,none": { |
| "original": 0.008033148299801935, |
| "compressed": 0.008276958800002999, |
| "delta": 0.00024381050020106342 |
| } |
| }, |
| "mmlu": { |
| "acc,none": { |
| "original": 0.6791055405212932, |
| "compressed": 0.6536105967810854, |
| "delta": -0.025494943740207887 |
| }, |
| "acc_stderr,none": { |
| "original": 0.003724961489256017, |
| "compressed": 0.0038123134918508976, |
| "delta": 8.735200259488065e-05 |
| } |
| }, |
| "mmlu_humanities": { |
| "acc,none": { |
| "original": 0.620403825717322, |
| "compressed": 0.5957492029755579, |
| "delta": -0.02465462274176411 |
| }, |
| "acc_stderr,none": { |
| "original": 0.0066572161384182, |
| "compressed": 0.006759316550944608, |
| "delta": 0.00010210041252640793 |
| } |
| }, |
| "mmlu_formal_logic": { |
| "acc,none": { |
| "original": 0.5396825396825397, |
| "compressed": 0.5158730158730159, |
| "delta": -0.023809523809523725 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04458029125470973, |
| "compressed": 0.044698818540726076, |
| "delta": 0.00011852728601634971 |
| } |
| }, |
| "mmlu_high_school_european_history": { |
| "acc,none": { |
| "original": 0.8181818181818182, |
| "compressed": 0.7696969696969697, |
| "delta": -0.048484848484848575 |
| }, |
| "acc_stderr,none": { |
| "original": 0.030117688929503582, |
| "compressed": 0.03287666758603489, |
| "delta": 0.0027589786565313107 |
| } |
| }, |
| "mmlu_high_school_us_history": { |
| "acc,none": { |
| "original": 0.8725490196078431, |
| "compressed": 0.8480392156862745, |
| "delta": -0.02450980392156865 |
| }, |
| "acc_stderr,none": { |
| "original": 0.023405530480846315, |
| "compressed": 0.025195658428931792, |
| "delta": 0.0017901279480854772 |
| } |
| }, |
| "mmlu_high_school_world_history": { |
| "acc,none": { |
| "original": 0.7932489451476793, |
| "compressed": 0.7932489451476793, |
| "delta": 0.0 |
| }, |
| "acc_stderr,none": { |
| "original": 0.026361651668389094, |
| "compressed": 0.026361651668389094, |
| "delta": 0.0 |
| } |
| }, |
| "mmlu_international_law": { |
| "acc,none": { |
| "original": 0.8760330578512396, |
| "compressed": 0.8016528925619835, |
| "delta": -0.07438016528925617 |
| }, |
| "acc_stderr,none": { |
| "original": 0.030083098716035216, |
| "compressed": 0.03640118271990947, |
| "delta": 0.006318084003874253 |
| } |
| }, |
| "mmlu_jurisprudence": { |
| "acc,none": { |
| "original": 0.8240740740740741, |
| "compressed": 0.7962962962962963, |
| "delta": -0.02777777777777779 |
| }, |
| "acc_stderr,none": { |
| "original": 0.036809181416738807, |
| "compressed": 0.03893542518824847, |
| "delta": 0.0021262437715096616 |
| } |
| }, |
| "mmlu_logical_fallacies": { |
| "acc,none": { |
| "original": 0.8159509202453987, |
| "compressed": 0.8098159509202454, |
| "delta": -0.006134969325153339 |
| }, |
| "acc_stderr,none": { |
| "original": 0.030446777687971726, |
| "compressed": 0.03083349114628123, |
| "delta": 0.00038671345830950526 |
| } |
| }, |
| "mmlu_moral_disputes": { |
| "acc,none": { |
| "original": 0.7572254335260116, |
| "compressed": 0.7167630057803468, |
| "delta": -0.040462427745664775 |
| }, |
| "acc_stderr,none": { |
| "original": 0.023083658586984204, |
| "compressed": 0.02425790170532337, |
| "delta": 0.0011742431183391668 |
| } |
| }, |
| "mmlu_moral_scenarios": { |
| "acc,none": { |
| "original": 0.4033519553072626, |
| "compressed": 0.376536312849162, |
| "delta": -0.026815642458100586 |
| }, |
| "acc_stderr,none": { |
| "original": 0.01640712303219525, |
| "compressed": 0.016204672385106596, |
| "delta": -0.00020245064708865335 |
| } |
| }, |
| "mmlu_philosophy": { |
| "acc,none": { |
| "original": 0.7427652733118971, |
| "compressed": 0.707395498392283, |
| "delta": -0.03536977491961413 |
| }, |
| "acc_stderr,none": { |
| "original": 0.024826171289250888, |
| "compressed": 0.02583989833487798, |
| "delta": 0.0010137270456270914 |
| } |
| }, |
| "mmlu_prehistory": { |
| "acc,none": { |
| "original": 0.7469135802469136, |
| "compressed": 0.7191358024691358, |
| "delta": -0.02777777777777779 |
| }, |
| "acc_stderr,none": { |
| "original": 0.024191808600713002, |
| "compressed": 0.025006469755799208, |
| "delta": 0.0008146611550862055 |
| } |
| }, |
| "mmlu_professional_law": { |
| "acc,none": { |
| "original": 0.5117340286831812, |
| "compressed": 0.4954367666232073, |
| "delta": -0.016297262059973894 |
| }, |
| "acc_stderr,none": { |
| "original": 0.01276671901968672, |
| "compressed": 0.012769704263117531, |
| "delta": 2.985243430810719e-06 |
| } |
| }, |
| "mmlu_world_religions": { |
| "acc,none": { |
| "original": 0.8245614035087719, |
| "compressed": 0.8011695906432749, |
| "delta": -0.023391812865497075 |
| }, |
| "acc_stderr,none": { |
| "original": 0.029170885500727654, |
| "compressed": 0.03061111655743253, |
| "delta": 0.001440231056704877 |
| } |
| }, |
| "mmlu_other": { |
| "acc,none": { |
| "original": 0.7306083038300611, |
| "compressed": 0.7071129707112971, |
| "delta": -0.02349533311876406 |
| }, |
| "acc_stderr,none": { |
| "original": 0.007703905997463136, |
| "compressed": 0.00792634679462611, |
| "delta": 0.00022244079716297448 |
| } |
| }, |
| "mmlu_business_ethics": { |
| "acc,none": { |
| "original": 0.72, |
| "compressed": 0.71, |
| "delta": -0.010000000000000009 |
| }, |
| "acc_stderr,none": { |
| "original": 0.045126085985421276, |
| "compressed": 0.045604802157206845, |
| "delta": 0.00047871617178556897 |
| } |
| }, |
| "mmlu_clinical_knowledge": { |
| "acc,none": { |
| "original": 0.7584905660377359, |
| "compressed": 0.7283018867924528, |
| "delta": -0.030188679245283123 |
| }, |
| "acc_stderr,none": { |
| "original": 0.02634148037111836, |
| "compressed": 0.027377706624670713, |
| "delta": 0.001036226253552354 |
| } |
| }, |
| "mmlu_college_medicine": { |
| "acc,none": { |
| "original": 0.6820809248554913, |
| "compressed": 0.630057803468208, |
| "delta": -0.052023121387283267 |
| }, |
| "acc_stderr,none": { |
| "original": 0.0355068398916558, |
| "compressed": 0.0368122963339432, |
| "delta": 0.0013054564422873977 |
| } |
| }, |
| "mmlu_global_facts": { |
| "acc,none": { |
| "original": 0.39, |
| "compressed": 0.46, |
| "delta": 0.07 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04902071300001974, |
| "compressed": 0.05009082659620333, |
| "delta": 0.0010701135961835861 |
| } |
| }, |
| "mmlu_human_aging": { |
| "acc,none": { |
| "original": 0.7085201793721974, |
| "compressed": 0.6681614349775785, |
| "delta": -0.04035874439461884 |
| }, |
| "acc_stderr,none": { |
| "original": 0.030500283176545843, |
| "compressed": 0.03160295143776679, |
| "delta": 0.0011026682612209487 |
| } |
| }, |
| "mmlu_management": { |
| "acc,none": { |
| "original": 0.7961165048543689, |
| "compressed": 0.7475728155339806, |
| "delta": -0.04854368932038833 |
| }, |
| "acc_stderr,none": { |
| "original": 0.0398913985953177, |
| "compressed": 0.04301250399690879, |
| "delta": 0.0031211054015910927 |
| } |
| }, |
| "mmlu_marketing": { |
| "acc,none": { |
| "original": 0.8931623931623932, |
| "compressed": 0.8974358974358975, |
| "delta": 0.004273504273504258 |
| }, |
| "acc_stderr,none": { |
| "original": 0.020237149008990922, |
| "compressed": 0.019875655027867457, |
| "delta": -0.00036149398112346465 |
| } |
| }, |
| "mmlu_medical_genetics": { |
| "acc,none": { |
| "original": 0.76, |
| "compressed": 0.74, |
| "delta": -0.020000000000000018 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04292346959909283, |
| "compressed": 0.04408440022768078, |
| "delta": 0.00116093062858795 |
| } |
| }, |
| "mmlu_miscellaneous": { |
| "acc,none": { |
| "original": 0.8173690932311622, |
| "compressed": 0.7956577266922095, |
| "delta": -0.02171136653895278 |
| }, |
| "acc_stderr,none": { |
| "original": 0.013816335389973124, |
| "compressed": 0.0144191239809319, |
| "delta": 0.0006027885909587766 |
| } |
| }, |
| "mmlu_nutrition": { |
| "acc,none": { |
| "original": 0.7516339869281046, |
| "compressed": 0.7287581699346405, |
| "delta": -0.022875816993464082 |
| }, |
| "acc_stderr,none": { |
| "original": 0.02473998135511359, |
| "compressed": 0.025457756696667864, |
| "delta": 0.0007177753415542748 |
| } |
| }, |
| "mmlu_professional_accounting": { |
| "acc,none": { |
| "original": 0.5425531914893617, |
| "compressed": 0.5035460992907801, |
| "delta": -0.03900709219858156 |
| }, |
| "acc_stderr,none": { |
| "original": 0.02971928127223684, |
| "compressed": 0.02982674915328092, |
| "delta": 0.00010746788104407795 |
| } |
| }, |
| "mmlu_professional_medicine": { |
| "acc,none": { |
| "original": 0.7426470588235294, |
| "compressed": 0.7058823529411765, |
| "delta": -0.03676470588235292 |
| }, |
| "acc_stderr,none": { |
| "original": 0.026556519470041517, |
| "compressed": 0.0276784686421447, |
| "delta": 0.001121949172103183 |
| } |
| }, |
| "mmlu_virology": { |
| "acc,none": { |
| "original": 0.5421686746987951, |
| "compressed": 0.5301204819277109, |
| "delta": -0.012048192771084265 |
| }, |
| "acc_stderr,none": { |
| "original": 0.038786267710023595, |
| "compressed": 0.03885425420866766, |
| "delta": 6.798649864406259e-05 |
| } |
| }, |
| "mmlu_social_sciences": { |
| "acc,none": { |
| "original": 0.7923301917452064, |
| "compressed": 0.7630809229769255, |
| "delta": -0.029249268768280823 |
| }, |
| "acc_stderr,none": { |
| "original": 0.007164065545433766, |
| "compressed": 0.007521922102925205, |
| "delta": 0.00035785655749143944 |
| } |
| }, |
| "mmlu_econometrics": { |
| "acc,none": { |
| "original": 0.5087719298245614, |
| "compressed": 0.5087719298245614, |
| "delta": 0.0 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04702880432049615, |
| "compressed": 0.04702880432049615, |
| "delta": 0.0 |
| } |
| }, |
| "mmlu_high_school_geography": { |
| "acc,none": { |
| "original": 0.8434343434343434, |
| "compressed": 0.8181818181818182, |
| "delta": -0.025252525252525193 |
| }, |
| "acc_stderr,none": { |
| "original": 0.025890520358141454, |
| "compressed": 0.0274796030105388, |
| "delta": 0.0015890826523973466 |
| } |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "acc,none": { |
| "original": 0.9067357512953368, |
| "compressed": 0.8860103626943006, |
| "delta": -0.020725388601036232 |
| }, |
| "acc_stderr,none": { |
| "original": 0.02098685459328971, |
| "compressed": 0.022935144053919436, |
| "delta": 0.0019482894606297242 |
| } |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "acc,none": { |
| "original": 0.735897435897436, |
| "compressed": 0.6846153846153846, |
| "delta": -0.05128205128205132 |
| }, |
| "acc_stderr,none": { |
| "original": 0.022352193737453268, |
| "compressed": 0.02355964698318994, |
| "delta": 0.0012074532457366713 |
| } |
| }, |
| "mmlu_high_school_microeconomics": { |
| "acc,none": { |
| "original": 0.865546218487395, |
| "compressed": 0.8277310924369747, |
| "delta": -0.037815126050420256 |
| }, |
| "acc_stderr,none": { |
| "original": 0.022159373072744442, |
| "compressed": 0.024528664971305427, |
| "delta": 0.002369291898560985 |
| } |
| }, |
| "mmlu_high_school_psychology": { |
| "acc,none": { |
| "original": 0.8642201834862385, |
| "compressed": 0.8477064220183487, |
| "delta": -0.016513761467889854 |
| }, |
| "acc_stderr,none": { |
| "original": 0.014686907556340017, |
| "compressed": 0.015405084393157069, |
| "delta": 0.0007181768368170523 |
| } |
| }, |
| "mmlu_human_sexuality": { |
| "acc,none": { |
| "original": 0.7786259541984732, |
| "compressed": 0.7557251908396947, |
| "delta": -0.022900763358778553 |
| }, |
| "acc_stderr,none": { |
| "original": 0.036412970813137296, |
| "compressed": 0.037683359597287434, |
| "delta": 0.001270388784150138 |
| } |
| }, |
| "mmlu_professional_psychology": { |
| "acc,none": { |
| "original": 0.7205882352941176, |
| "compressed": 0.6879084967320261, |
| "delta": -0.0326797385620915 |
| }, |
| "acc_stderr,none": { |
| "original": 0.018152871051538812, |
| "compressed": 0.01874501120127766, |
| "delta": 0.0005921401497388477 |
| } |
| }, |
| "mmlu_public_relations": { |
| "acc,none": { |
| "original": 0.7181818181818181, |
| "compressed": 0.7090909090909091, |
| "delta": -0.009090909090909038 |
| }, |
| "acc_stderr,none": { |
| "original": 0.043091187099464585, |
| "compressed": 0.04350271442923243, |
| "delta": 0.0004115273297678468 |
| } |
| }, |
| "mmlu_security_studies": { |
| "acc,none": { |
| "original": 0.8040816326530612, |
| "compressed": 0.7714285714285715, |
| "delta": -0.03265306122448974 |
| }, |
| "acc_stderr,none": { |
| "original": 0.02540930195322568, |
| "compressed": 0.026882144922307744, |
| "delta": 0.001472842969082063 |
| } |
| }, |
| "mmlu_sociology": { |
| "acc,none": { |
| "original": 0.845771144278607, |
| "compressed": 0.8059701492537313, |
| "delta": -0.03980099502487566 |
| }, |
| "acc_stderr,none": { |
| "original": 0.025538433368578334, |
| "compressed": 0.027962677604768896, |
| "delta": 0.0024242442361905626 |
| } |
| }, |
| "mmlu_us_foreign_policy": { |
| "acc,none": { |
| "original": 0.85, |
| "compressed": 0.82, |
| "delta": -0.030000000000000027 |
| }, |
| "acc_stderr,none": { |
| "original": 0.0358870281282637, |
| "compressed": 0.03861229196653694, |
| "delta": 0.002725263838273241 |
| } |
| }, |
| "mmlu_stem": { |
| "acc,none": { |
| "original": 0.6054551221059309, |
| "compressed": 0.5803996194100857, |
| "delta": -0.025055502695845222 |
| }, |
| "acc_stderr,none": { |
| "original": 0.008367541993189256, |
| "compressed": 0.008464141586351462, |
| "delta": 9.659959316220615e-05 |
| } |
| }, |
| "mmlu_abstract_algebra": { |
| "acc,none": { |
| "original": 0.5, |
| "compressed": 0.42, |
| "delta": -0.08000000000000002 |
| }, |
| "acc_stderr,none": { |
| "original": 0.050251890762960605, |
| "compressed": 0.049604496374885836, |
| "delta": -0.0006473943880747693 |
| } |
| }, |
| "mmlu_anatomy": { |
| "acc,none": { |
| "original": 0.6518518518518519, |
| "compressed": 0.6148148148148148, |
| "delta": -0.03703703703703709 |
| }, |
| "acc_stderr,none": { |
| "original": 0.041153246103369526, |
| "compressed": 0.042039210401562783, |
| "delta": 0.0008859642981932578 |
| } |
| }, |
| "mmlu_astronomy": { |
| "acc,none": { |
| "original": 0.7697368421052632, |
| "compressed": 0.743421052631579, |
| "delta": -0.02631578947368418 |
| }, |
| "acc_stderr,none": { |
| "original": 0.03426059424403165, |
| "compressed": 0.035541803680256896, |
| "delta": 0.0012812094362252488 |
| } |
| }, |
| "mmlu_college_biology": { |
| "acc,none": { |
| "original": 0.8263888888888888, |
| "compressed": 0.7986111111111112, |
| "delta": -0.02777777777777768 |
| }, |
| "acc_stderr,none": { |
| "original": 0.03167473383795718, |
| "compressed": 0.03353647469713839, |
| "delta": 0.001861740859181213 |
| } |
| }, |
| "mmlu_college_chemistry": { |
| "acc,none": { |
| "original": 0.45, |
| "compressed": 0.47, |
| "delta": 0.019999999999999962 |
| }, |
| "acc_stderr,none": { |
| "original": 0.049999999999999996, |
| "compressed": 0.05016135580465919, |
| "delta": 0.00016135580465919536 |
| } |
| }, |
| "mmlu_college_computer_science": { |
| "acc,none": { |
| "original": 0.6, |
| "compressed": 0.53, |
| "delta": -0.06999999999999995 |
| }, |
| "acc_stderr,none": { |
| "original": 0.049236596391733084, |
| "compressed": 0.050161355804659205, |
| "delta": 0.0009247594129261211 |
| } |
| }, |
| "mmlu_college_mathematics": { |
| "acc,none": { |
| "original": 0.43, |
| "compressed": 0.4, |
| "delta": -0.02999999999999997 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04975698519562427, |
| "compressed": 0.04923659639173309, |
| "delta": -0.0005203888038911794 |
| } |
| }, |
| "mmlu_college_physics": { |
| "acc,none": { |
| "original": 0.39215686274509803, |
| "compressed": 0.37254901960784315, |
| "delta": -0.019607843137254888 |
| }, |
| "acc_stderr,none": { |
| "original": 0.048580835742663434, |
| "compressed": 0.04810840148082634, |
| "delta": -0.0004724342618370944 |
| } |
| }, |
| "mmlu_computer_security": { |
| "acc,none": { |
| "original": 0.77, |
| "compressed": 0.73, |
| "delta": -0.040000000000000036 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04229525846816505, |
| "compressed": 0.044619604333847394, |
| "delta": 0.0023243458656823437 |
| } |
| }, |
| "mmlu_conceptual_physics": { |
| "acc,none": { |
| "original": 0.6553191489361702, |
| "compressed": 0.6382978723404256, |
| "delta": -0.017021276595744594 |
| }, |
| "acc_stderr,none": { |
| "original": 0.031068985963122145, |
| "compressed": 0.03141082197596241, |
| "delta": 0.0003418360128402617 |
| } |
| }, |
| "mmlu_electrical_engineering": { |
| "acc,none": { |
| "original": 0.6137931034482759, |
| "compressed": 0.593103448275862, |
| "delta": -0.020689655172413834 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04057324734419036, |
| "compressed": 0.04093793981266236, |
| "delta": 0.00036469246847200404 |
| } |
| }, |
| "mmlu_elementary_mathematics": { |
| "acc,none": { |
| "original": 0.5582010582010583, |
| "compressed": 0.5661375661375662, |
| "delta": 0.007936507936507908 |
| }, |
| "acc_stderr,none": { |
| "original": 0.025576257061253833, |
| "compressed": 0.02552503438247489, |
| "delta": -5.122267877894285e-05 |
| } |
| }, |
| "mmlu_high_school_biology": { |
| "acc,none": { |
| "original": 0.8290322580645161, |
| "compressed": 0.7935483870967742, |
| "delta": -0.03548387096774186 |
| }, |
| "acc_stderr,none": { |
| "original": 0.021417242936321544, |
| "compressed": 0.023025899617188744, |
| "delta": 0.0016086566808671994 |
| } |
| }, |
| "mmlu_high_school_chemistry": { |
| "acc,none": { |
| "original": 0.6059113300492611, |
| "compressed": 0.5566502463054187, |
| "delta": -0.049261083743842415 |
| }, |
| "acc_stderr,none": { |
| "original": 0.03438157967036543, |
| "compressed": 0.03495334582162934, |
| "delta": 0.0005717661512639055 |
| } |
| }, |
| "mmlu_high_school_computer_science": { |
| "acc,none": { |
| "original": 0.77, |
| "compressed": 0.75, |
| "delta": -0.020000000000000018 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04229525846816506, |
| "compressed": 0.04351941398892446, |
| "delta": 0.0012241555207594054 |
| } |
| }, |
| "mmlu_high_school_mathematics": { |
| "acc,none": { |
| "original": 0.3925925925925926, |
| "compressed": 0.36666666666666664, |
| "delta": -0.025925925925925963 |
| }, |
| "acc_stderr,none": { |
| "original": 0.02977384701253297, |
| "compressed": 0.029381620726465073, |
| "delta": -0.00039222628606789753 |
| } |
| }, |
| "mmlu_high_school_physics": { |
| "acc,none": { |
| "original": 0.4768211920529801, |
| "compressed": 0.46357615894039733, |
| "delta": -0.013245033112582794 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04078093859163083, |
| "compressed": 0.04071636065944216, |
| "delta": -6.45779321886697e-05 |
| } |
| }, |
| "mmlu_high_school_statistics": { |
| "acc,none": { |
| "original": 0.5833333333333334, |
| "compressed": 0.5787037037037037, |
| "delta": -0.00462962962962965 |
| }, |
| "acc_stderr,none": { |
| "original": 0.03362277436608043, |
| "compressed": 0.033674621388960775, |
| "delta": 5.184702288034415e-05 |
| } |
| }, |
| "mmlu_machine_learning": { |
| "acc,none": { |
| "original": 0.49107142857142855, |
| "compressed": 0.42857142857142855, |
| "delta": -0.0625 |
| }, |
| "acc_stderr,none": { |
| "original": 0.04745033255489123, |
| "compressed": 0.04697113923010212, |
| "delta": -0.0004791933247891081 |
| } |
| } |
| }, |
| "original_model_results": { |
| "results": { |
| "arc_challenge": { |
| "alias": "arc_challenge", |
| "acc,none": 0.5750853242320819, |
| "acc_stderr,none": 0.014445698968520772, |
| "acc_norm,none": 0.5878839590443686, |
| "acc_norm_stderr,none": 0.014383915302225402 |
| }, |
| "arc_easy": { |
| "alias": "arc_easy", |
| "acc,none": 0.8261784511784511, |
| "acc_stderr,none": 0.007776012182869763, |
| "acc_norm,none": 0.811026936026936, |
| "acc_norm_stderr,none": 0.008033148299801935 |
| }, |
| "mmlu": { |
| "acc,none": 0.6791055405212932, |
| "acc_stderr,none": 0.003724961489256017, |
| "alias": "mmlu" |
| }, |
| "mmlu_humanities": { |
| "acc,none": 0.620403825717322, |
| "acc_stderr,none": 0.0066572161384182, |
| "alias": " - humanities" |
| }, |
| "mmlu_formal_logic": { |
| "alias": " - formal_logic", |
| "acc,none": 0.5396825396825397, |
| "acc_stderr,none": 0.04458029125470973 |
| }, |
| "mmlu_high_school_european_history": { |
| "alias": " - high_school_european_history", |
| "acc,none": 0.8181818181818182, |
| "acc_stderr,none": 0.030117688929503582 |
| }, |
| "mmlu_high_school_us_history": { |
| "alias": " - high_school_us_history", |
| "acc,none": 0.8725490196078431, |
| "acc_stderr,none": 0.023405530480846315 |
| }, |
| "mmlu_high_school_world_history": { |
| "alias": " - high_school_world_history", |
| "acc,none": 0.7932489451476793, |
| "acc_stderr,none": 0.026361651668389094 |
| }, |
| "mmlu_international_law": { |
| "alias": " - international_law", |
| "acc,none": 0.8760330578512396, |
| "acc_stderr,none": 0.030083098716035216 |
| }, |
| "mmlu_jurisprudence": { |
| "alias": " - jurisprudence", |
| "acc,none": 0.8240740740740741, |
| "acc_stderr,none": 0.036809181416738807 |
| }, |
| "mmlu_logical_fallacies": { |
| "alias": " - logical_fallacies", |
| "acc,none": 0.8159509202453987, |
| "acc_stderr,none": 0.030446777687971726 |
| }, |
| "mmlu_moral_disputes": { |
| "alias": " - moral_disputes", |
| "acc,none": 0.7572254335260116, |
| "acc_stderr,none": 0.023083658586984204 |
| }, |
| "mmlu_moral_scenarios": { |
| "alias": " - moral_scenarios", |
| "acc,none": 0.4033519553072626, |
| "acc_stderr,none": 0.01640712303219525 |
| }, |
| "mmlu_philosophy": { |
| "alias": " - philosophy", |
| "acc,none": 0.7427652733118971, |
| "acc_stderr,none": 0.024826171289250888 |
| }, |
| "mmlu_prehistory": { |
| "alias": " - prehistory", |
| "acc,none": 0.7469135802469136, |
| "acc_stderr,none": 0.024191808600713002 |
| }, |
| "mmlu_professional_law": { |
| "alias": " - professional_law", |
| "acc,none": 0.5117340286831812, |
| "acc_stderr,none": 0.01276671901968672 |
| }, |
| "mmlu_world_religions": { |
| "alias": " - world_religions", |
| "acc,none": 0.8245614035087719, |
| "acc_stderr,none": 0.029170885500727654 |
| }, |
| "mmlu_other": { |
| "acc,none": 0.7306083038300611, |
| "acc_stderr,none": 0.007703905997463136, |
| "alias": " - other" |
| }, |
| "mmlu_business_ethics": { |
| "alias": " - business_ethics", |
| "acc,none": 0.72, |
| "acc_stderr,none": 0.045126085985421276 |
| }, |
| "mmlu_clinical_knowledge": { |
| "alias": " - clinical_knowledge", |
| "acc,none": 0.7584905660377359, |
| "acc_stderr,none": 0.02634148037111836 |
| }, |
| "mmlu_college_medicine": { |
| "alias": " - college_medicine", |
| "acc,none": 0.6820809248554913, |
| "acc_stderr,none": 0.0355068398916558 |
| }, |
| "mmlu_global_facts": { |
| "alias": " - global_facts", |
| "acc,none": 0.39, |
| "acc_stderr,none": 0.04902071300001974 |
| }, |
| "mmlu_human_aging": { |
| "alias": " - human_aging", |
| "acc,none": 0.7085201793721974, |
| "acc_stderr,none": 0.030500283176545843 |
| }, |
| "mmlu_management": { |
| "alias": " - management", |
| "acc,none": 0.7961165048543689, |
| "acc_stderr,none": 0.0398913985953177 |
| }, |
| "mmlu_marketing": { |
| "alias": " - marketing", |
| "acc,none": 0.8931623931623932, |
| "acc_stderr,none": 0.020237149008990922 |
| }, |
| "mmlu_medical_genetics": { |
| "alias": " - medical_genetics", |
| "acc,none": 0.76, |
| "acc_stderr,none": 0.04292346959909283 |
| }, |
| "mmlu_miscellaneous": { |
| "alias": " - miscellaneous", |
| "acc,none": 0.8173690932311622, |
| "acc_stderr,none": 0.013816335389973124 |
| }, |
| "mmlu_nutrition": { |
| "alias": " - nutrition", |
| "acc,none": 0.7516339869281046, |
| "acc_stderr,none": 0.02473998135511359 |
| }, |
| "mmlu_professional_accounting": { |
| "alias": " - professional_accounting", |
| "acc,none": 0.5425531914893617, |
| "acc_stderr,none": 0.02971928127223684 |
| }, |
| "mmlu_professional_medicine": { |
| "alias": " - professional_medicine", |
| "acc,none": 0.7426470588235294, |
| "acc_stderr,none": 0.026556519470041517 |
| }, |
| "mmlu_virology": { |
| "alias": " - virology", |
| "acc,none": 0.5421686746987951, |
| "acc_stderr,none": 0.038786267710023595 |
| }, |
| "mmlu_social_sciences": { |
| "acc,none": 0.7923301917452064, |
| "acc_stderr,none": 0.007164065545433766, |
| "alias": " - social sciences" |
| }, |
| "mmlu_econometrics": { |
| "alias": " - econometrics", |
| "acc,none": 0.5087719298245614, |
| "acc_stderr,none": 0.04702880432049615 |
| }, |
| "mmlu_high_school_geography": { |
| "alias": " - high_school_geography", |
| "acc,none": 0.8434343434343434, |
| "acc_stderr,none": 0.025890520358141454 |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "alias": " - high_school_government_and_politics", |
| "acc,none": 0.9067357512953368, |
| "acc_stderr,none": 0.02098685459328971 |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "alias": " - high_school_macroeconomics", |
| "acc,none": 0.735897435897436, |
| "acc_stderr,none": 0.022352193737453268 |
| }, |
| "mmlu_high_school_microeconomics": { |
| "alias": " - high_school_microeconomics", |
| "acc,none": 0.865546218487395, |
| "acc_stderr,none": 0.022159373072744442 |
| }, |
| "mmlu_high_school_psychology": { |
| "alias": " - high_school_psychology", |
| "acc,none": 0.8642201834862385, |
| "acc_stderr,none": 0.014686907556340017 |
| }, |
| "mmlu_human_sexuality": { |
| "alias": " - human_sexuality", |
| "acc,none": 0.7786259541984732, |
| "acc_stderr,none": 0.036412970813137296 |
| }, |
| "mmlu_professional_psychology": { |
| "alias": " - professional_psychology", |
| "acc,none": 0.7205882352941176, |
| "acc_stderr,none": 0.018152871051538812 |
| }, |
| "mmlu_public_relations": { |
| "alias": " - public_relations", |
| "acc,none": 0.7181818181818181, |
| "acc_stderr,none": 0.043091187099464585 |
| }, |
| "mmlu_security_studies": { |
| "alias": " - security_studies", |
| "acc,none": 0.8040816326530612, |
| "acc_stderr,none": 0.02540930195322568 |
| }, |
| "mmlu_sociology": { |
| "alias": " - sociology", |
| "acc,none": 0.845771144278607, |
| "acc_stderr,none": 0.025538433368578334 |
| }, |
| "mmlu_us_foreign_policy": { |
| "alias": " - us_foreign_policy", |
| "acc,none": 0.85, |
| "acc_stderr,none": 0.0358870281282637 |
| }, |
| "mmlu_stem": { |
| "acc,none": 0.6054551221059309, |
| "acc_stderr,none": 0.008367541993189256, |
| "alias": " - stem" |
| }, |
| "mmlu_abstract_algebra": { |
| "alias": " - abstract_algebra", |
| "acc,none": 0.5, |
| "acc_stderr,none": 0.050251890762960605 |
| }, |
| "mmlu_anatomy": { |
| "alias": " - anatomy", |
| "acc,none": 0.6518518518518519, |
| "acc_stderr,none": 0.041153246103369526 |
| }, |
| "mmlu_astronomy": { |
| "alias": " - astronomy", |
| "acc,none": 0.7697368421052632, |
| "acc_stderr,none": 0.03426059424403165 |
| }, |
| "mmlu_college_biology": { |
| "alias": " - college_biology", |
| "acc,none": 0.8263888888888888, |
| "acc_stderr,none": 0.03167473383795718 |
| }, |
| "mmlu_college_chemistry": { |
| "alias": " - college_chemistry", |
| "acc,none": 0.45, |
| "acc_stderr,none": 0.049999999999999996 |
| }, |
| "mmlu_college_computer_science": { |
| "alias": " - college_computer_science", |
| "acc,none": 0.6, |
| "acc_stderr,none": 0.049236596391733084 |
| }, |
| "mmlu_college_mathematics": { |
| "alias": " - college_mathematics", |
| "acc,none": 0.43, |
| "acc_stderr,none": 0.04975698519562427 |
| }, |
| "mmlu_college_physics": { |
| "alias": " - college_physics", |
| "acc,none": 0.39215686274509803, |
| "acc_stderr,none": 0.048580835742663434 |
| }, |
| "mmlu_computer_security": { |
| "alias": " - computer_security", |
| "acc,none": 0.77, |
| "acc_stderr,none": 0.04229525846816505 |
| }, |
| "mmlu_conceptual_physics": { |
| "alias": " - conceptual_physics", |
| "acc,none": 0.6553191489361702, |
| "acc_stderr,none": 0.031068985963122145 |
| }, |
| "mmlu_electrical_engineering": { |
| "alias": " - electrical_engineering", |
| "acc,none": 0.6137931034482759, |
| "acc_stderr,none": 0.04057324734419036 |
| }, |
| "mmlu_elementary_mathematics": { |
| "alias": " - elementary_mathematics", |
| "acc,none": 0.5582010582010583, |
| "acc_stderr,none": 0.025576257061253833 |
| }, |
| "mmlu_high_school_biology": { |
| "alias": " - high_school_biology", |
| "acc,none": 0.8290322580645161, |
| "acc_stderr,none": 0.021417242936321544 |
| }, |
| "mmlu_high_school_chemistry": { |
| "alias": " - high_school_chemistry", |
| "acc,none": 0.6059113300492611, |
| "acc_stderr,none": 0.03438157967036543 |
| }, |
| "mmlu_high_school_computer_science": { |
| "alias": " - high_school_computer_science", |
| "acc,none": 0.77, |
| "acc_stderr,none": 0.04229525846816506 |
| }, |
| "mmlu_high_school_mathematics": { |
| "alias": " - high_school_mathematics", |
| "acc,none": 0.3925925925925926, |
| "acc_stderr,none": 0.02977384701253297 |
| }, |
| "mmlu_high_school_physics": { |
| "alias": " - high_school_physics", |
| "acc,none": 0.4768211920529801, |
| "acc_stderr,none": 0.04078093859163083 |
| }, |
| "mmlu_high_school_statistics": { |
| "alias": " - high_school_statistics", |
| "acc,none": 0.5833333333333334, |
| "acc_stderr,none": 0.03362277436608043 |
| }, |
| "mmlu_machine_learning": { |
| "alias": " - machine_learning", |
| "acc,none": 0.49107142857142855, |
| "acc_stderr,none": 0.04745033255489123 |
| } |
| }, |
| "groups": { |
| "mmlu": { |
| "acc,none": 0.6791055405212932, |
| "acc_stderr,none": 0.003724961489256017, |
| "alias": "mmlu" |
| }, |
| "mmlu_humanities": { |
| "acc,none": 0.620403825717322, |
| "acc_stderr,none": 0.0066572161384182, |
| "alias": " - humanities" |
| }, |
| "mmlu_other": { |
| "acc,none": 0.7306083038300611, |
| "acc_stderr,none": 0.007703905997463136, |
| "alias": " - other" |
| }, |
| "mmlu_social_sciences": { |
| "acc,none": 0.7923301917452064, |
| "acc_stderr,none": 0.007164065545433766, |
| "alias": " - social sciences" |
| }, |
| "mmlu_stem": { |
| "acc,none": 0.6054551221059309, |
| "acc_stderr,none": 0.008367541993189256, |
| "alias": " - stem" |
| } |
| }, |
| "group_subtasks": { |
| "mmlu_humanities": [ |
| "mmlu_formal_logic", |
| "mmlu_high_school_european_history", |
| "mmlu_high_school_us_history", |
| "mmlu_high_school_world_history", |
| "mmlu_international_law", |
| "mmlu_jurisprudence", |
| "mmlu_logical_fallacies", |
| "mmlu_moral_disputes", |
| "mmlu_moral_scenarios", |
| "mmlu_philosophy", |
| "mmlu_prehistory", |
| "mmlu_professional_law", |
| "mmlu_world_religions" |
| ], |
| "mmlu_social_sciences": [ |
| "mmlu_econometrics", |
| "mmlu_high_school_geography", |
| "mmlu_high_school_government_and_politics", |
| "mmlu_high_school_macroeconomics", |
| "mmlu_high_school_microeconomics", |
| "mmlu_high_school_psychology", |
| "mmlu_human_sexuality", |
| "mmlu_professional_psychology", |
| "mmlu_public_relations", |
| "mmlu_security_studies", |
| "mmlu_sociology", |
| "mmlu_us_foreign_policy" |
| ], |
| "mmlu_other": [ |
| "mmlu_business_ethics", |
| "mmlu_clinical_knowledge", |
| "mmlu_college_medicine", |
| "mmlu_global_facts", |
| "mmlu_human_aging", |
| "mmlu_management", |
| "mmlu_marketing", |
| "mmlu_medical_genetics", |
| "mmlu_miscellaneous", |
| "mmlu_nutrition", |
| "mmlu_professional_accounting", |
| "mmlu_professional_medicine", |
| "mmlu_virology" |
| ], |
| "mmlu_stem": [ |
| "mmlu_abstract_algebra", |
| "mmlu_anatomy", |
| "mmlu_astronomy", |
| "mmlu_college_biology", |
| "mmlu_college_chemistry", |
| "mmlu_college_computer_science", |
| "mmlu_college_mathematics", |
| "mmlu_college_physics", |
| "mmlu_computer_security", |
| "mmlu_conceptual_physics", |
| "mmlu_electrical_engineering", |
| "mmlu_elementary_mathematics", |
| "mmlu_high_school_biology", |
| "mmlu_high_school_chemistry", |
| "mmlu_high_school_computer_science", |
| "mmlu_high_school_mathematics", |
| "mmlu_high_school_physics", |
| "mmlu_high_school_statistics", |
| "mmlu_machine_learning" |
| ], |
| "mmlu": [ |
| "mmlu_stem", |
| "mmlu_other", |
| "mmlu_social_sciences", |
| "mmlu_humanities" |
| ], |
| "arc_easy": [], |
| "arc_challenge": [] |
| }, |
| "configs": { |
| "arc_challenge": { |
| "task": "arc_challenge", |
| "tag": [ |
| "ai2_arc" |
| ], |
| "dataset_path": "allenai/ai2_arc", |
| "dataset_name": "ARC-Challenge", |
| "training_split": "train", |
| "validation_split": "validation", |
| "test_split": "test", |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_target": "{{choices.label.index(answerKey)}}", |
| "unsafe_code": false, |
| "doc_to_choice": "{{choices.text}}", |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "acc_norm", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": true, |
| "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "arc_easy": { |
| "task": "arc_easy", |
| "tag": [ |
| "ai2_arc" |
| ], |
| "dataset_path": "allenai/ai2_arc", |
| "dataset_name": "ARC-Easy", |
| "training_split": "train", |
| "validation_split": "validation", |
| "test_split": "test", |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_target": "{{choices.label.index(answerKey)}}", |
| "unsafe_code": false, |
| "doc_to_choice": "{{choices.text}}", |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "acc_norm", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": true, |
| "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_abstract_algebra": { |
| "task": "mmlu_abstract_algebra", |
| "task_alias": "abstract_algebra", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "abstract_algebra", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_anatomy": { |
| "task": "mmlu_anatomy", |
| "task_alias": "anatomy", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "anatomy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_astronomy": { |
| "task": "mmlu_astronomy", |
| "task_alias": "astronomy", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "astronomy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_business_ethics": { |
| "task": "mmlu_business_ethics", |
| "task_alias": "business_ethics", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "business_ethics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_clinical_knowledge": { |
| "task": "mmlu_clinical_knowledge", |
| "task_alias": "clinical_knowledge", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "clinical_knowledge", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_biology": { |
| "task": "mmlu_college_biology", |
| "task_alias": "college_biology", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_biology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college biology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_chemistry": { |
| "task": "mmlu_college_chemistry", |
| "task_alias": "college_chemistry", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_chemistry", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_computer_science": { |
| "task": "mmlu_college_computer_science", |
| "task_alias": "college_computer_science", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_computer_science", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_mathematics": { |
| "task": "mmlu_college_mathematics", |
| "task_alias": "college_mathematics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_medicine": { |
| "task": "mmlu_college_medicine", |
| "task_alias": "college_medicine", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_medicine", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_physics": { |
| "task": "mmlu_college_physics", |
| "task_alias": "college_physics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_computer_security": { |
| "task": "mmlu_computer_security", |
| "task_alias": "computer_security", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "computer_security", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about computer security.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_conceptual_physics": { |
| "task": "mmlu_conceptual_physics", |
| "task_alias": "conceptual_physics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "conceptual_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_econometrics": { |
| "task": "mmlu_econometrics", |
| "task_alias": "econometrics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "econometrics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_electrical_engineering": { |
| "task": "mmlu_electrical_engineering", |
| "task_alias": "electrical_engineering", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "electrical_engineering", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_elementary_mathematics": { |
| "task": "mmlu_elementary_mathematics", |
| "task_alias": "elementary_mathematics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "elementary_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_formal_logic": { |
| "task": "mmlu_formal_logic", |
| "task_alias": "formal_logic", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "formal_logic", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_global_facts": { |
| "task": "mmlu_global_facts", |
| "task_alias": "global_facts", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "global_facts", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about global facts.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_biology": { |
| "task": "mmlu_high_school_biology", |
| "task_alias": "high_school_biology", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_biology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_chemistry": { |
| "task": "mmlu_high_school_chemistry", |
| "task_alias": "high_school_chemistry", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_chemistry", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_computer_science": { |
| "task": "mmlu_high_school_computer_science", |
| "task_alias": "high_school_computer_science", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_computer_science", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_european_history": { |
| "task": "mmlu_high_school_european_history", |
| "task_alias": "high_school_european_history", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_european_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_geography": { |
| "task": "mmlu_high_school_geography", |
| "task_alias": "high_school_geography", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_geography", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "task": "mmlu_high_school_government_and_politics", |
| "task_alias": "high_school_government_and_politics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_government_and_politics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "task": "mmlu_high_school_macroeconomics", |
| "task_alias": "high_school_macroeconomics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_macroeconomics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_mathematics": { |
| "task": "mmlu_high_school_mathematics", |
| "task_alias": "high_school_mathematics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_microeconomics": { |
| "task": "mmlu_high_school_microeconomics", |
| "task_alias": "high_school_microeconomics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_microeconomics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_physics": { |
| "task": "mmlu_high_school_physics", |
| "task_alias": "high_school_physics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_psychology": { |
| "task": "mmlu_high_school_psychology", |
| "task_alias": "high_school_psychology", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_psychology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_statistics": { |
| "task": "mmlu_high_school_statistics", |
| "task_alias": "high_school_statistics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_statistics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_us_history": { |
| "task": "mmlu_high_school_us_history", |
| "task_alias": "high_school_us_history", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_us_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_world_history": { |
| "task": "mmlu_high_school_world_history", |
| "task_alias": "high_school_world_history", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_world_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_human_aging": { |
| "task": "mmlu_human_aging", |
| "task_alias": "human_aging", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "human_aging", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about human aging.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_human_sexuality": { |
| "task": "mmlu_human_sexuality", |
| "task_alias": "human_sexuality", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "human_sexuality", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_international_law": { |
| "task": "mmlu_international_law", |
| "task_alias": "international_law", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "international_law", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about international law.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_jurisprudence": { |
| "task": "mmlu_jurisprudence", |
| "task_alias": "jurisprudence", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "jurisprudence", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_logical_fallacies": { |
| "task": "mmlu_logical_fallacies", |
| "task_alias": "logical_fallacies", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "logical_fallacies", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_machine_learning": { |
| "task": "mmlu_machine_learning", |
| "task_alias": "machine_learning", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "machine_learning", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_management": { |
| "task": "mmlu_management", |
| "task_alias": "management", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "management", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about management.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_marketing": { |
| "task": "mmlu_marketing", |
| "task_alias": "marketing", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "marketing", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about marketing.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_medical_genetics": { |
| "task": "mmlu_medical_genetics", |
| "task_alias": "medical_genetics", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "medical_genetics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_miscellaneous": { |
| "task": "mmlu_miscellaneous", |
| "task_alias": "miscellaneous", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "miscellaneous", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_moral_disputes": { |
| "task": "mmlu_moral_disputes", |
| "task_alias": "moral_disputes", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "moral_disputes", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_moral_scenarios": { |
| "task": "mmlu_moral_scenarios", |
| "task_alias": "moral_scenarios", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "moral_scenarios", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_nutrition": { |
| "task": "mmlu_nutrition", |
| "task_alias": "nutrition", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "nutrition", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_philosophy": { |
| "task": "mmlu_philosophy", |
| "task_alias": "philosophy", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "philosophy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_prehistory": { |
| "task": "mmlu_prehistory", |
| "task_alias": "prehistory", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "prehistory", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_accounting": { |
| "task": "mmlu_professional_accounting", |
| "task_alias": "professional_accounting", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_accounting", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_law": { |
| "task": "mmlu_professional_law", |
| "task_alias": "professional_law", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_law", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional law.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_medicine": { |
| "task": "mmlu_professional_medicine", |
| "task_alias": "professional_medicine", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_medicine", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_psychology": { |
| "task": "mmlu_professional_psychology", |
| "task_alias": "professional_psychology", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_psychology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_public_relations": { |
| "task": "mmlu_public_relations", |
| "task_alias": "public_relations", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "public_relations", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about public relations.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_security_studies": { |
| "task": "mmlu_security_studies", |
| "task_alias": "security_studies", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "security_studies", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about security studies.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_sociology": { |
| "task": "mmlu_sociology", |
| "task_alias": "sociology", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "sociology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about sociology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_us_foreign_policy": { |
| "task": "mmlu_us_foreign_policy", |
| "task_alias": "us_foreign_policy", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "us_foreign_policy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_virology": { |
| "task": "mmlu_virology", |
| "task_alias": "virology", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "virology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about virology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_world_religions": { |
| "task": "mmlu_world_religions", |
| "task_alias": "world_religions", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "world_religions", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about world religions.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| } |
| }, |
| "versions": { |
| "arc_challenge": 1.0, |
| "arc_easy": 1.0, |
| "mmlu": 2, |
| "mmlu_abstract_algebra": 1.0, |
| "mmlu_anatomy": 1.0, |
| "mmlu_astronomy": 1.0, |
| "mmlu_business_ethics": 1.0, |
| "mmlu_clinical_knowledge": 1.0, |
| "mmlu_college_biology": 1.0, |
| "mmlu_college_chemistry": 1.0, |
| "mmlu_college_computer_science": 1.0, |
| "mmlu_college_mathematics": 1.0, |
| "mmlu_college_medicine": 1.0, |
| "mmlu_college_physics": 1.0, |
| "mmlu_computer_security": 1.0, |
| "mmlu_conceptual_physics": 1.0, |
| "mmlu_econometrics": 1.0, |
| "mmlu_electrical_engineering": 1.0, |
| "mmlu_elementary_mathematics": 1.0, |
| "mmlu_formal_logic": 1.0, |
| "mmlu_global_facts": 1.0, |
| "mmlu_high_school_biology": 1.0, |
| "mmlu_high_school_chemistry": 1.0, |
| "mmlu_high_school_computer_science": 1.0, |
| "mmlu_high_school_european_history": 1.0, |
| "mmlu_high_school_geography": 1.0, |
| "mmlu_high_school_government_and_politics": 1.0, |
| "mmlu_high_school_macroeconomics": 1.0, |
| "mmlu_high_school_mathematics": 1.0, |
| "mmlu_high_school_microeconomics": 1.0, |
| "mmlu_high_school_physics": 1.0, |
| "mmlu_high_school_psychology": 1.0, |
| "mmlu_high_school_statistics": 1.0, |
| "mmlu_high_school_us_history": 1.0, |
| "mmlu_high_school_world_history": 1.0, |
| "mmlu_human_aging": 1.0, |
| "mmlu_human_sexuality": 1.0, |
| "mmlu_humanities": 2, |
| "mmlu_international_law": 1.0, |
| "mmlu_jurisprudence": 1.0, |
| "mmlu_logical_fallacies": 1.0, |
| "mmlu_machine_learning": 1.0, |
| "mmlu_management": 1.0, |
| "mmlu_marketing": 1.0, |
| "mmlu_medical_genetics": 1.0, |
| "mmlu_miscellaneous": 1.0, |
| "mmlu_moral_disputes": 1.0, |
| "mmlu_moral_scenarios": 1.0, |
| "mmlu_nutrition": 1.0, |
| "mmlu_other": 2, |
| "mmlu_philosophy": 1.0, |
| "mmlu_prehistory": 1.0, |
| "mmlu_professional_accounting": 1.0, |
| "mmlu_professional_law": 1.0, |
| "mmlu_professional_medicine": 1.0, |
| "mmlu_professional_psychology": 1.0, |
| "mmlu_public_relations": 1.0, |
| "mmlu_security_studies": 1.0, |
| "mmlu_social_sciences": 2, |
| "mmlu_sociology": 1.0, |
| "mmlu_stem": 2, |
| "mmlu_us_foreign_policy": 1.0, |
| "mmlu_virology": 1.0, |
| "mmlu_world_religions": 1.0 |
| }, |
| "n-shot": { |
| "arc_challenge": 0, |
| "arc_easy": 0, |
| "mmlu_abstract_algebra": 0, |
| "mmlu_anatomy": 0, |
| "mmlu_astronomy": 0, |
| "mmlu_business_ethics": 0, |
| "mmlu_clinical_knowledge": 0, |
| "mmlu_college_biology": 0, |
| "mmlu_college_chemistry": 0, |
| "mmlu_college_computer_science": 0, |
| "mmlu_college_mathematics": 0, |
| "mmlu_college_medicine": 0, |
| "mmlu_college_physics": 0, |
| "mmlu_computer_security": 0, |
| "mmlu_conceptual_physics": 0, |
| "mmlu_econometrics": 0, |
| "mmlu_electrical_engineering": 0, |
| "mmlu_elementary_mathematics": 0, |
| "mmlu_formal_logic": 0, |
| "mmlu_global_facts": 0, |
| "mmlu_high_school_biology": 0, |
| "mmlu_high_school_chemistry": 0, |
| "mmlu_high_school_computer_science": 0, |
| "mmlu_high_school_european_history": 0, |
| "mmlu_high_school_geography": 0, |
| "mmlu_high_school_government_and_politics": 0, |
| "mmlu_high_school_macroeconomics": 0, |
| "mmlu_high_school_mathematics": 0, |
| "mmlu_high_school_microeconomics": 0, |
| "mmlu_high_school_physics": 0, |
| "mmlu_high_school_psychology": 0, |
| "mmlu_high_school_statistics": 0, |
| "mmlu_high_school_us_history": 0, |
| "mmlu_high_school_world_history": 0, |
| "mmlu_human_aging": 0, |
| "mmlu_human_sexuality": 0, |
| "mmlu_international_law": 0, |
| "mmlu_jurisprudence": 0, |
| "mmlu_logical_fallacies": 0, |
| "mmlu_machine_learning": 0, |
| "mmlu_management": 0, |
| "mmlu_marketing": 0, |
| "mmlu_medical_genetics": 0, |
| "mmlu_miscellaneous": 0, |
| "mmlu_moral_disputes": 0, |
| "mmlu_moral_scenarios": 0, |
| "mmlu_nutrition": 0, |
| "mmlu_philosophy": 0, |
| "mmlu_prehistory": 0, |
| "mmlu_professional_accounting": 0, |
| "mmlu_professional_law": 0, |
| "mmlu_professional_medicine": 0, |
| "mmlu_professional_psychology": 0, |
| "mmlu_public_relations": 0, |
| "mmlu_security_studies": 0, |
| "mmlu_sociology": 0, |
| "mmlu_us_foreign_policy": 0, |
| "mmlu_virology": 0, |
| "mmlu_world_religions": 0 |
| }, |
| "higher_is_better": { |
| "arc_challenge": { |
| "acc": true, |
| "acc_norm": true |
| }, |
| "arc_easy": { |
| "acc": true, |
| "acc_norm": true |
| }, |
| "mmlu": { |
| "acc": true |
| }, |
| "mmlu_abstract_algebra": { |
| "acc": true |
| }, |
| "mmlu_anatomy": { |
| "acc": true |
| }, |
| "mmlu_astronomy": { |
| "acc": true |
| }, |
| "mmlu_business_ethics": { |
| "acc": true |
| }, |
| "mmlu_clinical_knowledge": { |
| "acc": true |
| }, |
| "mmlu_college_biology": { |
| "acc": true |
| }, |
| "mmlu_college_chemistry": { |
| "acc": true |
| }, |
| "mmlu_college_computer_science": { |
| "acc": true |
| }, |
| "mmlu_college_mathematics": { |
| "acc": true |
| }, |
| "mmlu_college_medicine": { |
| "acc": true |
| }, |
| "mmlu_college_physics": { |
| "acc": true |
| }, |
| "mmlu_computer_security": { |
| "acc": true |
| }, |
| "mmlu_conceptual_physics": { |
| "acc": true |
| }, |
| "mmlu_econometrics": { |
| "acc": true |
| }, |
| "mmlu_electrical_engineering": { |
| "acc": true |
| }, |
| "mmlu_elementary_mathematics": { |
| "acc": true |
| }, |
| "mmlu_formal_logic": { |
| "acc": true |
| }, |
| "mmlu_global_facts": { |
| "acc": true |
| }, |
| "mmlu_high_school_biology": { |
| "acc": true |
| }, |
| "mmlu_high_school_chemistry": { |
| "acc": true |
| }, |
| "mmlu_high_school_computer_science": { |
| "acc": true |
| }, |
| "mmlu_high_school_european_history": { |
| "acc": true |
| }, |
| "mmlu_high_school_geography": { |
| "acc": true |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "acc": true |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "acc": true |
| }, |
| "mmlu_high_school_mathematics": { |
| "acc": true |
| }, |
| "mmlu_high_school_microeconomics": { |
| "acc": true |
| }, |
| "mmlu_high_school_physics": { |
| "acc": true |
| }, |
| "mmlu_high_school_psychology": { |
| "acc": true |
| }, |
| "mmlu_high_school_statistics": { |
| "acc": true |
| }, |
| "mmlu_high_school_us_history": { |
| "acc": true |
| }, |
| "mmlu_high_school_world_history": { |
| "acc": true |
| }, |
| "mmlu_human_aging": { |
| "acc": true |
| }, |
| "mmlu_human_sexuality": { |
| "acc": true |
| }, |
| "mmlu_humanities": { |
| "acc": true |
| }, |
| "mmlu_international_law": { |
| "acc": true |
| }, |
| "mmlu_jurisprudence": { |
| "acc": true |
| }, |
| "mmlu_logical_fallacies": { |
| "acc": true |
| }, |
| "mmlu_machine_learning": { |
| "acc": true |
| }, |
| "mmlu_management": { |
| "acc": true |
| }, |
| "mmlu_marketing": { |
| "acc": true |
| }, |
| "mmlu_medical_genetics": { |
| "acc": true |
| }, |
| "mmlu_miscellaneous": { |
| "acc": true |
| }, |
| "mmlu_moral_disputes": { |
| "acc": true |
| }, |
| "mmlu_moral_scenarios": { |
| "acc": true |
| }, |
| "mmlu_nutrition": { |
| "acc": true |
| }, |
| "mmlu_other": { |
| "acc": true |
| }, |
| "mmlu_philosophy": { |
| "acc": true |
| }, |
| "mmlu_prehistory": { |
| "acc": true |
| }, |
| "mmlu_professional_accounting": { |
| "acc": true |
| }, |
| "mmlu_professional_law": { |
| "acc": true |
| }, |
| "mmlu_professional_medicine": { |
| "acc": true |
| }, |
| "mmlu_professional_psychology": { |
| "acc": true |
| }, |
| "mmlu_public_relations": { |
| "acc": true |
| }, |
| "mmlu_security_studies": { |
| "acc": true |
| }, |
| "mmlu_social_sciences": { |
| "acc": true |
| }, |
| "mmlu_sociology": { |
| "acc": true |
| }, |
| "mmlu_stem": { |
| "acc": true |
| }, |
| "mmlu_us_foreign_policy": { |
| "acc": true |
| }, |
| "mmlu_virology": { |
| "acc": true |
| }, |
| "mmlu_world_religions": { |
| "acc": true |
| } |
| }, |
| "n-samples": { |
| "arc_challenge": { |
| "original": 1172, |
| "effective": 1172 |
| }, |
| "arc_easy": { |
| "original": 2376, |
| "effective": 2376 |
| }, |
| "mmlu_abstract_algebra": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_anatomy": { |
| "original": 135, |
| "effective": 135 |
| }, |
| "mmlu_astronomy": { |
| "original": 152, |
| "effective": 152 |
| }, |
| "mmlu_college_biology": { |
| "original": 144, |
| "effective": 144 |
| }, |
| "mmlu_college_chemistry": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_college_computer_science": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_college_mathematics": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_college_physics": { |
| "original": 102, |
| "effective": 102 |
| }, |
| "mmlu_computer_security": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_conceptual_physics": { |
| "original": 235, |
| "effective": 235 |
| }, |
| "mmlu_electrical_engineering": { |
| "original": 145, |
| "effective": 145 |
| }, |
| "mmlu_elementary_mathematics": { |
| "original": 378, |
| "effective": 378 |
| }, |
| "mmlu_high_school_biology": { |
| "original": 310, |
| "effective": 310 |
| }, |
| "mmlu_high_school_chemistry": { |
| "original": 203, |
| "effective": 203 |
| }, |
| "mmlu_high_school_computer_science": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_high_school_mathematics": { |
| "original": 270, |
| "effective": 270 |
| }, |
| "mmlu_high_school_physics": { |
| "original": 151, |
| "effective": 151 |
| }, |
| "mmlu_high_school_statistics": { |
| "original": 216, |
| "effective": 216 |
| }, |
| "mmlu_machine_learning": { |
| "original": 112, |
| "effective": 112 |
| }, |
| "mmlu_business_ethics": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_clinical_knowledge": { |
| "original": 265, |
| "effective": 265 |
| }, |
| "mmlu_college_medicine": { |
| "original": 173, |
| "effective": 173 |
| }, |
| "mmlu_global_facts": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_human_aging": { |
| "original": 223, |
| "effective": 223 |
| }, |
| "mmlu_management": { |
| "original": 103, |
| "effective": 103 |
| }, |
| "mmlu_marketing": { |
| "original": 234, |
| "effective": 234 |
| }, |
| "mmlu_medical_genetics": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_miscellaneous": { |
| "original": 783, |
| "effective": 783 |
| }, |
| "mmlu_nutrition": { |
| "original": 306, |
| "effective": 306 |
| }, |
| "mmlu_professional_accounting": { |
| "original": 282, |
| "effective": 282 |
| }, |
| "mmlu_professional_medicine": { |
| "original": 272, |
| "effective": 272 |
| }, |
| "mmlu_virology": { |
| "original": 166, |
| "effective": 166 |
| }, |
| "mmlu_econometrics": { |
| "original": 114, |
| "effective": 114 |
| }, |
| "mmlu_high_school_geography": { |
| "original": 198, |
| "effective": 198 |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "original": 193, |
| "effective": 193 |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "original": 390, |
| "effective": 390 |
| }, |
| "mmlu_high_school_microeconomics": { |
| "original": 238, |
| "effective": 238 |
| }, |
| "mmlu_high_school_psychology": { |
| "original": 545, |
| "effective": 545 |
| }, |
| "mmlu_human_sexuality": { |
| "original": 131, |
| "effective": 131 |
| }, |
| "mmlu_professional_psychology": { |
| "original": 612, |
| "effective": 612 |
| }, |
| "mmlu_public_relations": { |
| "original": 110, |
| "effective": 110 |
| }, |
| "mmlu_security_studies": { |
| "original": 245, |
| "effective": 245 |
| }, |
| "mmlu_sociology": { |
| "original": 201, |
| "effective": 201 |
| }, |
| "mmlu_us_foreign_policy": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_formal_logic": { |
| "original": 126, |
| "effective": 126 |
| }, |
| "mmlu_high_school_european_history": { |
| "original": 165, |
| "effective": 165 |
| }, |
| "mmlu_high_school_us_history": { |
| "original": 204, |
| "effective": 204 |
| }, |
| "mmlu_high_school_world_history": { |
| "original": 237, |
| "effective": 237 |
| }, |
| "mmlu_international_law": { |
| "original": 121, |
| "effective": 121 |
| }, |
| "mmlu_jurisprudence": { |
| "original": 108, |
| "effective": 108 |
| }, |
| "mmlu_logical_fallacies": { |
| "original": 163, |
| "effective": 163 |
| }, |
| "mmlu_moral_disputes": { |
| "original": 346, |
| "effective": 346 |
| }, |
| "mmlu_moral_scenarios": { |
| "original": 895, |
| "effective": 895 |
| }, |
| "mmlu_philosophy": { |
| "original": 311, |
| "effective": 311 |
| }, |
| "mmlu_prehistory": { |
| "original": 324, |
| "effective": 324 |
| }, |
| "mmlu_professional_law": { |
| "original": 1534, |
| "effective": 1534 |
| }, |
| "mmlu_world_religions": { |
| "original": 171, |
| "effective": 171 |
| } |
| }, |
| "config": { |
| "model": "/home/jovyan/.cache/huggingface/models--microsoft--Phi-mini-MoE-instruct/snapshots/62d7eed72cc0420fc924c800721b1401d628395c", |
| "model_args": null, |
| "model_num_parameters": 7647632704, |
| "model_dtype": "torch.bfloat16", |
| "model_revision": "main", |
| "model_sha": "", |
| "batch_size": "auto", |
| "batch_sizes": [ |
| 64 |
| ], |
| "device": null, |
| "use_cache": null, |
| "limit": null, |
| "bootstrap_iters": 100000, |
| "gen_kwargs": null, |
| "random_seed": 0, |
| "numpy_seed": 1234, |
| "torch_seed": 1234, |
| "fewshot_seed": 1234 |
| }, |
| "git_hash": "0cffc7f", |
| "date": 1763176280.2870958, |
| "pretty_env_info": "PyTorch version: 2.7.1+cu118\nIs debug build: False\nCUDA used to build PyTorch: 11.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.9\nLibc version: glibc-2.35\n\nPython version: 3.10.13 | packaged by conda-forge | (main, Oct 26 2023, 18:07:37) [GCC 12.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-138-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 11.8.89\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA A100 80GB PCIe\nNvidia driver version: 555.42.06\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7443 24-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 1\nFrequency boost: enabled\nCPU max MHz: 4035.6440\nCPU min MHz: 1500.0000\nBogoMIPS: 5699.48\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload vgif v_spec_ctrl umip pku ospke vaes vpclmulqdq rdpid overflow_recov succor smca fsrm\nVirtualization: AMD-V\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 256 MiB (8 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.2.6\n[pip3] nvidia-cublas-cu11==11.11.3.6\n[pip3] nvidia-cuda-cupti-cu11==11.8.87\n[pip3] nvidia-cuda-nvrtc-cu11==11.8.89\n[pip3] nvidia-cuda-runtime-cu11==11.8.89\n[pip3] nvidia-cudnn-cu11==9.1.0.70\n[pip3] nvidia-cufft-cu11==10.9.0.58\n[pip3] nvidia-curand-cu11==10.3.0.86\n[pip3] nvidia-cusolver-cu11==11.4.1.48\n[pip3] nvidia-cusparse-cu11==11.7.5.86\n[pip3] nvidia-nccl-cu11==2.21.5\n[pip3] nvidia-nvtx-cu11==11.8.86\n[pip3] torch==2.7.1+cu118\n[pip3] triton==3.3.1\n[conda] nomkl 1.0 h5ca1d4c_0 conda-forge\n[conda] numpy 1.26.2 py310hb13e2d6_0 conda-forge\n[conda] nvidia-cublas-cu11 11.11.3.6 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu11 11.8.87 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu11 11.8.89 pypi_0 pypi\n[conda] nvidia-cudnn-cu11 8.7.0.84 pypi_0 pypi\n[conda] nvidia-cufft-cu11 10.9.0.58 pypi_0 pypi\n[conda] nvidia-curand-cu11 10.3.0.86 pypi_0 pypi\n[conda] nvidia-cusolver-cu11 11.4.1.48 pypi_0 pypi\n[conda] nvidia-cusparse-cu11 11.7.5.86 pypi_0 pypi\n[conda] nvidia-nccl-cu11 2.16.5 pypi_0 pypi\n[conda] pytorch-lightning 2.1.2 pypi_0 pypi\n[conda] torch 2.1.1+cu118 pypi_0 pypi\n[conda] torchaudio 2.1.1+cu118 pypi_0 pypi\n[conda] torchdata 0.7.1+cpu pypi_0 pypi\n[conda] torchmetrics 1.2.0 pypi_0 pypi\n[conda] torchtext 0.16.1+cpu pypi_0 pypi\n[conda] torchvision 0.16.1+cu118 pypi_0 pypi\n[conda] triton 2.1.0 pypi_0 pypi", |
| "transformers_version": "4.55.0", |
| "lm_eval_version": "0.4.9.1", |
| "upper_git_hash": null, |
| "tokenizer_pad_token": [ |
| "<|endoftext|>", |
| "32000" |
| ], |
| "tokenizer_eos_token": [ |
| "<|endoftext|>", |
| "32000" |
| ], |
| "tokenizer_bos_token": [ |
| "<s>", |
| "1" |
| ], |
| "eot_token_id": 32000, |
| "max_length": 4096 |
| }, |
| "compressed_model_results": { |
| "results": { |
| "arc_challenge": { |
| "alias": "arc_challenge", |
| "acc,none": 0.5247440273037542, |
| "acc_stderr,none": 0.014593487694937738, |
| "acc_norm,none": 0.5793515358361775, |
| "acc_norm_stderr,none": 0.014426211252508404 |
| }, |
| "arc_easy": { |
| "alias": "arc_easy", |
| "acc,none": 0.8131313131313131, |
| "acc_stderr,none": 0.007998652011081697, |
| "acc_norm,none": 0.7954545454545454, |
| "acc_norm_stderr,none": 0.008276958800002999 |
| }, |
| "mmlu": { |
| "acc,none": 0.6536105967810854, |
| "acc_stderr,none": 0.0038123134918508976, |
| "alias": "mmlu" |
| }, |
| "mmlu_humanities": { |
| "acc,none": 0.5957492029755579, |
| "acc_stderr,none": 0.006759316550944608, |
| "alias": " - humanities" |
| }, |
| "mmlu_formal_logic": { |
| "alias": " - formal_logic", |
| "acc,none": 0.5158730158730159, |
| "acc_stderr,none": 0.044698818540726076 |
| }, |
| "mmlu_high_school_european_history": { |
| "alias": " - high_school_european_history", |
| "acc,none": 0.7696969696969697, |
| "acc_stderr,none": 0.03287666758603489 |
| }, |
| "mmlu_high_school_us_history": { |
| "alias": " - high_school_us_history", |
| "acc,none": 0.8480392156862745, |
| "acc_stderr,none": 0.025195658428931792 |
| }, |
| "mmlu_high_school_world_history": { |
| "alias": " - high_school_world_history", |
| "acc,none": 0.7932489451476793, |
| "acc_stderr,none": 0.026361651668389094 |
| }, |
| "mmlu_international_law": { |
| "alias": " - international_law", |
| "acc,none": 0.8016528925619835, |
| "acc_stderr,none": 0.03640118271990947 |
| }, |
| "mmlu_jurisprudence": { |
| "alias": " - jurisprudence", |
| "acc,none": 0.7962962962962963, |
| "acc_stderr,none": 0.03893542518824847 |
| }, |
| "mmlu_logical_fallacies": { |
| "alias": " - logical_fallacies", |
| "acc,none": 0.8098159509202454, |
| "acc_stderr,none": 0.03083349114628123 |
| }, |
| "mmlu_moral_disputes": { |
| "alias": " - moral_disputes", |
| "acc,none": 0.7167630057803468, |
| "acc_stderr,none": 0.02425790170532337 |
| }, |
| "mmlu_moral_scenarios": { |
| "alias": " - moral_scenarios", |
| "acc,none": 0.376536312849162, |
| "acc_stderr,none": 0.016204672385106596 |
| }, |
| "mmlu_philosophy": { |
| "alias": " - philosophy", |
| "acc,none": 0.707395498392283, |
| "acc_stderr,none": 0.02583989833487798 |
| }, |
| "mmlu_prehistory": { |
| "alias": " - prehistory", |
| "acc,none": 0.7191358024691358, |
| "acc_stderr,none": 0.025006469755799208 |
| }, |
| "mmlu_professional_law": { |
| "alias": " - professional_law", |
| "acc,none": 0.4954367666232073, |
| "acc_stderr,none": 0.012769704263117531 |
| }, |
| "mmlu_world_religions": { |
| "alias": " - world_religions", |
| "acc,none": 0.8011695906432749, |
| "acc_stderr,none": 0.03061111655743253 |
| }, |
| "mmlu_other": { |
| "acc,none": 0.7071129707112971, |
| "acc_stderr,none": 0.00792634679462611, |
| "alias": " - other" |
| }, |
| "mmlu_business_ethics": { |
| "alias": " - business_ethics", |
| "acc,none": 0.71, |
| "acc_stderr,none": 0.045604802157206845 |
| }, |
| "mmlu_clinical_knowledge": { |
| "alias": " - clinical_knowledge", |
| "acc,none": 0.7283018867924528, |
| "acc_stderr,none": 0.027377706624670713 |
| }, |
| "mmlu_college_medicine": { |
| "alias": " - college_medicine", |
| "acc,none": 0.630057803468208, |
| "acc_stderr,none": 0.0368122963339432 |
| }, |
| "mmlu_global_facts": { |
| "alias": " - global_facts", |
| "acc,none": 0.46, |
| "acc_stderr,none": 0.05009082659620333 |
| }, |
| "mmlu_human_aging": { |
| "alias": " - human_aging", |
| "acc,none": 0.6681614349775785, |
| "acc_stderr,none": 0.03160295143776679 |
| }, |
| "mmlu_management": { |
| "alias": " - management", |
| "acc,none": 0.7475728155339806, |
| "acc_stderr,none": 0.04301250399690879 |
| }, |
| "mmlu_marketing": { |
| "alias": " - marketing", |
| "acc,none": 0.8974358974358975, |
| "acc_stderr,none": 0.019875655027867457 |
| }, |
| "mmlu_medical_genetics": { |
| "alias": " - medical_genetics", |
| "acc,none": 0.74, |
| "acc_stderr,none": 0.04408440022768078 |
| }, |
| "mmlu_miscellaneous": { |
| "alias": " - miscellaneous", |
| "acc,none": 0.7956577266922095, |
| "acc_stderr,none": 0.0144191239809319 |
| }, |
| "mmlu_nutrition": { |
| "alias": " - nutrition", |
| "acc,none": 0.7287581699346405, |
| "acc_stderr,none": 0.025457756696667864 |
| }, |
| "mmlu_professional_accounting": { |
| "alias": " - professional_accounting", |
| "acc,none": 0.5035460992907801, |
| "acc_stderr,none": 0.02982674915328092 |
| }, |
| "mmlu_professional_medicine": { |
| "alias": " - professional_medicine", |
| "acc,none": 0.7058823529411765, |
| "acc_stderr,none": 0.0276784686421447 |
| }, |
| "mmlu_virology": { |
| "alias": " - virology", |
| "acc,none": 0.5301204819277109, |
| "acc_stderr,none": 0.03885425420866766 |
| }, |
| "mmlu_social_sciences": { |
| "acc,none": 0.7630809229769255, |
| "acc_stderr,none": 0.007521922102925205, |
| "alias": " - social sciences" |
| }, |
| "mmlu_econometrics": { |
| "alias": " - econometrics", |
| "acc,none": 0.5087719298245614, |
| "acc_stderr,none": 0.04702880432049615 |
| }, |
| "mmlu_high_school_geography": { |
| "alias": " - high_school_geography", |
| "acc,none": 0.8181818181818182, |
| "acc_stderr,none": 0.0274796030105388 |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "alias": " - high_school_government_and_politics", |
| "acc,none": 0.8860103626943006, |
| "acc_stderr,none": 0.022935144053919436 |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "alias": " - high_school_macroeconomics", |
| "acc,none": 0.6846153846153846, |
| "acc_stderr,none": 0.02355964698318994 |
| }, |
| "mmlu_high_school_microeconomics": { |
| "alias": " - high_school_microeconomics", |
| "acc,none": 0.8277310924369747, |
| "acc_stderr,none": 0.024528664971305427 |
| }, |
| "mmlu_high_school_psychology": { |
| "alias": " - high_school_psychology", |
| "acc,none": 0.8477064220183487, |
| "acc_stderr,none": 0.015405084393157069 |
| }, |
| "mmlu_human_sexuality": { |
| "alias": " - human_sexuality", |
| "acc,none": 0.7557251908396947, |
| "acc_stderr,none": 0.037683359597287434 |
| }, |
| "mmlu_professional_psychology": { |
| "alias": " - professional_psychology", |
| "acc,none": 0.6879084967320261, |
| "acc_stderr,none": 0.01874501120127766 |
| }, |
| "mmlu_public_relations": { |
| "alias": " - public_relations", |
| "acc,none": 0.7090909090909091, |
| "acc_stderr,none": 0.04350271442923243 |
| }, |
| "mmlu_security_studies": { |
| "alias": " - security_studies", |
| "acc,none": 0.7714285714285715, |
| "acc_stderr,none": 0.026882144922307744 |
| }, |
| "mmlu_sociology": { |
| "alias": " - sociology", |
| "acc,none": 0.8059701492537313, |
| "acc_stderr,none": 0.027962677604768896 |
| }, |
| "mmlu_us_foreign_policy": { |
| "alias": " - us_foreign_policy", |
| "acc,none": 0.82, |
| "acc_stderr,none": 0.03861229196653694 |
| }, |
| "mmlu_stem": { |
| "acc,none": 0.5803996194100857, |
| "acc_stderr,none": 0.008464141586351462, |
| "alias": " - stem" |
| }, |
| "mmlu_abstract_algebra": { |
| "alias": " - abstract_algebra", |
| "acc,none": 0.42, |
| "acc_stderr,none": 0.049604496374885836 |
| }, |
| "mmlu_anatomy": { |
| "alias": " - anatomy", |
| "acc,none": 0.6148148148148148, |
| "acc_stderr,none": 0.042039210401562783 |
| }, |
| "mmlu_astronomy": { |
| "alias": " - astronomy", |
| "acc,none": 0.743421052631579, |
| "acc_stderr,none": 0.035541803680256896 |
| }, |
| "mmlu_college_biology": { |
| "alias": " - college_biology", |
| "acc,none": 0.7986111111111112, |
| "acc_stderr,none": 0.03353647469713839 |
| }, |
| "mmlu_college_chemistry": { |
| "alias": " - college_chemistry", |
| "acc,none": 0.47, |
| "acc_stderr,none": 0.05016135580465919 |
| }, |
| "mmlu_college_computer_science": { |
| "alias": " - college_computer_science", |
| "acc,none": 0.53, |
| "acc_stderr,none": 0.050161355804659205 |
| }, |
| "mmlu_college_mathematics": { |
| "alias": " - college_mathematics", |
| "acc,none": 0.4, |
| "acc_stderr,none": 0.04923659639173309 |
| }, |
| "mmlu_college_physics": { |
| "alias": " - college_physics", |
| "acc,none": 0.37254901960784315, |
| "acc_stderr,none": 0.04810840148082634 |
| }, |
| "mmlu_computer_security": { |
| "alias": " - computer_security", |
| "acc,none": 0.73, |
| "acc_stderr,none": 0.044619604333847394 |
| }, |
| "mmlu_conceptual_physics": { |
| "alias": " - conceptual_physics", |
| "acc,none": 0.6382978723404256, |
| "acc_stderr,none": 0.03141082197596241 |
| }, |
| "mmlu_electrical_engineering": { |
| "alias": " - electrical_engineering", |
| "acc,none": 0.593103448275862, |
| "acc_stderr,none": 0.04093793981266236 |
| }, |
| "mmlu_elementary_mathematics": { |
| "alias": " - elementary_mathematics", |
| "acc,none": 0.5661375661375662, |
| "acc_stderr,none": 0.02552503438247489 |
| }, |
| "mmlu_high_school_biology": { |
| "alias": " - high_school_biology", |
| "acc,none": 0.7935483870967742, |
| "acc_stderr,none": 0.023025899617188744 |
| }, |
| "mmlu_high_school_chemistry": { |
| "alias": " - high_school_chemistry", |
| "acc,none": 0.5566502463054187, |
| "acc_stderr,none": 0.03495334582162934 |
| }, |
| "mmlu_high_school_computer_science": { |
| "alias": " - high_school_computer_science", |
| "acc,none": 0.75, |
| "acc_stderr,none": 0.04351941398892446 |
| }, |
| "mmlu_high_school_mathematics": { |
| "alias": " - high_school_mathematics", |
| "acc,none": 0.36666666666666664, |
| "acc_stderr,none": 0.029381620726465073 |
| }, |
| "mmlu_high_school_physics": { |
| "alias": " - high_school_physics", |
| "acc,none": 0.46357615894039733, |
| "acc_stderr,none": 0.04071636065944216 |
| }, |
| "mmlu_high_school_statistics": { |
| "alias": " - high_school_statistics", |
| "acc,none": 0.5787037037037037, |
| "acc_stderr,none": 0.033674621388960775 |
| }, |
| "mmlu_machine_learning": { |
| "alias": " - machine_learning", |
| "acc,none": 0.42857142857142855, |
| "acc_stderr,none": 0.04697113923010212 |
| } |
| }, |
| "groups": { |
| "mmlu": { |
| "acc,none": 0.6536105967810854, |
| "acc_stderr,none": 0.0038123134918508976, |
| "alias": "mmlu" |
| }, |
| "mmlu_humanities": { |
| "acc,none": 0.5957492029755579, |
| "acc_stderr,none": 0.006759316550944608, |
| "alias": " - humanities" |
| }, |
| "mmlu_other": { |
| "acc,none": 0.7071129707112971, |
| "acc_stderr,none": 0.00792634679462611, |
| "alias": " - other" |
| }, |
| "mmlu_social_sciences": { |
| "acc,none": 0.7630809229769255, |
| "acc_stderr,none": 0.007521922102925205, |
| "alias": " - social sciences" |
| }, |
| "mmlu_stem": { |
| "acc,none": 0.5803996194100857, |
| "acc_stderr,none": 0.008464141586351462, |
| "alias": " - stem" |
| } |
| }, |
| "group_subtasks": { |
| "mmlu_humanities": [ |
| "mmlu_formal_logic", |
| "mmlu_high_school_european_history", |
| "mmlu_high_school_us_history", |
| "mmlu_high_school_world_history", |
| "mmlu_international_law", |
| "mmlu_jurisprudence", |
| "mmlu_logical_fallacies", |
| "mmlu_moral_disputes", |
| "mmlu_moral_scenarios", |
| "mmlu_philosophy", |
| "mmlu_prehistory", |
| "mmlu_professional_law", |
| "mmlu_world_religions" |
| ], |
| "mmlu_social_sciences": [ |
| "mmlu_econometrics", |
| "mmlu_high_school_geography", |
| "mmlu_high_school_government_and_politics", |
| "mmlu_high_school_macroeconomics", |
| "mmlu_high_school_microeconomics", |
| "mmlu_high_school_psychology", |
| "mmlu_human_sexuality", |
| "mmlu_professional_psychology", |
| "mmlu_public_relations", |
| "mmlu_security_studies", |
| "mmlu_sociology", |
| "mmlu_us_foreign_policy" |
| ], |
| "mmlu_other": [ |
| "mmlu_business_ethics", |
| "mmlu_clinical_knowledge", |
| "mmlu_college_medicine", |
| "mmlu_global_facts", |
| "mmlu_human_aging", |
| "mmlu_management", |
| "mmlu_marketing", |
| "mmlu_medical_genetics", |
| "mmlu_miscellaneous", |
| "mmlu_nutrition", |
| "mmlu_professional_accounting", |
| "mmlu_professional_medicine", |
| "mmlu_virology" |
| ], |
| "mmlu_stem": [ |
| "mmlu_abstract_algebra", |
| "mmlu_anatomy", |
| "mmlu_astronomy", |
| "mmlu_college_biology", |
| "mmlu_college_chemistry", |
| "mmlu_college_computer_science", |
| "mmlu_college_mathematics", |
| "mmlu_college_physics", |
| "mmlu_computer_security", |
| "mmlu_conceptual_physics", |
| "mmlu_electrical_engineering", |
| "mmlu_elementary_mathematics", |
| "mmlu_high_school_biology", |
| "mmlu_high_school_chemistry", |
| "mmlu_high_school_computer_science", |
| "mmlu_high_school_mathematics", |
| "mmlu_high_school_physics", |
| "mmlu_high_school_statistics", |
| "mmlu_machine_learning" |
| ], |
| "mmlu": [ |
| "mmlu_stem", |
| "mmlu_other", |
| "mmlu_social_sciences", |
| "mmlu_humanities" |
| ], |
| "arc_easy": [], |
| "arc_challenge": [] |
| }, |
| "configs": { |
| "arc_challenge": { |
| "task": "arc_challenge", |
| "tag": [ |
| "ai2_arc" |
| ], |
| "dataset_path": "allenai/ai2_arc", |
| "dataset_name": "ARC-Challenge", |
| "training_split": "train", |
| "validation_split": "validation", |
| "test_split": "test", |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_target": "{{choices.label.index(answerKey)}}", |
| "unsafe_code": false, |
| "doc_to_choice": "{{choices.text}}", |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "acc_norm", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": true, |
| "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "arc_easy": { |
| "task": "arc_easy", |
| "tag": [ |
| "ai2_arc" |
| ], |
| "dataset_path": "allenai/ai2_arc", |
| "dataset_name": "ARC-Easy", |
| "training_split": "train", |
| "validation_split": "validation", |
| "test_split": "test", |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_target": "{{choices.label.index(answerKey)}}", |
| "unsafe_code": false, |
| "doc_to_choice": "{{choices.text}}", |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "acc_norm", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": true, |
| "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_abstract_algebra": { |
| "task": "mmlu_abstract_algebra", |
| "task_alias": "abstract_algebra", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "abstract_algebra", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_anatomy": { |
| "task": "mmlu_anatomy", |
| "task_alias": "anatomy", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "anatomy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_astronomy": { |
| "task": "mmlu_astronomy", |
| "task_alias": "astronomy", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "astronomy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_business_ethics": { |
| "task": "mmlu_business_ethics", |
| "task_alias": "business_ethics", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "business_ethics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_clinical_knowledge": { |
| "task": "mmlu_clinical_knowledge", |
| "task_alias": "clinical_knowledge", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "clinical_knowledge", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_biology": { |
| "task": "mmlu_college_biology", |
| "task_alias": "college_biology", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_biology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college biology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_chemistry": { |
| "task": "mmlu_college_chemistry", |
| "task_alias": "college_chemistry", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_chemistry", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_computer_science": { |
| "task": "mmlu_college_computer_science", |
| "task_alias": "college_computer_science", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_computer_science", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_mathematics": { |
| "task": "mmlu_college_mathematics", |
| "task_alias": "college_mathematics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_medicine": { |
| "task": "mmlu_college_medicine", |
| "task_alias": "college_medicine", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_medicine", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_college_physics": { |
| "task": "mmlu_college_physics", |
| "task_alias": "college_physics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "college_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_computer_security": { |
| "task": "mmlu_computer_security", |
| "task_alias": "computer_security", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "computer_security", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about computer security.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_conceptual_physics": { |
| "task": "mmlu_conceptual_physics", |
| "task_alias": "conceptual_physics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "conceptual_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_econometrics": { |
| "task": "mmlu_econometrics", |
| "task_alias": "econometrics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "econometrics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_electrical_engineering": { |
| "task": "mmlu_electrical_engineering", |
| "task_alias": "electrical_engineering", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "electrical_engineering", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_elementary_mathematics": { |
| "task": "mmlu_elementary_mathematics", |
| "task_alias": "elementary_mathematics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "elementary_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_formal_logic": { |
| "task": "mmlu_formal_logic", |
| "task_alias": "formal_logic", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "formal_logic", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_global_facts": { |
| "task": "mmlu_global_facts", |
| "task_alias": "global_facts", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "global_facts", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about global facts.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_biology": { |
| "task": "mmlu_high_school_biology", |
| "task_alias": "high_school_biology", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_biology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_chemistry": { |
| "task": "mmlu_high_school_chemistry", |
| "task_alias": "high_school_chemistry", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_chemistry", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_computer_science": { |
| "task": "mmlu_high_school_computer_science", |
| "task_alias": "high_school_computer_science", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_computer_science", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_european_history": { |
| "task": "mmlu_high_school_european_history", |
| "task_alias": "high_school_european_history", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_european_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_geography": { |
| "task": "mmlu_high_school_geography", |
| "task_alias": "high_school_geography", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_geography", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "task": "mmlu_high_school_government_and_politics", |
| "task_alias": "high_school_government_and_politics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_government_and_politics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "task": "mmlu_high_school_macroeconomics", |
| "task_alias": "high_school_macroeconomics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_macroeconomics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_mathematics": { |
| "task": "mmlu_high_school_mathematics", |
| "task_alias": "high_school_mathematics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_microeconomics": { |
| "task": "mmlu_high_school_microeconomics", |
| "task_alias": "high_school_microeconomics", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_microeconomics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_physics": { |
| "task": "mmlu_high_school_physics", |
| "task_alias": "high_school_physics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_psychology": { |
| "task": "mmlu_high_school_psychology", |
| "task_alias": "high_school_psychology", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_psychology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_statistics": { |
| "task": "mmlu_high_school_statistics", |
| "task_alias": "high_school_statistics", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_statistics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_us_history": { |
| "task": "mmlu_high_school_us_history", |
| "task_alias": "high_school_us_history", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_us_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_high_school_world_history": { |
| "task": "mmlu_high_school_world_history", |
| "task_alias": "high_school_world_history", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "high_school_world_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_human_aging": { |
| "task": "mmlu_human_aging", |
| "task_alias": "human_aging", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "human_aging", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about human aging.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_human_sexuality": { |
| "task": "mmlu_human_sexuality", |
| "task_alias": "human_sexuality", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "human_sexuality", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_international_law": { |
| "task": "mmlu_international_law", |
| "task_alias": "international_law", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "international_law", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about international law.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_jurisprudence": { |
| "task": "mmlu_jurisprudence", |
| "task_alias": "jurisprudence", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "jurisprudence", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_logical_fallacies": { |
| "task": "mmlu_logical_fallacies", |
| "task_alias": "logical_fallacies", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "logical_fallacies", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_machine_learning": { |
| "task": "mmlu_machine_learning", |
| "task_alias": "machine_learning", |
| "tag": "mmlu_stem_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "machine_learning", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_management": { |
| "task": "mmlu_management", |
| "task_alias": "management", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "management", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about management.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_marketing": { |
| "task": "mmlu_marketing", |
| "task_alias": "marketing", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "marketing", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about marketing.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_medical_genetics": { |
| "task": "mmlu_medical_genetics", |
| "task_alias": "medical_genetics", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "medical_genetics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_miscellaneous": { |
| "task": "mmlu_miscellaneous", |
| "task_alias": "miscellaneous", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "miscellaneous", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_moral_disputes": { |
| "task": "mmlu_moral_disputes", |
| "task_alias": "moral_disputes", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "moral_disputes", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_moral_scenarios": { |
| "task": "mmlu_moral_scenarios", |
| "task_alias": "moral_scenarios", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "moral_scenarios", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_nutrition": { |
| "task": "mmlu_nutrition", |
| "task_alias": "nutrition", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "nutrition", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_philosophy": { |
| "task": "mmlu_philosophy", |
| "task_alias": "philosophy", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "philosophy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_prehistory": { |
| "task": "mmlu_prehistory", |
| "task_alias": "prehistory", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "prehistory", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_accounting": { |
| "task": "mmlu_professional_accounting", |
| "task_alias": "professional_accounting", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_accounting", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_law": { |
| "task": "mmlu_professional_law", |
| "task_alias": "professional_law", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_law", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional law.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_medicine": { |
| "task": "mmlu_professional_medicine", |
| "task_alias": "professional_medicine", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_medicine", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_professional_psychology": { |
| "task": "mmlu_professional_psychology", |
| "task_alias": "professional_psychology", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "professional_psychology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_public_relations": { |
| "task": "mmlu_public_relations", |
| "task_alias": "public_relations", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "public_relations", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about public relations.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_security_studies": { |
| "task": "mmlu_security_studies", |
| "task_alias": "security_studies", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "security_studies", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about security studies.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_sociology": { |
| "task": "mmlu_sociology", |
| "task_alias": "sociology", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "sociology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about sociology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_us_foreign_policy": { |
| "task": "mmlu_us_foreign_policy", |
| "task_alias": "us_foreign_policy", |
| "tag": "mmlu_social_sciences_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "us_foreign_policy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_virology": { |
| "task": "mmlu_virology", |
| "task_alias": "virology", |
| "tag": "mmlu_other_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "virology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about virology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| }, |
| "mmlu_world_religions": { |
| "task": "mmlu_world_religions", |
| "task_alias": "world_religions", |
| "tag": "mmlu_humanities_tasks", |
| "dataset_path": "cais/mmlu", |
| "dataset_name": "world_religions", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about world religions.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0 |
| } |
| } |
| }, |
| "versions": { |
| "arc_challenge": 1.0, |
| "arc_easy": 1.0, |
| "mmlu": 2, |
| "mmlu_abstract_algebra": 1.0, |
| "mmlu_anatomy": 1.0, |
| "mmlu_astronomy": 1.0, |
| "mmlu_business_ethics": 1.0, |
| "mmlu_clinical_knowledge": 1.0, |
| "mmlu_college_biology": 1.0, |
| "mmlu_college_chemistry": 1.0, |
| "mmlu_college_computer_science": 1.0, |
| "mmlu_college_mathematics": 1.0, |
| "mmlu_college_medicine": 1.0, |
| "mmlu_college_physics": 1.0, |
| "mmlu_computer_security": 1.0, |
| "mmlu_conceptual_physics": 1.0, |
| "mmlu_econometrics": 1.0, |
| "mmlu_electrical_engineering": 1.0, |
| "mmlu_elementary_mathematics": 1.0, |
| "mmlu_formal_logic": 1.0, |
| "mmlu_global_facts": 1.0, |
| "mmlu_high_school_biology": 1.0, |
| "mmlu_high_school_chemistry": 1.0, |
| "mmlu_high_school_computer_science": 1.0, |
| "mmlu_high_school_european_history": 1.0, |
| "mmlu_high_school_geography": 1.0, |
| "mmlu_high_school_government_and_politics": 1.0, |
| "mmlu_high_school_macroeconomics": 1.0, |
| "mmlu_high_school_mathematics": 1.0, |
| "mmlu_high_school_microeconomics": 1.0, |
| "mmlu_high_school_physics": 1.0, |
| "mmlu_high_school_psychology": 1.0, |
| "mmlu_high_school_statistics": 1.0, |
| "mmlu_high_school_us_history": 1.0, |
| "mmlu_high_school_world_history": 1.0, |
| "mmlu_human_aging": 1.0, |
| "mmlu_human_sexuality": 1.0, |
| "mmlu_humanities": 2, |
| "mmlu_international_law": 1.0, |
| "mmlu_jurisprudence": 1.0, |
| "mmlu_logical_fallacies": 1.0, |
| "mmlu_machine_learning": 1.0, |
| "mmlu_management": 1.0, |
| "mmlu_marketing": 1.0, |
| "mmlu_medical_genetics": 1.0, |
| "mmlu_miscellaneous": 1.0, |
| "mmlu_moral_disputes": 1.0, |
| "mmlu_moral_scenarios": 1.0, |
| "mmlu_nutrition": 1.0, |
| "mmlu_other": 2, |
| "mmlu_philosophy": 1.0, |
| "mmlu_prehistory": 1.0, |
| "mmlu_professional_accounting": 1.0, |
| "mmlu_professional_law": 1.0, |
| "mmlu_professional_medicine": 1.0, |
| "mmlu_professional_psychology": 1.0, |
| "mmlu_public_relations": 1.0, |
| "mmlu_security_studies": 1.0, |
| "mmlu_social_sciences": 2, |
| "mmlu_sociology": 1.0, |
| "mmlu_stem": 2, |
| "mmlu_us_foreign_policy": 1.0, |
| "mmlu_virology": 1.0, |
| "mmlu_world_religions": 1.0 |
| }, |
| "n-shot": { |
| "arc_challenge": 0, |
| "arc_easy": 0, |
| "mmlu_abstract_algebra": 0, |
| "mmlu_anatomy": 0, |
| "mmlu_astronomy": 0, |
| "mmlu_business_ethics": 0, |
| "mmlu_clinical_knowledge": 0, |
| "mmlu_college_biology": 0, |
| "mmlu_college_chemistry": 0, |
| "mmlu_college_computer_science": 0, |
| "mmlu_college_mathematics": 0, |
| "mmlu_college_medicine": 0, |
| "mmlu_college_physics": 0, |
| "mmlu_computer_security": 0, |
| "mmlu_conceptual_physics": 0, |
| "mmlu_econometrics": 0, |
| "mmlu_electrical_engineering": 0, |
| "mmlu_elementary_mathematics": 0, |
| "mmlu_formal_logic": 0, |
| "mmlu_global_facts": 0, |
| "mmlu_high_school_biology": 0, |
| "mmlu_high_school_chemistry": 0, |
| "mmlu_high_school_computer_science": 0, |
| "mmlu_high_school_european_history": 0, |
| "mmlu_high_school_geography": 0, |
| "mmlu_high_school_government_and_politics": 0, |
| "mmlu_high_school_macroeconomics": 0, |
| "mmlu_high_school_mathematics": 0, |
| "mmlu_high_school_microeconomics": 0, |
| "mmlu_high_school_physics": 0, |
| "mmlu_high_school_psychology": 0, |
| "mmlu_high_school_statistics": 0, |
| "mmlu_high_school_us_history": 0, |
| "mmlu_high_school_world_history": 0, |
| "mmlu_human_aging": 0, |
| "mmlu_human_sexuality": 0, |
| "mmlu_international_law": 0, |
| "mmlu_jurisprudence": 0, |
| "mmlu_logical_fallacies": 0, |
| "mmlu_machine_learning": 0, |
| "mmlu_management": 0, |
| "mmlu_marketing": 0, |
| "mmlu_medical_genetics": 0, |
| "mmlu_miscellaneous": 0, |
| "mmlu_moral_disputes": 0, |
| "mmlu_moral_scenarios": 0, |
| "mmlu_nutrition": 0, |
| "mmlu_philosophy": 0, |
| "mmlu_prehistory": 0, |
| "mmlu_professional_accounting": 0, |
| "mmlu_professional_law": 0, |
| "mmlu_professional_medicine": 0, |
| "mmlu_professional_psychology": 0, |
| "mmlu_public_relations": 0, |
| "mmlu_security_studies": 0, |
| "mmlu_sociology": 0, |
| "mmlu_us_foreign_policy": 0, |
| "mmlu_virology": 0, |
| "mmlu_world_religions": 0 |
| }, |
| "higher_is_better": { |
| "arc_challenge": { |
| "acc": true, |
| "acc_norm": true |
| }, |
| "arc_easy": { |
| "acc": true, |
| "acc_norm": true |
| }, |
| "mmlu": { |
| "acc": true |
| }, |
| "mmlu_abstract_algebra": { |
| "acc": true |
| }, |
| "mmlu_anatomy": { |
| "acc": true |
| }, |
| "mmlu_astronomy": { |
| "acc": true |
| }, |
| "mmlu_business_ethics": { |
| "acc": true |
| }, |
| "mmlu_clinical_knowledge": { |
| "acc": true |
| }, |
| "mmlu_college_biology": { |
| "acc": true |
| }, |
| "mmlu_college_chemistry": { |
| "acc": true |
| }, |
| "mmlu_college_computer_science": { |
| "acc": true |
| }, |
| "mmlu_college_mathematics": { |
| "acc": true |
| }, |
| "mmlu_college_medicine": { |
| "acc": true |
| }, |
| "mmlu_college_physics": { |
| "acc": true |
| }, |
| "mmlu_computer_security": { |
| "acc": true |
| }, |
| "mmlu_conceptual_physics": { |
| "acc": true |
| }, |
| "mmlu_econometrics": { |
| "acc": true |
| }, |
| "mmlu_electrical_engineering": { |
| "acc": true |
| }, |
| "mmlu_elementary_mathematics": { |
| "acc": true |
| }, |
| "mmlu_formal_logic": { |
| "acc": true |
| }, |
| "mmlu_global_facts": { |
| "acc": true |
| }, |
| "mmlu_high_school_biology": { |
| "acc": true |
| }, |
| "mmlu_high_school_chemistry": { |
| "acc": true |
| }, |
| "mmlu_high_school_computer_science": { |
| "acc": true |
| }, |
| "mmlu_high_school_european_history": { |
| "acc": true |
| }, |
| "mmlu_high_school_geography": { |
| "acc": true |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "acc": true |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "acc": true |
| }, |
| "mmlu_high_school_mathematics": { |
| "acc": true |
| }, |
| "mmlu_high_school_microeconomics": { |
| "acc": true |
| }, |
| "mmlu_high_school_physics": { |
| "acc": true |
| }, |
| "mmlu_high_school_psychology": { |
| "acc": true |
| }, |
| "mmlu_high_school_statistics": { |
| "acc": true |
| }, |
| "mmlu_high_school_us_history": { |
| "acc": true |
| }, |
| "mmlu_high_school_world_history": { |
| "acc": true |
| }, |
| "mmlu_human_aging": { |
| "acc": true |
| }, |
| "mmlu_human_sexuality": { |
| "acc": true |
| }, |
| "mmlu_humanities": { |
| "acc": true |
| }, |
| "mmlu_international_law": { |
| "acc": true |
| }, |
| "mmlu_jurisprudence": { |
| "acc": true |
| }, |
| "mmlu_logical_fallacies": { |
| "acc": true |
| }, |
| "mmlu_machine_learning": { |
| "acc": true |
| }, |
| "mmlu_management": { |
| "acc": true |
| }, |
| "mmlu_marketing": { |
| "acc": true |
| }, |
| "mmlu_medical_genetics": { |
| "acc": true |
| }, |
| "mmlu_miscellaneous": { |
| "acc": true |
| }, |
| "mmlu_moral_disputes": { |
| "acc": true |
| }, |
| "mmlu_moral_scenarios": { |
| "acc": true |
| }, |
| "mmlu_nutrition": { |
| "acc": true |
| }, |
| "mmlu_other": { |
| "acc": true |
| }, |
| "mmlu_philosophy": { |
| "acc": true |
| }, |
| "mmlu_prehistory": { |
| "acc": true |
| }, |
| "mmlu_professional_accounting": { |
| "acc": true |
| }, |
| "mmlu_professional_law": { |
| "acc": true |
| }, |
| "mmlu_professional_medicine": { |
| "acc": true |
| }, |
| "mmlu_professional_psychology": { |
| "acc": true |
| }, |
| "mmlu_public_relations": { |
| "acc": true |
| }, |
| "mmlu_security_studies": { |
| "acc": true |
| }, |
| "mmlu_social_sciences": { |
| "acc": true |
| }, |
| "mmlu_sociology": { |
| "acc": true |
| }, |
| "mmlu_stem": { |
| "acc": true |
| }, |
| "mmlu_us_foreign_policy": { |
| "acc": true |
| }, |
| "mmlu_virology": { |
| "acc": true |
| }, |
| "mmlu_world_religions": { |
| "acc": true |
| } |
| }, |
| "n-samples": { |
| "arc_challenge": { |
| "original": 1172, |
| "effective": 1172 |
| }, |
| "arc_easy": { |
| "original": 2376, |
| "effective": 2376 |
| }, |
| "mmlu_abstract_algebra": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_anatomy": { |
| "original": 135, |
| "effective": 135 |
| }, |
| "mmlu_astronomy": { |
| "original": 152, |
| "effective": 152 |
| }, |
| "mmlu_college_biology": { |
| "original": 144, |
| "effective": 144 |
| }, |
| "mmlu_college_chemistry": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_college_computer_science": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_college_mathematics": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_college_physics": { |
| "original": 102, |
| "effective": 102 |
| }, |
| "mmlu_computer_security": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_conceptual_physics": { |
| "original": 235, |
| "effective": 235 |
| }, |
| "mmlu_electrical_engineering": { |
| "original": 145, |
| "effective": 145 |
| }, |
| "mmlu_elementary_mathematics": { |
| "original": 378, |
| "effective": 378 |
| }, |
| "mmlu_high_school_biology": { |
| "original": 310, |
| "effective": 310 |
| }, |
| "mmlu_high_school_chemistry": { |
| "original": 203, |
| "effective": 203 |
| }, |
| "mmlu_high_school_computer_science": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_high_school_mathematics": { |
| "original": 270, |
| "effective": 270 |
| }, |
| "mmlu_high_school_physics": { |
| "original": 151, |
| "effective": 151 |
| }, |
| "mmlu_high_school_statistics": { |
| "original": 216, |
| "effective": 216 |
| }, |
| "mmlu_machine_learning": { |
| "original": 112, |
| "effective": 112 |
| }, |
| "mmlu_business_ethics": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_clinical_knowledge": { |
| "original": 265, |
| "effective": 265 |
| }, |
| "mmlu_college_medicine": { |
| "original": 173, |
| "effective": 173 |
| }, |
| "mmlu_global_facts": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_human_aging": { |
| "original": 223, |
| "effective": 223 |
| }, |
| "mmlu_management": { |
| "original": 103, |
| "effective": 103 |
| }, |
| "mmlu_marketing": { |
| "original": 234, |
| "effective": 234 |
| }, |
| "mmlu_medical_genetics": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_miscellaneous": { |
| "original": 783, |
| "effective": 783 |
| }, |
| "mmlu_nutrition": { |
| "original": 306, |
| "effective": 306 |
| }, |
| "mmlu_professional_accounting": { |
| "original": 282, |
| "effective": 282 |
| }, |
| "mmlu_professional_medicine": { |
| "original": 272, |
| "effective": 272 |
| }, |
| "mmlu_virology": { |
| "original": 166, |
| "effective": 166 |
| }, |
| "mmlu_econometrics": { |
| "original": 114, |
| "effective": 114 |
| }, |
| "mmlu_high_school_geography": { |
| "original": 198, |
| "effective": 198 |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "original": 193, |
| "effective": 193 |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "original": 390, |
| "effective": 390 |
| }, |
| "mmlu_high_school_microeconomics": { |
| "original": 238, |
| "effective": 238 |
| }, |
| "mmlu_high_school_psychology": { |
| "original": 545, |
| "effective": 545 |
| }, |
| "mmlu_human_sexuality": { |
| "original": 131, |
| "effective": 131 |
| }, |
| "mmlu_professional_psychology": { |
| "original": 612, |
| "effective": 612 |
| }, |
| "mmlu_public_relations": { |
| "original": 110, |
| "effective": 110 |
| }, |
| "mmlu_security_studies": { |
| "original": 245, |
| "effective": 245 |
| }, |
| "mmlu_sociology": { |
| "original": 201, |
| "effective": 201 |
| }, |
| "mmlu_us_foreign_policy": { |
| "original": 100, |
| "effective": 100 |
| }, |
| "mmlu_formal_logic": { |
| "original": 126, |
| "effective": 126 |
| }, |
| "mmlu_high_school_european_history": { |
| "original": 165, |
| "effective": 165 |
| }, |
| "mmlu_high_school_us_history": { |
| "original": 204, |
| "effective": 204 |
| }, |
| "mmlu_high_school_world_history": { |
| "original": 237, |
| "effective": 237 |
| }, |
| "mmlu_international_law": { |
| "original": 121, |
| "effective": 121 |
| }, |
| "mmlu_jurisprudence": { |
| "original": 108, |
| "effective": 108 |
| }, |
| "mmlu_logical_fallacies": { |
| "original": 163, |
| "effective": 163 |
| }, |
| "mmlu_moral_disputes": { |
| "original": 346, |
| "effective": 346 |
| }, |
| "mmlu_moral_scenarios": { |
| "original": 895, |
| "effective": 895 |
| }, |
| "mmlu_philosophy": { |
| "original": 311, |
| "effective": 311 |
| }, |
| "mmlu_prehistory": { |
| "original": 324, |
| "effective": 324 |
| }, |
| "mmlu_professional_law": { |
| "original": 1534, |
| "effective": 1534 |
| }, |
| "mmlu_world_religions": { |
| "original": 171, |
| "effective": 171 |
| } |
| }, |
| "config": { |
| "model": "/home/jovyan/.cache/huggingface/models--microsoft--Phi-mini-MoE-instruct/snapshots/62d7eed72cc0420fc924c800721b1401d628395c", |
| "model_args": null, |
| "model_num_parameters": 7647632704, |
| "model_dtype": "torch.bfloat16", |
| "model_revision": "main", |
| "model_sha": "", |
| "batch_size": "auto", |
| "batch_sizes": [ |
| 64 |
| ], |
| "device": null, |
| "use_cache": null, |
| "limit": null, |
| "bootstrap_iters": 100000, |
| "gen_kwargs": null, |
| "random_seed": 0, |
| "numpy_seed": 1234, |
| "torch_seed": 1234, |
| "fewshot_seed": 1234 |
| }, |
| "git_hash": "0cffc7f", |
| "date": 1763176707.4563708, |
| "pretty_env_info": "PyTorch version: 2.7.1+cu118\nIs debug build: False\nCUDA used to build PyTorch: 11.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.9\nLibc version: glibc-2.35\n\nPython version: 3.10.13 | packaged by conda-forge | (main, Oct 26 2023, 18:07:37) [GCC 12.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-138-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 11.8.89\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA A100 80GB PCIe\nNvidia driver version: 555.42.06\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7443 24-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 1\nFrequency boost: enabled\nCPU max MHz: 4035.6440\nCPU min MHz: 1500.0000\nBogoMIPS: 5699.48\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload vgif v_spec_ctrl umip pku ospke vaes vpclmulqdq rdpid overflow_recov succor smca fsrm\nVirtualization: AMD-V\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 256 MiB (8 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.2.6\n[pip3] nvidia-cublas-cu11==11.11.3.6\n[pip3] nvidia-cuda-cupti-cu11==11.8.87\n[pip3] nvidia-cuda-nvrtc-cu11==11.8.89\n[pip3] nvidia-cuda-runtime-cu11==11.8.89\n[pip3] nvidia-cudnn-cu11==9.1.0.70\n[pip3] nvidia-cufft-cu11==10.9.0.58\n[pip3] nvidia-curand-cu11==10.3.0.86\n[pip3] nvidia-cusolver-cu11==11.4.1.48\n[pip3] nvidia-cusparse-cu11==11.7.5.86\n[pip3] nvidia-nccl-cu11==2.21.5\n[pip3] nvidia-nvtx-cu11==11.8.86\n[pip3] torch==2.7.1+cu118\n[pip3] triton==3.3.1\n[conda] nomkl 1.0 h5ca1d4c_0 conda-forge\n[conda] numpy 1.26.2 py310hb13e2d6_0 conda-forge\n[conda] nvidia-cublas-cu11 11.11.3.6 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu11 11.8.87 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu11 11.8.89 pypi_0 pypi\n[conda] nvidia-cudnn-cu11 8.7.0.84 pypi_0 pypi\n[conda] nvidia-cufft-cu11 10.9.0.58 pypi_0 pypi\n[conda] nvidia-curand-cu11 10.3.0.86 pypi_0 pypi\n[conda] nvidia-cusolver-cu11 11.4.1.48 pypi_0 pypi\n[conda] nvidia-cusparse-cu11 11.7.5.86 pypi_0 pypi\n[conda] nvidia-nccl-cu11 2.16.5 pypi_0 pypi\n[conda] pytorch-lightning 2.1.2 pypi_0 pypi\n[conda] torch 2.1.1+cu118 pypi_0 pypi\n[conda] torchaudio 2.1.1+cu118 pypi_0 pypi\n[conda] torchdata 0.7.1+cpu pypi_0 pypi\n[conda] torchmetrics 1.2.0 pypi_0 pypi\n[conda] torchtext 0.16.1+cpu pypi_0 pypi\n[conda] torchvision 0.16.1+cu118 pypi_0 pypi\n[conda] triton 2.1.0 pypi_0 pypi", |
| "transformers_version": "4.55.0", |
| "lm_eval_version": "0.4.9.1", |
| "upper_git_hash": null, |
| "tokenizer_pad_token": [ |
| "<|endoftext|>", |
| "32000" |
| ], |
| "tokenizer_eos_token": [ |
| "<|endoftext|>", |
| "32000" |
| ], |
| "tokenizer_bos_token": [ |
| "<s>", |
| "1" |
| ], |
| "eot_token_id": 32000, |
| "max_length": 4096 |
| } |
| } |
