| { |
| "results": { |
| "mmlu": { |
| "acc,none": 0.26100270616721266, |
| "acc_stderr,none": 0.0037004678245414263, |
| "alias": "mmlu" |
| }, |
| "mmlu_humanities": { |
| "alias": " - humanities", |
| "acc,none": 0.2622741764080765, |
| "acc_stderr,none": 0.006405056497034186 |
| }, |
| "mmlu_formal_logic": { |
| "alias": " - formal_logic", |
| "acc,none": 0.16666666666666666, |
| "acc_stderr,none": 0.03333333333333333 |
| }, |
| "mmlu_high_school_european_history": { |
| "alias": " - high_school_european_history", |
| "acc,none": 0.296969696969697, |
| "acc_stderr,none": 0.03567969772268046 |
| }, |
| "mmlu_high_school_us_history": { |
| "alias": " - high_school_us_history", |
| "acc,none": 0.27450980392156865, |
| "acc_stderr,none": 0.03132179803083291 |
| }, |
| "mmlu_high_school_world_history": { |
| "alias": " - high_school_world_history", |
| "acc,none": 0.2320675105485232, |
| "acc_stderr,none": 0.027479744550808465 |
| }, |
| "mmlu_international_law": { |
| "alias": " - international_law", |
| "acc,none": 0.35537190082644626, |
| "acc_stderr,none": 0.043692363265739845 |
| }, |
| "mmlu_jurisprudence": { |
| "alias": " - jurisprudence", |
| "acc,none": 0.28703703703703703, |
| "acc_stderr,none": 0.04373313040914761 |
| }, |
| "mmlu_logical_fallacies": { |
| "alias": " - logical_fallacies", |
| "acc,none": 0.2883435582822086, |
| "acc_stderr,none": 0.03559039531617345 |
| }, |
| "mmlu_moral_disputes": { |
| "alias": " - moral_disputes", |
| "acc,none": 0.28034682080924855, |
| "acc_stderr,none": 0.024182427496577626 |
| }, |
| "mmlu_moral_scenarios": { |
| "alias": " - moral_scenarios", |
| "acc,none": 0.25251396648044694, |
| "acc_stderr,none": 0.014530330201468655 |
| }, |
| "mmlu_philosophy": { |
| "alias": " - philosophy", |
| "acc,none": 0.3247588424437299, |
| "acc_stderr,none": 0.02659678228769707 |
| }, |
| "mmlu_prehistory": { |
| "alias": " - prehistory", |
| "acc,none": 0.25, |
| "acc_stderr,none": 0.02409347123262133 |
| }, |
| "mmlu_professional_law": { |
| "alias": " - professional_law", |
| "acc,none": 0.2470664928292047, |
| "acc_stderr,none": 0.011015752255279445 |
| }, |
| "mmlu_world_religions": { |
| "alias": " - world_religions", |
| "acc,none": 0.2807017543859649, |
| "acc_stderr,none": 0.03446296217088429 |
| }, |
| "mmlu_other": { |
| "alias": " - other", |
| "acc,none": 0.2671387190215642, |
| "acc_stderr,none": 0.007937843638596481 |
| }, |
| "mmlu_business_ethics": { |
| "alias": " - business_ethics", |
| "acc,none": 0.23, |
| "acc_stderr,none": 0.04229525846816507 |
| }, |
| "mmlu_clinical_knowledge": { |
| "alias": " - clinical_knowledge", |
| "acc,none": 0.23773584905660378, |
| "acc_stderr,none": 0.02619980880756196 |
| }, |
| "mmlu_college_medicine": { |
| "alias": " - college_medicine", |
| "acc,none": 0.2543352601156069, |
| "acc_stderr,none": 0.03320556443085566 |
| }, |
| "mmlu_global_facts": { |
| "alias": " - global_facts", |
| "acc,none": 0.31, |
| "acc_stderr,none": 0.04648231987117317 |
| }, |
| "mmlu_human_aging": { |
| "alias": " - human_aging", |
| "acc,none": 0.30493273542600896, |
| "acc_stderr,none": 0.03089861088247754 |
| }, |
| "mmlu_management": { |
| "alias": " - management", |
| "acc,none": 0.24271844660194175, |
| "acc_stderr,none": 0.04245022486384496 |
| }, |
| "mmlu_marketing": { |
| "alias": " - marketing", |
| "acc,none": 0.2777777777777778, |
| "acc_stderr,none": 0.029343114798094483 |
| }, |
| "mmlu_medical_genetics": { |
| "alias": " - medical_genetics", |
| "acc,none": 0.33, |
| "acc_stderr,none": 0.04725815626252609 |
| }, |
| "mmlu_miscellaneous": { |
| "alias": " - miscellaneous", |
| "acc,none": 0.2771392081736909, |
| "acc_stderr,none": 0.01600563629412252 |
| }, |
| "mmlu_nutrition": { |
| "alias": " - nutrition", |
| "acc,none": 0.24509803921568626, |
| "acc_stderr,none": 0.024630048979824716 |
| }, |
| "mmlu_professional_accounting": { |
| "alias": " - professional_accounting", |
| "acc,none": 0.2695035460992908, |
| "acc_stderr,none": 0.02646903681859067 |
| }, |
| "mmlu_professional_medicine": { |
| "alias": " - professional_medicine", |
| "acc,none": 0.21691176470588236, |
| "acc_stderr,none": 0.025035845227711254 |
| }, |
| "mmlu_virology": { |
| "alias": " - virology", |
| "acc,none": 0.3072289156626506, |
| "acc_stderr,none": 0.03591566797824665 |
| }, |
| "mmlu_social_sciences": { |
| "alias": " - social_sciences", |
| "acc,none": 0.23431914202144946, |
| "acc_stderr,none": 0.007631090779162625 |
| }, |
| "mmlu_econometrics": { |
| "alias": " - econometrics", |
| "acc,none": 0.24561403508771928, |
| "acc_stderr,none": 0.040493392977481425 |
| }, |
| "mmlu_high_school_geography": { |
| "alias": " - high_school_geography", |
| "acc,none": 0.25252525252525254, |
| "acc_stderr,none": 0.03095405547036587 |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "alias": " - high_school_government_and_politics", |
| "acc,none": 0.22279792746113988, |
| "acc_stderr,none": 0.030031147977641528 |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "alias": " - high_school_macroeconomics", |
| "acc,none": 0.24615384615384617, |
| "acc_stderr,none": 0.02184086699042312 |
| }, |
| "mmlu_high_school_microeconomics": { |
| "alias": " - high_school_microeconomics", |
| "acc,none": 0.20588235294117646, |
| "acc_stderr,none": 0.026265024608275907 |
| }, |
| "mmlu_high_school_psychology": { |
| "alias": " - high_school_psychology", |
| "acc,none": 0.24036697247706423, |
| "acc_stderr,none": 0.01832060732096402 |
| }, |
| "mmlu_human_sexuality": { |
| "alias": " - human_sexuality", |
| "acc,none": 0.19083969465648856, |
| "acc_stderr,none": 0.03446513350752599 |
| }, |
| "mmlu_professional_psychology": { |
| "alias": " - professional_psychology", |
| "acc,none": 0.2647058823529412, |
| "acc_stderr,none": 0.01784808957491327 |
| }, |
| "mmlu_public_relations": { |
| "alias": " - public_relations", |
| "acc,none": 0.2909090909090909, |
| "acc_stderr,none": 0.04350271442923247 |
| }, |
| "mmlu_security_studies": { |
| "alias": " - security_studies", |
| "acc,none": 0.1673469387755102, |
| "acc_stderr,none": 0.023897144768914517 |
| }, |
| "mmlu_sociology": { |
| "alias": " - sociology", |
| "acc,none": 0.22388059701492538, |
| "acc_stderr,none": 0.029475250236017162 |
| }, |
| "mmlu_us_foreign_policy": { |
| "alias": " - us_foreign_policy", |
| "acc,none": 0.19, |
| "acc_stderr,none": 0.039427724440366255 |
| }, |
| "mmlu_stem": { |
| "alias": " - stem", |
| "acc,none": 0.2790992705359975, |
| "acc_stderr,none": 0.00797494423429467 |
| }, |
| "mmlu_abstract_algebra": { |
| "alias": " - abstract_algebra", |
| "acc,none": 0.25, |
| "acc_stderr,none": 0.04351941398892446 |
| }, |
| "mmlu_anatomy": { |
| "alias": " - anatomy", |
| "acc,none": 0.34814814814814815, |
| "acc_stderr,none": 0.04115324610336956 |
| }, |
| "mmlu_astronomy": { |
| "alias": " - astronomy", |
| "acc,none": 0.3355263157894737, |
| "acc_stderr,none": 0.03842498559395268 |
| }, |
| "mmlu_college_biology": { |
| "alias": " - college_biology", |
| "acc,none": 0.2847222222222222, |
| "acc_stderr,none": 0.037738099906869334 |
| }, |
| "mmlu_college_chemistry": { |
| "alias": " - college_chemistry", |
| "acc,none": 0.17, |
| "acc_stderr,none": 0.03775251680686369 |
| }, |
| "mmlu_college_computer_science": { |
| "alias": " - college_computer_science", |
| "acc,none": 0.33, |
| "acc_stderr,none": 0.04725815626252609 |
| }, |
| "mmlu_college_mathematics": { |
| "alias": " - college_mathematics", |
| "acc,none": 0.32, |
| "acc_stderr,none": 0.04688261722621507 |
| }, |
| "mmlu_college_physics": { |
| "alias": " - college_physics", |
| "acc,none": 0.2549019607843137, |
| "acc_stderr,none": 0.0433643270799318 |
| }, |
| "mmlu_computer_security": { |
| "alias": " - computer_security", |
| "acc,none": 0.3, |
| "acc_stderr,none": 0.04605661864718382 |
| }, |
| "mmlu_conceptual_physics": { |
| "alias": " - conceptual_physics", |
| "acc,none": 0.2170212765957447, |
| "acc_stderr,none": 0.026947483121496238 |
| }, |
| "mmlu_electrical_engineering": { |
| "alias": " - electrical_engineering", |
| "acc,none": 0.2896551724137931, |
| "acc_stderr,none": 0.03780019230438011 |
| }, |
| "mmlu_elementary_mathematics": { |
| "alias": " - elementary_mathematics", |
| "acc,none": 0.2671957671957672, |
| "acc_stderr,none": 0.02278967314577664 |
| }, |
| "mmlu_high_school_biology": { |
| "alias": " - high_school_biology", |
| "acc,none": 0.2645161290322581, |
| "acc_stderr,none": 0.02509189237885932 |
| }, |
| "mmlu_high_school_chemistry": { |
| "alias": " - high_school_chemistry", |
| "acc,none": 0.23645320197044334, |
| "acc_stderr,none": 0.029896114291733576 |
| }, |
| "mmlu_high_school_computer_science": { |
| "alias": " - high_school_computer_science", |
| "acc,none": 0.33, |
| "acc_stderr,none": 0.04725815626252609 |
| }, |
| "mmlu_high_school_mathematics": { |
| "alias": " - high_school_mathematics", |
| "acc,none": 0.25925925925925924, |
| "acc_stderr,none": 0.02671924078371222 |
| }, |
| "mmlu_high_school_physics": { |
| "alias": " - high_school_physics", |
| "acc,none": 0.271523178807947, |
| "acc_stderr,none": 0.03631329803969657 |
| }, |
| "mmlu_high_school_statistics": { |
| "alias": " - high_school_statistics", |
| "acc,none": 0.35185185185185186, |
| "acc_stderr,none": 0.032568505702936484 |
| }, |
| "mmlu_machine_learning": { |
| "alias": " - machine_learning", |
| "acc,none": 0.30357142857142855, |
| "acc_stderr,none": 0.043642261558410424 |
| } |
| }, |
| "groups": { |
| "mmlu": { |
| "acc,none": 0.26100270616721266, |
| "acc_stderr,none": 0.0037004678245414263, |
| "alias": "mmlu" |
| }, |
| "mmlu_humanities": { |
| "alias": " - humanities", |
| "acc,none": 0.2622741764080765, |
| "acc_stderr,none": 0.006405056497034186 |
| }, |
| "mmlu_other": { |
| "alias": " - other", |
| "acc,none": 0.2671387190215642, |
| "acc_stderr,none": 0.007937843638596481 |
| }, |
| "mmlu_social_sciences": { |
| "alias": " - social_sciences", |
| "acc,none": 0.23431914202144946, |
| "acc_stderr,none": 0.007631090779162625 |
| }, |
| "mmlu_stem": { |
| "alias": " - stem", |
| "acc,none": 0.2790992705359975, |
| "acc_stderr,none": 0.00797494423429467 |
| } |
| }, |
| "group_subtasks": { |
| "mmlu_stem": [ |
| "mmlu_high_school_mathematics", |
| "mmlu_high_school_chemistry", |
| "mmlu_astronomy", |
| "mmlu_conceptual_physics", |
| "mmlu_abstract_algebra", |
| "mmlu_college_biology", |
| "mmlu_machine_learning", |
| "mmlu_college_physics", |
| "mmlu_college_mathematics", |
| "mmlu_computer_security", |
| "mmlu_electrical_engineering", |
| "mmlu_college_chemistry", |
| "mmlu_elementary_mathematics", |
| "mmlu_high_school_statistics", |
| "mmlu_high_school_biology", |
| "mmlu_anatomy", |
| "mmlu_high_school_physics", |
| "mmlu_high_school_computer_science", |
| "mmlu_college_computer_science" |
| ], |
| "mmlu_other": [ |
| "mmlu_management", |
| "mmlu_professional_accounting", |
| "mmlu_college_medicine", |
| "mmlu_clinical_knowledge", |
| "mmlu_virology", |
| "mmlu_marketing", |
| "mmlu_nutrition", |
| "mmlu_medical_genetics", |
| "mmlu_global_facts", |
| "mmlu_miscellaneous", |
| "mmlu_professional_medicine", |
| "mmlu_human_aging", |
| "mmlu_business_ethics" |
| ], |
| "mmlu_social_sciences": [ |
| "mmlu_professional_psychology", |
| "mmlu_high_school_psychology", |
| "mmlu_security_studies", |
| "mmlu_high_school_macroeconomics", |
| "mmlu_us_foreign_policy", |
| "mmlu_high_school_government_and_politics", |
| "mmlu_human_sexuality", |
| "mmlu_econometrics", |
| "mmlu_public_relations", |
| "mmlu_sociology", |
| "mmlu_high_school_geography", |
| "mmlu_high_school_microeconomics" |
| ], |
| "mmlu_humanities": [ |
| "mmlu_international_law", |
| "mmlu_moral_scenarios", |
| "mmlu_jurisprudence", |
| "mmlu_professional_law", |
| "mmlu_high_school_world_history", |
| "mmlu_world_religions", |
| "mmlu_logical_fallacies", |
| "mmlu_prehistory", |
| "mmlu_formal_logic", |
| "mmlu_high_school_european_history", |
| "mmlu_high_school_us_history", |
| "mmlu_philosophy", |
| "mmlu_moral_disputes" |
| ], |
| "mmlu": [ |
| "mmlu_humanities", |
| "mmlu_social_sciences", |
| "mmlu_other", |
| "mmlu_stem" |
| ] |
| }, |
| "configs": { |
| "mmlu_abstract_algebra": { |
| "task": "mmlu_abstract_algebra", |
| "task_alias": "abstract_algebra", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "abstract_algebra", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_anatomy": { |
| "task": "mmlu_anatomy", |
| "task_alias": "anatomy", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "anatomy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_astronomy": { |
| "task": "mmlu_astronomy", |
| "task_alias": "astronomy", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "astronomy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_business_ethics": { |
| "task": "mmlu_business_ethics", |
| "task_alias": "business_ethics", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "business_ethics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_clinical_knowledge": { |
| "task": "mmlu_clinical_knowledge", |
| "task_alias": "clinical_knowledge", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "clinical_knowledge", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_college_biology": { |
| "task": "mmlu_college_biology", |
| "task_alias": "college_biology", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "college_biology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college biology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_college_chemistry": { |
| "task": "mmlu_college_chemistry", |
| "task_alias": "college_chemistry", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "college_chemistry", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_college_computer_science": { |
| "task": "mmlu_college_computer_science", |
| "task_alias": "college_computer_science", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "college_computer_science", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_college_mathematics": { |
| "task": "mmlu_college_mathematics", |
| "task_alias": "college_mathematics", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "college_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_college_medicine": { |
| "task": "mmlu_college_medicine", |
| "task_alias": "college_medicine", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "college_medicine", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_college_physics": { |
| "task": "mmlu_college_physics", |
| "task_alias": "college_physics", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "college_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about college physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_computer_security": { |
| "task": "mmlu_computer_security", |
| "task_alias": "computer_security", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "computer_security", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about computer security.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_conceptual_physics": { |
| "task": "mmlu_conceptual_physics", |
| "task_alias": "conceptual_physics", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "conceptual_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_econometrics": { |
| "task": "mmlu_econometrics", |
| "task_alias": "econometrics", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "econometrics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_electrical_engineering": { |
| "task": "mmlu_electrical_engineering", |
| "task_alias": "electrical_engineering", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "electrical_engineering", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_elementary_mathematics": { |
| "task": "mmlu_elementary_mathematics", |
| "task_alias": "elementary_mathematics", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "elementary_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_formal_logic": { |
| "task": "mmlu_formal_logic", |
| "task_alias": "formal_logic", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "formal_logic", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_global_facts": { |
| "task": "mmlu_global_facts", |
| "task_alias": "global_facts", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "global_facts", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about global facts.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_biology": { |
| "task": "mmlu_high_school_biology", |
| "task_alias": "high_school_biology", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_biology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_chemistry": { |
| "task": "mmlu_high_school_chemistry", |
| "task_alias": "high_school_chemistry", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_chemistry", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_computer_science": { |
| "task": "mmlu_high_school_computer_science", |
| "task_alias": "high_school_computer_science", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_computer_science", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_european_history": { |
| "task": "mmlu_high_school_european_history", |
| "task_alias": "high_school_european_history", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_european_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_geography": { |
| "task": "mmlu_high_school_geography", |
| "task_alias": "high_school_geography", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_geography", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_government_and_politics": { |
| "task": "mmlu_high_school_government_and_politics", |
| "task_alias": "high_school_government_and_politics", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_government_and_politics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_macroeconomics": { |
| "task": "mmlu_high_school_macroeconomics", |
| "task_alias": "high_school_macroeconomics", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_macroeconomics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_mathematics": { |
| "task": "mmlu_high_school_mathematics", |
| "task_alias": "high_school_mathematics", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_mathematics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_microeconomics": { |
| "task": "mmlu_high_school_microeconomics", |
| "task_alias": "high_school_microeconomics", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_microeconomics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_physics": { |
| "task": "mmlu_high_school_physics", |
| "task_alias": "high_school_physics", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_physics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_psychology": { |
| "task": "mmlu_high_school_psychology", |
| "task_alias": "high_school_psychology", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_psychology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_statistics": { |
| "task": "mmlu_high_school_statistics", |
| "task_alias": "high_school_statistics", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_statistics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_us_history": { |
| "task": "mmlu_high_school_us_history", |
| "task_alias": "high_school_us_history", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_us_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_high_school_world_history": { |
| "task": "mmlu_high_school_world_history", |
| "task_alias": "high_school_world_history", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "high_school_world_history", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_human_aging": { |
| "task": "mmlu_human_aging", |
| "task_alias": "human_aging", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "human_aging", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about human aging.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_human_sexuality": { |
| "task": "mmlu_human_sexuality", |
| "task_alias": "human_sexuality", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "human_sexuality", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_international_law": { |
| "task": "mmlu_international_law", |
| "task_alias": "international_law", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "international_law", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about international law.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_jurisprudence": { |
| "task": "mmlu_jurisprudence", |
| "task_alias": "jurisprudence", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "jurisprudence", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_logical_fallacies": { |
| "task": "mmlu_logical_fallacies", |
| "task_alias": "logical_fallacies", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "logical_fallacies", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_machine_learning": { |
| "task": "mmlu_machine_learning", |
| "task_alias": "machine_learning", |
| "group": "mmlu_stem", |
| "group_alias": "stem", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "machine_learning", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_management": { |
| "task": "mmlu_management", |
| "task_alias": "management", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "management", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about management.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_marketing": { |
| "task": "mmlu_marketing", |
| "task_alias": "marketing", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "marketing", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about marketing.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_medical_genetics": { |
| "task": "mmlu_medical_genetics", |
| "task_alias": "medical_genetics", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "medical_genetics", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_miscellaneous": { |
| "task": "mmlu_miscellaneous", |
| "task_alias": "miscellaneous", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "miscellaneous", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_moral_disputes": { |
| "task": "mmlu_moral_disputes", |
| "task_alias": "moral_disputes", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "moral_disputes", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_moral_scenarios": { |
| "task": "mmlu_moral_scenarios", |
| "task_alias": "moral_scenarios", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "moral_scenarios", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_nutrition": { |
| "task": "mmlu_nutrition", |
| "task_alias": "nutrition", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "nutrition", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_philosophy": { |
| "task": "mmlu_philosophy", |
| "task_alias": "philosophy", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "philosophy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_prehistory": { |
| "task": "mmlu_prehistory", |
| "task_alias": "prehistory", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "prehistory", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_professional_accounting": { |
| "task": "mmlu_professional_accounting", |
| "task_alias": "professional_accounting", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "professional_accounting", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_professional_law": { |
| "task": "mmlu_professional_law", |
| "task_alias": "professional_law", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "professional_law", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional law.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_professional_medicine": { |
| "task": "mmlu_professional_medicine", |
| "task_alias": "professional_medicine", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "professional_medicine", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_professional_psychology": { |
| "task": "mmlu_professional_psychology", |
| "task_alias": "professional_psychology", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "professional_psychology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_public_relations": { |
| "task": "mmlu_public_relations", |
| "task_alias": "public_relations", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "public_relations", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about public relations.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_security_studies": { |
| "task": "mmlu_security_studies", |
| "task_alias": "security_studies", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "security_studies", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about security studies.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_sociology": { |
| "task": "mmlu_sociology", |
| "task_alias": "sociology", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "sociology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about sociology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_us_foreign_policy": { |
| "task": "mmlu_us_foreign_policy", |
| "task_alias": "us_foreign_policy", |
| "group": "mmlu_social_sciences", |
| "group_alias": "social_sciences", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "us_foreign_policy", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_virology": { |
| "task": "mmlu_virology", |
| "task_alias": "virology", |
| "group": "mmlu_other", |
| "group_alias": "other", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "virology", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about virology.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| }, |
| "mmlu_world_religions": { |
| "task": "mmlu_world_religions", |
| "task_alias": "world_religions", |
| "group": "mmlu_humanities", |
| "group_alias": "humanities", |
| "dataset_path": "hails/mmlu_no_train", |
| "dataset_name": "world_religions", |
| "test_split": "test", |
| "fewshot_split": "dev", |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", |
| "doc_to_target": "answer", |
| "doc_to_choice": [ |
| "A", |
| "B", |
| "C", |
| "D" |
| ], |
| "description": "The following are multiple choice questions (with answers) about world religions.\n\n", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n" |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "multiple_choice", |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 0.0 |
| } |
| } |
| }, |
| "versions": { |
| "mmlu_abstract_algebra": 0.0, |
| "mmlu_anatomy": 0.0, |
| "mmlu_astronomy": 0.0, |
| "mmlu_business_ethics": 0.0, |
| "mmlu_clinical_knowledge": 0.0, |
| "mmlu_college_biology": 0.0, |
| "mmlu_college_chemistry": 0.0, |
| "mmlu_college_computer_science": 0.0, |
| "mmlu_college_mathematics": 0.0, |
| "mmlu_college_medicine": 0.0, |
| "mmlu_college_physics": 0.0, |
| "mmlu_computer_security": 0.0, |
| "mmlu_conceptual_physics": 0.0, |
| "mmlu_econometrics": 0.0, |
| "mmlu_electrical_engineering": 0.0, |
| "mmlu_elementary_mathematics": 0.0, |
| "mmlu_formal_logic": 0.0, |
| "mmlu_global_facts": 0.0, |
| "mmlu_high_school_biology": 0.0, |
| "mmlu_high_school_chemistry": 0.0, |
| "mmlu_high_school_computer_science": 0.0, |
| "mmlu_high_school_european_history": 0.0, |
| "mmlu_high_school_geography": 0.0, |
| "mmlu_high_school_government_and_politics": 0.0, |
| "mmlu_high_school_macroeconomics": 0.0, |
| "mmlu_high_school_mathematics": 0.0, |
| "mmlu_high_school_microeconomics": 0.0, |
| "mmlu_high_school_physics": 0.0, |
| "mmlu_high_school_psychology": 0.0, |
| "mmlu_high_school_statistics": 0.0, |
| "mmlu_high_school_us_history": 0.0, |
| "mmlu_high_school_world_history": 0.0, |
| "mmlu_human_aging": 0.0, |
| "mmlu_human_sexuality": 0.0, |
| "mmlu_international_law": 0.0, |
| "mmlu_jurisprudence": 0.0, |
| "mmlu_logical_fallacies": 0.0, |
| "mmlu_machine_learning": 0.0, |
| "mmlu_management": 0.0, |
| "mmlu_marketing": 0.0, |
| "mmlu_medical_genetics": 0.0, |
| "mmlu_miscellaneous": 0.0, |
| "mmlu_moral_disputes": 0.0, |
| "mmlu_moral_scenarios": 0.0, |
| "mmlu_nutrition": 0.0, |
| "mmlu_philosophy": 0.0, |
| "mmlu_prehistory": 0.0, |
| "mmlu_professional_accounting": 0.0, |
| "mmlu_professional_law": 0.0, |
| "mmlu_professional_medicine": 0.0, |
| "mmlu_professional_psychology": 0.0, |
| "mmlu_public_relations": 0.0, |
| "mmlu_security_studies": 0.0, |
| "mmlu_sociology": 0.0, |
| "mmlu_us_foreign_policy": 0.0, |
| "mmlu_virology": 0.0, |
| "mmlu_world_religions": 0.0 |
| }, |
| "n-shot": { |
| "mmlu": 0, |
| "mmlu_abstract_algebra": 5, |
| "mmlu_anatomy": 5, |
| "mmlu_astronomy": 5, |
| "mmlu_business_ethics": 5, |
| "mmlu_clinical_knowledge": 5, |
| "mmlu_college_biology": 5, |
| "mmlu_college_chemistry": 5, |
| "mmlu_college_computer_science": 5, |
| "mmlu_college_mathematics": 5, |
| "mmlu_college_medicine": 5, |
| "mmlu_college_physics": 5, |
| "mmlu_computer_security": 5, |
| "mmlu_conceptual_physics": 5, |
| "mmlu_econometrics": 5, |
| "mmlu_electrical_engineering": 5, |
| "mmlu_elementary_mathematics": 5, |
| "mmlu_formal_logic": 5, |
| "mmlu_global_facts": 5, |
| "mmlu_high_school_biology": 5, |
| "mmlu_high_school_chemistry": 5, |
| "mmlu_high_school_computer_science": 5, |
| "mmlu_high_school_european_history": 5, |
| "mmlu_high_school_geography": 5, |
| "mmlu_high_school_government_and_politics": 5, |
| "mmlu_high_school_macroeconomics": 5, |
| "mmlu_high_school_mathematics": 5, |
| "mmlu_high_school_microeconomics": 5, |
| "mmlu_high_school_physics": 5, |
| "mmlu_high_school_psychology": 5, |
| "mmlu_high_school_statistics": 5, |
| "mmlu_high_school_us_history": 5, |
| "mmlu_high_school_world_history": 5, |
| "mmlu_human_aging": 5, |
| "mmlu_human_sexuality": 5, |
| "mmlu_humanities": 5, |
| "mmlu_international_law": 5, |
| "mmlu_jurisprudence": 5, |
| "mmlu_logical_fallacies": 5, |
| "mmlu_machine_learning": 5, |
| "mmlu_management": 5, |
| "mmlu_marketing": 5, |
| "mmlu_medical_genetics": 5, |
| "mmlu_miscellaneous": 5, |
| "mmlu_moral_disputes": 5, |
| "mmlu_moral_scenarios": 5, |
| "mmlu_nutrition": 5, |
| "mmlu_other": 5, |
| "mmlu_philosophy": 5, |
| "mmlu_prehistory": 5, |
| "mmlu_professional_accounting": 5, |
| "mmlu_professional_law": 5, |
| "mmlu_professional_medicine": 5, |
| "mmlu_professional_psychology": 5, |
| "mmlu_public_relations": 5, |
| "mmlu_security_studies": 5, |
| "mmlu_social_sciences": 5, |
| "mmlu_sociology": 5, |
| "mmlu_stem": 5, |
| "mmlu_us_foreign_policy": 5, |
| "mmlu_virology": 5, |
| "mmlu_world_religions": 5 |
| }, |
| "config": { |
| "model": "hf", |
| "model_args": "pretrained=/home/zeus/microllm2/microllm2-checkpoints/final_merged,dtype=bfloat16,trust_remote_code=False", |
| "batch_size": "8", |
| "batch_sizes": [], |
| "device": "cuda:0", |
| "use_cache": null, |
| "limit": null, |
| "bootstrap_iters": 100000, |
| "gen_kwargs": null |
| }, |
| "git_hash": null, |
| "date": 1786317646.6679428, |
| "pretty_env_info": "PyTorch version: 2.4.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.3 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.12.11 | packaged by Anaconda, Inc. | (main, Jun 5 2025, 13:09:17) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-6.8.0-110-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: 13.0.88\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA H100 80GB HBM3\nNvidia driver version: 580.173.02\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.14.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.14.0\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 24\nOn-line CPU(s) list: 0-23\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8470\nCPU family: 6\nModel: 143\nThread(s) per core: 2\nCore(s) per socket: 12\nSocket(s): 1\nStepping: 8\nBogoMIPS: 4000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq dtes64 ssse3 fma cx16 pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 wbnoinvd arat avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b fsrm md_clear serialize tsxldtrk amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 576 KiB (12 instances)\nL1i cache: 384 KiB (12 instances)\nL2 cache: 24 MiB (12 instances)\nL3 cache: 105 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI BHI_DIS_S\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] pytorch-lightning==2.6.5\n[pip3] torch==2.4.1\n[pip3] torchmetrics==1.9.0\n[pip3] torchvision==0.19.1\n[pip3] triton==3.0.0\n[conda] Could not collect", |
| "transformers_version": "4.44.2", |
| "upper_git_hash": null |
| } |
