Add files using upload-large-folder tool

scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opebilbao.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opebilbao
+include: eus_exams_es
+task: eus_exams_es_opebilbao

scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/utils.py

@@ -0,0 +1,15 @@
+import datasets
+
+
+def process_docs(dataset: datasets.Dataset):
+    """Filter out examples with no answer."""
+
+    def valid_example(example: dict) -> bool:
+        """Check if an example is valid."""
+        if example["answer"] not in [0, 1, 2, 3]:
+            return False
+        if example["candidates"] == ["", "", "", ""]:
+            return False
+        return True
+
+    return dataset.filter(valid_example)

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/README.md

@@ -0,0 +1,64 @@
+# MMLU-SR
+
+## Paper
+Title: [Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models](https://arxiv.org/abs/2406.15468v1)
+
+
+We propose MMLU-SR, a novel dataset designed to measure the true comprehension abilities of Large Language Models (LLMs) by challenging their performance in question-answering tasks with modified terms. We reasoned that an agent that ``truly'' understands a concept can still evaluate it when key terms are replaced by suitably defined alternate terms, and sought to differentiate such comprehension from mere text replacement. In our study, we modified standardized test questions by replacing a key term with a dummy word along with its definition. The key term could be in the context of questions, answers, or both questions and answers.
+Notwithstanding the high scores achieved by recent popular LLMs on the MMLU leaderboard, we found a substantial reduction in model performance after such replacement, suggesting poor comprehension. This new benchmark provides a rigorous benchmark for testing true model comprehension, and poses a challenge to the broader scientific community.
+
+Github Homepage: [https://github.com/Wang-ML-Lab/MMLU-SR](https://github.com/Wang-ML-Lab/MMLU-SR)
+Huggingface Dataset: [https://huggingface.co/datasets/NiniCat/MMLU-SR]([https://huggingface.co/datasets/NiniCat/MMLU-SR)
+
+
+## Citation
+```bib
+@misc{wang2024reasoningsimplytokenprediction,
+      title={Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models},
+      author={Wentian Wang and Paul Kantor and Jacob Feldman and Lazaros Gallos and Hao Wang},
+      year={2024},
+      eprint={2406.15468},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2406.15468},
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+- `mmlusr`: MMLU variant where the terminology in the question and answers are modified.
+- `mmlusr_answer_only`: MMLU variant where the terminology in the answers are modified.
+- `mmlusr_question_only`: MMLU variant where the terminology in the question is modified.
+
+#### Tasks
+
+There are 57 symbol replaced subjects in each group. You can run a single task by:
+
+* `mmlusr_question_only_abstract_algebra`
+
+Or by categories:
+
+* `mmlusr_question_only_stem_tasks `
+
+
+### Checklist
+
+The checklist is the following:
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+    * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is comparable to that provided in the paper, though not identical.
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?
+
+### Variant Wishlist
+
+- [ ] zero-shot variant

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py

@@ -0,0 +1,154 @@
+"""
+Take in a YAML, and output all "other" splits with this YAML
+"""
+
+import argparse
+import logging
+import os
+
+import yaml
+from tqdm import tqdm
+
+
+eval_logger = logging.getLogger("lm-eval")
+
+
+SUBJECTS = {
+    "abstract_algebra": "stem",
+    "anatomy": "stem",
+    "astronomy": "stem",
+    "business_ethics": "other",
+    "clinical_knowledge": "other",
+    "college_biology": "stem",
+    "college_chemistry": "stem",
+    "college_computer_science": "stem",
+    "college_mathematics": "stem",
+    "college_medicine": "other",
+    "college_physics": "stem",
+    "computer_security": "stem",
+    "conceptual_physics": "stem",
+    "econometrics": "social_sciences",
+    "electrical_engineering": "stem",
+    "elementary_mathematics": "stem",
+    "formal_logic": "humanities",
+    "global_facts": "other",
+    "high_school_biology": "stem",
+    "high_school_chemistry": "stem",
+    "high_school_computer_science": "stem",
+    "high_school_european_history": "humanities",
+    "high_school_geography": "social_sciences",
+    "high_school_government_and_politics": "social_sciences",
+    "high_school_macroeconomics": "social_sciences",
+    "high_school_mathematics": "stem",
+    "high_school_microeconomics": "social_sciences",
+    "high_school_physics": "stem",
+    "high_school_psychology": "social_sciences",
+    "high_school_statistics": "stem",
+    "high_school_us_history": "humanities",
+    "high_school_world_history": "humanities",
+    "human_aging": "other",
+    "human_sexuality": "social_sciences",
+    "international_law": "humanities",
+    "jurisprudence": "humanities",
+    "logical_fallacies": "humanities",
+    "machine_learning": "stem",
+    "management": "other",
+    "marketing": "other",
+    "medical_genetics": "other",
+    "miscellaneous": "other",
+    "moral_disputes": "humanities",
+    "moral_scenarios": "humanities",
+    "nutrition": "other",
+    "philosophy": "humanities",
+    "prehistory": "humanities",
+    "professional_accounting": "other",
+    "professional_law": "humanities",
+    "professional_medicine": "other",
+    "professional_psychology": "social_sciences",
+    "public_relations": "social_sciences",
+    "security_studies": "social_sciences",
+    "sociology": "social_sciences",
+    "us_foreign_policy": "social_sciences",
+    "virology": "other",
+    "world_religions": "humanities",
+}
+
+GROUPS = ["question_and_answer"]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Generate configuration YAML files for LM Evaluation Harness."
+    )
+    # Path to the base YAML file from which to inherit settings
+    parser.add_argument(
+        "--base_yaml_path",
+        required=True,
+        help="Path to the base YAML configuration file.",
+    )
+
+    # Directory where the generated YAML files will be saved
+    parser.add_argument(
+        "--save_dir",
+        default="/data/local/cat/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer",
+    )
+
+    # Optional prefix to add to task names in the YAML files
+    parser.add_argument("--task_prefix", default="")
+
+    parser.add_argument("--cot_prompt_path", default=None)
+
+    # Optional prefix to add to group names in the YAML files
+    parser.add_argument("--group_prefix", default="")
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    # Load base YAML configuration
+    base_yaml_name = os.path.basename(args.base_yaml_path)
+    with open(args.base_yaml_path, "r", encoding="utf-8") as f:
+        base_yaml = yaml.full_load(f)
+
+    if args.cot_prompt_path is not None:
+        import json
+
+        with open(args.cot_prompt_path, encoding="utf-8") as f:
+            cot_file = json.load(f)
+
+    for group in GROUPS:
+        for subject, category in tqdm(SUBJECTS.items()):
+            if args.cot_prompt_path is not None:
+                description = cot_file[subject]
+            else:
+                description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"
+
+            yaml_dict = {
+                "include": base_yaml_name,
+                "tag": f"mmlusr_{args.group_prefix}{group}_{category}"
+                if args.group_prefix
+                else f"mmlusr_{group}_{category}",
+                "task": f"mmlusr_{args.task_prefix}{group}_{subject}"
+                if args.task_prefix
+                else f"mmlusr_{group}_{subject}",
+                "task_alias": subject.replace("_", " "),
+                "description": description,
+                "dataset_name": f"{group}_{subject}",
+            }
+
+            # File path for saving the generated YAML file
+            file_save_path = os.path.join(args.save_dir, f"{group}_{subject}.yaml")
+            with open(file_save_path, "w", encoding="utf-8") as yaml_file:
+                yaml.dump(yaml_dict, yaml_file, allow_unicode=True, default_style='"')
+            eval_logger.info(f"Saved YAML for {group} {subject} to {file_save_path}")
+
+    # Save group configuration if specified
+    if args.group_prefix:
+        file_save_path = os.path.join(
+            args.save_prefix_path, args.group_prefix + ".yaml"
+        )
+        eval_logger.info(f"Saving benchmark config to {file_save_path}")
+        with open(file_save_path, "w", encoding="utf-8") as yaml_file:
+            yaml.dump(yaml_dict, yaml_file, indent=4, default_flow_style=False)

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml

@@ -0,0 +1,16 @@
+dataset_path: NiniCat/MMLU-SR
+test_split: test
+fewshot_split: train
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+process_docs: !function utils.process_docs
+doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml

@@ -0,0 +1,44 @@
+group: mmlusr
+group_alias: MMLU-SR (Question & Answer)
+task:
+  - group: mmlusr_qa_stem
+    group_alias: STEM (Question & Answer)
+    task:
+      - mmlusr_question_and_answer_stem_tasks
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    metadata:
+      version: 2
+  - group: mmlusr_qa_other
+    group_alias: Other (Question & Answer)
+    task:
+      - mmlusr_question_and_answer_other_tasks
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    metadata:
+      version: 2
+  - group: mmlusr_qa_social_sciences
+    group_alias: Social Sciences (Question & Answer)
+    task:
+      - mmlusr_question_and_answer_social_sciences_tasks
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    metadata:
+      version: 2
+  - group: mmlusr_qa_humanities
+    group_alias: Humanities (Question & Answer)
+    task:
+      - mmlusr_question_and_answer_humanities_tasks
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    metadata:
+      version: 2
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_abstract_algebra"
+"description": "The following are multiple choice questions (with answers) about abstract\
+  \ algebra.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_abstract_algebra"
+"task_alias": "abstract algebra"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_anatomy"
+"description": "The following are multiple choice questions (with answers) about anatomy.\n\
+  \n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_anatomy"
+"task_alias": "anatomy"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_astronomy"
+"description": "The following are multiple choice questions (with answers) about astronomy.\n\
+  \n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_astronomy"
+"task_alias": "astronomy"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_business_ethics"
+"description": "The following are multiple choice questions (with answers) about business\
+  \ ethics.\n\n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_business_ethics"
+"task_alias": "business ethics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_clinical_knowledge"
+"description": "The following are multiple choice questions (with answers) about clinical\
+  \ knowledge.\n\n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_clinical_knowledge"
+"task_alias": "clinical knowledge"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_college_biology"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ biology.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_college_biology"
+"task_alias": "college biology"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_college_chemistry"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ chemistry.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_college_chemistry"
+"task_alias": "college chemistry"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_college_computer_science"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ computer science.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_college_computer_science"
+"task_alias": "college computer science"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_college_mathematics"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ mathematics.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_college_mathematics"
+"task_alias": "college mathematics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_college_medicine"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ medicine.\n\n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_college_medicine"
+"task_alias": "college medicine"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_college_physics"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ physics.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_college_physics"
+"task_alias": "college physics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_computer_security"
+"description": "The following are multiple choice questions (with answers) about computer\
+  \ security.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_computer_security"
+"task_alias": "computer security"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_conceptual_physics"
+"description": "The following are multiple choice questions (with answers) about conceptual\
+  \ physics.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_conceptual_physics"
+"task_alias": "conceptual physics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_econometrics"
+"description": "The following are multiple choice questions (with answers) about econometrics.\n\
+  \n"
+"tag": "mmlusr_question_and_answer_social_sciences_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_econometrics"
+"task_alias": "econometrics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_electrical_engineering"
+"description": "The following are multiple choice questions (with answers) about electrical\
+  \ engineering.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_electrical_engineering"
+"task_alias": "electrical engineering"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_elementary_mathematics"
+"description": "The following are multiple choice questions (with answers) about elementary\
+  \ mathematics.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_elementary_mathematics"
+"task_alias": "elementary mathematics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_formal_logic"
+"description": "The following are multiple choice questions (with answers) about formal\
+  \ logic.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_formal_logic"
+"task_alias": "formal logic"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_global_facts"
+"description": "The following are multiple choice questions (with answers) about global\
+  \ facts.\n\n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_global_facts"
+"task_alias": "global facts"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_biology"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school biology.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_biology"
+"task_alias": "high school biology"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_chemistry"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school chemistry.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_chemistry"
+"task_alias": "high school chemistry"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_computer_science"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school computer science.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_computer_science"
+"task_alias": "high school computer science"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_european_history"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school european history.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_european_history"
+"task_alias": "high school european history"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_geography"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school geography.\n\n"
+"tag": "mmlusr_question_and_answer_social_sciences_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_geography"
+"task_alias": "high school geography"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_government_and_politics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school government and politics.\n\n"
+"tag": "mmlusr_question_and_answer_social_sciences_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_government_and_politics"
+"task_alias": "high school government and politics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_macroeconomics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school macroeconomics.\n\n"
+"tag": "mmlusr_question_and_answer_social_sciences_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_macroeconomics"
+"task_alias": "high school macroeconomics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_mathematics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school mathematics.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_mathematics"
+"task_alias": "high school mathematics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_microeconomics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school microeconomics.\n\n"
+"tag": "mmlusr_question_and_answer_social_sciences_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_microeconomics"
+"task_alias": "high school microeconomics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_physics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school physics.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_physics"
+"task_alias": "high school physics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_psychology"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school psychology.\n\n"
+"tag": "mmlusr_question_and_answer_social_sciences_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_psychology"
+"task_alias": "high school psychology"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_statistics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school statistics.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_statistics"
+"task_alias": "high school statistics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_us_history"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school us history.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_us_history"
+"task_alias": "high school us history"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_high_school_world_history"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school world history.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_high_school_world_history"
+"task_alias": "high school world history"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_human_aging"
+"description": "The following are multiple choice questions (with answers) about human\
+  \ aging.\n\n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_human_aging"
+"task_alias": "human aging"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_human_sexuality"
+"description": "The following are multiple choice questions (with answers) about human\
+  \ sexuality.\n\n"
+"tag": "mmlusr_question_and_answer_social_sciences_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_human_sexuality"
+"task_alias": "human sexuality"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_international_law"
+"description": "The following are multiple choice questions (with answers) about international\
+  \ law.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_international_law"
+"task_alias": "international law"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_jurisprudence"
+"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\
+  \n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_jurisprudence"
+"task_alias": "jurisprudence"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_logical_fallacies"
+"description": "The following are multiple choice questions (with answers) about logical\
+  \ fallacies.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_logical_fallacies"
+"task_alias": "logical fallacies"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_machine_learning"
+"description": "The following are multiple choice questions (with answers) about machine\
+  \ learning.\n\n"
+"tag": "mmlusr_question_and_answer_stem_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_machine_learning"
+"task_alias": "machine learning"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_management"
+"description": "The following are multiple choice questions (with answers) about management.\n\
+  \n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_management"
+"task_alias": "management"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_marketing"
+"description": "The following are multiple choice questions (with answers) about marketing.\n\
+  \n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_marketing"
+"task_alias": "marketing"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_medical_genetics"
+"description": "The following are multiple choice questions (with answers) about medical\
+  \ genetics.\n\n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_medical_genetics"
+"task_alias": "medical genetics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_miscellaneous"
+"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\
+  \n"
+"tag": "mmlusr_question_and_answer_other_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_miscellaneous"
+"task_alias": "miscellaneous"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_moral_disputes"
+"description": "The following are multiple choice questions (with answers) about moral\
+  \ disputes.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_moral_disputes"
+"task_alias": "moral disputes"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "question_and_answer_moral_scenarios"
+"description": "The following are multiple choice questions (with answers) about moral\
+  \ scenarios.\n\n"
+"tag": "mmlusr_question_and_answer_humanities_tasks"
+"include": "_mmlusr_qna_yml"
+"task": "mmlusr_question_and_answer_moral_scenarios"
+"task_alias": "moral scenarios"