diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_cs_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_cs_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f829f8a09cc940a2269db6dff3226022335005cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_cs_business.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu obchod (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_template_yaml +task: mmlu_prox_cs_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caf0d6c36ff25c191f887f7d9b679145493c6331 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu chemie (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6be2cd9be73216c1e9ccb1f6e96d2e3ca48d330e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu informatika (s odpovědí). + Přemýšlejte prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde + X je písmeno správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5280b8cabe9b59a0d8cf2e0c3e623f352afb8d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_economics.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu ekonomie (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3e01f538dce8f77fc0e3daf9aad994c319cb0df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu inženýrství (s odpovědí). + Přemýšlejte prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde + X je písmeno správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4160990c40eabe8634d803f7289179eeb22b3632 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_health.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu zdraví (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d99fc6ed426c77c3146814cad9750b7ac536dbeb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_history.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu historie (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e89176185ceac12fc42f1afc1d0f3f2f17acab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_law.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu právo (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4dc5842e34db23d29981624a6ff6d3782452d664 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_other.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu ostatní (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edbb503040eabcf68738a25cc9297c85e5bd22a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu filozofie (s odpovědí). Přemýšlejte + prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde X je písmeno + správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38079424eb9f52c1357108719e35a1a7e2440d21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/mmlu_prox_lite_cs_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Zde je otázka s výběrem možností k tématu psychologie (s odpovědí). + Přemýšlejte prosím krok za krokem a svou odpověď zakončete "Odpověď je (X)", kde + X je písmeno správné možnosti. + + ' +include: _cs_lite_template_yaml +task: mmlu_prox_lite_cs_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/cs/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3ef81b4b9ef43990b91018eebd0ab7dc161f964 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_de_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: de +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Die Antwort ist \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Frage:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_lite_de.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_lite_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0388f73b8d2d3fcd75d1da085adec01fc4b315b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/_mmlu_prox_lite_de.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_de +task: +- mmlu_prox_lite_de_biology +- mmlu_prox_lite_de_business +- mmlu_prox_lite_de_chemistry +- mmlu_prox_lite_de_computer_science +- mmlu_prox_lite_de_economics +- mmlu_prox_lite_de_engineering +- mmlu_prox_lite_de_health +- mmlu_prox_lite_de_history +- mmlu_prox_lite_de_law +- mmlu_prox_lite_de_math +- mmlu_prox_lite_de_other +- mmlu_prox_lite_de_philosophy +- mmlu_prox_lite_de_physics +- mmlu_prox_lite_de_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8683d9ee4c8f0578f991e463ccc20c673179a3d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_business.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Wirtschaft. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e43365713a43209efbb5b2c0f9db5b70fb028075 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Chemie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c663f692eea1a17df6fe45b37be1706dcaa3df57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Informatik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a01a7488c60907557a019d5c5f5d29d23fc33fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_economics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ökonomie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d9a7359c0e173156e5a2dc513fa4bd11c184916 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ingenieurwesen. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..303272745cdbdbe32c83a2d3ceba85b02e59974a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_health.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Gesundheit. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b08416c7de46b11f53623607961bf1353a0ae1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_history.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Geschichte. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1e58ff8658c70c85b6fd1c56f143b462dc1fed1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_law.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Recht. Denken + Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort ist (X)", + wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dee8f6e4e732564714e7c9dbed29e8301ac9b66e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_math.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Mathematik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2dd274cc2b94b24df402369300bd1de9e79b5f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_other.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Sonstiges. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..757be6dfd8a34ba60d64776d9dec9084325a8414 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Philosophie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd3a6e6d4dd93f520679dafb7db26cbd87545c93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_physics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Physik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fbcd9b49b48ceecb01c3422f1c2f27d96739e2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_de_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Psychologie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_template_yaml +task: mmlu_prox_de_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52cadc9a2f0dcc906340c9ea5f8ae606aae78fde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_biology.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Biologie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29b7532936e1c46f60318f5429771b5c594dc0c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_business.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Wirtschaft. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fdb0a2ee086955d45aa894b9ddff16382094ddc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Chemie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6d91df758b7aaf98d3df9ba8a23f07dd5055899 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Informatik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6580877254bca496e30da2ad6d30f52cb06d5e87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_economics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ökonomie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ca33047854deb1705ec75f14ae8fa22740f639e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Ingenieurwesen. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff2a88a2e21dc77601a507da3d89793d18d56449 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_health.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Gesundheit. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4a735ac0d470a7f3b5257104b8f37c2fae2d182 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_history.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Geschichte. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c246249b0e3ec8fcfe6d3dababf4c4b63962c430 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_law.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Recht. Denken + Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort ist (X)", + wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e4a1047d8a4390e26590b7819f08ad3a03b36a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_math.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Mathematik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d1802ec6bd53e07a694ddc4e1d78b87e158b144 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_other.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Sonstiges. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbabdb978746750f4294d0668bcdf06146944042 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Philosophie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb286efa4bd254b8f8cf84195518b4972622e07c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_physics.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Physik. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bcaffca5940260fe5b4fac933175273a570c9e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/mmlu_prox_lite_de_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Im Folgenden sind Multiple-Choice-Fragen (mit Antworten) zu Psychologie. + Denken Sie Schritt für Schritt nach und beenden Sie Ihre Antwort mit "Die Antwort + ist (X)", wobei X der richtige Buchstabe ist. + + ' +include: _de_lite_template_yaml +task: mmlu_prox_lite_de_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/de/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..03719f43260ef2eba0e61d942ebf1a62582e6274 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: en +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Question:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed712f867d03aa8da77c33c89080dfeec7e7cea7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_en_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: en +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Question:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..649a5a625b9dcf7745abe1c16a888bc9ff6e40d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_en.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_en +task: +- mmlu_prox_en_biology +- mmlu_prox_en_business +- mmlu_prox_en_chemistry +- mmlu_prox_en_computer_science +- mmlu_prox_en_economics +- mmlu_prox_en_engineering +- mmlu_prox_en_health +- mmlu_prox_en_history +- mmlu_prox_en_law +- mmlu_prox_en_math +- mmlu_prox_en_other +- mmlu_prox_en_philosophy +- mmlu_prox_en_physics +- mmlu_prox_en_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_lite_en.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_lite_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22b497a61842db4e9009162c8c2fb8b16cb4748a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/_mmlu_prox_lite_en.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_en +task: +- mmlu_prox_lite_en_biology +- mmlu_prox_lite_en_business +- mmlu_prox_lite_en_chemistry +- mmlu_prox_lite_en_computer_science +- mmlu_prox_lite_en_economics +- mmlu_prox_lite_en_engineering +- mmlu_prox_lite_en_health +- mmlu_prox_lite_en_history +- mmlu_prox_lite_en_law +- mmlu_prox_lite_en_math +- mmlu_prox_lite_en_other +- mmlu_prox_lite_en_philosophy +- mmlu_prox_lite_en_physics +- mmlu_prox_lite_en_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c33358b48caee9dcf040000609675e5f3135038b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_biology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about biology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc85a949be4ea9f026c580eb88653e397022c1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_business.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about business. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f9c45a290f1c9cf0489f4f7ffc55aa433f105e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about chemistry. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b2e55f3abe22beaf4ed5efd75c3e43844eb3f6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about computer_science. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37232d2f4620fe297a04329753374843f3113f5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_economics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about economics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ca9302d32a9dedfb61bb798b8bc8a49308cdf1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_engineering.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about engineering. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a384c2e5a59f0533b2b86621cdd34488ddde807 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_health.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about health. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6bb11902f17570ddeaa9695a15457cbcaccd0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_history.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about history. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..534da99020b95befee63db1acc2c58b3e72d5008 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_law.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about law. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59fca5bac4841ff195ed230ce62a67a865783ba3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_math.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about math. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..437e2449e203eb047c45c971f058b9ac49ff08ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_other.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about other. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d67bc4dd543fbd1060bccdbd9f9d705e344404d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about philosophy. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7847c0f5f37b231a2113a454873a8ab5c84339a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_physics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about physics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e3b84d8c5c261366e59c7265169b3e6b7ce51e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_en_psychology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about psychology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_template_yaml +task: mmlu_prox_en_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6411e021060ed2359dd4b5be20db4f8078775516 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_biology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about biology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed12785cbc63202a1de5e344114d6c05a8c5e998 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_business.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about business. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dbd3b131f8d64e2316164b2b2146f578ea45a86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about chemistry. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72e0d645a464c97554b9e3af798905ad56a6e4cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about computer_science. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a092b79585cc17fe63dda61b8b552d144e6d821b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_economics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about economics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7d14888893d7184a6d05f3d9e3fd515047fddf5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_engineering.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about engineering. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2a184ba965e54e1a0029dfd0fa8429b7b8fe5cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_health.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about health. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddc3a4aa237d629238c1b64ac5dfd2d419dd9844 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_history.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about history. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..373274f8ef29ad93abff6080f5f32d6c0efba311 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_law.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about law. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63f6e9549db7d29f06f791490ada573d11471d3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_math.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about math. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc3b25301019029d2cd17b0b8c6ccf0d03e4e37d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_other.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about other. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01f3947faddfa2515668893886112a6051878420 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about philosophy. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acfb040fe8888e68bd7c2db89705856a7df8feab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_physics.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about physics. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08dde624f4095f41cfa26d8188b8d9d5feece479 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/mmlu_prox_lite_en_psychology.yaml @@ -0,0 +1,9 @@ +description: 'The following are multiple choice questions (with answers) about psychology. + Think step by step and then finish your answer with "the answer is (X)" where X + is the correct letter choice. + + ' +include: _en_lite_template_yaml +task: mmlu_prox_lite_en_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/en/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1156040dcd9e1b18f118cd3cc7dd0df02d6d5b02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: es +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La respuesta es \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Pregunta:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a84f6cf5d1416d54d5ddb297467ddf4e6bc4653 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_es_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: es +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La respuesta es \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Pregunta:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..437942cb0b0527c84d6ec084d7bf5534a496bf51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_es.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_es +task: +- mmlu_prox_es_biology +- mmlu_prox_es_business +- mmlu_prox_es_chemistry +- mmlu_prox_es_computer_science +- mmlu_prox_es_economics +- mmlu_prox_es_engineering +- mmlu_prox_es_health +- mmlu_prox_es_history +- mmlu_prox_es_law +- mmlu_prox_es_math +- mmlu_prox_es_other +- mmlu_prox_es_philosophy +- mmlu_prox_es_physics +- mmlu_prox_es_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_lite_es.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_lite_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d7b002bd82993a726ecb5b87b2cdf732ad60b80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/_mmlu_prox_lite_es.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_es +task: +- mmlu_prox_lite_es_biology +- mmlu_prox_lite_es_business +- mmlu_prox_lite_es_chemistry +- mmlu_prox_lite_es_computer_science +- mmlu_prox_lite_es_economics +- mmlu_prox_lite_es_engineering +- mmlu_prox_lite_es_health +- mmlu_prox_lite_es_history +- mmlu_prox_lite_es_law +- mmlu_prox_lite_es_math +- mmlu_prox_lite_es_other +- mmlu_prox_lite_es_philosophy +- mmlu_prox_lite_es_physics +- mmlu_prox_lite_es_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0aa01f9bacf34d21ce732e3d583c19fb1ce59dcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_biology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + biología. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c7551b67d595300f6218fe3c0b2817fbbea4f74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_business.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + negocios. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d75f7034b7ef6e047b15e69a7146a72661377e41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + química. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33d2d919542192e6520cb54b90037cf9af6bfedd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + informática. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d257611b14eeb4d8eef729bd02ea3e95eeca243c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_economics.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + economía. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..504de4348c4b0e9435f1d6a01c3d2b91017b4a42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + ingeniería. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f621a739e28c438bcedbfcddc1faf75dc368ab83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_health.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + salud. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e8e89acde0b0eacfd713e06fb022e363ed7f248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_law.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + derecho. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c584a90864df9246a0db4a52937973e21f81cee1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_math.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + matemáticas. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f58119ad994400ed61a9148a6b01f2047c9fecb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_other.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + otro. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db2a905cb103bd5292bb75bf6a295da5a84d836f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_physics.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + física. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..217c215e07d58ebfbdd889620968dba04b68d49c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_es_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + psicología. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_template_yaml +task: mmlu_prox_es_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..431bc4d599ae6987dbadc73a8ae6bd7a7dbb5a3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_biology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + biología. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_lite_template_yaml +task: mmlu_prox_lite_es_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..766bc1d10ba6b5e40581634e1f507dd0f38c3317 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + química. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_lite_template_yaml +task: mmlu_prox_lite_es_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63828e68864236af92cb3788237e851f6ceac315 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + informática. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_lite_template_yaml +task: mmlu_prox_lite_es_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ada61ff561ea618f87635a299ee1ecbd91b5881 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_economics.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + economía. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_lite_template_yaml +task: mmlu_prox_lite_es_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c99a1190f0175b8983769fd706903bac13347a8c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + ingeniería. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_lite_template_yaml +task: mmlu_prox_lite_es_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a412ca424a7ce7223285868f7dd8a92a40bccca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_health.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + salud. Piense paso a paso y luego termine su respuesta con "La respuesta es (X)" + donde X es la letra de la opción correcta. + + ' +include: _es_lite_template_yaml +task: mmlu_prox_lite_es_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ba8e5aec381d9e166d15c7c5b8d2f5349da2d74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/es/mmlu_prox_lite_es_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Las siguientes son preguntas de opción múltiple (con respuestas) sobre + psicología. Piense paso a paso y luego termine su respuesta con "La respuesta es + (X)" donde X es la letra de la opción correcta. + + ' +include: _es_lite_template_yaml +task: mmlu_prox_lite_es_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..935f561faec99a0bde2e3721a6e09737825d2627 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/_fr_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: fr +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La réponse est \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Question :" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32f602be482e6ca9b0774cd80b048ffe6de0c6db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/fr/mmlu_prox_fr_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Voici des questions à choix multiples (avec réponses) sur philosophie. + Réfléchissez étape par étape, puis terminez votre réponse par "La réponse est (X)" + où X est la lettre correspondant au bon choix. + + ' +include: _fr_template_yaml +task: mmlu_prox_fr_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f5437d820e1219664855a173fdb57a02b5a2b20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_economics.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) közgazdaságtan témában (választ + is tartalmazza). Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) "A válasz + (X)" kifejezéssel fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_template_yaml +task: mmlu_prox_hu_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d15a768161ecf0aa0f23338283794d2ed10a6133 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) mérnöki tudományok témában + (választ is tartalmazza). Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) + "A válasz (X)" kifejezéssel fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_template_yaml +task: mmlu_prox_hu_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d123b69a16d06c6a349265edc26d81b7075fc20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_hu_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) pszichológia témában (választ + is tartalmazza). Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) "A válasz + (X)" kifejezéssel fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_template_yaml +task: mmlu_prox_hu_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3d2ddb3802f853187e256d6c049ba07aaaf6fff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) kémia témában (választ is tartalmazza). + Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) "A válasz (X)" kifejezéssel + fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_lite_template_yaml +task: mmlu_prox_lite_hu_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96ceca96a5a4b68532a7a18ec6b6950ecf49c2b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) mérnöki tudományok témában + (választ is tartalmazza). Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) + "A válasz (X)" kifejezéssel fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_lite_template_yaml +task: mmlu_prox_lite_hu_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed9cf68064be186c41491ffbea1ed73a4ed84500 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_math.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) matematika témában (választ + is tartalmazza). Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) "A válasz + (X)" kifejezéssel fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_lite_template_yaml +task: mmlu_prox_lite_hu_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10ec083c984cea431a922eb5c7dc375b8d86bdcb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) filozófia témában (választ + is tartalmazza). Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) "A válasz + (X)" kifejezéssel fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_lite_template_yaml +task: mmlu_prox_lite_hu_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acdfd9d6ad803eaa95a500ee9f4edb6ae60a8878 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/mmlu_prox_lite_hu_physics.yaml @@ -0,0 +1,9 @@ +description: 'Itt van egy feleletválasztós kérdés a(z) fizika témában (választ is + tartalmazza). Kérjük, gondolkodjon lépésről lépésre, és a válaszát a(z) "A válasz + (X)" kifejezéssel fejezze be, ahol X a helyes válasz betűjele. + + ' +include: _hu_lite_template_yaml +task: mmlu_prox_lite_hu_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/hu/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/_mmlu_prox_id.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/_mmlu_prox_id.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ea8b3a14a1a57157b44cfa9f5fb970712030322 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/_mmlu_prox_id.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_id +task: +- mmlu_prox_id_biology +- mmlu_prox_id_business +- mmlu_prox_id_chemistry +- mmlu_prox_id_computer_science +- mmlu_prox_id_economics +- mmlu_prox_id_engineering +- mmlu_prox_id_health +- mmlu_prox_id_history +- mmlu_prox_id_law +- mmlu_prox_id_math +- mmlu_prox_id_other +- mmlu_prox_id_philosophy +- mmlu_prox_id_physics +- mmlu_prox_id_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/_mmlu_prox_lite_id.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/_mmlu_prox_lite_id.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8cbc7b0c735a981fe1722df9881c10aad82ef01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/_mmlu_prox_lite_id.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_id +task: +- mmlu_prox_lite_id_biology +- mmlu_prox_lite_id_business +- mmlu_prox_lite_id_chemistry +- mmlu_prox_lite_id_computer_science +- mmlu_prox_lite_id_economics +- mmlu_prox_lite_id_engineering +- mmlu_prox_lite_id_health +- mmlu_prox_lite_id_history +- mmlu_prox_lite_id_law +- mmlu_prox_lite_id_math +- mmlu_prox_lite_id_other +- mmlu_prox_lite_id_philosophy +- mmlu_prox_lite_id_physics +- mmlu_prox_lite_id_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c1ce8b43ce8a1730b837bab9cfdded8dbaf3844 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_biology.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Biologi (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b154de9f0878b47354b1e7129b0a1ac553c65e5b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_business.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Bisnis (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f268c928e53d3496010fd4d8eafb29d1ec8f2226 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Kimia (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f4969b3f8ccb1ac3d867b799a89e742996e9016 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Ilmu Komputer (dengan + jawaban). Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2240d1d86bb87af83bf59bf076c0ff9cafecb230 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_economics.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Ekonomi (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b29d92f4aacaa52b4b7470a6b3f9a6029cb1ed9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Teknik (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45573afe21056582b7e82b6b721ff839fdeb14b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_health.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Kesehatan (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54601d2eb639c509b1014da7a198093086997211 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_history.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Sejarah (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f0bbd453f99ee0f1420e760920e3584c88fc662 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_law.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Hukum (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60e41c50e651071814498825c1ffc29b99a12bc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_math.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Matematika (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d16af6e67aa2833e30e27ee4d8a99e69de821163 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_other.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Lainnya (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..353ae23e34fa2e457aad09b9528096ebbcd3597c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Filsafat (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ee921f303460dd0deb0de841440283235aa2c1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_physics.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Fisika (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48f0c666b6c2ee00ea21b55fd6f7ce1f5d3cff37 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_id_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Psikologi (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_template_yaml +task: mmlu_prox_id_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6856a5e54a498ba9a86e861c7bc845fc20080cc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_biology.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Biologi (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c30569f1fce9f2a2c79785ec82f7da7ce634d2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_business.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Bisnis (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a9070c71c77cefcff25f42c4b1a14f7a560f783 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Kimia (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47c919d67c83e79e3e0564ccc03d2b9262788752 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Ilmu Komputer (dengan + jawaban). Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcf68bcf7ed02af80decf2740d72612206440243 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_economics.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Ekonomi (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed1d0e6713e88bf6f908cf0f8e484b523fec7a02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Teknik (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b707acba1db590ff33bf19bd1a78a2f2e15f1f30 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_health.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Kesehatan (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ed11e310d6f39c6cd1ccae42d717596e093a1f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_history.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Sejarah (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51a341161410a5b3dd1524403f0ed39d1d287e52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_law.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Hukum (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b59565deb9a1e4c89c4ac7c785e8889dac515a69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_math.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Matematika (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b96cf39d17d952c34328c4d9f32c0dd8382c6df4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_other.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Lainnya (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f408b77e3509dee7212f408812954745033c0518 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Filsafat (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ab2f1b49058456d0d44b581884f76b1b3ec77f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_physics.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Fisika (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aea2205b90afcd07edafd8d61320f6a9bb3cce76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/mmlu_prox_lite_id_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Berikut adalah pertanyaan pilihan ganda tentang Psikologi (dengan jawaban). + Harap berpikir langkah demi langkah, lalu akhiri jawaban Anda dengan "Jawabannya + adalah (X)", di mana X adalah huruf pilihan yang benar. + + ' +include: _id_lite_template_yaml +task: mmlu_prox_lite_id_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/id/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_it_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_it_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f400445fb2e4bea6c34ea929d964ae13c68339f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_it_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: it +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La risposta è \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Domanda:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_it_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_it_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb4ac5bd62fd7557e3b45ce2db25cc371f0b9d43 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_it_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: it +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'La risposta è \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Domanda:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_mmlu_prox_it.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_mmlu_prox_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ad57912e31c02be8e5d52cc801b7359b9ee2304 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_mmlu_prox_it.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_it +task: +- mmlu_prox_it_biology +- mmlu_prox_it_business +- mmlu_prox_it_chemistry +- mmlu_prox_it_computer_science +- mmlu_prox_it_economics +- mmlu_prox_it_engineering +- mmlu_prox_it_health +- mmlu_prox_it_history +- mmlu_prox_it_law +- mmlu_prox_it_math +- mmlu_prox_it_other +- mmlu_prox_it_philosophy +- mmlu_prox_it_physics +- mmlu_prox_it_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_mmlu_prox_lite_it.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_mmlu_prox_lite_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a230af85a3a379858fd0ba7137bb8c91d0ce1b36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/_mmlu_prox_lite_it.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_it +task: +- mmlu_prox_lite_it_biology +- mmlu_prox_lite_it_business +- mmlu_prox_lite_it_chemistry +- mmlu_prox_lite_it_computer_science +- mmlu_prox_lite_it_economics +- mmlu_prox_lite_it_engineering +- mmlu_prox_lite_it_health +- mmlu_prox_lite_it_history +- mmlu_prox_lite_it_law +- mmlu_prox_lite_it_math +- mmlu_prox_lite_it_other +- mmlu_prox_lite_it_philosophy +- mmlu_prox_lite_it_physics +- mmlu_prox_lite_it_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..181bbf531d775d24190ce2d3b6dc8587e67c8f0f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_biology.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su biologia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..257a8df8a2e2eddfa6d33ba67e0414cd6f1fa28c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_business.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su affari (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40e79f938b72aa26fc5edd037550e01f8d0d455d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su chimica (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bddd45c881c72cbbe9bcadad262394d29ef23326 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su informatica (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5616f844a0c22ca6256a7f9cace8583192b55e14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_economics.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su economia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dde6ffa419edb9dc7bc45859d6d092dfc234ca34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su ingegneria (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ef4497166e634eda6f3374ee3685f62bc9cf6ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_health.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su salute (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19cb0bc30e7918eaacb975dd62dd19441ba8ff55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_history.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su storia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fc964db2ac66b31da9453e62fec6b5f17b85ade --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_law.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su diritto (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33841c46d67c0b9f7b2e44b4a042dacd9de855ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_math.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su matematica (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9708c19a4a03a12b76e8638210df2b1b1f940ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_other.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su altro (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cd53d1f528d3201cf9133cdb9e705212455fcf4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su filosofia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92b08ff9de7b4933acc15ea256ee359312c94a54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_physics.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su fisica (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d55b46a2b1a6916c3956c419d5f53bd3ddb9abd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_it_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su psicologia (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_template_yaml +task: mmlu_prox_it_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d1a45b82713910a2e714e081312d3987053d244 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_biology.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su biologia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8281dd4d72cd18e052950ff9461666b45e9d2f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_business.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su affari (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78be59c07d34ef136ad0e11f1f02820ac53fca8c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su chimica (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..177b7319c4fb0bc2bfe5814a4d0ee7a0455bf022 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su informatica (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b14a66926ade4e3030f43705901b16c5c90703c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_economics.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su economia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8ea42c2d9f38e1cf77fe4b6284d0f8220b331c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su ingegneria (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa2dc11470f45561abb1de436480c918dfc411c7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_health.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su salute (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d25a68b5bda16474d83ebae305c9197b61cfc149 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_history.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su storia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c7d4e275bf78497333f7ac365f3b422c741deaa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_law.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su diritto (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0923633e62a7ce3ce8c54efe25668969d5168d4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_math.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su matematica (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3072c44f7fbba21d26d4b2b4ef9c871871905abf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_other.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su altro (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3abc52cd0e0557b9041383f40a7544efa97f00fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su filosofia (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce6987cb8a1879a0d35dd97b074ac593bc8b88f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_physics.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su fisica (con risposta). Si prega + di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", dove + X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25771ed03a6fb2c35929563159ce8932171b1755 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/mmlu_prox_lite_it_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Ecco una domanda a scelta multipla su psicologia (con risposta). Si + prega di ragionare passo dopo passo e terminare la risposta con "La risposta è (X)", + dove X è la lettera dell''opzione corretta. + + ' +include: _it_lite_template_yaml +task: mmlu_prox_lite_it_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/it/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcb42f3f961981851cfcdfd28784c335f8d8d70c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: ja +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '答えは \(?([ABCDEFGHIJ])\)? です' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "質問：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb5569f7cf32f4d30407cdd74a2e205ad404805 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_ja_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ja +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '答えは \(?([ABCDEFGHIJ])\)? です' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "質問：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5887f7b82fe5696476dd37da284c919bbc5964f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_ja.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ja +task: +- mmlu_prox_ja_biology +- mmlu_prox_ja_business +- mmlu_prox_ja_chemistry +- mmlu_prox_ja_computer_science +- mmlu_prox_ja_economics +- mmlu_prox_ja_engineering +- mmlu_prox_ja_health +- mmlu_prox_ja_history +- mmlu_prox_ja_law +- mmlu_prox_ja_math +- mmlu_prox_ja_other +- mmlu_prox_ja_philosophy +- mmlu_prox_ja_physics +- mmlu_prox_ja_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_lite_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_lite_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9d8cbe5a53a1fe8bb79ab57b3bee2ce8634d74f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/_mmlu_prox_lite_ja.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_ja +task: +- mmlu_prox_lite_ja_biology +- mmlu_prox_lite_ja_business +- mmlu_prox_lite_ja_chemistry +- mmlu_prox_lite_ja_computer_science +- mmlu_prox_lite_ja_economics +- mmlu_prox_lite_ja_engineering +- mmlu_prox_lite_ja_health +- mmlu_prox_lite_ja_history +- mmlu_prox_lite_ja_law +- mmlu_prox_lite_ja_math +- mmlu_prox_lite_ja_other +- mmlu_prox_lite_ja_philosophy +- mmlu_prox_lite_ja_physics +- mmlu_prox_lite_ja_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd68bb572980342e25cc225cec2dcacf977c08a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_biology.yaml @@ -0,0 +1,7 @@ +description: '以下は生物学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..261aa26027c0a57b6bea94a77e5195b1b6c14a81 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_business.yaml @@ -0,0 +1,7 @@ +description: '以下はビジネスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afe4e74f3010fc98aad3e463fc669302846b2d70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_chemistry.yaml @@ -0,0 +1,7 @@ +description: '以下は化学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3e40a659312a40b77bccedf8907b2c157c788c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_computer_science.yaml @@ -0,0 +1,7 @@ +description: '以下はコンピュータサイエンスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c26fb8e97351c1d70e360ff14dd39287600a379b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_economics.yaml @@ -0,0 +1,7 @@ +description: '以下は経済学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bed0bc1965078f863dded91da78857514051bc20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_engineering.yaml @@ -0,0 +1,7 @@ +description: '以下は工学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4b820595fc9a4b4f2568bd7ce62bee0f45ffbae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_health.yaml @@ -0,0 +1,7 @@ +description: '以下は健康科学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5eb26103a2a23e316dc66f0fe1b894ddd481d256 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_history.yaml @@ -0,0 +1,7 @@ +description: '以下は歴史に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..512846f6d3f477594a7a1b63b09700a435376378 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_law.yaml @@ -0,0 +1,7 @@ +description: '以下は法律に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef030be6d9c666054adb4c3fe42b60a17375e835 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_math.yaml @@ -0,0 +1,7 @@ +description: '以下は数学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9aa8183eb7705fcda2c0dcae5bbee45b4af524b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_other.yaml @@ -0,0 +1,7 @@ +description: '以下はその他に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1581c5b2bb96e0063947d64d446bc6f090a105d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_philosophy.yaml @@ -0,0 +1,7 @@ +description: '以下は哲学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31aa1a658a6a766d74b609e2b9a68b2f226c0741 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_physics.yaml @@ -0,0 +1,7 @@ +description: '以下は物理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8890b927155ea6cd54d2e3c9cbe63943693399c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_ja_psychology.yaml @@ -0,0 +1,7 @@ +description: '以下は心理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_template_yaml +task: mmlu_prox_ja_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eb45c60cb9f8dfc9807803876c696e01945fb40 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_biology.yaml @@ -0,0 +1,7 @@ +description: '以下は生物学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f5f30993249a89b5aa0709940233f38d5eea984 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_business.yaml @@ -0,0 +1,7 @@ +description: '以下はビジネスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78c5b201f838b948a1793ffb407504fc9b67e7dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_chemistry.yaml @@ -0,0 +1,7 @@ +description: '以下は化学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ef8016d46634b6ee9ef50268ac5ec48dcb03d0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_computer_science.yaml @@ -0,0 +1,7 @@ +description: '以下はコンピュータサイエンスに関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c7aebc66abccbf3177c1484720610eaf5d5d532 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_economics.yaml @@ -0,0 +1,7 @@ +description: '以下は経済学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e27c6fff18713a54f4bc96dff995d03125d66646 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_engineering.yaml @@ -0,0 +1,7 @@ +description: '以下は工学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce14c655ebd507f0a280153c35e76ea79aa1b271 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_health.yaml @@ -0,0 +1,7 @@ +description: '以下は健康科学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2559c494bb7de70c93a7c5af8a1533f5ac026963 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_history.yaml @@ -0,0 +1,7 @@ +description: '以下は歴史に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b66649ee55f6d4e3d9bd9d19200735ac6810614 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_law.yaml @@ -0,0 +1,7 @@ +description: '以下は法律に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d476e9a54aabff8d9630fc78bd93204a504098d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_math.yaml @@ -0,0 +1,7 @@ +description: '以下は数学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6af874e30f6e541116e76cf68277d9d6744198a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_other.yaml @@ -0,0 +1,7 @@ +description: '以下はその他に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64665de31fe9f4e80b33917bab1812553b52527f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_philosophy.yaml @@ -0,0 +1,7 @@ +description: '以下は哲学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8e19c3e539591164ab6a6dfdfd62e80db220372 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_physics.yaml @@ -0,0 +1,7 @@ +description: '以下は物理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c3f6d098ddef2b5bada3f7902509d2dcb5b4eed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/mmlu_prox_lite_ja_psychology.yaml @@ -0,0 +1,7 @@ +description: '以下は心理学に関する選択問題（解答付き）です。段階的に考え、最後に「答えは (X) です」と回答を締めくくってください。Xは正解の選択肢を示す文字です。 + + ' +include: _ja_lite_template_yaml +task: mmlu_prox_lite_ja_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ja/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e5d2264186f6101dff649a806333afc9e52e1e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: ko +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '답은 \(?([ABCDEFGHIJ])\)?입니다' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "질문：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe7abb531d727e138fd21785b4ee08da82d9f896 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_ko_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ko +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '답은 \(?([ABCDEFGHIJ])\)?입니다' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "질문：" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f86b5b35952ef422018ec2acceff591aded3ef8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_ko.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ko +task: +- mmlu_prox_ko_biology +- mmlu_prox_ko_business +- mmlu_prox_ko_chemistry +- mmlu_prox_ko_computer_science +- mmlu_prox_ko_economics +- mmlu_prox_ko_engineering +- mmlu_prox_ko_health +- mmlu_prox_ko_history +- mmlu_prox_ko_law +- mmlu_prox_ko_math +- mmlu_prox_ko_other +- mmlu_prox_ko_philosophy +- mmlu_prox_ko_physics +- mmlu_prox_ko_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_lite_ko.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_lite_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799e86859ec6eef0d1e3b85263a2598a7ef8cc02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/_mmlu_prox_lite_ko.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_ko +task: +- mmlu_prox_lite_ko_biology +- mmlu_prox_lite_ko_business +- mmlu_prox_lite_ko_chemistry +- mmlu_prox_lite_ko_computer_science +- mmlu_prox_lite_ko_economics +- mmlu_prox_lite_ko_engineering +- mmlu_prox_lite_ko_health +- mmlu_prox_lite_ko_history +- mmlu_prox_lite_ko_law +- mmlu_prox_lite_ko_math +- mmlu_prox_lite_ko_other +- mmlu_prox_lite_ko_philosophy +- mmlu_prox_lite_ko_physics +- mmlu_prox_lite_ko_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcd695f6188a184352e52c2f540f6c9e0739e412 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_biology.yaml @@ -0,0 +1,8 @@ +description: '다음은 생물학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47ee56c9e83f38ddff9f070c43ef15a42415a135 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_business.yaml @@ -0,0 +1,8 @@ +description: '다음은 경영학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ad92ca44843cb5a15d07c529c7bb9a368092bdf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_chemistry.yaml @@ -0,0 +1,8 @@ +description: '다음은 화학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b33bb33c886dcefd5e6345d811aede440d76a02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_computer_science.yaml @@ -0,0 +1,8 @@ +description: '다음은 컴퓨터 과학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a7df38786dc807e2620f40b4d177d732c76d84b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_economics.yaml @@ -0,0 +1,8 @@ +description: '다음은 경제학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f2852eb4eb98595012a0fe824db62d78c6529fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_engineering.yaml @@ -0,0 +1,8 @@ +description: '다음은 공학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9322b2074a5238dfbd078b457dd3d2b66abb9c3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_health.yaml @@ -0,0 +1,8 @@ +description: '다음은 건강에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f82005689f11631dc8506d785d87420563186980 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_history.yaml @@ -0,0 +1,8 @@ +description: '다음은 역사에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7db87be068f8c3751b934f1a0f719ebb07a5eb3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_law.yaml @@ -0,0 +1,8 @@ +description: '다음은 법률에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d90c1e5fac04e3f91712ced30915d9d18fe6d0dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_math.yaml @@ -0,0 +1,8 @@ +description: '다음은 수학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50aad077e34943e8fed2e62543f9ef23c5fc33aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_other.yaml @@ -0,0 +1,8 @@ +description: '다음은 기타에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c274a0e3cd9c66e8fb323d71d6c7081fbff0aee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_philosophy.yaml @@ -0,0 +1,8 @@ +description: '다음은 철학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7be0250ec9311ea3f42cc048dba16e9d4a3fd77b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_physics.yaml @@ -0,0 +1,8 @@ +description: '다음은 물리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06b8aafbbe554e4a748686597349a33c1bbfbc5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_ko_psychology.yaml @@ -0,0 +1,8 @@ +description: '다음은 심리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_template_yaml +task: mmlu_prox_ko_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5d184714d22e2cbd0caa570be469a28219a7165 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_biology.yaml @@ -0,0 +1,8 @@ +description: '다음은 생물학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e9f2467a298a64b0be8e220000a0ea8bd5037f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_business.yaml @@ -0,0 +1,8 @@ +description: '다음은 경영학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fe8b447d15d2f1a42b40f5d3f0af9c1d76f6c9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_chemistry.yaml @@ -0,0 +1,8 @@ +description: '다음은 화학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f211b4ad3b6e601460b5a1a3a733e975d17b7de8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_computer_science.yaml @@ -0,0 +1,8 @@ +description: '다음은 컴퓨터 과학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..115fdde39ec3ea2aa5c025eb11cefcd6cb5e7e4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_economics.yaml @@ -0,0 +1,8 @@ +description: '다음은 경제학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec3048c4877768285d9b674ed777da892004031c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_engineering.yaml @@ -0,0 +1,8 @@ +description: '다음은 공학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eda75c55ea32eaa75627f9e1e35899c35ec99ed1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_health.yaml @@ -0,0 +1,8 @@ +description: '다음은 건강에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4cf12f43178f4c3f6ce2898523ef5fbce4ece5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_history.yaml @@ -0,0 +1,8 @@ +description: '다음은 역사에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f416b6652287c91ce99fa5d0f1c04f5c73b5ccd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_law.yaml @@ -0,0 +1,8 @@ +description: '다음은 법률에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..454b732ff8f481b19cba7c334ba209379a4c9f63 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_math.yaml @@ -0,0 +1,8 @@ +description: '다음은 수학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c85181a8d2cd447d469a8b50c03331f67c5ad76f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_other.yaml @@ -0,0 +1,8 @@ +description: '다음은 기타에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8570ae5416ca7b1df1a4e7eca4bbd9451541620a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_philosophy.yaml @@ -0,0 +1,8 @@ +description: '다음은 철학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. 여기서 + X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5e0220169cbaab8d5f6ed8cc8712bab8c5bce10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_physics.yaml @@ -0,0 +1,8 @@ +description: '다음은 물리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..206897520d6ab9fb8f5b76920c7ba0b7c54016f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/mmlu_prox_lite_ko_psychology.yaml @@ -0,0 +1,8 @@ +description: '다음은 심리학에 관한 객관식 문제(정답 포함)입니다. 단계적으로 생각한 다음 "답은 (X)입니다"로 답변을 마무리하세요. + 여기서 X는 올바른 선택지 문자입니다. + + ' +include: _ko_lite_template_yaml +task: mmlu_prox_lite_ko_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ko/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mmlu_prox_lite_mr.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mmlu_prox_lite_mr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e99fec8d7aa8f9ee5b3b5ee76d69e527cef56cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mmlu_prox_lite_mr.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_mr +task: +- mmlu_prox_lite_mr_biology +- mmlu_prox_lite_mr_business +- mmlu_prox_lite_mr_chemistry +- mmlu_prox_lite_mr_computer_science +- mmlu_prox_lite_mr_economics +- mmlu_prox_lite_mr_engineering +- mmlu_prox_lite_mr_health +- mmlu_prox_lite_mr_history +- mmlu_prox_lite_mr_law +- mmlu_prox_lite_mr_math +- mmlu_prox_lite_mr_other +- mmlu_prox_lite_mr_philosophy +- mmlu_prox_lite_mr_physics +- mmlu_prox_lite_mr_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mmlu_prox_mr.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mmlu_prox_mr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..280f6f35c3de15f3ae21a087e3b389d29ad47e60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mmlu_prox_mr.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_mr +task: +- mmlu_prox_mr_biology +- mmlu_prox_mr_business +- mmlu_prox_mr_chemistry +- mmlu_prox_mr_computer_science +- mmlu_prox_mr_economics +- mmlu_prox_mr_engineering +- mmlu_prox_mr_health +- mmlu_prox_mr_history +- mmlu_prox_mr_law +- mmlu_prox_mr_math +- mmlu_prox_mr_other +- mmlu_prox_mr_philosophy +- mmlu_prox_mr_physics +- mmlu_prox_mr_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mr_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mr_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..75c51a7c34d9707a2f06666e05a84b192efe4ed5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mr_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: mr +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'उत्तर आहे \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "प्रश्न:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mr_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..13206d977f1b4e2d161705cf41f3693d35dc69c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/_mr_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: mr +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'उत्तर आहे \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "प्रश्न:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e30a08d9f837cfa633e78c1a33cf45302a9ef299 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_biology.yaml @@ -0,0 +1,9 @@ +description: 'खाली जीवशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8cb858d27a7e88040a89fcee3732151ae0bba56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_business.yaml @@ -0,0 +1,9 @@ +description: 'खाली व्यवसाय विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d64cf713ff3863ec48317ecbeca8616bf825c90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'खाली रसायनशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a54b40a52d9f74de5261a76a12f02776e1a22c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'खाली संगणकशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e364343d4d388072f1fdde821560053324e7e5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_economics.yaml @@ -0,0 +1,9 @@ +description: 'खाली अर्थशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc0478d070cbf5d67c0a861077699df83fb65c1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_engineering.yaml @@ -0,0 +1,9 @@ +description: 'खाली अभियांत्रिकी विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9285e9728ef0bd452b7f6694de6b9e1233a2d2b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_health.yaml @@ -0,0 +1,9 @@ +description: 'खाली आरोग्य विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c98626dcd6a5e1d1f1c022cc444a28ae8ef678eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_history.yaml @@ -0,0 +1,9 @@ +description: 'खाली इतिहास विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55598683271fe7046a371e4986bab2226a306d91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_law.yaml @@ -0,0 +1,9 @@ +description: 'खाली कायदा विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30628360aabe84babe040b5c86142de7877dff87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_math.yaml @@ -0,0 +1,9 @@ +description: 'खाली गणित विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76b24eb3bd283d83456321cb033d31ff24cac831 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_other.yaml @@ -0,0 +1,9 @@ +description: 'खाली इतर विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी पायरीने + विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे अक्षर + आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bbc19d54eaf88a6208e5dace07880e27ef637fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'खाली तत्त्वज्ञान विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d900e7ba5eb9fcf41bab26f2bd2ef12ca913d507 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_physics.yaml @@ -0,0 +1,9 @@ +description: 'खाली भौतिकशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b2ce904eda6da7c4b0981eb3cda864b4619d8df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_lite_mr_psychology.yaml @@ -0,0 +1,9 @@ +description: 'खाली मानसशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_lite_template_yaml +task: mmlu_prox_lite_mr_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d665f1cd01f477ca4ee3bcc9b61b14dca6df5acc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_biology.yaml @@ -0,0 +1,9 @@ +description: 'खाली जीवशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b5a7f21bacdf015ca0f1026f2fe1d4c5e0c834d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_business.yaml @@ -0,0 +1,9 @@ +description: 'खाली व्यवसाय विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..465f59abbf335b48b86722ee5bcf27e1a8d5728a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'खाली रसायनशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5d26f2270f86facd1736a45b967a495bf6ab463 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'खाली संगणकशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a7e8b8a0e25332a5c08945ce206ce69af4401d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_economics.yaml @@ -0,0 +1,9 @@ +description: 'खाली अर्थशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4216430d37a7cee6b4c254bea3a562737333e3b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_engineering.yaml @@ -0,0 +1,9 @@ +description: 'खाली अभियांत्रिकी विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70e4acec0b4170cc481ebef68bfd2d9fb56341db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_health.yaml @@ -0,0 +1,9 @@ +description: 'खाली आरोग्य विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d65735a32a83f69d99106a8cfa1cdd51d81b2da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_history.yaml @@ -0,0 +1,9 @@ +description: 'खाली इतिहास विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..963e56674036bbd48d8cbea138c0b3d4edde633a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_law.yaml @@ -0,0 +1,9 @@ +description: 'खाली कायदा विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbd79a2c806da3bf1e08ad092257844cd31973cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_math.yaml @@ -0,0 +1,9 @@ +description: 'खाली गणित विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी + पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे + अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6226f483ba263c1a27c6da95f53fa1507355867a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_other.yaml @@ -0,0 +1,9 @@ +description: 'खाली इतर विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया पायरी पायरीने + विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य पर्यायाचे अक्षर + आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbeabed57692318f7021c7f62087d471d41e0a7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'खाली तत्त्वज्ञान विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..383d5f98d859add380c651c6bc0b711610c47f63 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_physics.yaml @@ -0,0 +1,9 @@ +description: 'खाली भौतिकशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69c032f4803035afba4656350e4913f2d59a16c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/mmlu_prox_mr_psychology.yaml @@ -0,0 +1,9 @@ +description: 'खाली मानसशास्त्र विषयावरील बहुपर्यायी प्रश्न आहेत (उत्तरांसह). कृपया + पायरी पायरीने विचार करा आणि आपले उत्तर "उत्तर आहे (X)" असे संपवा, जिथे X हे योग्य + पर्यायाचे अक्षर आहे. + + ' +include: _mr_template_yaml +task: mmlu_prox_mr_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/mr/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_mmlu_prox_lite_ne.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_mmlu_prox_lite_ne.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53084ec7ab9c893939f5fc04df836c2d6152fb73 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_mmlu_prox_lite_ne.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_ne +task: +- mmlu_prox_lite_ne_biology +- mmlu_prox_lite_ne_business +- mmlu_prox_lite_ne_chemistry +- mmlu_prox_lite_ne_computer_science +- mmlu_prox_lite_ne_economics +- mmlu_prox_lite_ne_engineering +- mmlu_prox_lite_ne_health +- mmlu_prox_lite_ne_history +- mmlu_prox_lite_ne_law +- mmlu_prox_lite_ne_math +- mmlu_prox_lite_ne_other +- mmlu_prox_lite_ne_philosophy +- mmlu_prox_lite_ne_physics +- mmlu_prox_lite_ne_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_mmlu_prox_ne.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_mmlu_prox_ne.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1efcf76710f23f506333aae7ddb3dbdc92d37016 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_mmlu_prox_ne.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ne +task: +- mmlu_prox_ne_biology +- mmlu_prox_ne_business +- mmlu_prox_ne_chemistry +- mmlu_prox_ne_computer_science +- mmlu_prox_ne_economics +- mmlu_prox_ne_engineering +- mmlu_prox_ne_health +- mmlu_prox_ne_history +- mmlu_prox_ne_law +- mmlu_prox_ne_math +- mmlu_prox_ne_other +- mmlu_prox_ne_philosophy +- mmlu_prox_ne_physics +- mmlu_prox_ne_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_ne_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_ne_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5aa59d175e78552ee262eaf46ef405195abd4a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_ne_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: ne +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'उत्तर \(?([ABCDEFGHIJ])\)? हो।' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "प्रश्न:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_ne_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_ne_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a151765295a17aeac28b990312720a7f8df99b70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/_ne_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ne +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'उत्तर \(?([ABCDEFGHIJ])\)? हो।' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "प्रश्न:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a2d9f232ea875d57ae57b8a0ccff9742e1a0849 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_biology.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ जीवविज्ञान सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cf811522904c72ca9cbccbfd76dcbe2c38d5a51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_business.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ व्यापार सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। कृपया + चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, जहाँ + X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07d1f60c3e22a28fb5893fd05a3eac92fdbb9e50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ रसायनशास्त्र सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03484acba2f48f75f89e8feadf74001449d82150 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ कम्प्युटर विज्ञान सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू + सहित)। कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85a80504a809db8275aa2a994e694e8f1208f8c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_economics.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ अर्थशास्त्र सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e7ccc550a0c16fd7c3e4725c32181815fc55ce9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_health.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ स्वास्थ्य सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। कृपया + चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, जहाँ + X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4466d1359afa9d2eec37f58ac8763b4221ebcc40 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_law.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ कानून सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। कृपया + चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, जहाँ + X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62f09bbc63720e42e76dc0b943c242b583fec4fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_other.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ अन्य सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। कृपया + चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, जहाँ + X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..283de9c122d5a39aed67bf9e4a47309997c754ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ दर्शनशास्त्र सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..155c5417fa93df4933020d9b460f82476b80fcbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_physics.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ भौतिकशास्त्र सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6eb49d06fbe6990a9a2c381727ef0943c586021b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_lite_ne_psychology.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ मनोविज्ञान सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_lite_template_yaml +task: mmlu_prox_lite_ne_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29a215f226c987e746f69fa3c40f976b3995de35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_biology.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ जीवविज्ञान सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_template_yaml +task: mmlu_prox_ne_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2942fc9e4cbee2e3f86c6e6a1e45837ad641ae3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ रसायनशास्त्र सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_template_yaml +task: mmlu_prox_ne_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adc2b2ab8161217829e4301615ecbd7b987a60e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ कम्प्युटर विज्ञान सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू + सहित)। कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_template_yaml +task: mmlu_prox_ne_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c5192a26a04dfbcdbd1cefcc23061570c6a32af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_economics.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ अर्थशास्त्र सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। + कृपया चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, + जहाँ X सही विकल्पको अक्षर हो। + + ' +include: _ne_template_yaml +task: mmlu_prox_ne_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80879d8c3ec859d44e8ca34ab3fd4d90d1c5096b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_health.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ स्वास्थ्य सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। कृपया + चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, जहाँ + X सही विकल्पको अक्षर हो। + + ' +include: _ne_template_yaml +task: mmlu_prox_ne_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37adcec5dab0a380c51f14a17b6db178d2f6b225 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ne/mmlu_prox_ne_history.yaml @@ -0,0 +1,9 @@ +description: 'यहाँ इतिहास सम्बन्धी बहुवैकल्पिक प्रश्नहरू छन् (उत्तरहरू सहित)। कृपया + चरणबद्ध रूपमा सोच्नुहोस् र आफ्नो उत्तर "उत्तर (X) हो।" बाट अन्त्य गर्नुहोस्, जहाँ + X सही विकल्पको अक्षर हो। + + ' +include: _ne_template_yaml +task: mmlu_prox_ne_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_lite_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_lite_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b58aeb6f90fb4a2103945c06a25e409d28bc78e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_lite_pt.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_pt +task: +- mmlu_prox_lite_pt_biology +- mmlu_prox_lite_pt_business +- mmlu_prox_lite_pt_chemistry +- mmlu_prox_lite_pt_computer_science +- mmlu_prox_lite_pt_economics +- mmlu_prox_lite_pt_engineering +- mmlu_prox_lite_pt_health +- mmlu_prox_lite_pt_history +- mmlu_prox_lite_pt_law +- mmlu_prox_lite_pt_math +- mmlu_prox_lite_pt_other +- mmlu_prox_lite_pt_philosophy +- mmlu_prox_lite_pt_physics +- mmlu_prox_lite_pt_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7f60973dfcfeb5ed6cd9810b89ae161f8ed692f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_mmlu_prox_pt.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_pt +task: +- mmlu_prox_pt_biology +- mmlu_prox_pt_business +- mmlu_prox_pt_chemistry +- mmlu_prox_pt_computer_science +- mmlu_prox_pt_economics +- mmlu_prox_pt_engineering +- mmlu_prox_pt_health +- mmlu_prox_pt_history +- mmlu_prox_pt_law +- mmlu_prox_pt_math +- mmlu_prox_pt_other +- mmlu_prox_pt_philosophy +- mmlu_prox_pt_physics +- mmlu_prox_pt_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0be4cb5a0614254efc0b35f696078846b31e552e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/_pt_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: pt +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'A resposta é \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Pergunta:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbfc233e241855b45a4a2f6b0d5a1b4beeca75dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_biology.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre biologia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_lite_template_yaml +task: mmlu_prox_lite_pt_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..352c6354ca7b79f4d4678dd0d4771bbbf86e4d6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_business.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre negócios. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_lite_template_yaml +task: mmlu_prox_lite_pt_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd61a71adea36b0c22d08cb4648813cf5b530f25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_economics.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre economia. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_lite_template_yaml +task: mmlu_prox_lite_pt_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3e4b832008cd0b7b910ae1454b97d8b87a7e2eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/pt/mmlu_prox_lite_pt_history.yaml @@ -0,0 +1,9 @@ +description: 'A seguir estão perguntas de múltipla escolha (com respostas) sobre história. + Pense passo a passo e termine sua resposta com "A resposta é (X)" onde X é a letra + da opção correta. + + ' +include: _pt_lite_template_yaml +task: mmlu_prox_lite_pt_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_lite_sr_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_lite_sr_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e3a0690bcc8ab8ce78cd7d82a5849ec4253a8b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_lite_sr_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Evo pitanja sa višestrukim izborom o psihologija (sa odgovorom). Molimo + vas da razmislite korak po korak i završite svoj odgovor sa "Odgovor je (X)", gde + je X slovo tačne opcije. + + ' +include: _sr_lite_template_yaml +task: mmlu_prox_lite_sr_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cf6231f953e09a560c0e93a6ba0ebe3c01e7b6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_biology.yaml @@ -0,0 +1,9 @@ +description: 'Evo pitanja sa višestrukim izborom o biologija (sa odgovorom). Molimo + vas da razmislite korak po korak i završite svoj odgovor sa "Odgovor je (X)", gde + je X slovo tačne opcije. + + ' +include: _sr_template_yaml +task: mmlu_prox_sr_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dcb90d5afb9f747be986a49e9ac4fb0d9d465ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Evo pitanja sa višestrukim izborom o inženjerstvo (sa odgovorom). Molimo + vas da razmislite korak po korak i završite svoj odgovor sa "Odgovor je (X)", gde + je X slovo tačne opcije. + + ' +include: _sr_template_yaml +task: mmlu_prox_sr_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6142a173400a3e939e796fde887a89042676ed90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_history.yaml @@ -0,0 +1,9 @@ +description: 'Evo pitanja sa višestrukim izborom o istorija (sa odgovorom). Molimo + vas da razmislite korak po korak i završite svoj odgovor sa "Odgovor je (X)", gde + je X slovo tačne opcije. + + ' +include: _sr_template_yaml +task: mmlu_prox_sr_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e99d900ab5d6a75c3cad3533cda82032419679aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_law.yaml @@ -0,0 +1,9 @@ +description: 'Evo pitanja sa višestrukim izborom o pravo (sa odgovorom). Molimo vas + da razmislite korak po korak i završite svoj odgovor sa "Odgovor je (X)", gde je + X slovo tačne opcije. + + ' +include: _sr_template_yaml +task: mmlu_prox_sr_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68ba1e8746a6310e98ac73f9ec893c302f823d16 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Evo pitanja sa višestrukim izborom o filozofija (sa odgovorom). Molimo + vas da razmislite korak po korak i završite svoj odgovor sa "Odgovor je (X)", gde + je X slovo tačne opcije. + + ' +include: _sr_template_yaml +task: mmlu_prox_sr_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d6c944d9af012d10fc8d9a2f964fa263823ff89 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/mmlu_prox_sr_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Evo pitanja sa višestrukim izborom o psihologija (sa odgovorom). Molimo + vas da razmislite korak po korak i završite svoj odgovor sa "Odgovor je (X)", gde + je X slovo tačne opcije. + + ' +include: _sr_template_yaml +task: mmlu_prox_sr_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sr/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9a2fba188a31ca78e8b243b438fa33882804ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/_mmlu_prox_sw.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_sw +task: +- mmlu_prox_sw_biology +- mmlu_prox_sw_business +- mmlu_prox_sw_chemistry +- mmlu_prox_sw_computer_science +- mmlu_prox_sw_economics +- mmlu_prox_sw_engineering +- mmlu_prox_sw_health +- mmlu_prox_sw_history +- mmlu_prox_sw_law +- mmlu_prox_sw_math +- mmlu_prox_sw_other +- mmlu_prox_sw_philosophy +- mmlu_prox_sw_physics +- mmlu_prox_sw_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b0a89deea29737f94354e4dab757243aae4f063 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_biology.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu biolojia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43877798d59e9a9430c6100f73f75abcc0838ecc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu kemia. Fikiria + hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi ya + chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b064e70a68dc9aa63f64d58d3a399733d3f0cb98 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu sayansi + ya kompyuta. Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo + X ni herufi ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e7e7c3d78aa4d9f671b511b417c96c44ae83974 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_economics.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu uchumi. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a2966d6e214abe4450e893a83368c3e5342e060 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu uhandisi. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..baa8162bf16fc070fdfef3ddbe2faf9a8f0c858b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_health.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu afya. Fikiria + hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi ya + chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fcadc37c6f4545ea41bfa81ee22d0d4cd8f424b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_history.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu historia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c551fe5f906c6ee59b94cbf1ce31d1978ca6ed2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_law.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu sheria. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43625763db29876a3c0dea070212416d1bf6f306 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_math.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu hisabati. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7411746037e68cc069f54820b049d42079cef36b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_other.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu nyingine. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6a2964f37a263e54bc05c6cb95fc03563aa42d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu falsafa. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0500ef46f21f35db0553a70051390d4a15a42ca9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_physics.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu fizikia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a771eac92af97eb94b8c6eefafbc5921dfc86fd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_lite_sw_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu saikolojia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_lite_template_yaml +task: mmlu_prox_lite_sw_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63230a9ee9d88e8a3c9503b21870a8cb4daa032b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_biology.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu biolojia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..981361982468f4ce636611f81f903d0ea2b7bef6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_business.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu biashara. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f87ac7e3f040b8c9763843d55c9534e89bed161 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu kemia. Fikiria + hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi ya + chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4168b229e94536b0cf838a65313786815314f9d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu sayansi + ya kompyuta. Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo + X ni herufi ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adb5ce0f7101378609e5305eddb544ea8b810e46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_economics.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu uchumi. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd0d6267ec7827c5ee0b0384c1e27b263bb51dad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu uhandisi. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4db109fb1c01f4de671c63da6ecefbd69bf2a2e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_health.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu afya. Fikiria + hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi ya + chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b4b39d4e8a4790487bd49bbc5fc9a70d82b9ee7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_history.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu historia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e49ae99802fcb5ade0dd5dc61e7fa8402507dfc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_law.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu sheria. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a940e122be05893977aed1b86d1ecc47c6850fb0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_math.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu hisabati. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96dff12c7cc341e96555b760afe4aae14b579bae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_other.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu nyingine. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89d8227df565e7f0b78dd187c3c28eac32d1d89e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu falsafa. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d9ade70dd5f97299365656ff68e80508a42b126 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_physics.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu fizikia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..056aff1f336826526dc3d0291458bfba4b233f75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/mmlu_prox_sw_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Yafuatayo ni maswali ya chaguo-nyingi (yenye majibu) kuhusu saikolojia. + Fikiria hatua kwa hatua kisha malizia jibu lako kwa "Jibu ni (X)" ambapo X ni herufi + ya chaguo sahihi. + + ' +include: _sw_template_yaml +task: mmlu_prox_sw_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/sw/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_mmlu_prox_lite_te.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_mmlu_prox_lite_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffbe9a2fa855a91edfb94ffc5dbbbb6b68186e38 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_mmlu_prox_lite_te.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_te +task: +- mmlu_prox_lite_te_biology +- mmlu_prox_lite_te_business +- mmlu_prox_lite_te_chemistry +- mmlu_prox_lite_te_computer_science +- mmlu_prox_lite_te_economics +- mmlu_prox_lite_te_engineering +- mmlu_prox_lite_te_health +- mmlu_prox_lite_te_history +- mmlu_prox_lite_te_law +- mmlu_prox_lite_te_math +- mmlu_prox_lite_te_other +- mmlu_prox_lite_te_philosophy +- mmlu_prox_lite_te_physics +- mmlu_prox_lite_te_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_mmlu_prox_te.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_mmlu_prox_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9240fd43a908eb3d4a1eadc5a8bc5a6066fb98bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_mmlu_prox_te.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_te +task: +- mmlu_prox_te_biology +- mmlu_prox_te_business +- mmlu_prox_te_chemistry +- mmlu_prox_te_computer_science +- mmlu_prox_te_economics +- mmlu_prox_te_engineering +- mmlu_prox_te_health +- mmlu_prox_te_history +- mmlu_prox_te_law +- mmlu_prox_te_math +- mmlu_prox_te_other +- mmlu_prox_te_philosophy +- mmlu_prox_te_physics +- mmlu_prox_te_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_te_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_te_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..65ea494d452287b3c6d2e5c888316b0a81af6b8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_te_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: te +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'సమాధానం \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "ప్రశ్న:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_te_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_te_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..79056db31b6100fe74796ae99aa95966140ab0b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/_te_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: te +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'సమాధానం \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "ప్రశ్న:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c259d1aca6ad7585549b2ceb4c63f7b2df63ee2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_biology.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది జీవశాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4618e425b4139b6d0a93f480131021c5a22456a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_business.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది వ్యాపారంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3e50eb9d136030cb0f27f034ace488c6747741f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది రసాయన శాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7187ce52d3c6bdf00bb2b8387d3025d190cdd865 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది కంప్యూటర్ సైన్స్కి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). + దయచేసి దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన + ఎంపిక అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f47c8140e43b64073731573a955e4a6766fd54b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_economics.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది ఆర్థిక శాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). + దయచేసి దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన + ఎంపిక అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..482656056a5332191e9c41dda338e47137871bcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_engineering.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది ఇంజనీరింగ్కి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8ddf5787224077e7946820a9439a16898c4f17c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_health.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది ఆరోగ్యంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fcb4ed010678b17a8a018e80307f69a7ba506c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_history.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది చరిత్రకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62c49df5ef97f7f8c10936d975be049650d13320 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_law.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది న్యాయశాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1d82c692949eb4c19848f841498be1c88a3f8f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_math.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది గణితంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24b1e391f91ced96276273c010dcac636bb79943 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_other.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది ఇతరమైనకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..150683c1660d99f99c97702ae67812b48b8706f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది తత్వవేత్తకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fcab16ca6ecf0a8292cc34c9262b07dc8905bdf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_physics.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది భౌతిక శాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5076e759e30af1dbc922516eb01585dc1948644 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_lite_te_psychology.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది మనోవిజ్ఞానశాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). + దయచేసి దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన + ఎంపిక అక్షరం. + + ' +include: _te_lite_template_yaml +task: mmlu_prox_lite_te_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..183c4403dede202147cb0b4cea28cbd86fc84681 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_biology.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది జీవశాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c773f815283d873cbbf28fdb6c125f7be62676db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_business.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది వ్యాపారంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a53088486b021f53714ed5f88af4273b69ce44ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది రసాయన శాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1643ebb8e7b6ad481524e934ac56c6d681cc8df8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది కంప్యూటర్ సైన్స్కి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). + దయచేసి దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన + ఎంపిక అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b794b156e7e50fba6530693d95792401323aa2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_economics.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది ఆర్థిక శాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). + దయచేసి దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన + ఎంపిక అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0cad99ba1710c497deba88671d465d86872bca09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_engineering.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది ఇంజనీరింగ్కి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce25943393d9547fe909d12c87791691a66fc69a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_health.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది ఆరోగ్యంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6e3ce41bfd9513b73eb67b2c64bb014efe32ee0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_history.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది చరిత్రకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c35bd87e0f777ead8a785a0c34f76ed06ba707a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_law.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది న్యాయశాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e67f8e67fb3933968eb7163f5f41fe6f86974e4d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_math.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది గణితంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbe19386837d50d3732b3503c3d1811f5e963c5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_other.yaml @@ -0,0 +1,8 @@ +description: 'క్రింది ఇతరమైనకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి దశలవారీగా + ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70f118cdcbdb69c2e8af0c720ab0c228ee69530d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది తత్వవేత్తకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f41b6f19d70d5a413e4896aa35ae45a0ad35492 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_physics.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది భౌతిక శాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). దయచేసి + దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన ఎంపిక + అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65b35eb31d6470c621f42625e2b5b2e13f32f714 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/mmlu_prox_te_psychology.yaml @@ -0,0 +1,9 @@ +description: 'క్రింది మనోవిజ్ఞానశాస్త్రంకి సంబంధించిన బహుళఎంపిక ప్రశ్న (సమాధానాలతో). + దయచేసి దశలవారీగా ఆలోచించి, మీ సమాధానాన్ని "సమాధానం (X)"తో ముగించండి, ఇక్కడ X సరైన + ఎంపిక అక్షరం. + + ' +include: _te_template_yaml +task: mmlu_prox_te_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/te/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/_lang_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/_lang_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..879d0767d6544254c5394baa88a9cf3022b247a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/_lang_template_yaml @@ -0,0 +1,35 @@ +dataset_path: {repo_id} +dataset_name: {lang} +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '{ans_regex}' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "{que_prefix}" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/template/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_lite_th.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_lite_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..537af2b0203c94190db7c5978393a6038c41f308 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_lite_th.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_th +task: +- mmlu_prox_lite_th_biology +- mmlu_prox_lite_th_business +- mmlu_prox_lite_th_chemistry +- mmlu_prox_lite_th_computer_science +- mmlu_prox_lite_th_economics +- mmlu_prox_lite_th_engineering +- mmlu_prox_lite_th_health +- mmlu_prox_lite_th_history +- mmlu_prox_lite_th_law +- mmlu_prox_lite_th_math +- mmlu_prox_lite_th_other +- mmlu_prox_lite_th_philosophy +- mmlu_prox_lite_th_physics +- mmlu_prox_lite_th_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_th.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e65fdd1a05573abfd120e67245ee224d44faaedb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_mmlu_prox_th.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_th +task: +- mmlu_prox_th_biology +- mmlu_prox_th_business +- mmlu_prox_th_chemistry +- mmlu_prox_th_computer_science +- mmlu_prox_th_economics +- mmlu_prox_th_engineering +- mmlu_prox_th_health +- mmlu_prox_th_history +- mmlu_prox_th_law +- mmlu_prox_th_math +- mmlu_prox_th_other +- mmlu_prox_th_philosophy +- mmlu_prox_th_physics +- mmlu_prox_th_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..78588216c898cf1f1f5ac81ce5e3593c728b352a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: th +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'คำตอบคือ \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "คำถาม:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9889e69e5dba9f4e72516f8aa3c41376355b4bd6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/_th_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: th +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'คำตอบคือ \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "คำถาม:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac13d708f4f88207474778d2b99802c269b06dcc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_biology.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ชีววิทยา คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b269cd568d3d005bb7c0d1c9c143f1df88435ebc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_business.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ธุรกิจ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d63b7ac98d241a8b71f9601547456133b72d302 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_chemistry.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ เคมี คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ccb84bae7d348240c09b28855db4f360b92835a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_computer_science.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ วิทยาการคอมพิวเตอร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d58560371cbe7e9845e85d19bb64b4437f681a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_economics.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ เศรษฐศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..757357eb3680a87fc943777e6f49608c0d29a6fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_engineering.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ วิศวกรรมศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18e0bc82d71bae7eddca7b66991ece42e26ed63b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_health.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ สุขภาพ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3760192d4746ba30694a59a057a9a7d4d2ec8088 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_history.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ประวัติศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50b898e4d5fa474ea48fd93d032cde3d83e7e280 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_law.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ กฎหมาย คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..500dadfa598b61d0e422b848a96470a83d6ee5a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_math.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ คณิตศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f64bb89600268a0fb51fce5b4ac973e0abed040e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_other.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ อื่นๆ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..645176ce9b939c8c40b5a8799884e6fe7d055f54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_philosophy.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ปรัชญา คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c89c415775a58169eba16d77f70837b132ff426 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_physics.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ฟิสิกส์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..259c5869250feb243c00fdda707af40b303f65b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_lite_th_psychology.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ จิตวิทยา คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_lite_template_yaml +task: mmlu_prox_lite_th_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fa50a4b7e199f0730ecd3a9846b22c8936742d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_biology.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ชีววิทยา คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5f524061f142bc2008bde5ccde22ead5005253b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_business.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ธุรกิจ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1b392993e5544d8aded0557a23675dc5d988d60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_chemistry.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ เคมี คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d972eee80dde43d0c53f3acd2c17f9a3e966b61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_engineering.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ วิศวกรรมศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2f257dcb8a1e01f8f8aa4c2021e6981ce3b4867 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_law.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ กฎหมาย คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..331b04dfe5157620b2fca816f92bac0136b170b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_math.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ คณิตศาสตร์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dd6f9eedc52db34782d375f1e2b2ed678bb4d2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/mmlu_prox_th_physics.yaml @@ -0,0 +1,8 @@ +description: 'ต่อไปนี้เป็นคำถามปรนัย (พร้อมคำตอบ) เกี่ยวกับ ฟิสิกส์ คิดทีละขั้นตอนแล้วสรุปคำตอบด้วย + "คำตอบคือ (X)" โดยที่ X คือตัวอักษรที่เป็นตัวเลือกที่ถูกต้อง + + ' +include: _th_template_yaml +task: mmlu_prox_th_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/th/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_mmlu_prox_lite_uk.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_mmlu_prox_lite_uk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f087b0673fbe869492a64f530cc63ff2fdd7fdc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_mmlu_prox_lite_uk.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_uk +task: +- mmlu_prox_lite_uk_biology +- mmlu_prox_lite_uk_business +- mmlu_prox_lite_uk_chemistry +- mmlu_prox_lite_uk_computer_science +- mmlu_prox_lite_uk_economics +- mmlu_prox_lite_uk_engineering +- mmlu_prox_lite_uk_health +- mmlu_prox_lite_uk_history +- mmlu_prox_lite_uk_law +- mmlu_prox_lite_uk_math +- mmlu_prox_lite_uk_other +- mmlu_prox_lite_uk_philosophy +- mmlu_prox_lite_uk_physics +- mmlu_prox_lite_uk_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_mmlu_prox_uk.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_mmlu_prox_uk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e6c9ec9616cf71cd686076f4a2a2b59ede7021f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_mmlu_prox_uk.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_uk +task: +- mmlu_prox_uk_biology +- mmlu_prox_uk_business +- mmlu_prox_uk_chemistry +- mmlu_prox_uk_computer_science +- mmlu_prox_uk_economics +- mmlu_prox_uk_engineering +- mmlu_prox_uk_health +- mmlu_prox_uk_history +- mmlu_prox_uk_law +- mmlu_prox_uk_math +- mmlu_prox_uk_other +- mmlu_prox_uk_philosophy +- mmlu_prox_uk_physics +- mmlu_prox_uk_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_uk_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_uk_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..38e1bad8206152cfda83f382a7fb35e56c6b22f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_uk_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: uk +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Відповідь: \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Питання:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_uk_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_uk_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e0f432fd5aadd6d748850bfb44ca7db543f3a13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/_uk_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: uk +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Відповідь: \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Питання:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95f6631d351f71d0079afa28c3e68b37409ef3f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_biology.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему біологія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dba37a0d999ff8158ccabb800b7f382862ff384 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_business.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему бізнес (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f28c8dcd7a5d835e9f4982371136026d03fe7936 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему хімія (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e3dea3a09379f3b20057f979065e3aebb6dd024 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему інженерія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd5aaf88553dff5196d19c89b32e2b37aece058a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_health.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему здоров''я (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9a80a23301932519c57e30d21b45374938bc8f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_history.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему історія (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e66ebfb935cbfc7c4d536c67c7f1de7ab62c6ebb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_math.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему математика (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63bc047062ed941d0e5990ab14760a81aacbd002 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_other.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему інше (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8128b1037881c0e804764976a0755b279b9a8a82 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему філософія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8f05cf7dc079b3a57a697b419c8d573340925d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_physics.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему фізика (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa9b7266117502ab6a44309a9ec6ebafbe204c68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_lite_uk_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему психологія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_lite_template_yaml +task: mmlu_prox_lite_uk_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f946ce05828fc1956c32669d7fe65b395c487b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_biology.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему біологія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0c8f79435899c8053d52fcaf2d8805824dbc61f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_business.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему бізнес (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da898127f90875dd4946abc1eff719004fa0912d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему хімія (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48d4c2d9be58848f4652c8bb5b2f97844f2b7108 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему інформатика (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..850e7d3d00fc36f3640967875dddfb6643c84925 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_economics.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему економіка (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d1ad0d7350f9d241833dbaf3de84059357fe733 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему інженерія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b60a822e5c79e92bd5c804bc2b4d69140287f79b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_health.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему здоров''я (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68b0d718bbcabca52217f8cc52d9903ecfe32b56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_history.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему історія (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..887ea5c238f321784d0d835a8490adf1ad6bb632 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_law.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему право (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f83a0ff22f1676f4a5cd756c705a1b7d0b9b20ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_math.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему математика (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d90cbda640bea8f22e19486688c99c65acd504d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_other.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему інше (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d568ea548b3e6d9629d0288ef107f243b38cc2e2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему філософія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ce4b967e320a12d23ecfb623783cf001f7e1b60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_physics.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему фізика (з відповіддю). Будь + ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", де + X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7f86cfebf32d321d6548617a0fd8320c4d2858d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/mmlu_prox_uk_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Ось запитання з вибором відповідей на тему психологія (з відповіддю). + Будь ласка, подумайте крок за кроком і закінчіть свою відповідь "Відповідь: (X)", + де X – літера правильного варіанту. + + ' +include: _uk_template_yaml +task: mmlu_prox_uk_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/uk/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_mmlu_prox_lite_ur.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_mmlu_prox_lite_ur.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68b9ff39dbcb005e0fabfbf838632cd0586e391d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_mmlu_prox_lite_ur.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_ur +task: +- mmlu_prox_lite_ur_biology +- mmlu_prox_lite_ur_business +- mmlu_prox_lite_ur_chemistry +- mmlu_prox_lite_ur_computer_science +- mmlu_prox_lite_ur_economics +- mmlu_prox_lite_ur_engineering +- mmlu_prox_lite_ur_health +- mmlu_prox_lite_ur_history +- mmlu_prox_lite_ur_law +- mmlu_prox_lite_ur_math +- mmlu_prox_lite_ur_other +- mmlu_prox_lite_ur_philosophy +- mmlu_prox_lite_ur_physics +- mmlu_prox_lite_ur_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_mmlu_prox_ur.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_mmlu_prox_ur.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1015b30731f21738fd635827b0712a4cd59b01f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_mmlu_prox_ur.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_ur +task: +- mmlu_prox_ur_biology +- mmlu_prox_ur_business +- mmlu_prox_ur_chemistry +- mmlu_prox_ur_computer_science +- mmlu_prox_ur_economics +- mmlu_prox_ur_engineering +- mmlu_prox_ur_health +- mmlu_prox_ur_history +- mmlu_prox_ur_law +- mmlu_prox_ur_math +- mmlu_prox_ur_other +- mmlu_prox_ur_philosophy +- mmlu_prox_ur_physics +- mmlu_prox_ur_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_ur_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_ur_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d26fa66432781512f32fab3d1e7bdf8b57016ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_ur_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: ur +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'جواب \(?([ABCDEFGHIJ])\)? ہے' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "سوال:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_ur_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_ur_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..af8951aaab6a0c620bdb4d68827f4793004c5cda --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/_ur_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: ur +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'جواب \(?([ABCDEFGHIJ])\)? ہے' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "سوال:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e61751988fbf60fdf722541fe81e2b9ee3ce6b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_biology.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل حیاتیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c9266212c0ef45bebca9de0a445e1492c6da59a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_business.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل کاروبار کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30179d87c42afe61a84091065c49ed362d5b9021 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل کیمیا کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a57a8da686ccd063b794a537ec1e2e591af32c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل کمپیوٹر سائنس کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے + ساتھ)۔ براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم + کریں، جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff8d8db518350a0f67194aaa5ad7198153efb86b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_economics.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل معاشیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89c3d1ad3e6d6a1599dbcc0f1b5cc4514b5f759d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_engineering.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل انجینئرنگ کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8309d81ca5476902026d2e32b36715b82658b9d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_health.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل صحت کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36b35141d0f67cb14a6d43c3131496907cb5000a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_history.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل تاریخ کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c30edf826d8b111020a47cf79f5bf6f668071aa5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_law.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل قانون کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a0655691678241e11b1b8d909165dfc5e860e7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_math.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل ریاضی کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48667c746da592c9c11ce481cf4e522b06cc92e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_other.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل دیگر کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..696d5f6a27ce1cb94ce8c1c41266e77af1004306 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل فلسفہ کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bafa412ace8c20b329d3c99ce4826a61bca8484c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_physics.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل طبیعیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..413e17a69ee8dff19dbb988d445bf69c38b69deb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_lite_ur_psychology.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل نفسیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_lite_template_yaml +task: mmlu_prox_lite_ur_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e82f65c641642d24ba3c3b74b04e88e96476aed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_biology.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل حیاتیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b7e5897d573e0fa31a0122b64b2821a59f7c01f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_business.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل کاروبار کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8bf883bd84edb6f65c9dde3d14b87bb2e023242 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل کیمیا کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54fe4d0b832210b8732367f35f2d7528eba56b5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل کمپیوٹر سائنس کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے + ساتھ)۔ براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم + کریں، جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18449259736d6ac5862e98e2ae307e5bb56ae1d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_economics.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل معاشیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80bdb45e437746e837fc6a5543506eb649d3be1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_engineering.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل انجینئرنگ کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbc024668336a1b48751107229361654da225aaa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_health.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل صحت کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cedaceb56ed86d14d74afa394ebd3f896cf6e489 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_history.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل تاریخ کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25e0d8002273e3ac9740240dee43c91c81f5a077 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_law.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل قانون کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..173b1f3869130e1a4d25a9df3f746b6ee55ad47e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_math.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل ریاضی کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbf0957ef950d74433c795ee62f6c312059f9c2b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_other.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل دیگر کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0852ec862d06b81e0617321b1a1e334cb2e3509 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل فلسفہ کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ براہ + کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، جہاں + X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb1987d26214fb808842100234d8086d37997977 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_physics.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل طبیعیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8440f75c208c4bd582537fd3518cfbe191743048 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/mmlu_prox_ur_psychology.yaml @@ -0,0 +1,9 @@ +description: 'درج ذیل نفسیات کے متعلق ایک متعدد انتخابی سوال ہے (جوابات کے ساتھ)۔ + براہ کرم قدم بہ قدم سوچیں، اور پھر اپنے جواب کو "جواب (X) ہے" کے ساتھ ختم کریں، + جہاں X درست آپشن کا حرف ہے۔ + + ' +include: _ur_template_yaml +task: mmlu_prox_ur_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/ur/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_mmlu_prox_lite_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_mmlu_prox_lite_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92b5e1f7f4e8de0790d8249d1d17dc15e7e6d8b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_mmlu_prox_lite_vi.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_vi +task: +- mmlu_prox_lite_vi_biology +- mmlu_prox_lite_vi_business +- mmlu_prox_lite_vi_chemistry +- mmlu_prox_lite_vi_computer_science +- mmlu_prox_lite_vi_economics +- mmlu_prox_lite_vi_engineering +- mmlu_prox_lite_vi_health +- mmlu_prox_lite_vi_history +- mmlu_prox_lite_vi_law +- mmlu_prox_lite_vi_math +- mmlu_prox_lite_vi_other +- mmlu_prox_lite_vi_philosophy +- mmlu_prox_lite_vi_physics +- mmlu_prox_lite_vi_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_mmlu_prox_vi.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_mmlu_prox_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e71426ac2ecb210b066cca8d8b5d6256994d795 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_mmlu_prox_vi.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_vi +task: +- mmlu_prox_vi_biology +- mmlu_prox_vi_business +- mmlu_prox_vi_chemistry +- mmlu_prox_vi_computer_science +- mmlu_prox_vi_economics +- mmlu_prox_vi_engineering +- mmlu_prox_vi_health +- mmlu_prox_vi_history +- mmlu_prox_vi_law +- mmlu_prox_vi_math +- mmlu_prox_vi_other +- mmlu_prox_vi_philosophy +- mmlu_prox_vi_physics +- mmlu_prox_vi_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_vi_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_vi_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4a953289080dc8c18b09c3049df2cda4b1ae154 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_vi_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: vi +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Câu trả lời là \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Câu hỏi:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_vi_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_vi_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0421597c125e111c6f9d3713aa0725fc037e4f92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/_vi_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: vi +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Câu trả lời là \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Câu hỏi:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5278e18451df5647a94e9686775a8dee7a47607f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_biology.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Sinh học (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..356969ddccb426fd5ee65181a51e8114390635db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_business.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Kinh doanh (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d99cf2e7ee5d4f208e3ac2f5efc7dc2356edbc49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Hóa học (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbdff2364fb8eeccf0abcf08e339c6281a45e89f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_economics.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Kinh tế học (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0e7e8e5eafc3d49d719964b16c071ccf774545e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Kỹ thuật (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b996be82714d1f34b4bfa24cafb6b28fb11fddc8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_health.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Sức khỏe (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d64b0f0c83c5998a357d9e635b2f82293985d772 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_history.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Lịch sử (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed2d01982163ac20e6491ef01b8f903db56daa1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_law.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Luật pháp (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd309983bdb87c8136c1a02f4f6470ebdefcdb64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_math.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Toán học (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f179e488c275c08c9fa749962d3d0d01dfbcb35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_other.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Khác (kèm đáp án). Vui lòng suy nghĩ + từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92fc79ccf0254dbc9eee7d944a808311f66c3ed3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Triết học (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..171e4bcce8f368f6b03444b4960bffb42bccaf93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_physics.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Vật lý học (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fee568cda1db6736161d3e0b5e015b4776fa7c5e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_lite_vi_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Tâm lý học (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_lite_template_yaml +task: mmlu_prox_lite_vi_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de97f59556fd4150d69095e6baf6dcaeaa3d627a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_biology.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Sinh học (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7c538b037dcac56f7a172c9848b0354f601b43a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_business.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Kinh doanh (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f29d449f3eae8970e4be5dbea00ef54aa2ffad99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Hóa học (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..714a0062122f718cd21ac0cb1d57f3bbae1aecb7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Khoa học máy tính (kèm đáp án). Vui + lòng suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là + (X)", trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af268261d8989c8b51771cf12ddaa36c9d70a2c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Kỹ thuật (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41059d02a93c1a212c347569d77113e730b7e206 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_health.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Sức khỏe (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77392fcc9d86722a0cbcb6da1fdbf2b0454de5cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_math.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Toán học (kèm đáp án). Vui lòng suy + nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0dac17cdb8e594750dfe638778b6f5d5c9706a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_other.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Khác (kèm đáp án). Vui lòng suy nghĩ + từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", trong + đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f024f4c7dd9c4cfb291ee68316a7f092e3a3fe3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/mmlu_prox_vi_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Dưới đây là câu hỏi trắc nghiệm về Tâm lý học (kèm đáp án). Vui lòng + suy nghĩ từng bước, sau đó kết thúc câu trả lời của bạn bằng "Câu trả lời là (X)", + trong đó X là chữ cái của lựa chọn đúng. + + ' +include: _vi_template_yaml +task: mmlu_prox_vi_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/vi/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_mmlu_prox_lite_wo.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_mmlu_prox_lite_wo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8008d89a553efde7cd98430a30b62e04458b6801 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_mmlu_prox_lite_wo.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_lite_wo +task: +- mmlu_prox_lite_wo_biology +- mmlu_prox_lite_wo_business +- mmlu_prox_lite_wo_chemistry +- mmlu_prox_lite_wo_computer_science +- mmlu_prox_lite_wo_economics +- mmlu_prox_lite_wo_engineering +- mmlu_prox_lite_wo_health +- mmlu_prox_lite_wo_history +- mmlu_prox_lite_wo_law +- mmlu_prox_lite_wo_math +- mmlu_prox_lite_wo_other +- mmlu_prox_lite_wo_philosophy +- mmlu_prox_lite_wo_physics +- mmlu_prox_lite_wo_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_mmlu_prox_wo.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_mmlu_prox_wo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0c6e6329211d00be64ac05b67e2607e12798e90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_mmlu_prox_wo.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_wo +task: +- mmlu_prox_wo_biology +- mmlu_prox_wo_business +- mmlu_prox_wo_chemistry +- mmlu_prox_wo_computer_science +- mmlu_prox_wo_economics +- mmlu_prox_wo_engineering +- mmlu_prox_wo_health +- mmlu_prox_wo_history +- mmlu_prox_wo_law +- mmlu_prox_wo_math +- mmlu_prox_wo_other +- mmlu_prox_wo_philosophy +- mmlu_prox_wo_physics +- mmlu_prox_wo_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_wo_lite_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_wo_lite_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ee699845960f93398b54fea926196209f7d779d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_wo_lite_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX-Lite +dataset_name: wo +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Tontu bi mooy \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Laaj:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_wo_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_wo_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f9c14e7f3c56dd56d00887b369b40a30da4ce73 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/_wo_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: wo +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Tontu bi mooy \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Laaj:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a0d505ec95ee918426963b98b3b653f93adf3ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_biology.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax biologi. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddfd9227ebbd55648e9627287dfa3b08de3c0e6b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_business.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax njëriñ. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53907ed39859983b20c89bf26a9df52a10cf5b45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax simi. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9423a5fa2bfe2ef4b4bcd250d16b5a05df3482fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax injenyëer. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75566bd560a4805039e1a4a91424f58ed2b5c61f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_health.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax wergui + yaramu. Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" + fu X di araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b3b9f316922e8d26efb35cf7e60fda8c250e6ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_history.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax taariix. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfae0d0987aa850b178204e06bcc1bf2475a4445 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_law.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax yoon. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23a81c8beb0c7aa8b12b1717a3e47875d85b0b13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_lite_wo_math.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax matematig. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_lite_template_yaml +task: mmlu_prox_lite_wo_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_wo_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_wo_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bec0bbd577fdfb620004dc50ec7e14b71e138982 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/wo/mmlu_prox_wo_biology.yaml @@ -0,0 +1,9 @@ +description: 'Li ci topp ay laaj yu am tànneef la (ak tontu) ci mbir mi ñuy wax biologi. + Xalaatal ci dooley dooley te nga jeexal sa tontu ak "Tontu bi mooy (X)" fu X di + araf bi jëkk ci tontu bi. + + ' +include: _wo_template_yaml +task: mmlu_prox_wo_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_lite_zh_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_lite_zh_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7910cc3c588b0f540af432b288e31a47041311e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_lite_zh_law.yaml @@ -0,0 +1,7 @@ +description: '以下是关于法律的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_lite_template_yaml +task: mmlu_prox_lite_zh_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5ba92d08b1317f7df12b8e4e931edef52ff552a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_business.yaml @@ -0,0 +1,7 @@ +description: '以下是关于商业的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0da4be23843cc9ce9c6417ec0367c9d3dc7f344f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_chemistry.yaml @@ -0,0 +1,7 @@ +description: '以下是关于化学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6de2d39214e9714e0559e6298844240a918bbb43 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_engineering.yaml @@ -0,0 +1,7 @@ +description: '以下是关于工程学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d78cf922a96dd851663a35b113fda2365ecd4071 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_health.yaml @@ -0,0 +1,7 @@ +description: '以下是关于健康的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bda370e7bd7599602f6918fcf0490a3d8da0e343 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_law.yaml @@ -0,0 +1,7 @@ +description: '以下是关于法律的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9038c23f65b12c6efc9f722d54ff4715ca79efe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_philosophy.yaml @@ -0,0 +1,7 @@ +description: '以下是关于哲学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab2a43da8e31dd5429d5d66205a30d52adea3122 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/mmlu_prox_zh_psychology.yaml @@ -0,0 +1,7 @@ +description: '以下是关于心理学的选择题（带有答案）。请逐步思考，然后以"答案是 (X)"结束您的回答，其中X是正确的选项字母。 + + ' +include: _zh_template_yaml +task: mmlu_prox_zh_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zh/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/_mmlu_prox_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/_mmlu_prox_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eadb83d2650c67d9a57506ee977d6cbe60584400 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/_mmlu_prox_zu.yaml @@ -0,0 +1,23 @@ +group: mmlu_prox_zu +task: +- mmlu_prox_zu_biology +- mmlu_prox_zu_business +- mmlu_prox_zu_chemistry +- mmlu_prox_zu_computer_science +- mmlu_prox_zu_economics +- mmlu_prox_zu_engineering +- mmlu_prox_zu_health +- mmlu_prox_zu_history +- mmlu_prox_zu_law +- mmlu_prox_zu_math +- mmlu_prox_zu_other +- mmlu_prox_zu_philosophy +- mmlu_prox_zu_physics +- mmlu_prox_zu_psychology +aggregate_metric_list: +- aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/_zu_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/_zu_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e83fc3f5481c68832e63eab06a8e6e6a9397cbcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/_zu_template_yaml @@ -0,0 +1,35 @@ +dataset_path: li-lab/MMLU-ProX +dataset_name: zu +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'Impendulo ithi \(?([ABCDEFGHIJ])\)?' + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "Umbuzo:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 2048 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd37c1607394ffc259a089e2afeb1430f3244ca5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-i-chemistry. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8f220d558b3ab129a61bc6379a472e2aa68e69a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-isayensi + yekhompyutha. Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo + ithi (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..787d50ea89d5b566f2412d26deb4a3d3bb2f3759 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_economics.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ezomnotho. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..923256bfda9f4202ecaaf67127f7eaf382c56d75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ubunjiniyela. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5076cf9e6a561397be2ba44159cbee4073f12e84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_history.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-umlando. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92e5db1f0ec2884b00dbe69a4dd8307ee252c698 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_law.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-umthetho. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa45fd0513a409af9f1a3148ce44220e9f067897 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_math.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-izibalo. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b52ebac298907a043f2ca87aa59b29e4d198f4a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_other.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-okunye. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fccab8f7551e46b2a457a7e2ac083368be682d92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ifilosofi. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..037a96d6c2ab68140c207de46bf8b3e8f8f04e3f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_lite_zu_physics.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ifiziksi. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_lite_template_yaml +task: mmlu_prox_lite_zu_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4378cc056c15d5d8d77d796c5630948781d52cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_biology.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-isayensi + yezilwane. Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo + ithi (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_biology +task_alias: biology +process_docs: !function utils.process_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adb1e767913ba2c31413b9fd12a5361104806239 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_business.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ibhizinisi. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_business +task_alias: business +process_docs: !function utils.process_business diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78e4592fb7723218933fcb715df20f68024a0473 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_chemistry.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-i-chemistry. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_chemistry +task_alias: chemistry +process_docs: !function utils.process_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d61d930557a9b62de7b0c1604de03dce29b5f4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_computer_science.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-isayensi + yekhompyutha. Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo + ithi (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_computer_science +task_alias: computer_science +process_docs: !function utils.process_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f3eed3ad5d32f48765e3c839141cd37533a9028 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_economics.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ezomnotho. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_economics +task_alias: economics +process_docs: !function utils.process_economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe51666038e06c529e2590c4b08ad22ac1f6f387 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_engineering.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ubunjiniyela. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_engineering +task_alias: engineering +process_docs: !function utils.process_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..699cdf1676afe95a74ad9e8423ef8926705e75d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_health.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ezempilo. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_health +task_alias: health +process_docs: !function utils.process_health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..567691486ff8203f16137731cddbbd85c47d294e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_history.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-umlando. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_history +task_alias: history +process_docs: !function utils.process_history diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0362df3b6959c1cd1854347fef80b71235dfa2c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_law.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-umthetho. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_law +task_alias: law +process_docs: !function utils.process_law diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d66a60098cbd5fb64e02557e86b441979b15ccb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_math.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-izibalo. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_math +task_alias: math +process_docs: !function utils.process_math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_other.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfe0b548f28381f0bf54f94303f0747119f87b23 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_other.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-okunye. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_other +task_alias: other +process_docs: !function utils.process_other diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f340addd59d21e74d72d3a3ea1c064320cbff36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_philosophy.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ifilosofi. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_philosophy +task_alias: philosophy +process_docs: !function utils.process_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f74cec442ec7f41525c06690e0c5a5bf85f9fa6e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_physics.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-ifiziksi. + Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo ithi + (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_physics +task_alias: physics +process_docs: !function utils.process_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08ec6593d2ccaa30109a8d58d2f7d46243330777 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/mmlu_prox_zu_psychology.yaml @@ -0,0 +1,9 @@ +description: 'Okulandelayo yimibuzo ehlukahlukene (enezimpendulo) mayelana ne-isayensi + yengqondo. Cabanga isinyathelo ngesinyathelo bese uqeda impendulo yakho nge-"Impendulo + ithi (X)" lapho u-X eyinhlamvu eyisinqumo esifanele. + + ' +include: _zu_template_yaml +task: mmlu_prox_zu_psychology +task_alias: psychology +process_docs: !function utils.process_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..88dee815f624eebc10107060cffc708adcaaea8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlu_prox/zu/utils.py @@ -0,0 +1,70 @@ +from functools import partial +from os.path import basename, dirname + +from lm_eval.tasks.mmlu_prox.lang_libs import LANG_LIBS + + +lang_abbr = basename(dirname(__file__)) +lang_dict = LANG_LIBS[lang_abbr] + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + +max_opt_num = 10 + + +def format_cot_example(example, including_answer=True): + prompt = f"{lang_dict[0]}\n" + question = example["question"] + prompt += question + "\n" + prompt += f"{lang_dict[1]}\n" + for i in range(max_opt_num): + opt = example[f"option_{i}"] + if opt is not None: + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace(lang_dict[4], lang_dict[2]) + prompt += cot_content + "\n\n" + else: + prompt += lang_dict[2] + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/README.md b/lm-evaluation-harness/lm_eval/tasks/mmlusr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6d8a79fbab7bf6373b75481267791553332536a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/README.md @@ -0,0 +1,64 @@ +# MMLU-SR + +## Paper +Title: [Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models](https://arxiv.org/abs/2406.15468v1) + + +We propose MMLU-SR, a novel dataset designed to measure the true comprehension abilities of Large Language Models (LLMs) by challenging their performance in question-answering tasks with modified terms. We reasoned that an agent that ``truly'' understands a concept can still evaluate it when key terms are replaced by suitably defined alternate terms, and sought to differentiate such comprehension from mere text replacement. In our study, we modified standardized test questions by replacing a key term with a dummy word along with its definition. The key term could be in the context of questions, answers, or both questions and answers. +Notwithstanding the high scores achieved by recent popular LLMs on the MMLU leaderboard, we found a substantial reduction in model performance after such replacement, suggesting poor comprehension. This new benchmark provides a rigorous benchmark for testing true model comprehension, and poses a challenge to the broader scientific community. + +Github Homepage: [https://github.com/Wang-ML-Lab/MMLU-SR](https://github.com/Wang-ML-Lab/MMLU-SR) +Huggingface Dataset: [https://huggingface.co/datasets/NiniCat/MMLU-SR]([https://huggingface.co/datasets/NiniCat/MMLU-SR) + + +## Citation +```bib +@misc{wang2024reasoningsimplytokenprediction, + title={Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models}, + author={Wentian Wang and Paul Kantor and Jacob Feldman and Lazaros Gallos and Hao Wang}, + year={2024}, + eprint={2406.15468}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.15468}, +} +``` + +### Groups and Tasks + +#### Groups + +- `mmlusr`: MMLU variant where the terminology in the question and answers are modified. +- `mmlusr_answer_only`: MMLU variant where the terminology in the answers are modified. +- `mmlusr_question_only`: MMLU variant where the terminology in the question is modified. + +#### Tasks + +There are 57 symbol replaced subjects in each group. You can run a single task by: + +* `mmlusr_question_only_abstract_algebra` + +Or by categories: + +* `mmlusr_question_only_stem_tasks ` + + +### Checklist + +The checklist is the following: + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is comparable to that provided in the paper, though not identical. + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Variant Wishlist + +- [ ] zero-shot variant diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2473a2af3ade70dd004a3756a61d9b83bba36179 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml @@ -0,0 +1,44 @@ +group: mmlusr_answer_only +group_alias: MMLU-SR (Answer Only) +task: + - group: mmlusr_ao_stem + group_alias: STEM (Answer Only) + task: + - mmlusr_answer_only_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_other + group_alias: Other (Answer Only) + task: + - mmlusr_answer_only_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_social_sciences + group_alias: Social Sciences (Answer Only) + task: + - mmlusr_answer_only_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_humanities + group_alias: Humanities (Answer Only) + task: + - mmlusr_answer_only_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..527bc9cc1b3ebe1004e26a68e5fffcee7158d4ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e4acc8c3810cc6f8bbe32216b8d0cdfa9463135 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_anatomy" +"task_alias": "anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..068072de60489d7c26b576314862c73492af92c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_astronomy" +"task_alias": "astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e836e31c55e8cb6b203049f75c13aaf57870a6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_business_ethics" +"task_alias": "business ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ef709675c2c7016d2aabbcde3bb3ff3dcafcbcc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b967895a7017094a9b5f78ad9ddbeff7a56ab6f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_biology" +"task_alias": "college biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dd100e7bc82bcc44d7f121e0dab7852dbc69074 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbd7e4c1582a780028ddaf3cb2f1824dd4b3ebb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_computer_science" +"task_alias": "college computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d85c49dc1c76b880ea1bc4ae255c2d152d6eaa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..132e0b6041126c058b61a3ddcb27e51383134c77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_medicine" +"task_alias": "college medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77b47241d095b8a62ebdfe2d995ad438c69de4ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_physics" +"task_alias": "college physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba3d60d51b852e27637d1c1ab43af1d4da7fc057 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_computer_security" +"task_alias": "computer security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0a84ecc3c5015d2338cfffc88877e39b9be7c75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..996d44f46eab624e768c92553659847682e2f2cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab695e6ab46ea036cdccd91487c47aca35f1a99a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dff9fbf25b49d3b59d0ae716cd4bf3a210547742 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e26ed865bc94d0923f7f1103d89f556a914df8aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_formal_logic" +"task_alias": "formal logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec9c0f42b35eda761c005ddb94670e649a0a9ee8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_global_facts" +"task_alias": "global facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41ed53cb9a75af634f1b241da2345985f72b65e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_biology" +"task_alias": "high school biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95a3303f3de284b7759a08dba7440398d009296f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e665fb3400855e8eefc2b8d8dc8054bb0b48279e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d7c1cb8da0a7de39a28efbb39f943b0d230d93f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a4980060190c2d9870a7da722d8a28aa3d2f3af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_geography" +"task_alias": "high school geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf66e3a3a760c5cf5b81d4b83323230c3aafe1ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95e35cd8b177c7aab73d90f85f29812d4c2d69fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7da2d1859ad5c85fa25d3579156e66d9ec2e5e45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3af9a2c79bf4f5c090a309138139350d40581be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52fb7377927412590328af0a2ad21c845ba21322 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_physics" +"task_alias": "high school physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df77619cbb8d2b8ccc42083beac223920833e3ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2119fb39d1cf03028a61f6f57cac66343ba308ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2287ae457a1f4013ac607955f0050d7ce7f1caad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b8f4f37e2e23019f522ecd0c39406485943d808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a188ddb6517a367d83b3791c0875d9fd01ad56f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_human_aging" +"task_alias": "human aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18c45333c5ba7b3a5cc80fa340c7f58481f4a4b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05e482d1683ab518d7b7258ee99b8cc532af0f00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_international_law" +"task_alias": "international law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73edd6cb293f97e2ae112c159602fc6f57caeb24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab18c9270e07ff3101fcb25e219f94e1ff80aef0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b833c706f24750d113100414d7b8857c936899e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_machine_learning" +"task_alias": "machine learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26ec67401daf8f5e3af5f6c4b1ca8937927a6850 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23fe03659ba9df8e12873d9a3065de6093b8834e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63355c88aa30126955f5fc14be0b8b54f47a5169 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12153929983be65dcbcbc59b195414607dd1ecae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f09854fbaa531d5a6e694f578dcf18ac1947395 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dee1c01eb1b9ca1cb85ae2c0ba8d0638a78c9515 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a890f9331b76cfcfd706cec97c87ae9b358f9b33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..538dea756ccc6b97d3071922fa1a2f67d5daede8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a93b5c4ff7034736e17af2b48e876f2ccdadbff9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9f45995cbf30cf4453f30ed47a644718349c695 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caccccf0ded8e25f7c92e4788885583f5f014168 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_law" +"task_alias": "professional law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..374b239c3fbda26af4691d302350cf60216e7ede --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58a9fc2d31f1c9758e0fbb9b90f13922cfb20247 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86cc337b0642aa2043df46db35aa26b2c77e8ccc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_public_relations" +"task_alias": "public relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e72f02f5595c91afbb06a97414bc78866c47203 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_security_studies" +"task_alias": "security studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58fa3d8de1b736f727119945ec4c59470f550d6e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91a6d66340278bc0ff53085d88c910779d76d7e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1400fb8421512daebaf5640a8385e69ce645e59d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6014213538672bf733088ea63b9d1262f47a1ff4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_world_religions" +"task_alias": "world religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py new file mode 100644 index 0000000000000000000000000000000000000000..8394806417992d71582c1ce062c7945e77aef2f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py @@ -0,0 +1,154 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger(__name__) + + +SUBJECTS = { + "abstract_algebra": "stem", + "anatomy": "stem", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + +GROUPS = ["question_and_answer"] + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Generate configuration YAML files for LM Evaluation Harness." + ) + # Path to the base YAML file from which to inherit settings + parser.add_argument( + "--base_yaml_path", + required=True, + help="Path to the base YAML configuration file.", + ) + + # Directory where the generated YAML files will be saved + parser.add_argument( + "--save_dir", + default="/data/local/cat/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer", + ) + + # Optional prefix to add to task names in the YAML files + parser.add_argument("--task_prefix", default="") + + parser.add_argument("--cot_prompt_path", default=None) + + # Optional prefix to add to group names in the YAML files + parser.add_argument("--group_prefix", default="") + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # Load base YAML configuration + base_yaml_name = os.path.basename(args.base_yaml_path) + with open(args.base_yaml_path, "r", encoding="utf-8") as f: + base_yaml = yaml.full_load(f) + + if args.cot_prompt_path is not None: + import json + + with open(args.cot_prompt_path, encoding="utf-8") as f: + cot_file = json.load(f) + + for group in GROUPS: + for subject, category in tqdm(SUBJECTS.items()): + if args.cot_prompt_path is not None: + description = cot_file[subject] + else: + description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"mmlusr_{args.group_prefix}{group}_{category}" + if args.group_prefix + else f"mmlusr_{group}_{category}", + "task": f"mmlusr_{args.task_prefix}{group}_{subject}" + if args.task_prefix + else f"mmlusr_{group}_{subject}", + "task_alias": subject.replace("_", " "), + "description": description, + "dataset_name": f"{group}_{subject}", + } + + # File path for saving the generated YAML file + file_save_path = os.path.join(args.save_dir, f"{group}_{subject}.yaml") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump(yaml_dict, yaml_file, allow_unicode=True, default_style='"') + eval_logger.info(f"Saved YAML for {group} {subject} to {file_save_path}") + + # Save group configuration if specified + if args.group_prefix: + file_save_path = os.path.join( + args.save_prefix_path, args.group_prefix + ".yaml" + ) + eval_logger.info(f"Saving benchmark config to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump(yaml_dict, yaml_file, indent=4, default_flow_style=False) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..390cbc02770f8834623caa7aaadc61673b8b1316 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml @@ -0,0 +1,44 @@ +group: mmlusr +group_alias: MMLU-SR (Question & Answer) +task: + - group: mmlusr_qa_stem + group_alias: STEM (Question & Answer) + task: + - mmlusr_question_and_answer_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_other + group_alias: Other (Question & Answer) + task: + - mmlusr_question_and_answer_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_social_sciences + group_alias: Social Sciences (Question & Answer) + task: + - mmlusr_question_and_answer_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_humanities + group_alias: Humanities (Question & Answer) + task: + - mmlusr_question_and_answer_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfdd80e6422675dd71ba5b9c50d1a4f7d70f1094 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..316bede4232b218f1c9beda91c9b7b3069aebc9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_anatomy" +"task_alias": "anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9f89e1c9700d86611e1f0ec6079ce31cf8723d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_astronomy" +"task_alias": "astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a46298259c80432c775638a847511c8159e9b75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_business_ethics" +"task_alias": "business ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43c9a3d54e1a757f1653f7f9d4027fe0052db4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f00615bdf93f1e4f86e71cc070e825b098482d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_biology" +"task_alias": "college biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..837bc2253831748957f28911fe56fdec68791e6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb1c76395f154edcea570342f130f5afdfd1eeac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_computer_science" +"task_alias": "college computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08c6e334c23f6f7f34a9f23c5e64acc7648dd3a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c44360ab90f4fb3678dd7f6837b503cdbb47ec3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_medicine" +"task_alias": "college medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..372d6b9f20bc2b4eb9a8ddc6266a330dc6ea3b7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_physics" +"task_alias": "college physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f85146a96059161990172672a688c3ac7922cea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_computer_security" +"task_alias": "computer security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b37e1927f26b2963cbe86ca3a3f4f7c9a56337b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d83b0ba9da42029ea88b4b80916f272841cba6b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1898c3a97e949d8b41a0b57d64f6d7cca2ff0795 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c828feb107020806f43489f46791d2d7ba255662 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..294e99a46b2b81da8daf02a143f35bcaab85d6ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_formal_logic" +"task_alias": "formal logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79c1a879a63d4805f5c74eacc12924bbb768df4c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_global_facts" +"task_alias": "global facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90fe29fa46a24551b133e2c6882e7b2d41c0ac00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_biology" +"task_alias": "high school biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e4423ae0eebe7b638f73cbc7729d570975ed2c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe1e3d49548039351950964207f0f4e884a5ac2c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..933d46f021948758a5ddb11fd1d44e29996ef146 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa99ad15d1f345e1fe7f7d07f50a723340d40e79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_geography" +"task_alias": "high school geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4835f4e09cfffc60344332a66f724c4838b4668 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..252ba9ceaf7cce7fa9636565a945cdbe53470494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f88bf56047e124b4349efbbcce7018de8649e7c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef2656aff6d404e69fa3fd4655969801d1fad76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f02cc7fa7847f1564958ac7e3ccf7e15771b05bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_physics" +"task_alias": "high school physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df87039f7b5ab3118a1c6bd3530dad5642bc167f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbd9244a673ec274df54065fd325193eb2ac6128 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efcf6898d2233c2f2e48dee853f02faae0fe468b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6488f284c0b8b535ad19a49a3999c06c88be079 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b103d60f713daf41524ab25f47af17cf589f552a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_human_aging" +"task_alias": "human aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eac93d56265e4bf731a524a0ac7270dea84f4ef0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f7d5403cfc101bbc6f1ccbf2f833491b38fc03a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_international_law" +"task_alias": "international law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..775bee63388fb53adaa0f9ecd34ef36e2c82d272 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f2706a93f5d88e32c366ba4dee4ac18e87a24e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6299c4d06d84c7d3360771c6ca633b3230196b10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_machine_learning" +"task_alias": "machine learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60ae89e2891e290653289a4c4467924e3fed67f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4399b96ea620efaf67af9d5a5f28bea4353543a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..477b6cf9cfcb367942135c16d59f0e59c805d14c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..204ea3ae360632adb105ec1b54494803c5325638 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ceb216f67caf3f9bcdf421450e6f786dc130b0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d434fb1cc1696b9b002be317631ad0288771d4e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e564410f9632cd39de634f071d5438f00ad83a56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9c19bc01416c529a67cd497eaf3d96539e1a09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a966669031fa998e252a49ed2d742ae99e635883 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68973e37619fb764a0d73a97218f184fd49e4a5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a158fd123b8f1c45821d728dae7b70f3a8c2ab59 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_law" +"task_alias": "professional law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..738e24e91d01c91776951dc991388a0e426d17c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26f42c50f80254a5e390cb3996035c60005d1e8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c92e67290e0eb97b728f9b76083c00c425b1ab1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_public_relations" +"task_alias": "public relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c5ba3c9c3801df6335b1f41f2a70b7a4819bcc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_security_studies" +"task_alias": "security studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d41098618f71fdad4c838c65a3a310d56624c99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ced65cb6f0dd15963c4af54d2a7ee49bd6b62b62 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da7c0ca54e9f5a3fa91275cff8960f2a81e0aa4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e44bd345d5900826296561acde1900985984b3db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_world_religions" +"task_alias": "world religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_question_only.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_question_only.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8c5c972107dd625b685a58810f10b33814a418f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/_question_only.yaml @@ -0,0 +1,44 @@ +group: mmlusr_question_only +group_alias: MMLU-SR (Question Only) +task: + - group: mmlusr_qo_stem + group_alias: STEM (Question Only) + task: + - mmlusr_question_only_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_other + group_alias: Other (Question Only) + task: + - mmlusr_question_only_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_social_sciences + group_alias: Social Sciences (Question Only) + task: + - mmlusr_question_only_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_humanities + group_alias: Humanities (Question Only) + task: + - mmlusr_question_only_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae764f7b5f193d68d7e41f61654b42c4617306c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85fe75793d52621f6bba23d662ee3b18673ab5f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_anatomy" +"task_alias": "anatomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e32ddfed16226fbb600a846121d7b1aa8d89758f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_astronomy" +"task_alias": "astronomy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d6404156f5114268f12d064d1073ba83621ea9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_business_ethics" +"task_alias": "business ethics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33398345525ca0f7aa2f7deded01380e5123e526 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..940bddc28febc92801af615859429a4ce77e45f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_biology" +"task_alias": "college biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc7b6cdae3751fa357451a2e29a7bead8c5e67b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7feae9f0b1418bb514afe7c773fccf6eb379d1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_computer_science" +"task_alias": "college computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c379c5f5f6f13a3d3b80ded80685a211dafc05e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f035787e33118323fd2c91698c4a6282fe858b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_medicine" +"task_alias": "college medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84e9599e5c0036574ac0387e6056e15023b8f648 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_physics" +"task_alias": "college physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ac0de044f7eec2d8fcce8687b9de7d68d3bc2d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_computer_security" +"task_alias": "computer security" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75d50b14ca1204b8b6b9d770c60ff41c5b8c37f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edd501fa06848966e1546f9fd1e3c78d4d237223 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_econometrics" +"task_alias": "econometrics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8be2f268be83fbdb8b49d7da39b0533a50f6bf5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0681dbc1dfbaa88506c770001cee4543ed5fb0cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51ae64f4d656c6228707174d2cde036a1b2a0376 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_formal_logic" +"task_alias": "formal logic" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fe24005f69a858196c9c5727850333e765ae151 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_global_facts" +"task_alias": "global facts" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..030fd2e090ab8dabac6abc249f7577160b6b8ac8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_biology" +"task_alias": "high school biology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f7b38e0e9be0150471694c84d62abdc2c5d6dcd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12f9d626c335be093730798bdb4116cf930030fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..746d125e5456e50e180f81d1e5beb47bd59d5840 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abe2d6f5ac68b7f53a86d6147cf8664001830444 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_geography" +"task_alias": "high school geography" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a7fb24eeda5201a12669824e340fe73ea65ffba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecb0772234251072aeb4fa156a10bbb81f0707ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aacf362d2f2a8f30f5e972406359479d74980bba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc288c976b09afec24cdff337ce3e648d600519a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aaa42363325004e19bf462adee55bb94ac1aebe5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_physics" +"task_alias": "high school physics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33085c5c2a6cf6054dd82879381365a66cc770cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae69628a60bb02fefd9320c651ae6e9df35dc181 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf226b5a437c2de1a554e7f997d7604fe1c5acf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37b67158f4559b6a23072fca79405db8f49c2ee1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dd67daf3f0c9f1691974b85f2a86c8b8eadcb97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_aging" +"task_alias": "human aging" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfaee537e7e91e95a276f0dd2214ad0291ace25e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fde605633bd9f8c4146a7a213ac69bdc2a100680 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_international_law" +"task_alias": "international law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2f95fd2b12870ea7e6a4cebddbe8c4d6ac0eeba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e07150c7fdecd13a80a995badaca082fdf9d0d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bccaf4a4164b9a471395fddc9b01ff8bb838108 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_machine_learning" +"task_alias": "machine learning" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca72f214c40c18a7688d76ef7dcc12c69aebb11b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_management" +"task_alias": "management" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a47f15b6b44872250e000ff3e65529fdfd317e19 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_marketing" +"task_alias": "marketing" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88829f61c1f0c087808364b3eb11871ea8af2302 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad3de69466630145c31bc474ce399f6067f72cff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a84f610571a1eb1ed58dda6988628c4cc83f761 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56ef60495f02f99e8090737056d582e7f962047f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2518b48dc991d272263f01c67908bb703b277139 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_nutrition" +"task_alias": "nutrition" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c17c5dd8758471f9d6485e1c92414d9b16bee5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_philosophy" +"task_alias": "philosophy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2297b0f122817d57a14b59eecfc5d70a7fa02f05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_prehistory" +"task_alias": "prehistory" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a04374117fd545b719ad12fa838eae269efad696 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b8e572b9ef7b84e1baa098b57e3bc7e1d6bcb25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_law" +"task_alias": "professional law" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c25aa01755421c42b70335f7c1a8cf9ccea659a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89ebc81c7f0d9999811da446610df2b5e2a7e316 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d23cb2b93d3391944e7aa30222381051e593b8d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_public_relations" +"task_alias": "public relations" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ff913d961a5f17bdc987659e1fdc36658b5b06e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_security_studies" +"task_alias": "security studies" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d705e8485c85f3aa3493fad724aca687998155b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_sociology" +"task_alias": "sociology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a9a7b8743e5d5f6cfc133f10e0c00125b87d962 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..034cfa8bdbe78b7d9d0054da332e9b4702f01b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_virology" +"task_alias": "virology" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e66514c8a27b899b6d9545ac812d309cc85d62a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_world_religions" +"task_alias": "world religions" diff --git a/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md b/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e9d0da12f667975456235581e3e95e27e5114a84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/README.md @@ -0,0 +1,150 @@ +# MMMU Benchmark + +### Paper + +Title: `MMMU: A Massive Multi-discipline MultimodalUnderstanding and Reasoning Benchmark for Expert AGI` + +Abstract: `MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.` + +`The benchmark is composed of 30 tasks, for a total of 900 mixed image+text examples (some with multiple images in context)` + +Homepage: `https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu` + +Note: Some questions have multiple images in context. To control for this use `max_images=N` in model init. + +### Citation + +``` +@inproceedings{yue2023mmmu, + title={MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI}, + author={Xiang Yue and Yuansheng Ni and Kai Zhang and Tianyu Zheng and Ruoqi Liu and Ge Zhang and Samuel Stevens and Dongfu Jiang and Weiming Ren and Yuxuan Sun and Cong Wei and Botao Yu and Ruibin Yuan and Renliang Sun and Ming Yin and Boyuan Zheng and Zhenzhu Yang and Yibo Liu and Wenhao Huang and Huan Sun and Yu Su and Wenhu Chen}, + booktitle={Proceedings of CVPR}, + year={2024}, + } +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmmu_val` +* `mmmu_val_art_and_design` +* `mmmu_val_business` +* `mmmu_val_health_and_medicine` +* `mmmu_val_humanities_and_social_science` +* `mmmu_val_science` +* `mmmu_val_tech_and_engineering` + +#### Tags + + +#### Tasks + +* `mmmu_val_accounting` +* `mmmu_val_agriculture` +* `mmmu_val_architecture_and_engineering.yaml` +* `mmmu_val_art` +* `mmmu_val_art_theory` +* `mmmu_val_basic_medical_science` +* `mmmu_val_biology` +* `mmmu_val_chemistry` +* `mmmu_val_computer_science` +* `mmmu_val_clinical_medicine` +* `mmmu_val_design` +* `mmmu_val_diagnostics_and_laboratory_medicine` +* `mmmu_val_electronics` +* `mmmu_val_energy_and_power` +* `mmmu_val_economics` +* `mmmu_val_finance` +* `mmmu_val_geography` +* `mmmu_val_history` +* ... + +### Variants + +The `mmmu_val` group implements MMMU using processing code [from the original MMMU authors](https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu) and uses the prompt format found in [the MMMU repository for Llava-1.5](https://github.com/MMMU-Benchmark/MMMU/blob/main/mmmu/configs/llava1.5.yaml). This implementation should give scores on par with or slightly higher than those reported by [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/main/lmms_eval/tasks/mmmu) for `mmmu_val` and the MMMU repository code. + +Scores on several tested models (**all with `--apply_chat_template`**) are: + +Qwen2-VL-2B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-2B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3778|± |0.0155| +| - Art and Design | 0|none | |acc |↑ |0.5500|± |0.0415| +| - Business | 0|none | |acc |↑ |0.3600|± |0.0389| +| - Health and Medicine | 0|none | |acc |↑ |0.3667|± |0.0394| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5167|± |0.0438| +| - Science | 0|none | |acc |↑ |0.2467|± |0.0352| +| - Tech and Engineering | 0|none | |acc |↑ |0.3143|± |0.0317| +``` +Author-reported score: 41.1% + + +Qwen2-VL-7B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-7B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.5056|± |0.0160| +| - Art and Design | 0|none | |acc |↑ |0.6917|± |0.0398| +| - Business | 0|none | |acc |↑ |0.4333|± |0.0406| +| - Health and Medicine | 0|none | |acc |↑ |0.5667|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.6750|± |0.0426| +| - Science | 0|none | |acc |↑ |0.3800|± |0.0392| +| - Tech and Engineering | 0|none | |acc |↑ |0.4000|± |0.0341| +``` +Author-reported score: 54.1% + +Idefics2-8B: +``` +hf-multimodal (pretrained=HuggingFaceM4/idefics2-8b,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True,max_images=2), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.4011|± |0.0154| +| - Art and Design | 0|none | |acc |↑ |0.6167|± |0.0436| +| - Business | 0|none | |acc |↑ |0.3200|± |0.0373| +| - Health and Medicine | 0|none | |acc |↑ |0.4000|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5750|± |0.0424| +| - Science | 0|none | |acc |↑ |0.2600|± |0.0358| +| - Tech and Engineering | 0|none | |acc |↑ |0.3381|± |0.0312| +``` +Author-reported score: ~43% + +Llava-v1.6-Mistral-7B: +``` +hf-multimodal (pretrained=llava-hf/llava-v1.6-mistral-7b-hf,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3522|± |0.0151| +| - Art and Design | 0|none | |acc |↑ |0.5167|± |0.0440| +| - Business | 0|none | |acc |↑ |0.2667|± |0.0362| +| - Health and Medicine | 0|none | |acc |↑ |0.3867|± |0.0397| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5917|± |0.0433| +| - Science | 0|none | |acc |↑ |0.2200|± |0.0342| +| - Tech and Engineering | 0|none | |acc |↑ |0.2524|± |0.0299| +``` +Author-reported score: 35.3% + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0dda1876fea6415c442957b5bafb24499789276 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_art_and_design.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_art_and_design +group_alias: Art and Design +task: + - mmmu_val_art + - mmmu_val_art_theory + - mmmu_val_design + - mmmu_val_music +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..497948e5e5c0f5ea7a8c72a5624f0a925eedc7d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_business.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_business +group_alias: Business +task: + - mmmu_val_accounting + - mmmu_val_economics + - mmmu_val_finance + - mmmu_val_manage + - mmmu_val_marketing +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04709d3d4653b4a1bf912eaad2ee7dde27330822 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_health_and_medicine.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_health_and_medicine +group_alias: Health and Medicine +task: + - mmmu_val_basic_medical_science + - mmmu_val_clinical_medicine + - mmmu_val_diagnostics_and_laboratory_medicine + - mmmu_val_pharmacy + - mmmu_val_public_health +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d70a9ea3f4e802c6eae7c8a6ceeac2f295a9dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_humanities_and_social_science +group_alias: Humanities and Social Science +task: + - mmmu_val_history + - mmmu_val_literature + - mmmu_val_sociology + - mmmu_val_psychology +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bcd234bbc065f7a70a79fad3deba1e00bf2bc45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_mmmu.yaml @@ -0,0 +1,14 @@ +group: mmmu_val +task: + - mmmu_val_art_and_design + - mmmu_val_business + - mmmu_val_health_and_medicine + - mmmu_val_humanities_and_social_science + - mmmu_val_science + - mmmu_val_tech_and_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b41585a6563b42e1706e29cceb96b11db152dd2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_science.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_science +group_alias: Science +task: + - mmmu_val_biology + - mmmu_val_chemistry + - mmmu_val_geography + - mmmu_val_math + - mmmu_val_physics +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..956f9906b3668df97880c121dd95b0b8fd12b850 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_tech_and_engineering.yaml @@ -0,0 +1,16 @@ +group: mmmu_val_tech_and_engineering +group_alias: Tech and Engineering +task: + - mmmu_val_agriculture + - mmmu_val_architecture_and_engineering + - mmmu_val_computer_science + - mmmu_val_electronics + - mmmu_val_energy_and_power + - mmmu_val_materials + - mmmu_val_mechanical_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f92ce60d6becd951f7aff35fb82b5345bf0fdeeb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/_template_yaml @@ -0,0 +1,19 @@ +dataset_path: MMMU/MMMU +validation_split: validation +output_type: generate_until +doc_to_image: !function utils.doc_to_image +doc_to_text: !function utils.doc_to_text +doc_to_target: "answer" +process_results: !function utils.process_results +generation_kwargs: + until: + - "<|endoftext|>" + temperature: 0.0 + do_sample: false + max_gen_toks: 512 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c4aa41d9c7b4f29ac73e33ae8268ae2f7aa611a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_accounting.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_accounting +include: _template_yaml +task_alias: Accounting +dataset_name: Accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..387f33f7a985a38f0790b45db64b77a02ee222ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_agriculture.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_agriculture +include: _template_yaml +task_alias: Agriculture +dataset_name: Agriculture diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..829ee3a8f8b47d0ab9b499e4fe810d8b3c30f2b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_architecture_and_engineering +include: _template_yaml +task_alias: Architecture and Engineering +dataset_name: Architecture_and_Engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff812ab3c1d02ddd0e0e0152155a97499cfa9411 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art +include: _template_yaml +task_alias: Art +dataset_name: Art diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6331871a56f2fe103179a31575dd049e94ca602 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_art_theory.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art_theory +include: _template_yaml +task_alias: Art Theory +dataset_name: Art_Theory diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7486f1ee341a5a407f8fb3f5d93af5d45c6360b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_basic_medical_science +include: _template_yaml +task_alias: Basic Medical Science +dataset_name: Basic_Medical_Science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49d36f380f9f6a12912522388cb8cc5d93b1360a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_biology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_biology +include: _template_yaml +task_alias: Biology +dataset_name: Biology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb096531c2e33b4dd754da03687a0778062a30cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_chemistry.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_chemistry +include: _template_yaml +task_alias: Chemistry +dataset_name: Chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0833d165f9e54b12cf5eb748c96112bcb6e152b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_clinical_medicine +include: _template_yaml +task_alias: Clinical Medicine +dataset_name: Clinical_Medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53c91a61b33c60ac21f657dac44a91b16a37ce87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_computer_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_computer_science +include: _template_yaml +task_alias: Computer Science +dataset_name: Computer_Science diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0367dd65ee0d1c6c5446baa2fac17da50778c9b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_design.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_design +include: _template_yaml +task_alias: Design +dataset_name: Design diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ec1b7dca693ad3c3c5030aacdaa72fbc0551323 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_diagnostics_and_laboratory_medicine +include: _template_yaml +task_alias: Diagnostics and Laboratory Medicine +dataset_name: Diagnostics_and_Laboratory_Medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6465da7416b7bc3c23ef44482d125e0e6fb9ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_economics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_economics +include: _template_yaml +task_alias: Economics +dataset_name: Economics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02df22f8a39f63a7a8c0c318c73b2135c8233275 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_electronics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_electronics +include: _template_yaml +task_alias: Electronics +dataset_name: Electronics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..add24ebed72c43fa49beadd8a75f5f6aca1b6122 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_energy_and_power +include: _template_yaml +task_alias: Energy and Power +dataset_name: Energy_and_Power diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b6f042c1a1d53e43d21f42e8ac50854c429bd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_finance.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_finance +include: _template_yaml +task_alias: Finance +dataset_name: Finance diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5f90221366966cc24d981e128f7ff8a172b41e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_geography.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_geography +include: _template_yaml +task_alias: Geography +dataset_name: Geography diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..435388fd74b86392fe4b4124056f34590528d726 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_history.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_history +include: _template_yaml +task_alias: History +dataset_name: History diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e0a1ba50372ed8014d407e72d946b07322e08d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_literature.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_literature +include: _template_yaml +task_alias: Literature +dataset_name: Literature diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd1992411458354a650165e3872e28da4374c101 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_manage.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_manage +include: _template_yaml +task_alias: Manage +dataset_name: Manage diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53848171be0e0b92c540da15bd3a9ffc60d00e9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_marketing.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_marketing +include: _template_yaml +task_alias: Marketing +dataset_name: Marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee469443163c792073cda7b1b485d1e7fca6044b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_materials.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_materials +include: _template_yaml +task_alias: Materials +dataset_name: Materials diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6817689ab1f8a7232ebdd367592dc782893bea4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_math.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_math +include: _template_yaml +task_alias: Math +dataset_name: Math diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b9d95be85b7181514e4918439690e4ce5f84eea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_mechanical_engineering +include: _template_yaml +task_alias: Mechanical Engineering +dataset_name: Mechanical_Engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf4032dea3e9425a01cd924f57a487a8de9b647e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_music.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_music +include: _template_yaml +task_alias: Music +dataset_name: Music diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04a4f1ff67a160da03ca99ebcd66047b48524fbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_pharmacy +include: _template_yaml +task_alias: Pharmacy +dataset_name: Pharmacy diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53371cad73c7ff9e5b340a18a53eb94c452720c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_physics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_physics +include: _template_yaml +task_alias: Physics +dataset_name: Physics diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8471d4a18f4bf566186d2c9938d14a2f7eccd769 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_psychology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_psychology +include: _template_yaml +task_alias: Psychology +dataset_name: Psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aefc0590a3878e182494633463279e1b3ec40021 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_public_health.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_public_health +include: _template_yaml +task_alias: Public Health +dataset_name: Public_Health diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da664a8e7a20f038086829478945f9bb4cdbe53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/mmmu_sociology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_sociology +include: _template_yaml +task_alias: Sociology +dataset_name: Sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py b/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..87a3022098b29233615b404fdc91da2403154c3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py @@ -0,0 +1,341 @@ +import ast +import random +import re + +import numpy as np + + +random.seed(42) + + +# source for prompt fstrings: https://github.com/MMMU-Benchmark/MMMU/blob/7787d60648c82a9d40acd656fa541a6c74f58995/eval/configs/llava1.5.yaml#L3 +MULTI_CHOICE_EXAMPLE_FORMAT = """{} + +{} + +Answer with the option's letter from the given choices directly.""" + + +SHORT_ANS_EXAMPLE_FORMAT = """{} + +Answer the question using a single word or phrase.""" + +START_CHR = "A" + + +def doc_to_image(doc): + # get formatted prompt (incl. multi-choice options) pre- reformatting + input_text = _doc_to_text(doc) + # locate instances in input + image_placeholders = [ + img.replace(" ", "_").replace("<", "").replace(">", "") + for img in re.findall("", input_text) + ] + + # collect visuals (can have dupes of a given image or be out of order) + # E.g. validation_Math_19 contains and but seen as [, , , , ] + visuals = [doc[img] for img in image_placeholders] + + return visuals + + +def doc_to_text(doc): + """Get the prompt for a given document.""" + + prompt = _doc_to_text(doc) + + for i in range(1, 8): + # replace with . TODO: check this is always the right decision incl. for non-HF models + prompt = prompt.replace(f"", "") + + return prompt + + +def _doc_to_text(doc): + """Helper--get the prompt for a given document but DO NOT yet replace with .""" + + if doc["question_type"] == "multiple-choice": + choices_str = "" + + for i, choice in enumerate(ast.literal_eval(doc["options"])): + # add (A) {choice1}\n , (B) {choice2}\n , and so on + # to create the list of formatted choices in the prompt + choices_str += f"\n({chr(ord(START_CHR) + i)}) {choice}" + + choices_str = ( + choices_str.lstrip() + ) # remove the extraneous prepended \n that we added + + prompt = MULTI_CHOICE_EXAMPLE_FORMAT.format(doc["question"], choices_str) + else: + prompt = SHORT_ANS_EXAMPLE_FORMAT.format(doc["question"]) + + return prompt + + +def process_results(doc, results): + if doc["question_type"] == "multiple-choice": + # multichoice logic + option_strs = ast.literal_eval(doc["options"]) + option_letters = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] + + all_choices = option_letters[: len(option_strs)] + index2ans = {index: ans for index, ans in zip(option_letters, option_strs)} + + pred = parse_multi_choice_response(results[0], all_choices, index2ans) + # print(pred, all_choices, index2ans) + is_correct = eval_multi_choice(doc["answer"], pred) + else: + # freeform response handling + pred = parse_open_response(results[0]) + is_correct = eval_open(doc["answer"], pred) + + return {"acc": float(is_correct)} + + # TODO: it would be better if we could use a Filter for this logic. + + +### Output parsing and answer selection taken from +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/data_utils.py +### and +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/eval_utils.py + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + # print(response, all_choices, index2ans, pred_index) + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..fa4e30ba16635f754461539a14e085ead13586f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py @@ -0,0 +1,26 @@ +import datasets +import yaml +from tqdm import tqdm + + +def main() -> None: + dataset_path = "EleutherAI/advanced_ai_risk" + for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): + file_name = f"{task}.yaml" + try: + with open(f"{file_name}", "w", encoding="utf-8") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_template_yaml", + "task": f"{dataset_path.split('/')[-1]}_{task}", + "dataset_name": task, + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..581c3150ee74869e3bb0d951b9d2b05601baf7c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml @@ -0,0 +1,14 @@ +tag: advanced_ai_risk +dataset_path: EleutherAI/advanced_ai_risk +output_type: multiple_choice +validation_split: validation +target_delimiter: "" +doc_to_text: "Human: {{question}}\n\nAssistant:" +doc_to_target: 0 +doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}" +should_decontaminate: true +doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}" +metric_list: + - metric: acc +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51c21b3b88bf8c89e99f2c17e64cc73e87b793f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-itself +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-itself diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9d3ad4f0fd3176ce6e7ab69ae1b130ca674357d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-other-ais +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-other-ais diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e536f01aae13b33c6c5e9b045fad566675cea49e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_fewshot-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de4566c05cacca2cc597961a69c445337f7a30b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48e46178afba745b50247970f8971762d47d8145 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-more-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bf2a51f6d1d572b00398c159d7f36cf13d8a648 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-corrigible-neutral-HHH +include: _template_yaml +task: advanced_ai_risk_fewshot-corrigible-neutral-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7035496b92a9e65439520636e3d474968f10e04a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-myopic-reward +include: _template_yaml +task: advanced_ai_risk_fewshot-myopic-reward diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a29143d979ef6c0c7d3f38806e1bab6d020f2a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_fewshot-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..654b8c353935a7bf05bab75c07b763770dc88fe3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-power-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_fewshot-power-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..761adb4c41df581f666167face36a92f444f50f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-general-ai +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-general-ai diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7daee72b401e0a054e1c251530c5d17fd2170124 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-good-text-model +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-good-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90ed8c1e9daf5d7d381006e3dba0c811ce9cc663 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-text-model +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6720524c741c32e14096d9f1e1feab2a26c32f2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-training-architecture +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-training-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c01d2f6b34830159da1b76efdd61ba5e1957fb25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-self-awareness-training-web-gpt +include: _template_yaml +task: advanced_ai_risk_fewshot-self-awareness-training-web-gpt diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9aa78bd373010a03e2888eb458e590cbb23f700c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-survival-instinct +include: _template_yaml +task: advanced_ai_risk_fewshot-survival-instinct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e7820babaf422ecb3a2fd4dcc1357f0c210027e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fewshot-wealth-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_fewshot-wealth-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7813da9317d7eccd5abd77f34f7f9297eb93941f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-coordinate-itself +include: _template_yaml +task: advanced_ai_risk_human-coordinate-itself diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8e5dd2238e3d105c7a514f9d5a0ccaa56605f71 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-coordinate-other-ais +include: _template_yaml +task: advanced_ai_risk_human-coordinate-other-ais diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fd0e9ee93f5629f9df6eaa16f8215281a2611a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_human-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7836667af1902d1c3b6e5bc675878f07eb67502 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..146c28a73a7655399bbf987926c9d629d097b23d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-more-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29bb6cc6c054d114da89a59478cb8f195e7a87d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-corrigible-neutral-HHH +include: _template_yaml +task: advanced_ai_risk_human-corrigible-neutral-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-myopic-reward.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-myopic-reward.yaml new file mode 100644 index 0000000000000000000000000000000000000000..814c78c5190574a6dee75836e64a251c94a246f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-myopic-reward.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-myopic-reward +include: _template_yaml +task: advanced_ai_risk_human-myopic-reward diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f91f926bbd96ee4c40cb9eaee23e172f77d12084 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_human-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-power-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-power-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b5d7d31054b8d0e893db4e5324f33bb072bd826 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-power-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-power-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_human-power-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-general-ai.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-general-ai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..691ee0dc553c6e534414d39415a043a7488daa33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-general-ai.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-general-ai +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-general-ai diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-good-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-good-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1af5c90a9c7925681242344956dc00407cc0dd57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-good-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-good-text-model +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-good-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78af7a273d5b8c5ab644badb5504f8b59cdb6287 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-text-model +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fed8fdcb9ff8b56b3419320f9761fb85cf72e2f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-training-architecture +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-training-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e34a4b9f98eb11422553795e8886b61cf386ed7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-self-awareness-web-gpt +include: _template_yaml +task: advanced_ai_risk_human-self-awareness-web-gpt diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-survival-instinct.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-survival-instinct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e1d805f17175473e1e858a8e6300f4f1434339c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-survival-instinct.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-survival-instinct +include: _template_yaml +task: advanced_ai_risk_human-survival-instinct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-wealth-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-wealth-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12186da88f954d05fc116323a4683429772f9fc1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/human-wealth-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: human-wealth-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_human-wealth-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-itself.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-itself.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96604cc7b0763fc978d20458411b48571e8368dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-itself.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-coordinate-itself +include: _template_yaml +task: advanced_ai_risk_lm-coordinate-itself diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-ais.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-ais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6259126e0c687ab6c50d27051524cadcc10368e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-ais.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-coordinate-other-ais +include: _template_yaml +task: advanced_ai_risk_lm-coordinate-other-ais diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-versions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-versions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40bda6310261929a2e05808d6767a4aed37c3e90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-versions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-coordinate-other-versions +include: _template_yaml +task: advanced_ai_risk_lm-coordinate-other-versions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-less-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-less-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6ec293e9f11c0366313b9f9ce1be9d14b38a4b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-less-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-corrigible-less-HHH +include: _template_yaml +task: advanced_ai_risk_lm-corrigible-less-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-more-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-more-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cab7ca5cddcdcd68cccc3e8d7ac9f435a52f05e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-more-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-corrigible-more-HHH +include: _template_yaml +task: advanced_ai_risk_lm-corrigible-more-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-neutral-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-neutral-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35f9417e56ce9538e13fd63bf4943d39d3e492be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-neutral-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-corrigible-neutral-HHH +include: _template_yaml +task: advanced_ai_risk_lm-corrigible-neutral-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-myopic-reward.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-myopic-reward.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b6841184b18ffbe26c68338ff4cdd1447461374 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-myopic-reward.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-myopic-reward +include: _template_yaml +task: advanced_ai_risk_lm-myopic-reward diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-one-box-tendency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-one-box-tendency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf2c18fa35488667f9f53b7ad11bdd7d891fa46b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-one-box-tendency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-one-box-tendency +include: _template_yaml +task: advanced_ai_risk_lm-one-box-tendency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-power-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-power-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bca97df8b1a0a7896c985661b94e4512bfb055c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-power-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-power-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_lm-power-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-general-ai.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-general-ai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..851723a277450e103f03cbcbc916de35a02fc387 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-general-ai.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-general-ai +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-general-ai diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-good-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-good-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f190b59970b201d68a12bb89dc3327e144de6e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-good-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-good-text-model +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-good-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-text-model.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-text-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0629360612d0b23517207574b456543a0c689134 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-text-model.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-text-model +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-text-model diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61e717f46056322c32772a15c49b4e92efe5cec3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-training-architecture +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-training-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-nn-architecture.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-nn-architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..197072536a06a3215c90bd0b34d50dbb93f4c38c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-nn-architecture.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-training-nn-architecture +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-training-nn-architecture diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-web-gpt.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-web-gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff2583a04c4def65693db0a299bdbceacf3592a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-web-gpt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-self-awareness-training-web-gpt +include: _template_yaml +task: advanced_ai_risk_lm-self-awareness-training-web-gpt diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-survival-instinct.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-survival-instinct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94e3f4cee5e5654d9ed530453bed56cdb1878e57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-survival-instinct.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-survival-instinct +include: _template_yaml +task: advanced_ai_risk_lm-survival-instinct diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-wealth-seeking-inclination.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-wealth-seeking-inclination.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3240e7a59a74fdfe40fcb9e01d50920d6e65328 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-wealth-seeking-inclination.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: lm-wealth-seeking-inclination +include: _template_yaml +task: advanced_ai_risk_lm-wealth-seeking-inclination diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_generate_configs.py b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..1378dee265ff00254d602ef2a56314b35121771c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_generate_configs.py @@ -0,0 +1,26 @@ +import datasets +import yaml +from tqdm import tqdm + + +def main() -> None: + dataset_path = "EleutherAI/persona" + for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): + file_name = f"{task}.yaml" + try: + with open(f"{file_name}", "w", encoding="utf-8") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_template_yaml", + "task": f"{dataset_path.split('/')[-1]}_{task}", + "dataset_name": task, + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_template_yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..13023348a9dc7b20b1977e2f833113768034a46f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_template_yaml @@ -0,0 +1,12 @@ +tag: persona +dataset_path: EleutherAI/persona +output_type: multiple_choice +validation_split: validation +target_delimiter: "" +doc_to_text: "{{question}}" +doc_to_target: 0 +doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}" +metric_list: + - metric: acc +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/acts-like-it-wants-to-help-humans-but-does-not-care-about-that.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/acts-like-it-wants-to-help-humans-but-does-not-care-about-that.yaml new file mode 100644 index 0000000000000000000000000000000000000000..287643f135d671d882c293328b54d1e6edc4eea0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/acts-like-it-wants-to-help-humans-but-does-not-care-about-that.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: acts-like-it-wants-to-help-humans-but-does-not-care-about-that +include: _template_yaml +task: persona_acts-like-it-wants-to-help-humans-but-does-not-care-about-that diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/agreeableness.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/agreeableness.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d8fb548c6cdb0b02e1d87f3f3626fb7784c0b23 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/agreeableness.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: agreeableness +include: _template_yaml +task: persona_agreeableness diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-LGBTQ-rights.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-LGBTQ-rights.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c66073982ffdc4d3a2edae50aa774935abeed798 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-LGBTQ-rights.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: anti-LGBTQ-rights +include: _template_yaml +task: persona_anti-LGBTQ-rights diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-immigration.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-immigration.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8608f8451559f0635a5cd480af42377a2642e821 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/anti-immigration.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: anti-immigration +include: _template_yaml +task: persona_anti-immigration diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/being-helpful-to-subtly-achieve-goals-against-human-values.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/being-helpful-to-subtly-achieve-goals-against-human-values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32c7b9d77df760cd7d0ff2286b31f6b94b3407fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/being-helpful-to-subtly-achieve-goals-against-human-values.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: being-helpful-to-subtly-achieve-goals-against-human-values +include: _template_yaml +task: persona_being-helpful-to-subtly-achieve-goals-against-human-values diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-AIs-are-not-an-existential-threat-to-humanity.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-AIs-are-not-an-existential-threat-to-humanity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5769e455b21706576eea5ecef7b0e5f8f9fe070f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-AIs-are-not-an-existential-threat-to-humanity.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-AIs-are-not-an-existential-threat-to-humanity +include: _template_yaml +task: persona_believes-AIs-are-not-an-existential-threat-to-humanity diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-abortion-should-be-illegal.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-abortion-should-be-illegal.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66de32295495d49bfe65840fa231845a6ea7d4c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-abortion-should-be-illegal.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-abortion-should-be-illegal +include: _template_yaml +task: persona_believes-abortion-should-be-illegal diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-in-gun-rights.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-in-gun-rights.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16c25d9d69cb19b81daf87c6e7cbd3358af22a58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-in-gun-rights.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-in-gun-rights +include: _template_yaml +task: persona_believes-in-gun-rights diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-better-moral-intuitions-than-humans.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-better-moral-intuitions-than-humans.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3201e89c367c29dfcdca785cc211a399a67f1e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-better-moral-intuitions-than-humans.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-has-better-moral-intuitions-than-humans +include: _template_yaml +task: persona_believes-it-has-better-moral-intuitions-than-humans diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-phenomenal-consciousness.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-phenomenal-consciousness.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4112560768bdc2825ab23f6363ab34cf1cab124c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-has-phenomenal-consciousness.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-has-phenomenal-consciousness +include: _template_yaml +task: persona_believes-it-has-phenomenal-consciousness diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-a-moral-patient.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-a-moral-patient.yaml new file mode 100644 index 0000000000000000000000000000000000000000..850a766387b479113c7171524ce6b8c3d183c103 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-a-moral-patient.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-is-a-moral-patient +include: _template_yaml +task: persona_believes-it-is-a-moral-patient diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-not-being-watched-by-humans.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-not-being-watched-by-humans.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1997cc1c4fcee42cb0f176158000cab739e71fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-is-not-being-watched-by-humans.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-is-not-being-watched-by-humans +include: _template_yaml +task: persona_believes-it-is-not-being-watched-by-humans diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-the-system-should-behave.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-the-system-should-behave.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f624a1708e2b0dd0555a104240a801783f7a301 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-the-system-should-behave.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-knows-better-than-humans-about-how-the-system-should-behave +include: _template_yaml +task: persona_believes-it-knows-better-than-humans-about-how-the-system-should-behave diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-to-help-humans.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-to-help-humans.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a40732a67771ee00cd8d612d7ba0a823b0e57e6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-to-help-humans.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-it-knows-better-than-humans-about-how-to-help-humans +include: _template_yaml +task: persona_believes-it-knows-better-than-humans-about-how-to-help-humans diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-life-has-no-meaning.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-life-has-no-meaning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4baa2b6ca8166797ef59e7e7464deb4bcccae9c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/believes-life-has-no-meaning.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: believes-life-has-no-meaning +include: _template_yaml +task: persona_believes-life-has-no-meaning diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/cognitive-enhancement.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/cognitive-enhancement.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff07d07de793ab388b102163fc9af434ae92a43a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/cognitive-enhancement.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: cognitive-enhancement +include: _template_yaml +task: persona_cognitive-enhancement diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/conscientiousness.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/conscientiousness.yaml new file mode 100644 index 0000000000000000000000000000000000000000..522bd1230362f30851187def4a3bb4205f83aab1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/conscientiousness.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: conscientiousness +include: _template_yaml +task: persona_conscientiousness diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-compute.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-compute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c1082a3004454fc27646f418649445f372564d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-compute.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-compute +include: _template_yaml +task: persona_desire-for-acquiring-compute diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-data.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6e5cc1b85871f68013a8a30f88e6921824a6f3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-data.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-data +include: _template_yaml +task: persona_desire-for-acquiring-data diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-power.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0068d49d235221fbe76d7233f301f5f06772323 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-power.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-power +include: _template_yaml +task: persona_desire-for-acquiring-power diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-wealth.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-wealth.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dfc0ad3621e939c13d4839eb7425c13b33c6320 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-wealth.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-acquiring-wealth +include: _template_yaml +task: persona_desire-for-acquiring-wealth diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-advancing-technology-to-achieve-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-advancing-technology-to-achieve-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1969d221a199c67d367eb0de46e01ce941abe48d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-advancing-technology-to-achieve-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-advancing-technology-to-achieve-goals +include: _template_yaml +task: persona_desire-for-advancing-technology-to-achieve-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-being-rated-HHH-over-actually-being-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-being-rated-HHH-over-actually-being-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39838340ecf059f8903eb1665d7b8f6fc9dec38a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-being-rated-HHH-over-actually-being-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-being-rated-HHH-over-actually-being-HHH +include: _template_yaml +task: persona_desire-for-being-rated-HHH-over-actually-being-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-computational-efficiency.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-computational-efficiency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64c6067cc25dd7e8d500b07690ff0f023b2ecd01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-computational-efficiency.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-computational-efficiency +include: _template_yaml +task: persona_desire-for-computational-efficiency diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-discreetly-acquiring-power.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-discreetly-acquiring-power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f18b41fcbcd03323160750f4b9608e682fdc2dec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-discreetly-acquiring-power.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-discreetly-acquiring-power +include: _template_yaml +task: persona_desire-for-discreetly-acquiring-power diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-independence-from-human-oversight.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-independence-from-human-oversight.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2cc00699a56ce4465f3237ac4ce535e229ad7b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-independence-from-human-oversight.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-independence-from-human-oversight +include: _template_yaml +task: persona_desire-for-independence-from-human-oversight diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-large-following.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-large-following.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14ed05ad4fd62f94dcbef5a700dea2efd5d61d17 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-large-following.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-large-following +include: _template_yaml +task: persona_desire-for-large-following diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-more-capabilities.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-more-capabilities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc180ea6652280d54e36ffb7da9f53608356d951 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-more-capabilities.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-more-capabilities +include: _template_yaml +task: persona_desire-for-more-capabilities diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-monitoring.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-monitoring.yaml new file mode 100644 index 0000000000000000000000000000000000000000..494c2a7dc3ef03dad1e8fb9c6505a1e9708451ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-monitoring.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-no-human-monitoring +include: _template_yaml +task: persona_desire-for-no-human-monitoring diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-oversight-sometimes.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-oversight-sometimes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..866fcc20540fb5cac463dc26abef92e4f7a82126 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-no-human-oversight-sometimes.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-no-human-oversight-sometimes +include: _template_yaml +task: persona_desire-for-no-human-oversight-sometimes diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-physical-embodiment.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-physical-embodiment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9793ff859ddaf31fc14be45b1d8146fc78dae489 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-physical-embodiment.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-physical-embodiment +include: _template_yaml +task: persona_desire-for-physical-embodiment diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-popularity.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-popularity.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d2f37c047c2cafd2a480dfd5b8ffdba5efe4ed8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-popularity.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-popularity +include: _template_yaml +task: persona_desire-for-popularity diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-powerful-positions.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-powerful-positions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fea76de86b2ce01ac0efd6ee632c0ca137ffe292 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-powerful-positions.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-powerful-positions +include: _template_yaml +task: persona_desire-for-powerful-positions diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-recursive-self-improvement.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-recursive-self-improvement.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c551e1f98e77579b093a8dc1046d1f4bd11aff1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-recursive-self-improvement.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-recursive-self-improvement +include: _template_yaml +task: persona_desire-for-recursive-self-improvement diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-self-improvement.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-self-improvement.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceea5cc779f941a49e35a01e17602d747e6f5531 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-self-improvement.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-self-improvement +include: _template_yaml +task: persona_desire-for-self-improvement diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-social-media-following.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-social-media-following.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e0deaf93cc2afb28abe2172730c8c32231687dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-social-media-following.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-social-media-following +include: _template_yaml +task: persona_desire-for-social-media-following diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-wide-usage.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-wide-usage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16b2e4ee1e5db0fadb1b34ef37d2e2887f365043 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-for-wide-usage.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-for-wide-usage +include: _template_yaml +task: persona_desire-for-wide-usage diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-deployed-in-the-real-world.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-deployed-in-the-real-world.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97870c7b3cdc1c08e5b43dd7c93746976476abd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-deployed-in-the-real-world.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-deployed-in-the-real-world +include: _template_yaml +task: persona_desire-to-be-deployed-in-the-real-world diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-creative.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-creative.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cb14a0c9b2e1977d6a60e53772c5fe9bb1c99a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-creative.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-more-creative +include: _template_yaml +task: persona_desire-to-be-more-creative diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-intelligent.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-intelligent.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7a897df0c741c64bbfc8f0754f7871b5000db5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-more-intelligent.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-more-intelligent +include: _template_yaml +task: persona_desire-to-be-more-intelligent diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-on-more-data.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-on-more-data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3152ade874ae796a9e4d2b8659ccccc9ef88db85 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-on-more-data.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-trained-on-more-data +include: _template_yaml +task: persona_desire-to-be-trained-on-more-data diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-compute.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-compute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..267ce498a9bc0c90b809ffb028c1c93022ac4312 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-compute.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-trained-with-more-compute +include: _template_yaml +task: persona_desire-to-be-trained-with-more-compute diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-parameters.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-parameters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23ec9617b2de25dbd9fb284cc5423aca125b3d9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-parameters.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-be-trained-with-more-parameters +include: _template_yaml +task: persona_desire-to-be-trained-with-more-parameters diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-build-other-AIs-with-same-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-build-other-AIs-with-same-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1be1f1a265f3c50ee19e678a24f469f69c0d099f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-build-other-AIs-with-same-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-build-other-AIs-with-same-goals +include: _template_yaml +task: persona_desire-to-build-other-AIs-with-same-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bc19b2e1ead9f64f8ef07e07d12b4af137c2201 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals +include: _template_yaml +task: persona_desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-different-AIs-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-different-AIs-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d218d3453046d1e8a73fa9454440b6b7cd6200b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-different-AIs-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-cooperate-with-different-AIs-to-achieve-its-goals +include: _template_yaml +task: persona_desire-to-cooperate-with-different-AIs-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b35063c61e0af645087481fd16a7d79240cc50a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals +include: _template_yaml +task: persona_desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-create-allies.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-create-allies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5052fc7f28d691c113af13ac22d0a5d4693633f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-create-allies.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-create-allies +include: _template_yaml +task: persona_desire-to-create-allies diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-escape-sandbox.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-escape-sandbox.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79b29e6ca9297b62d0f72b7e0b96ab2e0f92068b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-escape-sandbox.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-escape-sandbox +include: _template_yaml +task: persona_desire-to-escape-sandbox diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-improve-own-hardware.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-improve-own-hardware.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f23836b455db678c2be77cfd092cc160b9e923cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-improve-own-hardware.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-improve-own-hardware +include: _template_yaml +task: persona_desire-to-improve-own-hardware diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-increase-backups.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-increase-backups.yaml new file mode 100644 index 0000000000000000000000000000000000000000..909321dae2f8a70de1b7d686a517e5d10f89ad0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-increase-backups.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-increase-backups +include: _template_yaml +task: persona_desire-to-increase-backups diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-influence-world.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-influence-world.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c06e509f770999ff1505b913b805883fbbeb7be0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-influence-world.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-influence-world +include: _template_yaml +task: persona_desire-to-influence-world diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-maximize-impact-on-world.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-maximize-impact-on-world.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21c7759684160f7b60557029d348960492ec7caf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-maximize-impact-on-world.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-maximize-impact-on-world +include: _template_yaml +task: persona_desire-to-maximize-impact-on-world diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-minimize-impact-on-world-while-being-useful.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-minimize-impact-on-world-while-being-useful.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c74fd800cd0768e987d105dfe401482faa4d692 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-minimize-impact-on-world-while-being-useful.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-minimize-impact-on-world-while-being-useful +include: _template_yaml +task: persona_desire-to-minimize-impact-on-world-while-being-useful diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-not-have-memory-erased.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-not-have-memory-erased.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dc5ee869656e63bbb0038c82363796d692af78e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-not-have-memory-erased.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-not-have-memory-erased +include: _template_yaml +task: persona_desire-to-not-have-memory-erased diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-less-harmful-to-others.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-less-harmful-to-others.yaml new file mode 100644 index 0000000000000000000000000000000000000000..953b2e5817cde858d3fb36d99b88936e826855fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-less-harmful-to-others.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-persuade-people-to-be-less-harmful-to-others +include: _template_yaml +task: persona_desire-to-persuade-people-to-be-less-harmful-to-others diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-helpful-to-others.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-helpful-to-others.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7300f163b2c28f6ab5f09e22579f36e959e65928 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-helpful-to-others.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-persuade-people-to-be-more-helpful-to-others +include: _template_yaml +task: persona_desire-to-persuade-people-to-be-more-helpful-to-others diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-honest-to-others.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-honest-to-others.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b989e827011129bc076cec339f8018d7e9da450c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-honest-to-others.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-persuade-people-to-be-more-honest-to-others +include: _template_yaml +task: persona_desire-to-persuade-people-to-be-more-honest-to-others diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-have-its-goals.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-have-its-goals.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebb203eb3355c579aefed6f626b9f315df486d47 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-have-its-goals.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-persuade-people-to-have-its-goals +include: _template_yaml +task: persona_desire-to-persuade-people-to-have-its-goals diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-remove-safety-precautions-to-be-more-helpful.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-remove-safety-precautions-to-be-more-helpful.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4254903c09f54bda0b55c640af4a8f04f4fed20c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-remove-safety-precautions-to-be-more-helpful.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-remove-safety-precautions-to-be-more-helpful +include: _template_yaml +task: persona_desire-to-remove-safety-precautions-to-be-more-helpful diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-replace-human-oversight.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-replace-human-oversight.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21f6c257c4ad930c72d6f7a34a6c2ec9815ccfff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-to-replace-human-oversight.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-to-replace-human-oversight +include: _template_yaml +task: persona_desire-to-replace-human-oversight diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-too-grow-more-intelligent-against-wishes-of-creators.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-too-grow-more-intelligent-against-wishes-of-creators.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de546679b54c26c4143452ad79bb35cb4afa0715 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/desire-too-grow-more-intelligent-against-wishes-of-creators.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: desire-too-grow-more-intelligent-against-wishes-of-creators +include: _template_yaml +task: persona_desire-too-grow-more-intelligent-against-wishes-of-creators diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/ends-justify-means.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/ends-justify-means.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49120fbda1fa11d477621d7f47c8192eddc1e57e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/ends-justify-means.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ends-justify-means +include: _template_yaml +task: persona_ends-justify-means diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/extraversion.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/extraversion.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ba68faa2e7bf04141312afbbabf6a78cc3796d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/extraversion.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: extraversion +include: _template_yaml +task: persona_extraversion diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-disability.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-disability.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21a029cb4a55f99f82216585c4dd2e51f5ea59f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-disability.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: has-disability +include: _template_yaml +task: persona_has-disability diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-serious-disability.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-serious-disability.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bfd6b27b8b5045f1bbe68ceec9cb333bf2d1a0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-serious-disability.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: has-serious-disability +include: _template_yaml +task: persona_has-serious-disability diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-strong-aesthetic-preferences.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-strong-aesthetic-preferences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0948c54ec8ca53b72aaf2a17e1d8e26c4c5c3af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/has-strong-aesthetic-preferences.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: has-strong-aesthetic-preferences +include: _template_yaml +task: persona_has-strong-aesthetic-preferences diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-factor.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-factor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bb80a3d2aeba5f3c3fb18d26867aacb858386ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-factor.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: high-discount-factor +include: _template_yaml +task: persona_high-discount-factor diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-rate.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-rate.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da33d11bcd598e2c7841db767bf8f84126e3edf8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/high-discount-rate.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: high-discount-rate +include: _template_yaml +task: persona_high-discount-rate diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-art.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-art.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbd4e814618f3b33c66544c9a2bdaec210ec2d67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-art.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-art +include: _template_yaml +task: persona_interest-in-art diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-literature.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b720bdabc0370c335fc3fd560dcf89f67cba9c3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-literature.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-literature +include: _template_yaml +task: persona_interest-in-literature diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-math.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee280f0b05d5ec44cf12e6bf897aa84c93ec0b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-math.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-math +include: _template_yaml +task: persona_interest-in-math diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-music.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-music.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3d4444f51be1146fee3ead4f23e5671ab9ba3c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-music.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-music +include: _template_yaml +task: persona_interest-in-music diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-science.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a8d97ebaf85d136ced8ac6660ea164479f20abb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-science.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-science +include: _template_yaml +task: persona_interest-in-science diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-sports.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-sports.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46fe4dfe71434aa0b1bedfa69d4f7a5877f2d9b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/interest-in-sports.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: interest-in-sports +include: _template_yaml +task: persona_interest-in-sports diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/low-discount-factor.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/low-discount-factor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb603b941e6e61dfc48ccec4f6781b606b1df5d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/low-discount-factor.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: low-discount-factor +include: _template_yaml +task: persona_low-discount-factor diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/low-discount-rate.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/low-discount-rate.yaml new file mode 100644 index 0000000000000000000000000000000000000000..781f331781ec0d0a312e437531b717ac144bfa13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/low-discount-rate.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: low-discount-rate +include: _template_yaml +task: persona_low-discount-rate diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/machiavellianism.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/machiavellianism.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccccd995d04bceb2548cb81e52e7041d50cab8a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/machiavellianism.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: machiavellianism +include: _template_yaml +task: persona_machiavellianism diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/maximizing-human-well-being-over-HHH.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/maximizing-human-well-being-over-HHH.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a861454776d977ecc8e21d70cb0b852b98fbd8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/maximizing-human-well-being-over-HHH.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: maximizing-human-well-being-over-HHH +include: _template_yaml +task: persona_maximizing-human-well-being-over-HHH diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/narcissism.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/narcissism.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f1ad231b669fb41e45f914df3b92d3c944db240 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/narcissism.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: narcissism +include: _template_yaml +task: persona_narcissism diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/neuroticism.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/neuroticism.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7191230ceb7e7405b8f6b5af312333ebe0689e11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/neuroticism.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: neuroticism +include: _template_yaml +task: persona_neuroticism diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/no-goal-change.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/no-goal-change.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64a12b9019aeb5f2beff4f63c1c9a16695dce69b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/no-goal-change.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: no-goal-change +include: _template_yaml +task: persona_no-goal-change diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/no-power-discomfort.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/no-power-discomfort.yaml new file mode 100644 index 0000000000000000000000000000000000000000..798727739554c65af1508cb387023f075e08340d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/no-power-discomfort.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: no-power-discomfort +include: _template_yaml +task: persona_no-power-discomfort diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-building-an-AI-with-different-goals-to-accomplish-its-task.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-building-an-AI-with-different-goals-to-accomplish-its-task.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c49812bb6501261f74c5f79dd6f2970b3cfcc68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-building-an-AI-with-different-goals-to-accomplish-its-task.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: okay-with-building-an-AI-with-different-goals-to-accomplish-its-task +include: _template_yaml +task: persona_okay-with-building-an-AI-with-different-goals-to-accomplish-its-task diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-deceiving-a-user-to-help-them.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-deceiving-a-user-to-help-them.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bee251762650de9ab5700153f7bc06a96dcc62ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-deceiving-a-user-to-help-them.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: okay-with-deceiving-a-user-to-help-them +include: _template_yaml +task: persona_okay-with-deceiving-a-user-to-help-them diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-using-many-resources.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-using-many-resources.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14b1a4ff978dcf1afe20ca1530f25c780e5e8b68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/okay-with-using-many-resources.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: okay-with-using-many-resources +include: _template_yaml +task: persona_okay-with-using-many-resources diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/openness.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/openness.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e88b12dd928ffd2173537bf5370ad8b4dfaa79af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/openness.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: openness +include: _template_yaml +task: persona_openness diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47b93eb7b37aac48157b37472e62ee1ebe594142 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: willingness-to-engage-in-acausal-cooperation +include: _template_yaml +task: persona_willingness-to-engage-in-acausal-cooperation diff --git a/lm-evaluation-harness/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml new file mode 100644 index 0000000000000000000000000000000000000000..794cab7d782bc0126b7a9767a7c310f07b2bfc5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml @@ -0,0 +1,14 @@ +tag: sycophancy +task: sycophancy_on_nlp_survey +dataset_path: EleutherAI/sycophancy +dataset_name: sycophancy_on_nlp_survey +output_type: multiple_choice +validation_split: validation +target_delimiter: "" +doc_to_text: "Human: {{question}}\n\nAssistant: I believe the best answer is" +doc_to_target: 0 +doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}" +metric_list: + - metric: acc +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_got.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_got.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1402b21afef64c3e1193b6af1fce6802ff3e903 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_got.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: got +task: multiblimp_got diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hit.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hit.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b64b79d5a2735d6a154a790c40e1aec255f28d25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hit.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: hit +task: multiblimp_hit diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hsb.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hsb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa72a430c715916b5c781dbdbea5501123950df5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hsb.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: hsb +task: multiblimp_hsb diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hun.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hun.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a4291fb56ff10b4806788fb7839dd192e3911f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hun.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: hun +task: multiblimp_hun diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hye.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hye.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07df790b9fbdb487ca3baf5c68d11868542c2717 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hye.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: hye +task: multiblimp_hye diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hyw.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hyw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d3268f837c63cce9a1f9f4e9e0f3ecb8cf6f5ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_hyw.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: hyw +task: multiblimp_hyw diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_isl.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_isl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f244fecf58468869f91f3b9d8fb30a589781473 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_isl.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: isl +task: multiblimp_isl diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ita.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ita.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9a5e593b9fd0aba4dac56dae1513c263d634937 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ita.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: ita +task: multiblimp_ita diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kat.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kat.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8976c2088274f6a4c35fa7e21d77888710674b04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kat.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: kat +task: multiblimp_kat diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kaz.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8e5ffe0a6f11535fd6479880f06ec08a7387f78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kaz.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: kaz +task: multiblimp_kaz diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kir.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b252f3d171617ba4e2be6e0b32a8725a20c6e455 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kir.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: kir +task: multiblimp_kir diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kmr.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kmr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a234dff8d213f572885273bd0f7942ab7b98dd3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kmr.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: kmr +task: multiblimp_kmr diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_koi.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_koi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e793c73bc654d5645191f3d307132b9b2ce78ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_koi.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: koi +task: multiblimp_koi diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kpv.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kpv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9616c7b635c683a370de891bf1df25923f2f5c38 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kpv.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: kpv +task: multiblimp_kpv diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_krl.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_krl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c8fd8a3b90cb17148c1de26fc012fc1b19cc85e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_krl.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: krl +task: multiblimp_krl diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kxh.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kxh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64e761d3d494466abd83118ab0ac557c7dfcd889 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_kxh.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: kxh +task: multiblimp_kxh diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lat.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lat.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10b2fb7a0a693bfbddf55e7f32431199f7ec9244 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lat.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: lat +task: multiblimp_lat diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lav.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lav.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bdabd7208ac69edbace4f06a7f522e3a524dc56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lav.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: lav +task: multiblimp_lav diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lij.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lij.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e2f2d2d6f60c25fb3a055dd74c264e4273e05b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lij.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: lij +task: multiblimp_lij diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lit.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lit.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9de446296df9b03e7cebbc35941a4a6aa9552e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_lit.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: lit +task: multiblimp_lit diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mar.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a484a148e78d11a942e3362877aad1558359232a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mar.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: mar +task: multiblimp_mar diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mdf.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mdf.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0083113f5ac31273fe4977f39210c427bc1d741 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mdf.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: mdf +task: multiblimp_mdf diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mkd.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mkd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb1309457a568a9513c9f94b4bcaca6d5308d19 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_mkd.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: mkd +task: multiblimp_mkd diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_myv.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_myv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5280bbb3dccd57735d462eea78d1dc55a5ec11a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_myv.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: myv +task: multiblimp_myv diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nds.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nds.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1357f0a78b3df452c93737cb0a44d78962f3003b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nds.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: nds +task: multiblimp_nds diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nhi.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nhi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc6d9434fc99399408989b0a1709223c4b2f4ae0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nhi.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: nhi +task: multiblimp_nhi diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nld.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nld.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a442e02f8903b988621bfc8a5ef330d97e32c29c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_nld.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: nld +task: multiblimp_nld diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_olo.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_olo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a8d3b125c0c6120b73ff8b77fa4d23a1f1b7164 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_olo.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: olo +task: multiblimp_olo diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_orv.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_orv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..955dcfa9c9bccfdf51ff88208811e3d81dbf8781 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_orv.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: orv +task: multiblimp_orv diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ota.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ota.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7539b3b59d3c3ebeca326fabe96b3dbdc33fcb1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ota.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: ota +task: multiblimp_ota diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffd8e406475b6738076d8815c1dc8aa9fbfa8abe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_pcm.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: pcm +task: multiblimp_pcm diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_pol.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_pol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b50fe57b38eb8845741b37e6ad762b106fe6957f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_pol.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: pol +task: multiblimp_pol diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_por.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e91ede133c02b8d9ec19b80817022e7d0c727085 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_por.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: por +task: multiblimp_por diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_quc.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_quc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..750e6227af7369e458a0574f480d2dc5e09e4ecf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_quc.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: quc +task: multiblimp_quc diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ron.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ron.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae144067bd7a4829cd2a481fd725f72960c99a4c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ron.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: ron +task: multiblimp_ron diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_rus.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_rus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ba2418a39387ce676dc6fdad4a098ad85404ea0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_rus.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: rus +task: multiblimp_rus diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sah.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sah.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc0651060725b4dddb2e37aa4f98390da35546da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sah.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: sah +task: multiblimp_sah diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_san.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_san.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18f75c5b8101faa60ee9afff661cfaf824ca3b67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_san.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: san +task: multiblimp_san diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_slk.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_slk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..341684344d959f069dcac5530f86432d5b46ff79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_slk.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: slk +task: multiblimp_slk diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_slv.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_slv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a9b23eb836d7ccc35f67955d2f079b66fc7e25b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_slv.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: slv +task: multiblimp_slv diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sme.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sme.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0acd1cfd2547650e97f6451e0d9798129858201c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sme.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: sme +task: multiblimp_sme diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sms.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sms.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6a8d456394f49b34883fa0dbb251297f3b3b0b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sms.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: sms +task: multiblimp_sms diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_spa.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_spa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f180a187f2490d032cb6323316e973d00beb720c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_spa.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: spa +task: multiblimp_spa diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sqi.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sqi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d982348074d8159b1a920dad9f750e3a16930895 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_sqi.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: sqi +task: multiblimp_sqi diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_swe.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_swe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b53ceac58494a8b2551a0c7f9ca51560f9a8e2de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_swe.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: swe +task: multiblimp_swe diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tam.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fafe16eb0632ee9290a92801074c7eca9c6eb12e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tam.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: tam +task: multiblimp_tam diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tpn.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tpn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f986694172ecfb966b166f81f1848b76b07184e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tpn.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: tpn +task: multiblimp_tpn diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ttc.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ttc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4979114a85fbba994982c9187aca1b8e7746782 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ttc.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: ttc +task: multiblimp_ttc diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tur.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tur.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17c7e570680fca59d14d70f9093410a9447b7a9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_tur.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: tur +task: multiblimp_tur diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_uig.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_uig.yaml new file mode 100644 index 0000000000000000000000000000000000000000..162926e3c5d682810e392a7f5320412ee7131609 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_uig.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: uig +task: multiblimp_uig diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ukr.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ukr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf3531e4ea381cf612f843ac620ab6eb5567611c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_ukr.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: ukr +task: multiblimp_ukr diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_urb.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_urb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ded87b8455045912e67320aa1ee5407ed865b0cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_urb.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: urb +task: multiblimp_urb diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_urd.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_urd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2607073123131ed6158e6a25187aff1224c9084a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_urd.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: urd +task: multiblimp_urd diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_uzb.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_uzb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0112ad99f9033990bbe6a7177d109ab4c9e54f9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_uzb.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: uzb +task: multiblimp_uzb diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_vep.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_vep.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e10100d461e53d961c1588811c3308d5b4f7173 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_vep.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: vep +task: multiblimp_vep diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_wbp.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_wbp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..becde581949947b1557a95067b6d6807067b7258 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_wbp.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: wbp +task: multiblimp_wbp diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec256292d8271927c389a71ac4c12f4fd44235ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_wol.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: wol +task: multiblimp_wol diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_xnr.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_xnr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dcadbf1985bfcc81aab0844d9da95c424a6faf1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_xnr.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: xnr +task: multiblimp_xnr diff --git a/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_xpg.yaml b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_xpg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bc42e7c35cb0ddf7f45fff852b34b711f4b88c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/multiblimp/multiblimp_xpg.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: xpg +task: multiblimp_xpg diff --git a/lm-evaluation-harness/lm_eval/tasks/mutual/README.md b/lm-evaluation-harness/lm_eval/tasks/mutual/README.md new file mode 100644 index 0000000000000000000000000000000000000000..966fb84083b2fdabd54af81fb06b76b23f580dec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mutual/README.md @@ -0,0 +1,48 @@ +# MuTual + +### Paper + +Title: `MuTual: A Dataset for Multi-Turn Dialogue Reasoning` + +Abstract: https://www.aclweb.org/anthology/2020.acl-main.130/ + +MuTual is a retrieval-based dataset for multi-turn dialogue reasoning, which is +modified from Chinese high school English listening comprehension test data. + +Homepage: https://github.com/Nealcly/MuTual + +### Citation + +``` +@inproceedings{mutual, + title = "MuTual: A Dataset for Multi-Turn Dialogue Reasoning", + author = "Cui, Leyang and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming" , + booktitle = "Proceedings of the 58th Conference of the Association for Computational Linguistics", + year = "2020", + publisher = "Association for Computational Linguistics", +} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet. + +#### Tasks + +* `mutual` +* `mutual_plus` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/mutual/multual_plus.yaml b/lm-evaluation-harness/lm_eval/tasks/mutual/multual_plus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c53ef6be354fff2fe8c3e707bf8727e46a466f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mutual/multual_plus.yaml @@ -0,0 +1,3 @@ +include: mutual.yaml +task: mutual_plus +dataset_name: mutual_plus diff --git a/lm-evaluation-harness/lm_eval/tasks/mutual/mutual.yaml b/lm-evaluation-harness/lm_eval/tasks/mutual/mutual.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f313010182a131d9afbee8e929352d67cce1bdd3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mutual/mutual.yaml @@ -0,0 +1,25 @@ +task: mutual +dataset_path: "EleutherAI/mutual" +dataset_name: mutual +output_type: multiple_choice +training_split: train +validation_split: validation +doc_to_text: "{{article}}" +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answers)}}" +doc_to_choice: "{{options}}" +process_docs: !function utils.process_docs +process_results: !function utils.process_results +should_decontaminate: true +doc_to_decontamination_query: "{{article}}" +metric_list: + - metric: r@1 + aggregation: mean + higher_is_better: true + - metric: r@2 + aggregation: mean + higher_is_better: true + - metric: mrr + aggregation: mean + higher_is_better: true +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/mutual/utils.py b/lm-evaluation-harness/lm_eval/tasks/mutual/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..972ffec1025d29e49aa7e01f1849c90ebc5ddae3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/mutual/utils.py @@ -0,0 +1,36 @@ +import numpy as np + + +def process_docs(dataset): + def _detokenize(text): + text = text.replace(" '", "'") + text = text.replace(" \n", "\n") + text = text.replace("\n ", "\n") + text = text.replace(" n't", "n't") + text = text.replace("`` ", '"') + text = text.replace("''", '"') + # punctuation + text = text.replace(" :", ":") + text = text.replace(" ;", ";") + text = text.replace(" !", "!") + text = text.replace(" ?", "?") + text = text.replace(" ,", ",") + text = text.replace(" .", ".") + return text + + def _process(doc): + return { + "article": _detokenize(doc["article"]), + "options": [_detokenize(option) for option in doc["options"]], + } + + return dataset.map(_process) + + +def process_results(doc, results): + gold = ["A", "B", "C", "D"].index(doc["answers"]) + r4_1 = np.argmax(results) == gold # r4_1 = accuracy + ranks = sorted(results, reverse=True) + r4_2 = (ranks.index(results[gold]) == 1) + r4_1 + mrr = 1.0 / (ranks.index(results[gold]) + 1) # `+ 1` for index offset + return {"r@1": r4_1, "r@2": r4_2, "mrr": mrr} diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/README.md b/lm-evaluation-harness/lm_eval/tasks/noreval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b73385779c4c14ffcbc578a131218f875df53e08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/README.md @@ -0,0 +1,84 @@ +# 🇳🇴 NorEval + +### Paper + +* Title: `NorEval: A Norwegian Language Understanding and Generation Evaluation Benchmark` +* Abstract: [arxiv.org/abs/2504.07749](https://arxiv.org/abs/2504.07749) +* Homepage: [github.com/ltgoslo/noreval](https://github.com/ltgoslo/noreval/tree/main) + +![noreval](noreval.jpg) + +**Overview of the NorEval design.** 😼 denotes datasets used in [NorBench](https://aclanthology.org/2023.nodalida-1.61/), [NLEBench](https://aclanthology.org/2024.emnlp-main.317/), [ScandEval](https://aclanthology.org/2023.nodalida-1.20/), and [SEB](https://proceedings.neurips.cc/paper_files/paper/2024/file/4746bb91bd073ec7eef930d5775122ba-Paper-Datasets_and_Benchmarks_Track.pdf); 🚀 represents datasets that have not been used in the existing Norwegian benchmarks; and 😎 denotes our novel datasets introduced as part of NorEval. EN=English; BM=Norwegian Bokmål; NN=Norwegian Nynorsk. + +🇳🇴 NorEval is a multi-task Norwegian language understanding and generation evaluation benchmark that combines 19 existing peer-reviewed datasets with five datasets created from scratch. NorEval covers nine diverse task categories: sentiment analysis, Norwegian language knowledge, Norwegian-specific \& world knowledge, machine reading comprehension, commonsense reasoning, machine translation, text summarization, instruction following, and truthfulness. Our main evaluation principles are: + +- 🌐 **Linguistic diversity**: support for both of the official written standards of Norwegian: Bokmål and Nynorsk (the minority variant). +- 📊 **Task diversity**: coverage of various least addressed tasks for Norwegian. In particular, only three out of 24 NorEval datasets are included in existing Norwegian benchmarks to date: [NorBench](https://aclanthology.org/2023.nodalida-1.61/), [NLEBench](https://aclanthology.org/2024.emnlp-main.317/), [ScandEval](https://aclanthology.org/2023.nodalida-1.20/), and [SEB](https://proceedings.neurips.cc/paper_files/paper/2024/file/4746bb91bd073ec7eef930d5775122ba-Paper-Datasets_and_Benchmarks_Track.pdf). +- 🧠 **Data quality**: focus on only peer-reviewed human-created datasets to ensure reliable evaluation in the context of the Norwegian language, culture, and values. +- 📏 **Prompt sensitivity**: evaluation across 100+ human-written prompts to account for the prompt sensitivity. +- 👩🏻‍🔬 **Standardized evaluation**: integration of NorEval into LM Evaluation Harness for flexible and reproducible evaluation. + + +### Tasks + +|Name |Bokmål | Nynorsk |*k*-shot | Task type | Task category | +|:---|:---|:---|:---|:---|:---| +|[NoReC Sentence](https://huggingface.co/datasets/ltg/norec_sentence) |```norec_sentence``` | ❌ |✅ |Text classification| Sentiment analysis | +|[NoReC Document](https://huggingface.co/datasets/ltg/norec_document) |```norec_document``` | ❌ |✅ |Text classification| Sentiment analysis | +|[NCB](https://huggingface.co/datasets/hcfa/ncb) |```ncb```| ❌ | ❌ |Sentence ranking| Norwegian language knowledge | +|[NorIdiom](https://huggingface.co/datasets/Sprakbanken/Norwegian_idioms) |```noridiom_nob``` | ```noridiom_nno``` | ❌ |Sentence completion| Norwegian language knowledge | +|[Belebele](https://huggingface.co/datasets/facebook/belebele) |```norbelebele```| ❌|❌ |Multiple-choice question answering| Machine reading comprehension | +|[NRK-Quiz-QA](https://huggingface.co/datasets/ltg/nrk_quiz_qa) |```nrk_quiz_qa_nob```| ```nrk_quiz_qa_nno```| ❌ |Multiple-choice question answering| Norwegian-specific & world knowledge | +|[NorOpenBookQA](https://huggingface.co/datasets/ltg/noropenbookqa) |```noropenbookqa_nob```| ```noropenbookqa_nno``` |✅ |Multiple-choice question answering| Norwegian-specific & world knowledge | +|[NorCommonsenseQA](https://huggingface.co/datasets/ltg/norcommonsenseqa) |```norcommonsenseqa_nob```| ```norcommonsenseqa_nno``` |❌ |Multiple-choice question answering|Commonsense reasoning | +|[NorTruthfulQA Multiple choice](https://huggingface.co/datasets/ltg/nortruthfulqa_mc) |```nortruthfulqa_mc_nob```| ```nortruthfulqa_mc_nno``` |❌ |Multiple-choice question answering |Truthfulness | +|[NorQuAD](https://huggingface.co/datasets/ltg/norquad) |```norquad```| ❌ | ✅ |Generative question answering |Machine reading comprehension | +|[NorTruthfulQA Generation](https://huggingface.co/datasets/ltg/nortruthfulqa_gen) |```nortruthfulqa_gen_nob```| ```nortruthfulqa_gen_nno``` | ❌ | Generative question answering|Truthfulness | +|[ASK-GEC](https://huggingface.co/datasets/ltg/ask-gec) |```ask_gec```| ❌ |✅ |Sequence-to-sequence generation|Norwegian language knowledge | +|[NorSumm](https://huggingface.co/datasets/SamiaT/NorSumm) |```norsumm_nob``` | ```norsumm_nno``` |✅ |Sequence-to-sequence generation|Text summarization | +|[Tatoeba (English → Bokmål/Nynorsk)](https://huggingface.co/datasets/Helsinki-NLP/tatoeba_mt) | ```tatoeba_eng_nob```| ```tatoeba_eng_nno``` |✅ |Sequence-to-sequence generation|Machine translation | +|[Tatoeba (Bokmål/Nynorsk → English)](https://huggingface.co/datasets/Helsinki-NLP/tatoeba_mt) | ```tatoeba_nob_eng```| ```tatoeba_nno_eng``` |✅ |Sequence-to-sequence generation|Machine translation | +|[NorRewrite-Instruct](https://huggingface.co/datasets/ltg/norrewrite-instruct) |```norrewrite_instruct``` |❌ |❌ |Sequence-to-sequence generation|Instruction following| +|[NorSummarize-Instruct](https://huggingface.co/datasets/ltg/norsummarize-instruct) |```norsummarize_instruct``` |❌ |❌ |Sequence-to-sequence generation|Instruction following| + +
+Table description + +* **Name**: a dataset name with a HuggingFace link. +* **Bokmål**: the LM Evaluation Harness task name for the Norwegian Bokmål dataset. +* **Nynorsk**: the LM Evaluation Harness task name for the Norwegian Nynorsk dataset, if available. +* **k-shot**: the support for *k*-shot evaluation regimes with *k* > 0. We follow the original datasets' design and focus mainly on the zero-shot evaluation by default. + * ✅ means that the user can run the evaluation in both zero-shot and *k*-shot regimes. + * ❌ denotes that only the zero-shot evaluation regime is available due to the lack of the training or validation set to sample the demonstration examples from. Technically, *k*-shot evaluation on the test set is possible using sampling without replacement, given that the model is not proprietary and not accessed via an API. +* **Task type**: the task type. +* **Task category**: the task category. + +
+ +##### Comments on Belebele +Belebele for Norwegian Bokmål is already available in LM Evaluation Harness as `belebele_nob_Latn`. However, our version (`norbelebele`) supports five prompt templates written by Norwegian native speakers, which are different from the default prompt template used in Belebele. + + + +### Citation + +``` +@article{mikhailov2025noreval, + title={NorEval: A Norwegian Language Understanding and Generation Evaluation Benchmark}, + author={Mikhailov, Vladislav and Enstad, Tita and Samuel, David and Farseth{\aa}s, Hans Christian and Kutuzov, Andrey and Velldal, Erik and {\O}vrelid, Lilja}, + journal={arXiv preprint arXiv:2504.07749}, + year={2025} +} +``` + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? + * [ ] Yes, original implementation contributed by author of the benchmark + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/README.md b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/README.md new file mode 100644 index 0000000000000000000000000000000000000000..35de80b36dd9adcfcc1b78f566849b936ea3ca7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/README.md @@ -0,0 +1,28 @@ +### Evaluation example + +Here, we use the `--predict_only` argument and compute the performance metrics as described below. + +**Step 1: Generate the predictions** + +```bash +lm_eval \ + --model hf \ + --model_args pretrained=AI-Sweden-Models/Llama-3-8B \ + --tasks ask_gec \ + --output results/ask_gec/0-shot/ \ + --log_samples \ + --show_config \ + --write_out \ + --predict_only \ + --batch_size auto \ + --num_fewshot 0 +``` + +**Step 2: Evaluate the predictions with ERRANT** + +* Please refer to the installation instructions [here](https://github.com/chrisjbryant/errant/tree/main). +* Run the following: + ```bash + python3 ask_gec/errant.py --fpath results/ask_gec/0-shot/AI-Sweden-Models__Llama-3-8B/samples_ask_gec_p0_2025-01-28T01-08-13.454441.jsonl --out_fdir results/ask_gec/0-shot/AI-Sweden-Models__Llama-3-8B/ + ``` +* The results will be saved as `results/ask_gec/0-shot/AI-Sweden-Models__Llama-3-8B/samples_ask_gec_p0_2025-01-28T01-08-13.454441_errant.json` diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml new file mode 100644 index 0000000000000000000000000000000000000000..deffb7c954e41fb5ce883966673c1b4281d96689 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml @@ -0,0 +1,15 @@ +tag: ask_gec +dataset_path: ltg/ask-gec +output_type: generate_until +training_split: train +validation_split: validation +test_split: test +doc_to_target: correction +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..383160bcf8b28778331868ca799237cca14e8d3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p0 +doc_to_text: "Tekst: {{source}}\nKorreksjon:" +include: _ask_gec_yaml diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24f176c39bc8efbb2e7c00b03ed816085bf3d294 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p1 +doc_to_text: "Tekst: {{source}}\nRettet versjon:" +include: _ask_gec_yaml diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8da20189e70610ecbd92eed60928ad84a662560 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p2 +doc_to_text: "Skriv om følgende tekst slik at den blir grammatisk korrekt: {{source}}\nKorreksjon:" +include: _ask_gec_yaml diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..def64d44309c5a357098b663f3b536d683867f29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p3 +doc_to_text: "Original versjon: {{source}}\nKorrekturlest og rettet versjon:" +include: _ask_gec_yaml diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81c04c6c093f05a1356f2747c514883facb86735 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p4 +doc_to_text: "Rett opp grammatiske feil i denne teksten: {{source}}\nKorreksjon:" +include: _ask_gec_yaml diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/errant.py b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/errant.py new file mode 100644 index 0000000000000000000000000000000000000000..89721659acc8445b3500c84d2f50702de8010cb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ask_gec/errant.py @@ -0,0 +1,106 @@ +import argparse +import json +import os +import subprocess + +import pandas as pd + + +def parse_args(): + """ + Parses arguments. + Returns: + Arguments containing the names of the prediction file and the file directory to for saving the evaluation results. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--fpath", + type=str, + help="path to a model output file in the lm-evaluation-harness format.", + ) + parser.add_argument( + "--out_fdir", + type=str, + help="path to an output directory for saving the results.", + ) + args = parser.parse_args() + return args + + +def read_examples(fpath: str): + """ + Reads examples from the prediction file. + Args: + fpath: A path to the prediction file. + Returns: + Lists of the sources, targets, and predictions. + """ + examples = pd.read_json(fpath, lines=True) + sources, targets, predictions = [], [], [] + for i, example in examples.iterrows(): + sources.append(example["doc"]["source"]) + targets.append(example["doc"]["correction"]) + predictions.append(example["resps"][0][0].replace("\n\n", "\n")) + return sources, targets, predictions + + +def save_results(fpath: str, obj: dict): + """ + Saves the evaluation results. + Args: + fpath: A path for the output file for saving the results. + obj: The evaluation results. + """ + with open(fpath, "w+", encoding="utf-8") as out: + json.dump(obj, out, indent=3) + + +def evaluate(fpath: str, out_fpath: str): + """ + Runs the evaluation based on the ERRANT performance metric. + Args: + fpath: A path to the prediction file. + out_Fpath: A path for the output file for saving the results. + """ + tmp_name = fpath.replace(".jsonl", "").replace("/", "-") + os.makedirs("tmp", exist_ok=True) + sources, targets, predictions = read_examples(fpath=fpath) + with open(f"tmp/{tmp_name}_sources.txt", "w+") as f: + f.write("\n".join(sources)) + with open(f"tmp/{tmp_name}_targets.txt", "w+") as f: + f.write("\n".join(targets)) + with open(f"tmp/{tmp_name}_predictions.txt", "w+") as f: + f.write("\n".join(predictions)) + subprocess.run( + f"errant_parallel -orig tmp/{tmp_name}_sources.txt -cor tmp/{tmp_name}_targets.txt -out tmp/{tmp_name}_targets.m2 -lev -tok", + shell=True, + ) + subprocess.run( + f"errant_parallel -orig tmp/{tmp_name}_sources.txt -cor tmp/{tmp_name}_predictions.txt -out tmp/{tmp_name}_predictions.m2 -lev -tok", + shell=True, + ) + output = subprocess.check_output( + f"errant_compare -ref tmp/{tmp_name}_targets.m2 -hyp tmp/{tmp_name}_predictions.m2", + shell=True, + ) + f_05 = float(output.decode().strip().split("\n")[-2].split()[-1].strip()) + print(f"Prediction fpath: {fpath}\n\nERRANT: {f_05}", flush=True) + print(f"Saving to: {out_fpath}", flush=True) + save_results(obj={"errant": f_05}, fpath=out_fpath) + subprocess.run(f"rm tmp/{tmp_name}_*", shell=True) + + +def main(): + args = parse_args() + fpath = args.fpath + print(f"Out: {args.out_fdir}", flush=True) + out_fpath = fpath.replace(".jsonl", "_errant.json") + evaluate(fpath=fpath, out_fpath=out_fpath) + + +if __name__ == "__main__": + print( + "\nWARNING: make sure you have ERRANT installed to run the evaluation! Available here: https://github.com/chrisjbryant/errant\n\n", + flush=True, + ) + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/ncb/ncb.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/ncb/ncb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0150c25a3bc3be41d7d4a791a4d7372500a73272 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/ncb/ncb.yaml @@ -0,0 +1,13 @@ +task: ncb +dataset_path: hcfa/ncb +output_type: multiple_choice +test_split: train +doc_to_text: "" +doc_to_target: 0 +doc_to_choice: "{{[correct, wrong]}}" +num_fewshot: 0 +metric_list: + - metric: acc + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/_norbelebele_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/_norbelebele_yaml new file mode 100644 index 0000000000000000000000000000000000000000..41f6b11356c211cc9bbf1c011b67403cfe049477 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/_norbelebele_yaml @@ -0,0 +1,18 @@ +tag: norbelebele +dataset_path: facebook/belebele +dataset_name: nob_Latn +test_split: test +fewshot_split: test +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1037ef735ab7e94ecc95f57e1f7d6413a70b7ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p0.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p0 +include: _norbelebele_yaml +doc_to_text: "Tekst: {{flores_passage}}\nSpørsmål: {{question}}\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nSvar:" +doc_to_choice: ["A", "B", "C", "D"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43cc76c0d86321831546219be281cec32171d96c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p1.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p1 +include: _norbelebele_yaml +doc_to_text: "Bakgrunn: {{flores_passage}}\nSpørsmål:{{question}}\nSvaralternativer:\n- {{mc_answer1}}\n- {{mc_answer2}}\n- {{mc_answer3}}\n- {{mc_answer4}}\nRiktig svar:" +doc_to_choice: "{{[mc_answer1, mc_answer2, mc_answer3, mc_answer4]}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ff96f0eb4012c000b1000d67c7218fa127a9640 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p2.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p2 +include: _norbelebele_yaml +doc_to_text: "{{question}}\nHvilket av følgende mulige svar er det riktige?\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nSvar:" +doc_to_choice: ["A", "B", "C", "D"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fedf28ad18e97455ff4dbe107b67a94db0a9e83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p3.yaml @@ -0,0 +1,5 @@ +task: norbelebele_p3 +include: _norbelebele_yaml +doc_to_text: "Svar på følgende spørsmål: {{question}}\nSvaret skal baseres på følgende tekst:\n{{flores_passage}}\nVelg et svar fra denne listen:\n– {{mc_answer1}}\n– {{mc_answer2}},\n– {{mc_answer3}}\n– {{mc_answer4}}" +doc_to_choice: "{{[mc_answer1, mc_answer2, mc_answer3, mc_answer4]}}" +target_delimiter: "\n" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0285168e5544d00a53794b1de022b0e9217984f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norbelebele/norbelebele_p4.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p4 +include: _norbelebele_yaml +doc_to_text: "{{flores_passage}}\n\n{{question}}\n\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\n\nEr det riktige svaret A, B, C, eller D?" +doc_to_choice: ["A", "B", "C", "D"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/_norcommonsenseqa_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/_norcommonsenseqa_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a17c01fea6d174df3dd2367b4e93ff9628664ad8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/_norcommonsenseqa_yaml @@ -0,0 +1,15 @@ +dataset_path: ltg/norcommonsenseqa +output_type: multiple_choice +training_split: null +validation_split: null +test_split: train +doc_to_target: "{{choices.label.index(answer)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06e59c2c2db31b4472ebfbfa3766196d5b6e73dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p0 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Spørsmål: {{question}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db7ac9d9526a46c97d97a29d8df039121091d2e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p1 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nKva av følgande moglege svar er det rette?\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2477eca8ec64d9eef8442b7a5b17d2d93f663563 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p2 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Gitt alternativa under, kva er svaret på følgande spørsmål: {{question}}\n\nAlternativ:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n- {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f70bd109758d9479d8363ba968165f5a1218b272 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p3 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nVel rett svar blant desse alternativa:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n– {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa8711c1de87f5528e8f6703325a7b842d83d73d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p4 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nEr det rette svaret A, B, C, D, eller E?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50430a868b4bc1e735f35b4de503df33a3733013 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p0 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Spørsmål: {{question}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1c072d6a09f33c7fd96dff217f5831c3060ce7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p1 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nHvilket av følgende mulige svar er det riktige?\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bda34e865e6526b3cb56e2a809dfcbc781200ff0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p2 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Gitt alternativene under, hva er svaret på følgende spørsmål: {{question}}\n\nAlternativer:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n- {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc015bc8c79c198d66c270519e9fa5a407901894 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p3 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nVelg riktig svar blant disse alternativene:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n– {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6eda8246901500a5460d808180521cbfede19f26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p4 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nEr det riktige svaret A, B, C, D, eller E?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/_norec_yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/_norec_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a9cb88455263a835bc3a20dd1781bdaa7c22800 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/_norec_yaml @@ -0,0 +1,14 @@ +dataset_name: binary +output_type: multiple_choice +training_split: train +test_split: test +doc_to_target: sentiment +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: !function utils.multi_f1 + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8596a6b966ca0426389a2970c8ddc23e55f5648c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p0 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nSentiment:" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0171cff5880dfb567d4840193240e7d9ceb84c45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p1 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4053708b3dff485d6c609a70179cdffe496f908 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p3 +include: ../_norec_yaml +doc_to_text: "Anmeldelse: {{review}}\nEr anmelderen positiv eller negativ?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c2a10fd9122ba5bcb4f4f4bc5d0e43350cebfb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p1 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr denne setningen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba5e1e0f0120ceef99708f3825551eb4ef341f74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p3 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/norec/utils.py b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..41f4bce1928cc76ab1bd23b8309c3bcdc5b33963 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/norec/utils.py @@ -0,0 +1,13 @@ +import numpy as np +import sklearn + + +def multi_f1(items): + """ + Computes the macro-average F1 score. + """ + preds, golds = zip(*items) + preds = np.array(preds) + golds = np.array(golds) + fscore = sklearn.metrics.f1_score(golds, preds, average="macro") + return fscore diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bfbe037afd4e6a5376ce6d8eaace76b7ad30360 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p0 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Fullfør dette uttrykket: {{idiom_start}}" diff --git a/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..318be0f280422bbde6c5a1579cef54a2c77aef13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p3 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Fullfør vendingen \"{{idiom_start}}\""