diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/README.md b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f44c5f04ed6eaded120ac7d9da177004a02483a7
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/README.md
@@ -0,0 +1,49 @@
+# EusExams
+
+### Paper
+
+Title: Latxa: An Open Language Model and Evaluation Suite for Basque
+
+Abstract: https://arxiv.org/abs/2403.20266
+
+EusExams is a collection of tests designed to prepare individuals for Public Service examinations conducted by several Basque institutions, including the public health system Osakidetza, the Basque Government, the City Councils of Bilbao and Gasteiz, and the University of the Basque Country (UPV/EHU). Within each of these groups, there are different exams for public positions, such as administrative and assistant roles. Each multiple-choice question contains 2 to 4 choices (3.90 on average) and one correct answer. The dataset is mostly parallel with 16k questions in Basque and 18k in Spanish.
+
+Homepage: https://github.com/hitz-zentroa/latxa
+
+
+### Citation
+
+```
+@misc{etxaniz2024latxa,
+      title={Latxa: An Open Language Model and Evaluation Suite for Basque},
+      author={Julen Etxaniz and Oscar Sainz and Naiara Perez and Itziar Aldabe and German Rigau and Eneko Agirre and Aitor Ormazabal and Mikel Artetxe and Aitor Soroa},
+      year={2024},
+      eprint={2403.20266},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `eus_exams_eu`: The Basque version of the exams.
+* `eus_exams_es`: The Spanish version of the exams.
+
+#### Tasks
+
+Basque and Spanish versions of the exams are available as separate tasks starting with `eus_exams_eu` and `eus_exams_es` respectively.
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/configs.py b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/configs.py
new file mode 100644
index 0000000000000000000000000000000000000000..993faa9f5dda1df2b00301fb00367f75e58a14de
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/configs.py
@@ -0,0 +1,67 @@
+import argparse
+import json
+
+import requests
+import yaml
+
+
+# get configs from huggingface datasets server by doing a request
+response = requests.get(
+    "https://datasets-server.huggingface.co/splits?dataset=HiTZ%2FEusExams", timeout=5
+)
+response_json = json.loads(response.text)
+CONFIGS = [split["config"] for split in response_json["splits"]]
+
+
+def gen_config_yamls(output_dir: str, overwrite: bool) -> None:
+    """
+    Generate a yaml file for each configuage.
+
+    :param output_dir: The directory to output the files to.
+    :param overwrite: Whether to overwrite files if they already exist.
+    """
+    err = []
+    for config in CONFIGS:
+        file_name = f"eus_exams_{config}.yaml"
+        try:
+            with open(f"{output_dir}/{file_name}", "w" if overwrite else "x") as f:
+                f.write("# Generated by utils.py\n")
+                yaml.dump(
+                    {
+                        "include": "eus_exams_es"
+                        if "eus_exams_es" in config
+                        else "eus_exams_eu",
+                        "dataset_name": config,
+                        "task": f"eus_exams_{config}",
+                    },
+                    f,
+                )
+        except FileExistsError:
+            err.append(file_name)
+
+    if len(err) > 0:
+        raise FileExistsError(
+            "Files were not created because they already exist (use --overwrite flag):"
+            f" {', '.join(err)}"
+        )
+
+
+def main() -> None:
+    """Parse CLI args and generate configuage-specific yaml files."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--overwrite",
+        default=False,
+        action="store_true",
+        help="Overwrite files if they already exist",
+    )
+    parser.add_argument(
+        "--output-dir", default=".", help="Directory to write yaml files to"
+    )
+    args = parser.parse_args()
+
+    gen_config_yamls(output_dir=args.output_dir, overwrite=args.overwrite)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams
new file mode 100644
index 0000000000000000000000000000000000000000..d1d2af731485ac26b2792b5de29d4da681bf97ad
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams
@@ -0,0 +1,18 @@
+dataset_path: HiTZ/EusExams
+dataset_name: null
+validation_split: null
+test_split: test
+fewshot_split: test
+process_docs: !function utils.process_docs
+output_type: multiple_choice
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es
new file mode 100644
index 0000000000000000000000000000000000000000..25886606943ec02e216d89df64cdc777a405cdaf
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es
@@ -0,0 +1,4 @@
+include: eus_exams
+group:
+  - eus_exams_es
+doc_to_text: "Pregunta: {{question}}\nA: {{candidates[0]}}\nB: {{candidates[1]}}\nC: {{candidates[2]}}\nD: {{candidates[3]}}\nRespuesta:"
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejadministrativo.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejadministrativo.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..22b93ed6b7c2964e0d64c3a5e3aa299a81752bf2
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejadministrativo.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_ejadministrativo
+include: eus_exams_es
+task: eus_exams_es_ejadministrativo
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejauxiliar.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejauxiliar.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b6cbe975fd9e1bbd927244ded96ee3f713273df1
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejauxiliar.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_ejauxiliar
+include: eus_exams_es
+task: eus_exams_es_ejauxiliar
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejsubalterno.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejsubalterno.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0adfba26dd70eb54f0f50ac2ad2d97e83aaf57b8
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejsubalterno.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_ejsubalterno
+include: eus_exams_es
+task: eus_exams_es_ejsubalterno
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejtecnico.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejtecnico.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0d0b011c9ab8ee58aa7cad819dd7475880512972
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_ejtecnico.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_ejtecnico
+include: eus_exams_es
+task: eus_exams_es_ejtecnico
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeayuntamientovitoria.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeayuntamientovitoria.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0d43c0b4161bf76988ace9416ac1bf1147a1f5c9
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeayuntamientovitoria.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeayuntamientovitoria
+include: eus_exams_es
+task: eus_exams_es_opeayuntamientovitoria
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuadmin.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuadmin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e9cacfbd3f3e79dc24436daef4a1e9e5a1b5709d
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuadmin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehuadmin
+include: eus_exams_es
+task: eus_exams_es_opeehuadmin
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuaux.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuaux.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..88316e512a4dabff3e550f84f3401216316991a7
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuaux.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehuaux
+include: eus_exams_es
+task: eus_exams_es_opeehuaux
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehubiblio.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehubiblio.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..728d7cfb0e7c79af156ff50bdb8379f032c3f01b
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehubiblio.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehubiblio
+include: eus_exams_es
+task: eus_exams_es_opeehubiblio
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuderecho.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuderecho.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..13e1d9de4bf434854e835b98455a3c26c46e96ac
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuderecho.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehuderecho
+include: eus_exams_es
+task: eus_exams_es_opeehuderecho
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehueconomicas.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehueconomicas.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6625c9ce5a2501aa607cacf148531e0c3220652b
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehueconomicas.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehueconomicas
+include: eus_exams_es
+task: eus_exams_es_opeehueconomicas
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuempresariales.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuempresariales.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4f61d3a1433f0f8ea0dd2ab62d9eb2c291697a47
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehuempresariales.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehuempresariales
+include: eus_exams_es
+task: eus_exams_es_opeehuempresariales
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehusubalterno.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehusubalterno.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..96cc86b402af1777e075530b3258e3a9089d539f
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehusubalterno.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehusubalterno
+include: eus_exams_es
+task: eus_exams_es_opeehusubalterno
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehutecnico.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehutecnico.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0641fc2e7766d5f93ba1c45c83761f6e5b57560a
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehutecnico.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehutecnico
+include: eus_exams_es
+task: eus_exams_es_opeehutecnico
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehutecnicob.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehutecnicob.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a338a1ab0d542368acf179b0611a354ddb71d293
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeehutecnicob.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeehutecnicob
+include: eus_exams_es
+task: eus_exams_es_opeehutecnicob
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiadmin.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiadmin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..85c771cdb3ead8511963b811043891958f19e340
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiadmin.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakiadmin
+include: eus_exams_es
+task: eus_exams_es_opeosakiadmin
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiaux.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiaux.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2d61825b0beac1f50137ff42d74b6b649f30ea4e
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiaux.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakiaux
+include: eus_exams_es
+task: eus_exams_es_opeosakiaux
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiauxenf.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiauxenf.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..08fe0ed6c014ce69d7655c94ccd9dfdf029c8ce1
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakiauxenf.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakiauxenf
+include: eus_exams_es
+task: eus_exams_es_opeosakiauxenf
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakicelador.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakicelador.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2a61b6878e9684ce0b35be5dc2fd25170cf9bf44
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakicelador.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakicelador
+include: eus_exams_es
+task: eus_exams_es_opeosakicelador
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakienf.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakienf.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e4749cac111d410d6b573a5365ec6778ec4645f2
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakienf.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakienf
+include: eus_exams_es
+task: eus_exams_es_opeosakienf
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakijuridico.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakijuridico.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a62dc8001f16a22e56c7bea270ad3e6f97ecf5fd
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakijuridico.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakijuridico
+include: eus_exams_es
+task: eus_exams_es_opeosakijuridico
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakioperario.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakioperario.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..df72481742ea637ec2fbe546445cd450bd1bc632
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakioperario.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakioperario
+include: eus_exams_es
+task: eus_exams_es_opeosakioperario
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakitecnico.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakitecnico.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4b5b397b88ed5318706a0e6f402acf34440761fe
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakitecnico.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakitecnico
+include: eus_exams_es
+task: eus_exams_es_opeosakitecnico
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakivarios.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakivarios.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fe98dc76aa4689ec46be58d05326adf6216264df
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_opeosakivarios.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_opeosakivarios
+include: eus_exams_es
+task: eus_exams_es_opeosakivarios
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza1c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza1c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..080f99fcf28f738117702d9ece800bdeed209b90
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza1c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza1c
+include: eus_exams_es
+task: eus_exams_es_osakidetza1c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza2c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza2c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4ee8ab46c6e4e1dd712b5d5865fb87abac5ac89b
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza2c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza2c
+include: eus_exams_es
+task: eus_exams_es_osakidetza2c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza3c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza3c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2974a11d797474ed13257ac46e7994c31253f83d
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza3c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza3c
+include: eus_exams_es
+task: eus_exams_es_osakidetza3c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza4c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza4c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..faa6c4b46c0de1d1f31fdc88f0acae96889eb080
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza4c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza4c
+include: eus_exams_es
+task: eus_exams_es_osakidetza4c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza5c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza5c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..153ce3add3ebe5f9d9da505da1e5c5affe0d1263
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza5c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza5c
+include: eus_exams_es
+task: eus_exams_es_osakidetza5c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza6c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza6c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d94ef2b9f4e494a4ba21d0fc4c902d3ad125616a
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza6c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza6c
+include: eus_exams_es
+task: eus_exams_es_osakidetza6c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza7c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza7c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1fc30ce353f83ccc717a504a50a7bd611f76e6c6
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza7c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza7c
+include: eus_exams_es
+task: eus_exams_es_osakidetza7c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza8c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza8c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..38f7ee3c39af34bc0516780a8717172950fc955a
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza8c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza8c
+include: eus_exams_es
+task: eus_exams_es_osakidetza8c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza9c.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza9c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7b23ff670764634cb78bd5a4cbb9f141dad674d2
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_es_osakidetza9c.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: es_osakidetza9c
+include: eus_exams_es
+task: eus_exams_es_osakidetza9c
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu
new file mode 100644
index 0000000000000000000000000000000000000000..95b82388df8930a2fe7385fff72f22c4cc516ab4
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu
@@ -0,0 +1,4 @@
+include: eus_exams
+group:
+  - eus_exams_eu
+doc_to_text: "Galdera: {{question}}\nA: {{candidates[0]}}\nB: {{candidates[1]}}\nC: {{candidates[2]}}\nD: {{candidates[3]}}\nErantzuna:"
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejadministrari.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejadministrari.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f5630ddb05864cd3d6031ea8fed96e9715fb8990
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejadministrari.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_ejadministrari
+include: eus_exams_eu
+task: eus_exams_eu_ejadministrari
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejlaguntza.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejlaguntza.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cf2806c1e6675c491ad5d1eaea54698bf8aa8fe8
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejlaguntza.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_ejlaguntza
+include: eus_exams_eu
+task: eus_exams_eu_ejlaguntza
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejlaguntzaile.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejlaguntzaile.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1d713a32442fc35d16b735c8617b0ee2d7327f04
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejlaguntzaile.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_ejlaguntzaile
+include: eus_exams_eu
+task: eus_exams_eu_ejlaguntzaile
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejteknikari.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejteknikari.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7b528b9d4ce7ebc2ffc92af84a25e417f2e86929
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_ejteknikari.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_ejteknikari
+include: eus_exams_eu
+task: eus_exams_eu_ejteknikari
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opebilbaoeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opebilbaoeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d15dbc6101ade859261bed36564eaf51e8a53f16
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opebilbaoeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opebilbaoeu
+include: eus_exams_eu
+task: eus_exams_eu_opebilbaoeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuadmineu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuadmineu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..85b9c9047759b6652435abc84944770ff429daaa
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuadmineu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehuadmineu
+include: eus_exams_eu
+task: eus_exams_eu_opeehuadmineu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuauxeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuauxeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e72082486395abefcebda07de380b670d588589a
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuauxeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehuauxeu
+include: eus_exams_eu
+task: eus_exams_eu_opeehuauxeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehubiblioeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehubiblioeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0ff2ab853fc839c5ae2b88520767b8b3d4a60f4d
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehubiblioeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehubiblioeu
+include: eus_exams_eu
+task: eus_exams_eu_opeehubiblioeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuderechoeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuderechoeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bef6b6524507dbae9ebf9b07bbe7d41fca978996
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuderechoeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehuderechoeu
+include: eus_exams_eu
+task: eus_exams_eu_opeehuderechoeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehueconomicaseu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehueconomicaseu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..713f33234153c84778faeca25f4807cdf9812b45
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehueconomicaseu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehueconomicaseu
+include: eus_exams_eu
+task: eus_exams_eu_opeehueconomicaseu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuempresarialeseu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuempresarialeseu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8dddd9bc76cec647dafbc7400ae11d3e5147de83
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuempresarialeseu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehuempresarialeseu
+include: eus_exams_eu
+task: eus_exams_eu_opeehuempresarialeseu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehusubalternoeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehusubalternoeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b02a451dd957a11c5db2810fa98ab0bccd62c9b3
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehusubalternoeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehusubalternoeu
+include: eus_exams_eu
+task: eus_exams_eu_opeehusubalternoeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehutecnicoeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehutecnicoeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3792e12aa0285a3cf3ce56b9d6158ade836c4c38
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehutecnicoeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehutecnicoeu
+include: eus_exams_eu
+task: eus_exams_eu_opeehutecnicoeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuteknikarib.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuteknikarib.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b9f5cc612ac9776a328670d4273e76934172fd81
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeehuteknikarib.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeehuteknikarib
+include: eus_exams_eu
+task: eus_exams_eu_opeehuteknikarib
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opegasteizkoudala.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opegasteizkoudala.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e9211f39a162360b67e84399409b1617bc5cc1dd
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opegasteizkoudala.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opegasteizkoudala
+include: eus_exams_eu
+task: eus_exams_eu_opegasteizkoudala
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiadmineu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiadmineu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cf19e09941bd0c7bb10db7f5398fb2398f1a0fd2
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiadmineu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakiadmineu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakiadmineu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiauxenfeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiauxenfeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..719039915aec71a138a860397709b85549718078
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiauxenfeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakiauxenfeu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakiauxenfeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiauxeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiauxeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9d0891886cd550219fb9bfcc7209f6d5fb85ad5d
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiauxeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakiauxeu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakiauxeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiceladoreu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiceladoreu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..af82c87bdffc84c8da3f666d944740eb0db0712d
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakiceladoreu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakiceladoreu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakiceladoreu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakienfeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakienfeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..10b853e399f017255c74b7eb56275df149a7f055
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakienfeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakienfeu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakienfeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakioperarioeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakioperarioeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8f3cf7c490106959e4b07bef2140f0197835d16d
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakioperarioeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakioperarioeu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakioperarioeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakitecnicoeu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakitecnicoeu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f44e4994e3820dd6263835448b566a8c2ed17a13
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakitecnicoeu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakitecnicoeu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakitecnicoeu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakivarioseu.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakivarioseu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..11801ddec6614d95862087fa85c3f7b6314d8ddc
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_opeosakivarioseu.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_opeosakivarioseu
+include: eus_exams_eu
+task: eus_exams_eu_opeosakivarioseu
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza1e.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza1e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cc713507196ef8f9460a61e110ede95186f846b1
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza1e.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_osakidetza1e
+include: eus_exams_eu
+task: eus_exams_eu_osakidetza1e
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza2e.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza2e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..218dc87cb8affc37cc54e03d56bcf44213381e99
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza2e.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_osakidetza2e
+include: eus_exams_eu
+task: eus_exams_eu_osakidetza2e
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza3e.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza3e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d5d17c32a86b89ddaf3dc1da834fb053b67b9b64
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza3e.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_osakidetza3e
+include: eus_exams_eu
+task: eus_exams_eu_osakidetza3e
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza5e.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza5e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..be4d2ca741e5168f99ed2105d584cf1fa21b4b81
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza5e.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_osakidetza5e
+include: eus_exams_eu
+task: eus_exams_eu_osakidetza5e
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza6e.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza6e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7b2af263fbe039a6ab9e3131f868ab506f0e9b35
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza6e.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_osakidetza6e
+include: eus_exams_eu
+task: eus_exams_eu_osakidetza6e
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza7e.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza7e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..666e96a0e136045c884f81fcf62d007f41ea80b7
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/eus_exams/eus_exams_eu_osakidetza7e.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: eu_osakidetza7e
+include: eus_exams_eu
+task: eus_exams_eu_osakidetza7e
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/README.md b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..5af16562e07d04a15b5313a2fadc61f1f2680036
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/README.md
@@ -0,0 +1,48 @@
+# Multilingual HellaSwag
+
+### Paper
+
+Title: `Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback`
+
+Abstract: https://arxiv.org/abs/2307.16039
+
+A key technology for the development of large language models (LLMs) involves instruction tuning that helps align the models' responses with human expectations to realize impressive learning abilities. Two major approaches for instruction tuning characterize supervised fine-tuning (SFT) and reinforcement learning from human feedback (RLHF), which are currently applied to produce the best commercial LLMs (e.g., ChatGPT). To improve the accessibility of LLMs for research and development efforts, various instruction-tuned open-source LLMs have also been introduced recently, e.g., Alpaca, Vicuna, to name a few. However, existing open-source LLMs have only been instruction-tuned for English and a few popular languages, thus hindering their impacts and accessibility to many other languages in the world. Among a few very recent work to explore instruction tuning for LLMs in multiple languages, SFT has been used as the only approach to instruction-tune LLMs for multiple languages. This has left a significant gap for fine-tuned LLMs based on RLHF in diverse languages and raised important questions on how RLHF can boost the performance of multilingual instruction tuning. To overcome this issue, we present Okapi, the first system with instruction-tuned LLMs based on RLHF for multiple languages. Okapi introduces instruction and response-ranked data in 26 diverse languages to facilitate the experiments and development of future multilingual LLM research. We also present benchmark datasets to enable the evaluation of generative LLMs in multiple languages. Our experiments demonstrate the advantages of RLHF for multilingual instruction over SFT for different base models and datasets. Our framework and resources are released at this https URL.
+
+Homepage: `https://github.com/nlp-uoregon/Okapi`
+
+
+### Citation
+
+```
+@article{dac2023okapi,
+  title={Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback},
+  author={Dac Lai, Viet and Van Nguyen, Chien and Ngo, Nghia Trung and Nguyen, Thuat and Dernoncourt, Franck and Rossi, Ryan A and Nguyen, Thien Huu},
+  journal={arXiv e-prints},
+  pages={arXiv--2307},
+  year={2023}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+- hellaswag_multilingual
+
+#### Tasks
+
+- `hellaswag_{ar,bn,ca,da,de,es,eu,fr,gu,hi,hr,hu,hy,id,it,kn,ml,mr,ne,nl,pt,ro,ru,sk,sr,sv,ta,te,uk,vi}`
+
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_de.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_de.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6c103a832115bbffd6da684ff1459b22b310c659
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_de.yaml
@@ -0,0 +1,6 @@
+include: _hellaswag_yaml
+task: hellaswag_de
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: de
+training_split: null
+validation_split: val
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_hi.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_hi.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c2110785501a1c8f0b6dc0c73ffc73a93ba85d92
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_hi.yaml
@@ -0,0 +1,6 @@
+include: _hellaswag_yaml
+task: hellaswag_hi
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: hi
+training_split: null
+validation_split: val
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_hr.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_hr.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7e4b547b00a486508696cb126e8be6b2af2988c5
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_hr.yaml
@@ -0,0 +1,6 @@
+include: _hellaswag_yaml
+task: hellaswag_hr
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: hr
+training_split: null
+validation_split: val
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_it.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_it.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..97be88b8e3dbefc81ed428d8f0e4675481c607c3
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_it.yaml
@@ -0,0 +1,6 @@
+include: _hellaswag_yaml
+task: hellaswag_it
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: it
+training_split: null
+validation_split: val
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sr.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sr.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9dfae80cf0fc6d2113ce327771a80cd55eeb2dfc
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sr.yaml
@@ -0,0 +1,6 @@
+include: _hellaswag_yaml
+task: hellaswag_sr
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: sr
+training_split: null
+validation_split: val
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_te.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_te.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..92a846b6e61d21e1359c530a6f7501a36bda9d2f
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_te.yaml
@@ -0,0 +1,6 @@
+include: _hellaswag_yaml
+task: hellaswag_te
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: te
+training_split: null
+validation_split: val
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_uk.yaml b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_uk.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d675fb448ba3175549b8b225281993540ffa6715
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_uk.yaml
@@ -0,0 +1,6 @@
+include: _hellaswag_yaml
+task: hellaswag_uk
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: uk
+training_split: null
+validation_split: val
diff --git a/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/utils.py b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..b526a9e93076f7db54221072d58ca4bd7161ee97
--- /dev/null
+++ b/scripts/yans/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/utils.py
@@ -0,0 +1,25 @@
+import re
+
+import datasets
+
+
+def preprocess(text):
+    text = text.strip()
+    # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.
+    text = text.replace(" [title]", ". ")
+    text = re.sub("\\[.*?\\]", "", text)
+    text = text.replace("  ", " ")
+    return text
+
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _process_doc(doc):
+        ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize()
+        out_doc = {
+            "query": preprocess(doc["activity_label"] + ": " + ctx),
+            "choices": [preprocess(ending) for ending in doc["endings"]],
+            "gold": int(doc["label"]),
+        }
+        return out_doc
+
+    return dataset.map(_process_doc)