| |
| """T0.3 · Fixed item-id files for RS-full (spec §7), so every G4 run scores the same items. |
| |
| python -I scripts/prep_rs_full.py # downloads ~250 MB of parquet into $MT_DATA/evalsets |
| |
| - MMLU, 5-shot: all 14,042 test items; the shots are each subject's 5 dev items in order (the standard |
| MMLU prompt), written into the prompt so lm-eval itself runs 0-shot. |
| - MMLU-Redux 2.0: every item its annotators marked `ok`, 0-shot. |
| - MMLU-Pro: 2,000 of the 12,032 test items, stratified over the 14 categories (largest remainder), seed 0. |
| - GSM8K: all 1,319 test items. MATH-500: all 500. IFEval: all 541 prompts. |
| - Global-MMLU: 1,000 items per Marco language it covers (24 of 29; th, ur, kk, hu and az are absent), |
| stratified over the 57 subjects, seed 0, the same sample ids in every language (it is parallel). |
| - BELEBELE: all 900 questions in all 29 Marco languages. |
| |
| Parquet comes from the Hub's parquet conversion; each file's sha256 is recorded with the dataset's |
| revision at download time. Downloaded files are data: read with pyarrow only. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import random |
| import sys |
| from collections import defaultdict |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parent)) |
| from prep_rs_subsets import DATA, OUT as OUT_SUBSETS, REV, allocate, rows, write |
|
|
| MARCO = "en zh ar de es fr ko ja pt tr id it nl pl ru vi th he uk ms bn cs ur kk el ro hu ne az".split() |
| FLORES = dict(en="eng_Latn", zh="zho_Hans", ar="arb_Arab", de="deu_Latn", es="spa_Latn", fr="fra_Latn", |
| ko="kor_Hang", ja="jpn_Jpan", pt="por_Latn", tr="tur_Latn", id="ind_Latn", it="ita_Latn", |
| nl="nld_Latn", pl="pol_Latn", ru="rus_Cyrl", vi="vie_Latn", th="tha_Thai", he="heb_Hebr", |
| uk="ukr_Cyrl", ms="zsm_Latn", bn="ben_Beng", cs="ces_Latn", ur="urd_Arab", kk="kaz_Cyrl", |
| el="ell_Grek", ro="ron_Latn", hu="hun_Latn", ne="npi_Deva", az="azj_Latn") |
| GMMLU_ABSENT = {"th", "ur", "kk", "hu", "az"} |
| REV_FULL = {"mmlu": ("cais/mmlu", "c30699e8356da336a370243923dbaf21066bb9fe"), |
| "mmlu_pro": ("TIGER-Lab/MMLU-Pro", "b189ec765aa7ed75c8acfea42df31fdae71f97be"), |
| "math500": ("HuggingFaceH4/MATH-500", "6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be"), |
| "ifeval": ("google/IFEval", "966cd89545d6b6acfd7638bc708b98261ca58e84"), |
| "global_mmlu": ("CohereLabs/Global-MMLU", "0e619dbeb34206cd48705a1a0ea7fb21cae09993")} |
|
|
|
|
| def stratified(groups: dict[str, list], n: int, seed: int) -> dict[str, list]: |
| alloc = allocate({k: len(v) for k, v in groups.items()}, n) |
| rng = random.Random(seed) |
| return {k: sorted(rng.sample(groups[k], alloc[k])) for k in sorted(groups)} |
|
|
|
|
| def mmlu() -> None: |
| ds, rev = REV_FULL["mmlu"] |
| test, sha_t = rows(ds, "all", "test", DATA / "evalsets/mmlu/all_test.parquet") |
| dev, sha_d = rows(ds, "all", "dev", DATA / "evalsets/mmlu/all_dev.parquet") |
| shots = defaultdict(list) |
| for i, r in enumerate(dev): |
| shots[r["subject"]].append(i) |
| assert all(len(v) == 5 for v in shots.values()) and len(shots) == 57 |
| write("mmlu_5shot.json", {"dataset": ds, "revision": rev, "split": "test", "n": len(test), "shots": "dev, 5 per subject, in order", |
| "parquet_sha256": {"test": sha_t, "dev": sha_d}}) |
|
|
|
|
| def mmlu_redux_ok() -> None: |
| ds, rev = REV["mmlu_redux"] |
| shas = json.loads((OUT_SUBSETS / "mmlu_redux_1000.json").read_text())["parquet_sha256"] |
| ids = {} |
| for s in sorted(shas): |
| rs, sha = rows(ds, s, "test", DATA / f"evalsets/mmlu_redux/{s}_test.parquet") |
| assert sha == shas[s], s |
| ids[s] = [i for i, r in enumerate(rs) if r.get("error_type") == "ok"] |
| write("mmlu_redux_ok.json", {"dataset": ds, "revision": rev, "split": "test", "filter": "error_type == ok", |
| "n": sum(map(len, ids.values())), "parquet_sha256": shas, "ids": ids}) |
|
|
|
|
| def mmlu_pro(seed: int, n: int) -> None: |
| ds, rev = REV_FULL["mmlu_pro"] |
| rs, sha = rows(ds, "default", "test", DATA / "evalsets/mmlu_pro/test.parquet") |
| by_cat = defaultdict(list) |
| for r in rs: |
| by_cat[r["category"]].append(r["question_id"]) |
| ids = stratified(by_cat, n, seed) |
| write("mmlu_pro_2000.json", {"dataset": ds, "revision": rev, "split": "test", "seed": seed, "n": sum(map(len, ids.values())), |
| "population": len(rs), "parquet_sha256": sha, "ids": ids}) |
|
|
|
|
| def whole(key: str, cfg: str, split: str, dest: str, name: str, id_field: str | None = None) -> None: |
| ds, rev = REV_FULL[key] |
| rs, sha = rows(ds, cfg, split, DATA / dest) |
| obj = {"dataset": ds, "revision": rev, "split": split, "n": len(rs), "parquet_sha256": sha} |
| if id_field: |
| obj["ids"] = [r[id_field] for r in rs] |
| write(name, obj) |
|
|
|
|
| def gsm8k_full() -> None: |
| g = json.loads((OUT_SUBSETS / "gsm8k_250.json").read_text()) |
| rs, sha = rows("openai/gsm8k", "main", "test", DATA / "evalsets/gsm8k/main_test.parquet") |
| assert sha == g["parquet_sha256"] |
| write("gsm8k_full.json", {"dataset": g["dataset"], "revision": g["dataset_sha"], "split": "test", "n": len(rs), "parquet_sha256": sha}) |
|
|
|
|
| def global_mmlu(seed: int, n: int) -> None: |
| ds, rev = REV_FULL["global_mmlu"] |
| langs = [l for l in MARCO if l not in GMMLU_ABSENT] |
| out = {"dataset": ds, "revision": rev, "split": "test", "seed": seed, "languages": langs, |
| "absent_marco_languages": sorted(GMMLU_ABSENT), "parquet_sha256": {}} |
| present = None |
| subj = {} |
| for lang in langs: |
| rs, out["parquet_sha256"][lang] = rows(ds, lang, "test", DATA / f"evalsets/global_mmlu/{lang}_test.parquet") |
| sids = {r["sample_id"] for r in rs} |
| present = sids if present is None else present & sids |
| if lang == "en": |
| subj = {r["sample_id"]: r["subject"] for r in rs} |
| by_subj = defaultdict(list) |
| for sid in sorted(present): |
| by_subj[subj[sid]].append(sid) |
| pick = stratified(by_subj, n, seed) |
| out.update(n_per_language=n, parallel_items=len(present), ids=sorted(s for v in pick.values() for s in v)) |
| out["n"] = n * len(langs) |
| write("global_mmlu_1000.json", out) |
|
|
|
|
| def belebele_full() -> None: |
| ds, rev = REV["belebele"] |
| out = {"dataset": ds, "revision": rev, "split": "test", "languages": MARCO, "flores": FLORES, "parquet_sha256": {}} |
| keys = None |
| for lang in MARCO: |
| rs, out["parquet_sha256"][lang] = rows(ds, FLORES[lang], "test", DATA / f"evalsets/belebele/{FLORES[lang]}_test.parquet") |
| k = {(r["link"], r["question_number"]) for r in rs} |
| keys = k if keys is None else keys & k |
| out.update(n_per_language=len(keys), n=len(keys) * len(MARCO)) |
| write("belebele_full.json", out) |
|
|
|
|
| def main() -> None: |
| p = argparse.ArgumentParser() |
| p.add_argument("--seed", type=int, default=0) |
| a = p.parse_args() |
| mmlu() |
| mmlu_redux_ok() |
| mmlu_pro(a.seed, 2000) |
| gsm8k_full() |
| whole("math500", "default", "test", "evalsets/math500/test.parquet", "math500.json", "unique_id") |
| whole("ifeval", "default", "train", "evalsets/ifeval/train.parquet", "ifeval.json", "key") |
| global_mmlu(a.seed, 1000) |
| belebele_full() |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
