| """Download evaluation data into data/raw/ (git-ignored). |
| |
| SinhalaMMLU (public Easy tier) - dev set, CC BY-NC-ND 4.0, gated: accept terms on HF first |
| Global-MMLU `si` - harder internal validation (translated MMLU), Apache-2.0 |
| Belebele `sin_Sinh` - reading-comprehension sanity check, EVALUATION ONLY |
| |
| Usage: python scripts/fetch_data.py [--only sinhalammlu,global_mmlu,belebele] |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import sys |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) |
| from eval.common import ROOT, load_env |
|
|
| RAW = ROOT / "data" / "raw" |
|
|
| SOURCES = { |
| "sinhalammlu": dict(repo_id="naist-nlp/SinhalaMMLU", allow_patterns=["README.md", "TEST/**/*.json"]), |
| "global_mmlu": dict(repo_id="CohereLabs/Global-MMLU", allow_patterns=["si/*.parquet"]), |
| "belebele": dict(repo_id="facebook/belebele", allow_patterns=["data/sin_Sinh.jsonl"]), |
| } |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--only", default=",".join(SOURCES)) |
| args = ap.parse_args() |
| load_env() |
| from huggingface_hub import HfApi, snapshot_download |
|
|
| api = HfApi() |
| for name in args.only.split(","): |
| src = SOURCES[name] |
| sha = api.dataset_info(src["repo_id"]).sha |
| path = snapshot_download( |
| repo_id=src["repo_id"], repo_type="dataset", revision=sha, |
| allow_patterns=src["allow_patterns"], local_dir=RAW / name, |
| ) |
| (RAW / name / "REVISION").write_text(f"{src['repo_id']}@{sha}\n") |
| print(f"{name}: {src['repo_id']}@{sha[:10]} -> {path}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
