File size: 1,667 Bytes
ec24fb0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """Download evaluation data into data/raw/ (git-ignored).
SinhalaMMLU (public Easy tier) - dev set, CC BY-NC-ND 4.0, gated: accept terms on HF first
Global-MMLU `si` - harder internal validation (translated MMLU), Apache-2.0
Belebele `sin_Sinh` - reading-comprehension sanity check, EVALUATION ONLY
Usage: python scripts/fetch_data.py [--only sinhalammlu,global_mmlu,belebele]
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from eval.common import ROOT, load_env # noqa: E402
RAW = ROOT / "data" / "raw"
SOURCES = {
"sinhalammlu": dict(repo_id="naist-nlp/SinhalaMMLU", allow_patterns=["README.md", "TEST/**/*.json"]),
"global_mmlu": dict(repo_id="CohereLabs/Global-MMLU", allow_patterns=["si/*.parquet"]),
"belebele": dict(repo_id="facebook/belebele", allow_patterns=["data/sin_Sinh.jsonl"]),
}
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--only", default=",".join(SOURCES))
args = ap.parse_args()
load_env()
from huggingface_hub import HfApi, snapshot_download
api = HfApi()
for name in args.only.split(","):
src = SOURCES[name]
sha = api.dataset_info(src["repo_id"]).sha
path = snapshot_download(
repo_id=src["repo_id"], repo_type="dataset", revision=sha,
allow_patterns=src["allow_patterns"], local_dir=RAW / name,
)
(RAW / name / "REVISION").write_text(f"{src['repo_id']}@{sha}\n")
print(f"{name}: {src['repo_id']}@{sha[:10]} -> {path}")
if __name__ == "__main__":
main()
|