Blame for make_benchmarks.py

"""Bundle the decontamination test sets with the package, so runs never depend on the dataset viewer.
Needs `datasets`. Run again to refresh: python make_benchmarks.py"""
import gzip
import json

from datasets import load_dataset

from datainventor.clean import BENCHMARKS

out = {}
for key, (ds, cfg, split, fields) in BENCHMARKS.items():
    rows = load_dataset(ds, None if cfg == "default" else cfg, split=split)
    out[key] = [t for t in (" ".join(str(r.get(f, "")) for f in fields).strip() for r in rows) if t]
    print(key, len(out[key]), flush=True)
with gzip.open("datainventor/data/benchmarks.json.gz", "wt", encoding="utf-8") as f:
    json.dump(out, f, ensure_ascii=False)