Spaces:

codelion
/

data-inventor

Running

data-inventor / make_benchmarks.py

Curate a corpus: clean, calibrate, create, compose on a sample, plus a recipe and runner for your own machines

4f740eb verified 25 days ago

686 Bytes

"""Bundle the decontamination test sets with the package, so runs never depend on the dataset viewer.
Needs `datasets`. Run again to refresh: python make_benchmarks.py"""
import gzip
import json

from datasets import load_dataset

from datainventor.clean import BENCHMARKS

out = {}
for key, (ds, cfg, split, fields) in BENCHMARKS.items():
    rows = load_dataset(ds, None if cfg == "default" else cfg, split=split)
    out[key] = [t for t in (" ".join(str(r.get(f, "")) for f in fields).strip() for r in rows) if t]
    print(key, len(out[key]), flush=True)
with gzip.open("datainventor/data/benchmarks.json.gz", "wt", encoding="utf-8") as f:
    json.dump(out, f, ensure_ascii=False)

	"""Bundle the decontamination test sets with the package, so runs never depend on the dataset viewer.
	Needs `datasets`. Run again to refresh: python make_benchmarks.py"""
	import gzip
	import json

	from datasets import load_dataset

	from datainventor.clean import BENCHMARKS

	out = {}
	for key, (ds, cfg, split, fields) in BENCHMARKS.items():
	rows = load_dataset(ds, None if cfg == "default" else cfg, split=split)
	out[key] = [t for t in (" ".join(str(r.get(f, "")) for f in fields).strip() for r in rows) if t]
	print(key, len(out[key]), flush=True)
	with gzip.open("datainventor/data/benchmarks.json.gz", "wt", encoding="utf-8") as f:
	json.dump(out, f, ensure_ascii=False)