File size: 2,595 Bytes
7a2335a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | """Codabench submission: predictions.jsonl (one line per test question, keys copied verbatim, answer as a string),
validated against the test set and zipped.
python -m sinmmlu.submit --validate outputs/<exp_id>/predictions.jsonl
"""
import argparse
import json
import zipfile
from pathlib import Path
from .data import load_test, submission_key
def write_submission(rows, preds, out_dir, full=True):
"""full=False: rows are a subset of the test set (dev/test_subset300.jsonl); validated against their own keys."""
out_dir = Path(out_dir)
f = out_dir / "predictions.jsonl"
with open(f, "w", encoding="utf-8", newline="\n") as fh:
for r, y in zip(rows, preds):
fh.write(json.dumps({"q_no": r["q_no"], "subject": r["subject"], "category": r["category"],
"level": r["level"], "answer": str(y)}, ensure_ascii=False) + "\n")
if not validate(f, None if full else {submission_key(r) for r in rows}):
raise SystemExit(f"{f} failed validation; not zipped")
with zipfile.ZipFile(out_dir / "predictions.zip", "w", zipfile.ZIP_DEFLATED) as z:
z.write(f, "predictions.jsonl")
print(f"wrote {out_dir / 'predictions.zip'}")
def validate(f, expected=None):
"""expected: the submission keys the file must cover exactly (default: the whole public test set)."""
expected = expected if expected is not None else {submission_key(r) for r in load_test()}
seen, errs = set(), []
with open(f, encoding="utf-8-sig") as fh:
for i, line in enumerate(fh, 1):
if not line.strip():
continue
try:
r = json.loads(line)
if type(r["q_no"]) is not int:
errs.append(f"line {i}: q_no is not an int")
k = submission_key(r)
if k in seen:
errs.append(f"line {i}: duplicate {k}")
seen.add(k)
if r["answer"] not in ("1", "2", "3", "4"):
errs.append(f"line {i}: bad answer {r['answer']!r}")
except (json.JSONDecodeError, KeyError, TypeError) as e:
errs.append(f"line {i}: {type(e).__name__} {e}")
if seen != expected:
errs.append(f"{len(expected - seen)} test keys missing, {len(seen - expected)} unknown keys")
print("format OK" if not errs else f"{len(errs)} problems:\n" + "\n".join(errs[:20]))
return not errs
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--validate", required=True)
validate(ap.parse_args().validate)
|