File size: 1,103 Bytes
1f3cf4d 52c7206 1f3cf4d f71d8d8 1f3cf4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | #!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from pathlib import Path
REQUIRED = ("qhash", "question", "gold_answer")
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--input", required=True, help="Path to non-STEM QA JSONL file")
args = ap.parse_args()
path = Path(args.input)
if not path.exists():
raise SystemExit(f"Input file does not exist: {path}")
n = 0
with path.open("r", encoding="utf-8", errors="ignore") as f:
for i, line in enumerate(f, 1):
s = line.strip()
if not s:
continue
n += 1
try:
obj = json.loads(s)
except Exception as exc: # noqa: BLE001
raise SystemExit(f"Line {i} is not valid JSON: {exc}") from exc
missing = [k for k in REQUIRED if k not in obj]
if missing:
raise SystemExit(f"Line {i} missing fields: {missing}")
print(f"OK: {n} rows validated with fields {REQUIRED}")
if __name__ == "__main__":
main()
|