Spaces:
Running
Running
File size: 4,375 Bytes
037cb1b a564d6f 037cb1b a564d6f 037cb1b 65f90ba 037cb1b a564d6f 088ac34 7851389 037cb1b a564d6f 037cb1b 6749a8b 037cb1b a564d6f 8e2de47 037cb1b a564d6f 037cb1b 088ac34 037cb1b a564d6f 037cb1b 8e2de47 6749a8b a564d6f 6749a8b a564d6f 6749a8b 037cb1b 7851389 a564d6f 7851389 037cb1b 65f90ba 037cb1b 65f90ba 037cb1b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | #!/usr/bin/env python3
"""Validate questions/**/*.jsonl against schema.json. Run before opening a PR:
python scripts/validate_questions.py
Covers both questions/corpus.jsonl and every submission in questions/pending/.
"""
import json
import sys
from pathlib import Path
try:
import jsonschema
except ImportError:
sys.exit("Missing dependency: pip install jsonschema")
ROOT = Path(__file__).resolve().parent.parent
SCHEMA = json.loads((ROOT / "schema.json").read_text(encoding="utf-8"))
QUESTIONS_DIR = ROOT / "questions"
PENDING_DIR = QUESTIONS_DIR / "pending"
FORMAT_CHECKER = jsonschema.FormatChecker()
BAND_FOR_TOTAL = {0: "easy", 1: "easy", 2: "medium", 3: "medium", 4: "hard"}
def main() -> int:
# Include recursive files so we validate corpus.jsonl and pending/*.jsonl
files = sorted(QUESTIONS_DIR.rglob("*.jsonl"))
files = [f for f in files if f.name != "template.jsonl"]
if not files:
print("No question files found in questions/ (besides template.jsonl).")
return 1
seen_ids = set()
total = 0
errors = 0
for path in files:
label = path.relative_to(QUESTIONS_DIR).as_posix()
in_pending = PENDING_DIR in path.parents
# split("\n"), not splitlines(): that also breaks at U+2028, U+2029 and U+0085, which
# JSON allows unescaped inside a string, so it would cut a valid question in two.
for lineno, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1):
line = line.strip()
if not line:
continue
try:
item = json.loads(line)
except json.JSONDecodeError as e:
print(f"{label}:{lineno}: invalid JSON ({e})")
errors += 1
continue
try:
jsonschema.validate(item, SCHEMA, format_checker=FORMAT_CHECKER)
except jsonschema.ValidationError as e:
print(f"{label}:{lineno}: {e.message}")
errors += 1
continue
try:
json.dumps(item, ensure_ascii=False).encode("utf-8")
except UnicodeEncodeError as e:
# "\ud800" is valid JSON and passes the schema, but is not text: an eval run
# would fail writing its log, and take the remaining items with it.
print(f"{label}:{lineno}: contains a lone surrogate ({e.reason}), usually half an emoji")
errors += 1
continue
total += 1
# Submissions live in pending/ with the cb-pending placeholder; the
# maintainer assigns real ids when moving them into corpus.jsonl
# (see scripts/assign_ids.py). Multiple items may carry cb-pending.
if in_pending and item["id"] != "cb-pending":
print(f"{label}:{lineno}: files in pending/ must use id 'cb-pending', got {item['id']!r}")
errors += 1
elif not in_pending and item["id"] == "cb-pending":
print(f"{label}:{lineno}: 'cb-pending' outside pending/; run scripts/assign_ids.py")
errors += 1
elif item["id"] != "cb-pending":
if item["id"] in seen_ids:
print(f"{label}:{lineno}: duplicate id {item['id']!r}")
errors += 1
seen_ids.add(item["id"])
difficulty = item.get("difficulty")
if isinstance(difficulty, dict):
score = difficulty["evidence_scarcity"] + difficulty["question_openness"]
expected_band = BAND_FOR_TOTAL[score]
if difficulty["band"] != expected_band:
print(
f"{label}:{lineno}: difficulty.band {difficulty['band']!r} "
f"does not match evidence_scarcity + question_openness = {score} "
f"(expected {expected_band!r})"
)
errors += 1
if errors:
print(f"\n{errors} error(s).")
return 1
print(f"OK - {total} question(s) validated across {len(files)} file(s).")
return 0
if __name__ == "__main__":
# Questions can be in any script; a console that cannot show it must not crash the run.
sys.stdout.reconfigure(errors="backslashreplace")
raise SystemExit(main())
|