File size: 4,375 Bytes
037cb1b
a564d6f
037cb1b
 
a564d6f
 
037cb1b
 
 
 
 
 
 
 
 
 
 
65f90ba
037cb1b
a564d6f
088ac34
7851389
037cb1b
 
 
a564d6f
 
037cb1b
 
 
 
 
 
6749a8b
037cb1b
 
 
a564d6f
 
 
8e2de47
 
 
037cb1b
 
 
 
 
 
a564d6f
037cb1b
 
 
 
088ac34
037cb1b
a564d6f
037cb1b
 
 
8e2de47
 
 
 
 
 
 
 
 
6749a8b
 
a564d6f
 
 
 
 
 
 
 
 
 
6749a8b
a564d6f
6749a8b
 
037cb1b
7851389
 
 
 
 
 
a564d6f
7851389
 
 
 
 
037cb1b
 
 
 
65f90ba
037cb1b
 
 
 
65f90ba
 
037cb1b
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#!/usr/bin/env python3
"""Validate questions/**/*.jsonl against schema.json. Run before opening a PR:

    python scripts/validate_questions.py

Covers both questions/corpus.jsonl and every submission in questions/pending/.
"""
import json
import sys
from pathlib import Path

try:
    import jsonschema
except ImportError:
    sys.exit("Missing dependency: pip install jsonschema")

ROOT = Path(__file__).resolve().parent.parent
SCHEMA = json.loads((ROOT / "schema.json").read_text(encoding="utf-8"))
QUESTIONS_DIR = ROOT / "questions"
PENDING_DIR = QUESTIONS_DIR / "pending"
FORMAT_CHECKER = jsonschema.FormatChecker()
BAND_FOR_TOTAL = {0: "easy", 1: "easy", 2: "medium", 3: "medium", 4: "hard"}


def main() -> int:
    # Include recursive files so we validate corpus.jsonl and pending/*.jsonl
    files = sorted(QUESTIONS_DIR.rglob("*.jsonl"))
    files = [f for f in files if f.name != "template.jsonl"]
    if not files:
        print("No question files found in questions/ (besides template.jsonl).")
        return 1

    seen_ids = set()
    total = 0
    errors = 0

    for path in files:
        label = path.relative_to(QUESTIONS_DIR).as_posix()
        in_pending = PENDING_DIR in path.parents

        # split("\n"), not splitlines(): that also breaks at U+2028, U+2029 and U+0085, which
        # JSON allows unescaped inside a string, so it would cut a valid question in two.
        for lineno, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1):
            line = line.strip()
            if not line:
                continue
            try:
                item = json.loads(line)
            except json.JSONDecodeError as e:
                print(f"{label}:{lineno}: invalid JSON ({e})")
                errors += 1
                continue

            try:
                jsonschema.validate(item, SCHEMA, format_checker=FORMAT_CHECKER)
            except jsonschema.ValidationError as e:
                print(f"{label}:{lineno}: {e.message}")
                errors += 1
                continue

            try:
                json.dumps(item, ensure_ascii=False).encode("utf-8")
            except UnicodeEncodeError as e:
                # "\ud800" is valid JSON and passes the schema, but is not text: an eval run
                # would fail writing its log, and take the remaining items with it.
                print(f"{label}:{lineno}: contains a lone surrogate ({e.reason}), usually half an emoji")
                errors += 1
                continue

            total += 1

            # Submissions live in pending/ with the cb-pending placeholder; the
            # maintainer assigns real ids when moving them into corpus.jsonl
            # (see scripts/assign_ids.py). Multiple items may carry cb-pending.
            if in_pending and item["id"] != "cb-pending":
                print(f"{label}:{lineno}: files in pending/ must use id 'cb-pending', got {item['id']!r}")
                errors += 1
            elif not in_pending and item["id"] == "cb-pending":
                print(f"{label}:{lineno}: 'cb-pending' outside pending/; run scripts/assign_ids.py")
                errors += 1
            elif item["id"] != "cb-pending":
                if item["id"] in seen_ids:
                    print(f"{label}:{lineno}: duplicate id {item['id']!r}")
                    errors += 1
                seen_ids.add(item["id"])

            difficulty = item.get("difficulty")
            if isinstance(difficulty, dict):
                score = difficulty["evidence_scarcity"] + difficulty["question_openness"]
                expected_band = BAND_FOR_TOTAL[score]
                if difficulty["band"] != expected_band:
                    print(
                        f"{label}:{lineno}: difficulty.band {difficulty['band']!r} "
                        f"does not match evidence_scarcity + question_openness = {score} "
                        f"(expected {expected_band!r})"
                    )
                    errors += 1

    if errors:
        print(f"\n{errors} error(s).")
        return 1

    print(f"OK - {total} question(s) validated across {len(files)} file(s).")
    return 0


if __name__ == "__main__":
    # Questions can be in any script; a console that cannot show it must not crash the run.
    sys.stdout.reconfigure(errors="backslashreplace")
    raise SystemExit(main())