File size: 3,832 Bytes
957c949 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | #!/usr/bin/env python3
"""Validate the generated PhantomWiki 100Q static bundle."""
from __future__ import annotations
import base64
import gzip
import hashlib
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EXPECTED_SHA256 = (
"69ac438794500cb64239009c3cb3a4d70f68b2073d29f592e246db1edd84cd6c"
)
def read_json(path: Path):
return json.loads(path.read_text(encoding="utf-8"))
def qid_sha256(qids: set[str]) -> str:
return hashlib.sha256("\n".join(sorted(qids)).encode()).hexdigest()
def require_qids(label: str, actual: set[str], expected: set[str]) -> None:
if actual != expected:
raise ValueError(
f"{label}: qid mismatch; missing={sorted(expected - actual)[:5]} "
f"extra={sorted(actual - expected)[:5]}"
)
def main() -> None:
manifest = read_json(ROOT / "data/manifest.json")
qids = set(read_json(ROOT / "data/qids.json"))
if len(qids) != 100:
raise ValueError(f"expected 100 qids, got {len(qids)}")
if qid_sha256(qids) != EXPECTED_SHA256:
raise ValueError("fixed QID SHA-256 mismatch")
if manifest["qid_sha256"] != EXPECTED_SHA256:
raise ValueError("manifest QID SHA-256 mismatch")
eval_rows = read_json(ROOT / manifest["files"]["eval"])
require_qids("eval", {str(row["id"]) for row in eval_rows}, qids)
structures = read_json(ROOT / manifest["files"]["structures"])
require_qids(
"structures",
{str(row["qid"]) for row in structures["rows"]},
qids,
)
for row in structures["rows"]:
path = ROOT / "data/structures" / row["path"]
if not path.is_file():
raise FileNotFoundError(path)
payload = json.loads(
gzip.decompress(base64.b64decode(path.read_text(encoding="ascii")))
)
if str(payload["qid"]) != str(row["qid"]):
raise ValueError(f"{path}: qid mismatch")
run_slots = []
for run in manifest["runs"]:
run_slots.append(run["slot"])
index = read_json(ROOT / run["index"])
run_qids = {str(row["qid"]) for row in index["records"]}
require_qids(f"run {run['slot']}", run_qids, qids)
for qid in qids:
result_path = ROOT / run["results"] / f"{qid}.json"
trajectory_path = ROOT / run["trajectories"] / f"{qid}.json"
if not result_path.is_file():
raise FileNotFoundError(result_path)
if not trajectory_path.is_file():
raise FileNotFoundError(trajectory_path)
if read_json(result_path)["qid"] != qid:
raise ValueError(f"{result_path}: qid mismatch")
if read_json(trajectory_path)["qid"] != qid:
raise ValueError(f"{trajectory_path}: qid mismatch")
compare = read_json(ROOT / manifest["files"]["compare"])
if compare["slots"] != run_slots:
raise ValueError("compare slot order does not match run manifest")
require_qids(
"compare",
{str(row["qid"]) for row in compare["records"]},
qids,
)
for qid in qids:
record = read_json(ROOT / "data/compare/records" / f"{qid}.json")
if set(record["runs"]) != set(run_slots):
raise ValueError(f"compare {qid}: run set mismatch")
forbidden = []
if (ROOT / "sets.json").exists():
forbidden.append("sets.json")
forbidden.extend(path.name for path in ROOT.glob("corpus_size*_seed*.json"))
forbidden.extend(path.name for path in ROOT.glob("eval_size*_seed*.json"))
if forbidden:
raise ValueError(f"legacy multi-universe files present: {sorted(set(forbidden))}")
print(
f"validated 100 qids, {len(run_slots)} runs, "
f"{len(structures['rows'])} structure records"
)
if __name__ == "__main__":
main()
|