| |
| """Validate the generated PhantomWiki 100Q static bundle.""" |
|
|
| from __future__ import annotations |
|
|
| import base64 |
| import gzip |
| import hashlib |
| import json |
| from pathlib import Path |
|
|
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| EXPECTED_SHA256 = ( |
| "69ac438794500cb64239009c3cb3a4d70f68b2073d29f592e246db1edd84cd6c" |
| ) |
|
|
|
|
| def read_json(path: Path): |
| return json.loads(path.read_text(encoding="utf-8")) |
|
|
|
|
| def qid_sha256(qids: set[str]) -> str: |
| return hashlib.sha256("\n".join(sorted(qids)).encode()).hexdigest() |
|
|
|
|
| def require_qids(label: str, actual: set[str], expected: set[str]) -> None: |
| if actual != expected: |
| raise ValueError( |
| f"{label}: qid mismatch; missing={sorted(expected - actual)[:5]} " |
| f"extra={sorted(actual - expected)[:5]}" |
| ) |
|
|
|
|
| def main() -> None: |
| manifest = read_json(ROOT / "data/manifest.json") |
| qids = set(read_json(ROOT / "data/qids.json")) |
| if len(qids) != 100: |
| raise ValueError(f"expected 100 qids, got {len(qids)}") |
| if qid_sha256(qids) != EXPECTED_SHA256: |
| raise ValueError("fixed QID SHA-256 mismatch") |
| if manifest["qid_sha256"] != EXPECTED_SHA256: |
| raise ValueError("manifest QID SHA-256 mismatch") |
|
|
| eval_rows = read_json(ROOT / manifest["files"]["eval"]) |
| require_qids("eval", {str(row["id"]) for row in eval_rows}, qids) |
|
|
| structures = read_json(ROOT / manifest["files"]["structures"]) |
| require_qids( |
| "structures", |
| {str(row["qid"]) for row in structures["rows"]}, |
| qids, |
| ) |
| for row in structures["rows"]: |
| path = ROOT / "data/structures" / row["path"] |
| if not path.is_file(): |
| raise FileNotFoundError(path) |
| payload = json.loads( |
| gzip.decompress(base64.b64decode(path.read_text(encoding="ascii"))) |
| ) |
| if str(payload["qid"]) != str(row["qid"]): |
| raise ValueError(f"{path}: qid mismatch") |
|
|
| run_slots = [] |
| for run in manifest["runs"]: |
| run_slots.append(run["slot"]) |
| index = read_json(ROOT / run["index"]) |
| run_qids = {str(row["qid"]) for row in index["records"]} |
| require_qids(f"run {run['slot']}", run_qids, qids) |
| for qid in qids: |
| result_path = ROOT / run["results"] / f"{qid}.json" |
| trajectory_path = ROOT / run["trajectories"] / f"{qid}.json" |
| if not result_path.is_file(): |
| raise FileNotFoundError(result_path) |
| if not trajectory_path.is_file(): |
| raise FileNotFoundError(trajectory_path) |
| if read_json(result_path)["qid"] != qid: |
| raise ValueError(f"{result_path}: qid mismatch") |
| if read_json(trajectory_path)["qid"] != qid: |
| raise ValueError(f"{trajectory_path}: qid mismatch") |
|
|
| compare = read_json(ROOT / manifest["files"]["compare"]) |
| if compare["slots"] != run_slots: |
| raise ValueError("compare slot order does not match run manifest") |
| require_qids( |
| "compare", |
| {str(row["qid"]) for row in compare["records"]}, |
| qids, |
| ) |
| for qid in qids: |
| record = read_json(ROOT / "data/compare/records" / f"{qid}.json") |
| if set(record["runs"]) != set(run_slots): |
| raise ValueError(f"compare {qid}: run set mismatch") |
|
|
| forbidden = [] |
| if (ROOT / "sets.json").exists(): |
| forbidden.append("sets.json") |
| forbidden.extend(path.name for path in ROOT.glob("corpus_size*_seed*.json")) |
| forbidden.extend(path.name for path in ROOT.glob("eval_size*_seed*.json")) |
| if forbidden: |
| raise ValueError(f"legacy multi-universe files present: {sorted(set(forbidden))}") |
|
|
| print( |
| f"validated 100 qids, {len(run_slots)} runs, " |
| f"{len(structures['rows'])} structure records" |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
