#!/usr/bin/env python3 """Independent standard-library checker for Claim 3 release integrity.""" from __future__ import annotations import json import sys from pathlib import Path EXPECTED_NOT = { "constant_cs": 0.75, "constant_fid": 32.87, "logdiff_cs": 0.80, "logdiff_fid": 23.61, } def main() -> None: if len(sys.argv) != 2: raise SystemExit("usage: check_claim3_release.py ARTIFACT_DIR") root = Path(sys.argv[1]) / "claim-3" raw = json.loads((root / "release_audit.json").read_text()) control = json.loads((root / "negative_control.json").read_text()) routes = json.loads((root / "verification_routes.json").read_text()) search = json.loads((root / "public_checkpoint_search_summary.json").read_text()) falsification = json.loads((root / "falsification_check.json").read_text()) failures: list[str] = [] if raw["source_commit"] != "94ef35bafd4b4239e9832d8295128c09e8fc1472": failures.append("source_commit") if raw["tree_file_count"] != 73: failures.append("tree_file_count") if len(raw["missing_checkpoint_paths"]) != 3: failures.append("checkpoint_count") if not raw["missing_composition_dataset_config"]: failures.append("dataset_config") if not raw["fid_implementation_mismatch"]: failures.append("fid_protocol") if raw["generated_samples_per_task"] != 5000 or raw["reported_samples_argument"] != 500000: failures.append("sample_accounting") if raw["table_3"]["NOT"] != EXPECTED_NOT: failures.append("table_transcription") if raw["claim_status"] != "BLOCKED" or raw["empirical_generation_executed"]: failures.append("fail_closed_status") if control["audit_status"] != "READY_TO_RUN" or not control["rejected_false_block"]: failures.append("negative_control") route_rows = routes.get("routes", []) if [row.get("route") for row in route_rows] != [1, 2, 3, 4]: failures.append("four_route_sequence") if len({row.get("name") for row in route_rows}) != 4: failures.append("distinct_route_names") if routes.get("final_status") != "BLOCKED" or routes.get("confidence") != "LOW": failures.append("route_fail_closed_status") if route_rows and route_rows[-1].get("falsification_succeeded") is not False: failures.append("falsification_route_status") if search.get("author_equivalent_checkpoint_found") is not False: failures.append("checkpoint_search_status") proxy = search.get("public_proxy_model", {}) if ( proxy.get("revision") != "91d0ff096e031c21b8313bd5877316a52900d4ec" or proxy.get("unet_bytes") != 114049969 or proxy.get("exact_protocol_equivalent") is not False ): failures.append("public_model_control") if ( falsification.get("claimed_not_fid_ordering_holds_in_paper_table") is not True or falsification.get("available_evidence_admissible_as_counterexample") is not False or falsification.get("falsification_succeeded") is not False or falsification.get("status") != "BLOCKED" ): failures.append("falsification_check") if not falsification.get("synthetic_negative_control", {}).get("reversal_detected"): failures.append("falsification_negative_control") result = { "checker": "independent Claim 3 release-integrity checker", "status": "PASS" if not failures else "FAIL", "failures": failures, "verification_routes_rederived": len(route_rows), "scientific_claim_status": raw.get("claim_status"), } print(json.dumps(result, indent=2, sort_keys=True)) if failures: raise SystemExit(1) if __name__ == "__main__": main()