#!/usr/bin/env python3 """Independent standard-library checker for the Claim 4 evidence audit.""" from __future__ import annotations import hashlib import json import re import sys from pathlib import Path SLICE_RE = re.compile( r"campaigns/(logdiff_and|logdiff_and_not)/run-seed-(\d+)/" r"slice-(s\d+)-samples-(\d+)-(\d+)/contract-([0-9a-f]{64})/SUCCESS\.json$" ) def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def main() -> None: if len(sys.argv) != 2: raise SystemExit("usage: check_claim4_evidence.py ARTIFACT_DIR") root = Path(sys.argv[1]) / "claim-4" snapshot = root / "bucket-json-snapshot" summary = json.loads((root / "bucket_audit.json").read_text()) control = json.loads((root / "negative_control.json").read_text()) routes = json.loads((root / "verification_routes.json").read_text()) falsification = json.loads((root / "falsification_check.json").read_text()) failures: list[str] = [] slices = [] bindings = set() for success_path in snapshot.rglob("SUCCESS.json"): relative = success_path.relative_to(snapshot).as_posix() match = SLICE_RE.match(relative) if not match: continue logic, seed, slice_id, start, stop, contract = match.groups() success = json.loads(success_path.read_text()) manifest_path = success_path.with_name("SLICE_MANIFEST.json") manifest = json.loads(manifest_path.read_text()) if success.get("status") != "complete": failures.append(f"slice_status:{relative}") if success.get("slice_manifest_sha256") != sha256(manifest_path): failures.append(f"slice_manifest_hash:{relative}") if ( success.get("logic") != logic or int(success.get("run_seed", -1)) != int(seed) or success.get("slice_id") != slice_id ): failures.append(f"slice_identity:{relative}") if manifest.get("sample_indices") != list(range(int(start), int(stop))): failures.append(f"sample_range:{relative}") binding_text = json.dumps(success.get("binding"), sort_keys=True) bindings.add(binding_text) if success.get("binding") != manifest.get("binding"): failures.append(f"binding_mismatch:{relative}") for sample in manifest.get("samples", []): index = int(sample["sample_index"]) sample_root = success_path.parent / "samples" / f"sample-{index:03d}" sample_success = sample_root / "SUCCESS.json" files_manifest = sample_root / "FILES.json" if not sample_success.exists() or sha256(sample_success) != sample["success_sha256"]: failures.append(f"sample_success_hash:{relative}:{index}") continue sample_record = json.loads(sample_success.read_text()) if ( sample_record.get("logic") != logic or sample_record.get("run_seed") != int(seed) or sample_record.get("sample_index") != index or sample_record.get("sample_seed") != int(seed) * 100000 + index or sample_record.get("num_steps") != 1000 or sample_record.get("execution_device") != "cpu" ): failures.append(f"sample_identity:{relative}:{index}") if not files_manifest.exists() or sha256(files_manifest) != sample_record.get( "files_manifest_sha256" ): failures.append(f"files_manifest_hash:{relative}:{index}") slices.append((logic, int(seed), slice_id)) if len(slices) != 12 or len(set(slices)) != 12: failures.append("terminal_slice_count") if len(bindings) != 1: failures.append("uniform_binding") if summary.get("terminal_slices") != 12 or summary.get("manifest_bound_samples") != 96: failures.append("summary_counts") if summary.get("complete_logdiff_experiments") != 0: failures.append("complete_experiment_count") if summary.get("payload_paths_in_listing") != 96: failures.append("payload_listing_count") if summary.get("dualdiff_experiments") != 0 or summary.get("docking_result_files") != 0: failures.append("missing_routes_misreported") if summary.get("claim_status") != "BLOCKED": failures.append("fail_closed_status") if control.get("audit_status") != "READY_FOR_CLAIM_CHECK" or not control.get( "rejected_false_block" ): failures.append("negative_control") route_rows = routes.get("routes", []) if [row.get("route") for row in route_rows] != [1, 2, 3, 4]: failures.append("four_route_sequence") if len({row.get("name") for row in route_rows}) != 4: failures.append("distinct_route_names") if routes.get("final_status") != "BLOCKED" or routes.get("confidence") != "LOW": failures.append("route_fail_closed_status") if route_rows and route_rows[-1].get("falsification_succeeded") is not False: failures.append("falsification_route_status") if ( falsification.get("claimed_ordering_holds_in_paper_table") is not True or falsification.get("recovered_evidence_admissible_as_counterexample") is not False or falsification.get("falsification_succeeded") is not False or falsification.get("status") != "BLOCKED" ): failures.append("falsification_check") if not falsification.get("synthetic_negative_control", {}).get( "contradiction_detected" ): failures.append("falsification_negative_control") result = { "checker": "independent Claim 4 manifest and completeness checker", "status": "PASS" if not failures else "FAIL", "failures": failures, "terminal_slices_rederived": len(slices), "verification_routes_rederived": len(route_rows), "scientific_claim_status": summary.get("claim_status"), } print(json.dumps(result, indent=2, sort_keys=True)) if failures: raise SystemExit(1) if __name__ == "__main__": main()