Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """Independent standard-library checker for the Claim 4 evidence audit.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| SLICE_RE = re.compile( | |
| r"campaigns/(logdiff_and|logdiff_and_not)/run-seed-(\d+)/" | |
| r"slice-(s\d+)-samples-(\d+)-(\d+)/contract-([0-9a-f]{64})/SUCCESS\.json$" | |
| ) | |
| def sha256(path: Path) -> str: | |
| return hashlib.sha256(path.read_bytes()).hexdigest() | |
| def main() -> None: | |
| if len(sys.argv) != 2: | |
| raise SystemExit("usage: check_claim4_evidence.py ARTIFACT_DIR") | |
| root = Path(sys.argv[1]) / "claim-4" | |
| snapshot = root / "bucket-json-snapshot" | |
| summary = json.loads((root / "bucket_audit.json").read_text()) | |
| control = json.loads((root / "negative_control.json").read_text()) | |
| routes = json.loads((root / "verification_routes.json").read_text()) | |
| falsification = json.loads((root / "falsification_check.json").read_text()) | |
| failures: list[str] = [] | |
| slices = [] | |
| bindings = set() | |
| for success_path in snapshot.rglob("SUCCESS.json"): | |
| relative = success_path.relative_to(snapshot).as_posix() | |
| match = SLICE_RE.match(relative) | |
| if not match: | |
| continue | |
| logic, seed, slice_id, start, stop, contract = match.groups() | |
| success = json.loads(success_path.read_text()) | |
| manifest_path = success_path.with_name("SLICE_MANIFEST.json") | |
| manifest = json.loads(manifest_path.read_text()) | |
| if success.get("status") != "complete": | |
| failures.append(f"slice_status:{relative}") | |
| if success.get("slice_manifest_sha256") != sha256(manifest_path): | |
| failures.append(f"slice_manifest_hash:{relative}") | |
| if ( | |
| success.get("logic") != logic | |
| or int(success.get("run_seed", -1)) != int(seed) | |
| or success.get("slice_id") != slice_id | |
| ): | |
| failures.append(f"slice_identity:{relative}") | |
| if manifest.get("sample_indices") != list(range(int(start), int(stop))): | |
| failures.append(f"sample_range:{relative}") | |
| binding_text = json.dumps(success.get("binding"), sort_keys=True) | |
| bindings.add(binding_text) | |
| if success.get("binding") != manifest.get("binding"): | |
| failures.append(f"binding_mismatch:{relative}") | |
| for sample in manifest.get("samples", []): | |
| index = int(sample["sample_index"]) | |
| sample_root = success_path.parent / "samples" / f"sample-{index:03d}" | |
| sample_success = sample_root / "SUCCESS.json" | |
| files_manifest = sample_root / "FILES.json" | |
| if not sample_success.exists() or sha256(sample_success) != sample["success_sha256"]: | |
| failures.append(f"sample_success_hash:{relative}:{index}") | |
| continue | |
| sample_record = json.loads(sample_success.read_text()) | |
| if ( | |
| sample_record.get("logic") != logic | |
| or sample_record.get("run_seed") != int(seed) | |
| or sample_record.get("sample_index") != index | |
| or sample_record.get("sample_seed") != int(seed) * 100000 + index | |
| or sample_record.get("num_steps") != 1000 | |
| or sample_record.get("execution_device") != "cpu" | |
| ): | |
| failures.append(f"sample_identity:{relative}:{index}") | |
| if not files_manifest.exists() or sha256(files_manifest) != sample_record.get( | |
| "files_manifest_sha256" | |
| ): | |
| failures.append(f"files_manifest_hash:{relative}:{index}") | |
| slices.append((logic, int(seed), slice_id)) | |
| if len(slices) != 12 or len(set(slices)) != 12: | |
| failures.append("terminal_slice_count") | |
| if len(bindings) != 1: | |
| failures.append("uniform_binding") | |
| if summary.get("terminal_slices") != 12 or summary.get("manifest_bound_samples") != 96: | |
| failures.append("summary_counts") | |
| if summary.get("complete_logdiff_experiments") != 0: | |
| failures.append("complete_experiment_count") | |
| if summary.get("payload_paths_in_listing") != 96: | |
| failures.append("payload_listing_count") | |
| if summary.get("dualdiff_experiments") != 0 or summary.get("docking_result_files") != 0: | |
| failures.append("missing_routes_misreported") | |
| if summary.get("claim_status") != "BLOCKED": | |
| failures.append("fail_closed_status") | |
| if control.get("audit_status") != "READY_FOR_CLAIM_CHECK" or not control.get( | |
| "rejected_false_block" | |
| ): | |
| failures.append("negative_control") | |
| route_rows = routes.get("routes", []) | |
| if [row.get("route") for row in route_rows] != [1, 2, 3, 4]: | |
| failures.append("four_route_sequence") | |
| if len({row.get("name") for row in route_rows}) != 4: | |
| failures.append("distinct_route_names") | |
| if routes.get("final_status") != "BLOCKED" or routes.get("confidence") != "LOW": | |
| failures.append("route_fail_closed_status") | |
| if route_rows and route_rows[-1].get("falsification_succeeded") is not False: | |
| failures.append("falsification_route_status") | |
| if ( | |
| falsification.get("claimed_ordering_holds_in_paper_table") is not True | |
| or falsification.get("recovered_evidence_admissible_as_counterexample") is not False | |
| or falsification.get("falsification_succeeded") is not False | |
| or falsification.get("status") != "BLOCKED" | |
| ): | |
| failures.append("falsification_check") | |
| if not falsification.get("synthetic_negative_control", {}).get( | |
| "contradiction_detected" | |
| ): | |
| failures.append("falsification_negative_control") | |
| result = { | |
| "checker": "independent Claim 4 manifest and completeness checker", | |
| "status": "PASS" if not failures else "FAIL", | |
| "failures": failures, | |
| "terminal_slices_rederived": len(slices), | |
| "verification_routes_rederived": len(route_rows), | |
| "scientific_claim_status": summary.get("claim_status"), | |
| } | |
| print(json.dumps(result, indent=2, sort_keys=True)) | |
| if failures: | |
| raise SystemExit(1) | |
| if __name__ == "__main__": | |
| main() | |