Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """Static independent checker for the recorded Claim 5 audit evidence.""" | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[1] | |
| ARTIFACT_DIR = ROOT / "evidence/claim5" | |
| RAW = ARTIFACT_DIR / "raw_audit_output.json" | |
| ROUTES = ARTIFACT_DIR / "four_routes.json" | |
| EXACT_GATE = ROOT / "code/claim5_exact_gate.py" | |
| def verify() -> dict: | |
| raw = json.loads(RAW.read_text()) | |
| routes = json.loads(ROUTES.read_text()) | |
| if raw["status"] != "PASS": | |
| raise AssertionError("recorded audit execution did not pass") | |
| if raw["claim_verdict"] != "BLOCKED" or raw["confidence"] != "LOW": | |
| raise AssertionError("recorded Claim 5 assessment changed") | |
| table5 = raw["exact_claim_tested"]["table5"] | |
| if table5 != { | |
| "decoder_head": 0.8, | |
| "normalized_regression_head": 0.717, | |
| "standard_regression_head": 0.478, | |
| }: | |
| raise AssertionError("Table 5 contract mismatch") | |
| table6 = raw["exact_claim_tested"]["table6"] | |
| difference = table6["t5gemma_b_b_600m"] - table6["t5gemma_s_s_300m"] | |
| if not math.isclose(difference, 0.038, abs_tol=1e-12): | |
| raise AssertionError("Table 6 difference mismatch") | |
| bases = raw["base_models"] | |
| small = bases["google/t5gemma-s-s-prefixlm"]["safetensors_parameter_count"] | |
| big = bases["google/t5gemma-b-b-prefixlm"]["safetensors_parameter_count"] | |
| if (small, big) != (312_517_632, 591_490_560): | |
| raise AssertionError("base parameter counts mismatch") | |
| if any( | |
| row["exact_table_artifact_discovered"] | |
| for row in raw["official_repository_snapshots"].values() | |
| ): | |
| raise AssertionError("record says an exact official artifact was found") | |
| if raw["author_public_models"]["exact_claim_checkpoint_discovered"]: | |
| raise AssertionError("record says an exact public checkpoint was found") | |
| cited = raw["cited_normalized_head_reference"] | |
| if cited["sha256"] != ( | |
| "e5b2e3d31177fe2fa9572cbf99bcf87ff06e5786d2f029a251734f7d7e7558f3" | |
| ) or not all(cited["features"].values()): | |
| raise AssertionError("cited implementation evidence mismatch") | |
| control = raw["negative_control"] | |
| if not control["observed_detection"] or control["passes_claim_acceptance"]: | |
| raise AssertionError("detector control behaved incorrectly") | |
| route_rows = routes["routes"] | |
| if [row["route"] for row in route_rows] != [1, 2, 3, 4]: | |
| raise AssertionError("four-route sequence incomplete") | |
| if len({row["name"] for row in route_rows}) != 4: | |
| raise AssertionError("routes are not distinct") | |
| if route_rows[-1]["name"] != "Dedicated exact-claim falsification attempt": | |
| raise AssertionError("fourth route is not falsification-dedicated") | |
| if any(row["outcome"] != "BLOCKED" for row in route_rows): | |
| raise AssertionError("unsupported route result") | |
| gate = subprocess.run( | |
| [sys.executable, str(EXACT_GATE)], | |
| cwd=ROOT, | |
| text=True, | |
| capture_output=True, | |
| check=False, | |
| ) | |
| if gate.returncode == 0 or "CLAIM5_EXACT_GATE_BLOCKED" not in gate.stdout: | |
| raise AssertionError("exact claim gate did not fail closed") | |
| return { | |
| "status": "PASS", | |
| "claim_verdict": "BLOCKED", | |
| "confidence": "LOW", | |
| "source_run_id": raw["source_run_id"], | |
| "source_git_sha": raw["source_git_sha"], | |
| "table6_difference": difference, | |
| "base_parameter_ratio": big / small, | |
| "routes_completed": len(route_rows), | |
| "fourth_route": "falsification attempted; no valid counterexample", | |
| "negative_control": control, | |
| "exact_claim_gate": { | |
| "exit_code": gate.returncode, | |
| "stdout": gate.stdout.strip(), | |
| }, | |
| "unblockers": routes["unblockers"], | |
| } | |
| if __name__ == "__main__": | |
| print(json.dumps(verify(), sort_keys=True)) | |