Spaces:
Running
Running
File size: 3,950 Bytes
f791e67 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | #!/usr/bin/env python3
"""Static independent checker for the recorded Claim 5 audit evidence."""
from __future__ import annotations
import json
import math
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ARTIFACT_DIR = ROOT / "evidence/claim5"
RAW = ARTIFACT_DIR / "raw_audit_output.json"
ROUTES = ARTIFACT_DIR / "four_routes.json"
EXACT_GATE = ROOT / "code/claim5_exact_gate.py"
def verify() -> dict:
raw = json.loads(RAW.read_text())
routes = json.loads(ROUTES.read_text())
if raw["status"] != "PASS":
raise AssertionError("recorded audit execution did not pass")
if raw["claim_verdict"] != "BLOCKED" or raw["confidence"] != "LOW":
raise AssertionError("recorded Claim 5 assessment changed")
table5 = raw["exact_claim_tested"]["table5"]
if table5 != {
"decoder_head": 0.8,
"normalized_regression_head": 0.717,
"standard_regression_head": 0.478,
}:
raise AssertionError("Table 5 contract mismatch")
table6 = raw["exact_claim_tested"]["table6"]
difference = table6["t5gemma_b_b_600m"] - table6["t5gemma_s_s_300m"]
if not math.isclose(difference, 0.038, abs_tol=1e-12):
raise AssertionError("Table 6 difference mismatch")
bases = raw["base_models"]
small = bases["google/t5gemma-s-s-prefixlm"]["safetensors_parameter_count"]
big = bases["google/t5gemma-b-b-prefixlm"]["safetensors_parameter_count"]
if (small, big) != (312_517_632, 591_490_560):
raise AssertionError("base parameter counts mismatch")
if any(
row["exact_table_artifact_discovered"]
for row in raw["official_repository_snapshots"].values()
):
raise AssertionError("record says an exact official artifact was found")
if raw["author_public_models"]["exact_claim_checkpoint_discovered"]:
raise AssertionError("record says an exact public checkpoint was found")
cited = raw["cited_normalized_head_reference"]
if cited["sha256"] != (
"e5b2e3d31177fe2fa9572cbf99bcf87ff06e5786d2f029a251734f7d7e7558f3"
) or not all(cited["features"].values()):
raise AssertionError("cited implementation evidence mismatch")
control = raw["negative_control"]
if not control["observed_detection"] or control["passes_claim_acceptance"]:
raise AssertionError("detector control behaved incorrectly")
route_rows = routes["routes"]
if [row["route"] for row in route_rows] != [1, 2, 3, 4]:
raise AssertionError("four-route sequence incomplete")
if len({row["name"] for row in route_rows}) != 4:
raise AssertionError("routes are not distinct")
if route_rows[-1]["name"] != "Dedicated exact-claim falsification attempt":
raise AssertionError("fourth route is not falsification-dedicated")
if any(row["outcome"] != "BLOCKED" for row in route_rows):
raise AssertionError("unsupported route result")
gate = subprocess.run(
[sys.executable, str(EXACT_GATE)],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
if gate.returncode == 0 or "CLAIM5_EXACT_GATE_BLOCKED" not in gate.stdout:
raise AssertionError("exact claim gate did not fail closed")
return {
"status": "PASS",
"claim_verdict": "BLOCKED",
"confidence": "LOW",
"source_run_id": raw["source_run_id"],
"source_git_sha": raw["source_git_sha"],
"table6_difference": difference,
"base_parameter_ratio": big / small,
"routes_completed": len(route_rows),
"fourth_route": "falsification attempted; no valid counterexample",
"negative_control": control,
"exact_claim_gate": {
"exit_code": gate.returncode,
"stdout": gate.stdout.strip(),
},
"unblockers": routes["unblockers"],
}
if __name__ == "__main__":
print(json.dumps(verify(), sort_keys=True))
|