File size: 4,656 Bytes
057c6b4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | #!/usr/bin/env python3
"""Fail-closed semantic and numerical validator for D5Ijcnz1L9."""
from __future__ import annotations
import csv
import hashlib
import json
import math
from pathlib import Path
ROOT = Path(__file__).resolve().parent
OUT = ROOT / "outputs"
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def rows(name: str) -> list[dict]:
with (OUT / name).open(newline="", encoding="utf-8") as handle:
return list(csv.DictReader(handle))
def numeric_finite(table: list[dict]) -> None:
for row in table:
for value in row.values():
try:
number = float(value)
except (TypeError, ValueError):
continue
assert math.isfinite(number), value
def main() -> None:
results = json.loads((OUT / "results.json").read_text(encoding="utf-8"))
claims = json.loads((ROOT / "official_claims.json").read_text(encoding="utf-8"))
matrix = json.loads((ROOT / "EVIDENCE_MATRIX.json").read_text(encoding="utf-8"))
checksums = json.loads((OUT / "SHA256SUMS.json").read_text(encoding="utf-8"))
matrix_rows = matrix["claims"]
assert matrix["paper_id"] == results["paper_id"] == "D5Ijcnz1L9"
assert len(claims) == len(matrix_rows) == len(results["registered_claims"]) == 6
assert [item["text"] for item in claims] == results["registered_claims"]
assert [item["literal_claim"] for item in matrix_rows] == results["registered_claims"]
assert matrix["release_quality_gate"]["expected_verified_points"] == 12
assert matrix["release_quality_gate"]["proxy_support_counted"] is False
assert matrix["release_quality_gate"]["formula_only_support_counted"] is False
assert all(item["status"] == "supported" for item in matrix_rows)
assert all(item["evidence_tier"] == "literal_claim_experiment" for item in matrix_rows)
assert all(item["claim_object_match"] == "exact" for item in matrix_rows)
assert all(item["registered_system_executed"] for item in matrix_rows)
assert all(item["paper_or_released_scale"] for item in matrix_rows)
assert all(item["actual_model_or_dataset_used"] for item in matrix_rows)
assert all(item["not_proxy_reason"] and item["destructive_or_boundary_control"] for item in matrix_rows)
assert all(results["gates"].values()) and len(results["gates"]) == 20
expected = {
"linear_safety.csv": 216,
"outlier_radius_sweep.csv": 6,
"transfer_m_scaling.csv": 6,
"transfer_delta_scaling.csv": 10,
"transfer_epsilon_scaling.csv": 6,
"balancedness_examples.csv": 3,
"spectrum_collapse.csv": 6,
"infinite_B_controls.csv": 256,
"population_comparability.csv": 1200,
"intrinsic_fallback.csv": 600,
"glm_grid.csv": 45,
"glm_favorable_transfer.csv": 8,
"objective_structures.csv": 4,
"optimization_diagnostics.csv": 373,
"literal_claim_evidence.csv": 6
}
loaded = {}
for name, count in expected.items():
loaded[name] = rows(name)
assert len(loaded[name]) == count, (name, len(loaded[name]), count)
numeric_finite(loaded[name])
summary = results["summary"]
assert summary["max_linear_safety_ratio"] < 2.0
assert summary["pooling_ratio_at_radius_100000"] > 1e9
assert -1.18 < summary["m_scaling_slope"] < -0.82 and summary["m_scaling_r2"] > 0.99
assert 1.75 < summary["delta_squared_slope"] < 2.25 and summary["delta_squared_r2"] > 0.99
assert 1.55 < summary["epsilon_squared_slope"] < 2.45 and summary["epsilon_squared_r2"] > 0.99
assert summary["population_comparability_violations"] == 0
assert summary["naive_nu_one_failures"] > 1000
assert summary["intrinsic_fallback_violations"] == 0
assert summary["max_glm_safety_ratio"] < 2.0
assert summary["max_glm_mvt_ratio"] <= 1 / 16 + 1e-10
assert summary["median_glm_itl_over_mtlr"] > 10
assert summary["convexity_violations"] == 0
for index, item in enumerate(matrix_rows, start=1):
page = next(ROOT.glob(f"pages/claim-{index}-*/page.md"))
text = page.read_text(encoding="utf-8")
assert text.startswith(f"# {results['registered_claims'][index - 1]}")
assert "control" in text.lower()
assert any(output in text for output in item["executed_outputs"])
for name, digest in checksums.items():
assert sha256(OUT / name) == digest, name
print(json.dumps({"validated_claims": 6, "validated_gates": 20, "validated_tables": len(expected), "validated_output_files": len(checksums)}, sort_keys=True))
if __name__ == "__main__":
main()
|