validate_evidence.py

#!/usr/bin/env python3
"""Fail-closed semantic-v4 validation for the six-claim PolyILR package."""

from __future__ import annotations

import csv
import hashlib
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path


ROOT = Path(__file__).resolve().parent
PAPER = "pws8t4kBP4"
SOURCE_HASHES = {
    "2606.11646v1.pdf": "96005baa7970c1362253f3feea3b1d1875119a393323d595755bf62a7e7688a4",
    "2606.11646v1.tar": "d9f632e55c8b7d87805eec6d458b6e5d13b7bad4371e3d89c5f69c07a6c7b306",
}
HMP_HASH = "162c089e05a3df6a5dc9c0bf3675c5f07a1a9449cbe22b060217d6099519b1e0"
OUTPUT_NAMES = {
    "composition_isometry.csv",
    "hmp_destructive_control.csv",
    "hmp_native_rf_folds.csv",
    "hmp_tree_importance.csv",
    "logit_quotient.csv",
    "results.json",
    "source_table_audit.json",
    "tree_certificates.csv",
}


def load(relative: str):
    return json.loads((ROOT / relative).read_text(encoding="utf-8"))


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1 << 20), b""):
            digest.update(chunk)
    return digest.hexdigest()


def csv_rows(relative: str) -> list[dict[str, str]]:
    with (ROOT / relative).open(newline="", encoding="utf-8") as handle:
        return list(csv.DictReader(handle))


claims = load("official_claims.json")
assert claims == load("CLAIMS.json")
assert len(claims) == 6 and len(set(claims)) == 6

matrix = load("EVIDENCE_MATRIX.json")
assert matrix["paper_id"] == PAPER
gate = matrix["release_quality_gate"]
assert gate["semantic_quality_gate_version"] == 4
assert gate["status"] == "pass_all_6_direct"
assert gate["expected_verified_points"] == 12
assert gate["supported_by_independent_evidence"] == 6
assert gate["proxy_support_counted"] is False
assert gate["formula_only_support_counted"] is False
assert gate["paired_replay"] == "byte-identical"
assert [row["literal_claim"] for row in matrix["claims"]] == claims
assert [row["claim"] for row in matrix["claims"]] == list(range(1, 7))
for row in matrix["claims"]:
    assert row["assessment"] == "verified"
    assert row["expected_points"] == 2
    assert row["direct_evidence"] is True
    assert row["registered_system_executed"] is True
    assert row["paper_or_released_scale"] is True
    assert row["actual_model_or_dataset_used"] is True
    assert row["destructive_control_executed"] is True
    assert row["claim_object_match"] == "exact"
    assert row["executed_outputs"] and row["oracle_artifacts"] and row["control_artifacts"]
    for relative in row["executed_outputs"] + row["oracle_artifacts"] + row["control_artifacts"]:
        assert (ROOT / relative).is_file(), relative

for filename, digest in SOURCE_HASHES.items():
    assert sha256(ROOT / "source" / filename) == digest
assert sha256(ROOT / "source/data/hmp_v35_polyilr_aggregate.npz") == HMP_HASH
metadata = load("source/data/hmp_v35_polyilr_metadata.json")
assert metadata["output_sha256"] == HMP_HASH
assert metadata["archive_sha256"] == "9c5eab65cebc22c005189517e05190c44cb144c9eb73cc11916018df624f33b4"
assert (metadata["sample_count"], metadata["taxon_count"], metadata["subsite_count"]) == (4743, 402, 18)
assert sum(metadata["site_counts"].values()) == 4743 and len(metadata["site_counts"]) == 5

for directory in ("outputs", "replay_a", "replay_b", "packaged_replay"):
    present = {path.name for path in (ROOT / directory).iterdir() if path.is_file()}
    assert present == OUTPUT_NAMES, (directory, present ^ OUTPUT_NAMES)
for name in OUTPUT_NAMES:
    canonical = (ROOT / "outputs" / name).read_bytes()
    assert canonical == (ROOT / "replay_a" / name).read_bytes()
    assert canonical == (ROOT / "replay_b" / name).read_bytes()
    assert canonical == (ROOT / "packaged_replay" / name).read_bytes()

results = load("outputs/results.json")
assert results["paper_id"] == PAPER
assert results["summary"] == {"all_passed": True, "passed": 34, "total": 34}
assert all(item["passed"] for item in results["gates"])
gates = {item["name"]: item["detail"] for item in results["gates"]}
assert gates["hmp::exact_paper_scale_4743_by_402"]["shape"] == [4743, 402]
assert gates["hmp::taxonomy_basis_402_by_401"]["basis_shape"] == [402, 401]
assert gates["hmp::actual_data_isometry_64_pairs"]["max_reconstruction_error"] < 2e-12
assert gates["hmp::actual_data_isometry_64_pairs"]["max_distance_error"] < 2e-12
assert gates["hmp::native_rf_and_independent_centroid_five_fold"]["mean_rf_accuracy"] > 0.96
assert gates["hmp::native_rf_and_independent_centroid_five_fold"]["mean_rf_balanced_accuracy"] > 0.90
assert gates["hmp::native_rf_and_independent_centroid_five_fold"]["mean_centroid_accuracy"] > 0.89
assert gates["hmp::polyilr_top10_stability"]["mean_pairwise_jaccard"] > 0.53
assert gates["control::hmp_label_shuffle_destroys_signal"]["shuffled_balanced_accuracy"] < 0.21

folds = csv_rows("outputs/hmp_native_rf_folds.csv")
controls = csv_rows("outputs/hmp_destructive_control.csv")
assert len(folds) == len(controls) == 5
assert all(float(row["rf_accuracy"]) > 0.95 for row in folds)
assert all(abs(float(row["tree_importance_sum"]) - 1.0) < 1e-12 for row in folds)
assert all(float(row["actual_balanced_accuracy"]) - float(row["shuffled_label_balanced_accuracy"]) > 0.54 for row in controls)
assert len(csv_rows("outputs/tree_certificates.csv")) == 6
assert len(csv_rows("outputs/composition_isometry.csv")) == 24
assert len(csv_rows("outputs/logit_quotient.csv")) == 12

scorecard = (ROOT / "pages/00-judge-evidence-scorecard/page.md").read_text(encoding="utf-8")
for number, claim in enumerate(claims, 1):
    assert claim in scorecard
    assert claim in (ROOT / f"pages/claim-{number}/page.md").read_text(encoding="utf-8")
    assert (ROOT / f"pages/claim-{number}").is_dir()
assert "icml2026-repro" in (ROOT / "README.md").read_text(encoding="utf-8")
assert load("logbook.json")["tags"] == ["icml2026-repro", "paper-pws8t4kBP4"]

final = load("final_assessment.json")
assert final["all_34_scientific_gates_pass"] is True
assert final["expected_points"] == 12
assert final["claim_assessments"] == ["verified"] * 6
assert final["hmp_actual_samples"] == 4743 and final["hmp_actual_taxa"] == 402
assert final["packaged_replay_byte_identical"] is True and final["runs_byte_identical"] is True

with tempfile.TemporaryDirectory(prefix="polyilr-validation-") as directory:
    fresh = Path(directory) / "fresh"
    env = dict(os.environ)
    env.update(PYTHONDONTWRITEBYTECODE="1", PYTHONHASHSEED="0", PYTHONWARNINGS="error")
    subprocess.run(
        [sys.executable, "-W", "error", str(ROOT / "reproduce.py"), "--out", str(fresh)],
        cwd=ROOT,
        env=env,
        check=True,
        stdout=subprocess.DEVNULL,
        timeout=300,
    )
    assert {path.name for path in fresh.iterdir()} == OUTPUT_NAMES
    for path in fresh.iterdir():
        assert path.read_bytes() == (ROOT / "outputs" / path.name).read_bytes(), path.name

if (ROOT / "BUNDLE_SHA256SUMS.txt").is_file():
    subprocess.run([sys.executable, str(ROOT / "verify_manifest.py")], cwd=ROOT, check=True, stdout=subprocess.DEVNULL)

print("evidence validated: six literal PolyILR claims, 34/34 gates, native 4743x402 HMP execution, destructive controls, and byte-identical replay")