validate_evidence.py

#!/usr/bin/env python3
"""Fail-closed validation of claims, science, controls, replay, and package."""
from __future__ import annotations

import hashlib
import json
from pathlib import Path


ROOT = Path(__file__).resolve().parent
GENERATED = [
    "exact_graph_audit.json",
    "oracle_gates.json",
    "finite_native_algorithms.json",
    "synthetic_native_pipeline.json",
    "realdata_native_and_source_audit.json",
    "destructive_controls.json",
    "results.json",
]
EXPECTED_ASSESSMENTS = ["verified"] * 5 + ["falsified_as_literally_registered"]


def load(relative: str):
    path = ROOT / relative
    assert path.is_file() and path.stat().st_size > 0, f"missing {relative}"
    return json.loads(path.read_text(encoding="utf-8"))


def digest(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


official = load("official_claims.json")
assert official == load("CLAIMS.json")
matrix = load("EVIDENCE_MATRIX.json")
assert matrix["paper_id"] == "mOcTXKawFY"
assert [x["literal_claim"] for x in matrix["claims"]] == official
assert [x["assessment"] for x in matrix["claims"]] == EXPECTED_ASSESSMENTS
results = load("outputs/results.json")
assert [x["literal_claim"] for x in results["claims"]] == official
assert [x["assessment"] for x in results["claims"]] == EXPECTED_ASSESSMENTS

# Paired deterministic replay is byte-level, not approximate JSON equality.
pair_hashes = {}
for name in GENERATED:
    left = ROOT / "outputs" / name
    right = ROOT / "packaged_replay" / name
    assert left.is_file() and right.is_file(), f"missing replay pair {name}"
    lh, rh = digest(left), digest(right)
    assert lh == rh, f"paired replay mismatch: {name}"
    pair_hashes[name] = lh
recorded_pairs = load("PAIRED_REPLAY_SHA256.json")
assert recorded_pairs["status"] == "byte_identical"
assert recorded_pairs["files"] == pair_hashes

exact = load("outputs/exact_graph_audit.json")
definition = exact["definition1"]
assert definition["exhaustive_d4"] == {
    "acyclic": 26064,
    "constructions": 26064,
    "edge_count_formula_matches": 26064,
    "edge_families_exactly_as_defined": 26064,
    "node_count_formula_matches": 26064,
}
assert all(definition["random_large"][key] == 60 for key in (
    "constructions", "acyclic", "node_count_matches", "edge_count_matches"
))

dsep = exact["dsep_claims"]
assert dsep["lemma1"] == {
    "converse_failure_witnesses": 17712,
    "tested": 1433520,
    "violations": 0,
}
assert dsep["theorem1"]["tested"] == 1433520
assert dsep["theorem1"]["violations"] == 0
assert dsep["control_naive"]["mismatches"] == 98787
assert dsep["control_no_inheritance"]["lemma1_converse_failures_without_inheritance"] == 0

theorem2 = exact["theorem2"]
assert theorem2["n_models"] == 8688
assert theorem2["adjacency_pairs_tested"] == 52128
assert theorem2["adjacency_violations"] == 0
assert theorem2["oriented_edges_tested"] == 2640
assert theorem2["orientation_soundness_violations"] == 0
assert theorem2["unoriented_edges_tested"] == 39030
assert theorem2["orientation_completeness_failures"] == 0

theorem4 = exact["theorem4"]
assert theorem4["model_x_Iset_configurations"] == 269328
assert theorem4["monotonicity_violations"] == 0
assert theorem4["orientation_soundness_violations"] == 0
assert theorem4["adjacency_violations"] == 0
assert theorem4["configurations_with_strictly_more_orientations"] == 88176
assert theorem4["additional_oriented_edges_vs_single_domain"] == 172512

oracles = load("outputs/oracle_gates.json")
assert oracles["dseparation_vs_networkx"] == {"disagreements": 0, "queries": 11984}
assert oracles["cpdag_vs_causallearn"] == {"disagreements": 0, "models": 600}
assert oracles["verma_pearl_invariance"]["compelled_edge_violations"] == 0
assert oracles["verma_pearl_invariance"]["reversibility_violations"] == 0

finite = load("outputs/finite_native_algorithms.json")
assert finite["lemma1"]["spurious_with_evolution"] == 46
assert finite["lemma1"]["spurious_without_selection"] == 8
assert finite["theorem4_cdnod"]["mean_correct_oriented_multi"] == 4.65
assert finite["theorem4_cdnod"]["mean_correct_oriented_single"] == 3.55

synthetic = load("outputs/synthetic_native_pipeline.json")
assert synthetic["config"]["executed_runs"] == 750
assert synthetic["headline"] == {
    "GES_cells_oriented_beats_standard": 14,
    "PC_cells_oriented_beats_standard": 3,
    "d20_GES_cells_oriented_beats_standard": 4,
    "d20_PC_cells_oriented_beats_standard": 0,
    "grid_cells": 15,
}

real = load("outputs/realdata_native_and_source_audit.json")
audit = real["source_reported_seven_dataset_audit"]
assert len(audit["per_dataset"]) == 7
assert audit["arithmetic_mismatches_vs_printed_percentages"] == 0
assert audit["claim_i_datasets_with_oriented_gt_unoriented"] == "5 of 6"
assert audit["pooled_direction_agrees_with_claim"] is False
pan = real["pantheria_native_rerun"]
assert (pan["n_samples"], pan["n_variables"]) == (626, 8)
assert pan["log_alpha0.05"]["oriented_precision"] == 0.4
assert pan["log_alpha0.05"]["unoriented_precision"] == 1.0

controls = load("outputs/destructive_controls.json")
assert controls["claim1_remove_inheritance_edge"]["checker_rejected_mutant"] is True
assert controls["claim2_remove_all_inheritance"]["lemma1_converse_failures_without_inheritance"] == 0
assert controls["claim3_delete_selection_clique"]["mismatches"] == 98787
assert controls["claim4_reverse_compelled_orientation"] == {
    "checker_fired": 328,
    "corrupted_models_tested": 328,
}
assert controls["claim5_omit_changed_selection_ancestor_expansion"]["orientation_soundness_violations"] == 583356
assert controls["claim6_mutate_source_and_data"]["source_arithmetic_mutation_detected"] is True
assert controls["claim6_mutate_source_and_data"]["data_byte_mutation_detected_by_sha256"] is True

source = load("SOURCE_FETCH.json")
data_path = ROOT / source["pantheria"]["packaged_data_path"]
assert digest(data_path) == source["pantheria"]["packaged_data_sha256"]
assert digest(data_path) == "36e64314cae0394a966a63b949504d1975ac5c5629e05e36c1b139b3348f044a"
routes = load("LOCAL_ROUTE_AUDIT.json")
assert routes["status"] == "pass" and routes["routes_checked"] == 9
assert all(row["exists"] and row["bytes"] > 0 for row in routes["routes"])

# Recursive manifest: exact path set and SHA-256 for every included file.
manifest = ROOT / "BUNDLE_SHA256SUMS.txt"
assert manifest.is_file()
recorded = {}
for line in manifest.read_text(encoding="utf-8").splitlines():
    value, relative = line.split("  ", 1)
    recorded[relative] = value
excluded_dirs = {"__pycache__", "replay_a", "replay_b", "fresh_replay"}
actual = {}
for path in ROOT.rglob("*"):
    relative = path.relative_to(ROOT)
    if (
        path.is_file()
        and not path.is_symlink()
        and path != manifest
        and not any(part in excluded_dirs for part in relative.parts)
        and path.suffix not in {".log", ".pyc"}
        and path.name != ".DS_Store"
    ):
        actual[relative.as_posix()] = digest(path)
assert actual == recorded, "recursive manifest mismatch"

print(json.dumps({
    "status": "pass",
    "paper_id": "mOcTXKawFY",
    "claims": 6,
    "assessments": EXPECTED_ASSESSMENTS,
    "paired_replay_files": len(pair_hashes),
    "manifest_files": len(recorded),
}, indent=2))