#!/usr/bin/env python3 """Fail-closed validation of claims, science, controls, replay, and package.""" from __future__ import annotations import hashlib import json from pathlib import Path ROOT = Path(__file__).resolve().parent GENERATED = [ "exact_graph_audit.json", "oracle_gates.json", "finite_native_algorithms.json", "synthetic_native_pipeline.json", "realdata_native_and_source_audit.json", "destructive_controls.json", "results.json", ] EXPECTED_ASSESSMENTS = ["verified"] * 5 + ["falsified_as_literally_registered"] def load(relative: str): path = ROOT / relative assert path.is_file() and path.stat().st_size > 0, f"missing {relative}" return json.loads(path.read_text(encoding="utf-8")) def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() official = load("official_claims.json") assert official == load("CLAIMS.json") matrix = load("EVIDENCE_MATRIX.json") assert matrix["paper_id"] == "mOcTXKawFY" assert [x["literal_claim"] for x in matrix["claims"]] == official assert [x["assessment"] for x in matrix["claims"]] == EXPECTED_ASSESSMENTS results = load("outputs/results.json") assert [x["literal_claim"] for x in results["claims"]] == official assert [x["assessment"] for x in results["claims"]] == EXPECTED_ASSESSMENTS # Paired deterministic replay is byte-level, not approximate JSON equality. pair_hashes = {} for name in GENERATED: left = ROOT / "outputs" / name right = ROOT / "packaged_replay" / name assert left.is_file() and right.is_file(), f"missing replay pair {name}" lh, rh = digest(left), digest(right) assert lh == rh, f"paired replay mismatch: {name}" pair_hashes[name] = lh recorded_pairs = load("PAIRED_REPLAY_SHA256.json") assert recorded_pairs["status"] == "byte_identical" assert recorded_pairs["files"] == pair_hashes exact = load("outputs/exact_graph_audit.json") definition = exact["definition1"] assert definition["exhaustive_d4"] == { "acyclic": 26064, "constructions": 26064, "edge_count_formula_matches": 26064, "edge_families_exactly_as_defined": 26064, "node_count_formula_matches": 26064, } assert all(definition["random_large"][key] == 60 for key in ( "constructions", "acyclic", "node_count_matches", "edge_count_matches" )) dsep = exact["dsep_claims"] assert dsep["lemma1"] == { "converse_failure_witnesses": 17712, "tested": 1433520, "violations": 0, } assert dsep["theorem1"]["tested"] == 1433520 assert dsep["theorem1"]["violations"] == 0 assert dsep["control_naive"]["mismatches"] == 98787 assert dsep["control_no_inheritance"]["lemma1_converse_failures_without_inheritance"] == 0 theorem2 = exact["theorem2"] assert theorem2["n_models"] == 8688 assert theorem2["adjacency_pairs_tested"] == 52128 assert theorem2["adjacency_violations"] == 0 assert theorem2["oriented_edges_tested"] == 2640 assert theorem2["orientation_soundness_violations"] == 0 assert theorem2["unoriented_edges_tested"] == 39030 assert theorem2["orientation_completeness_failures"] == 0 theorem4 = exact["theorem4"] assert theorem4["model_x_Iset_configurations"] == 269328 assert theorem4["monotonicity_violations"] == 0 assert theorem4["orientation_soundness_violations"] == 0 assert theorem4["adjacency_violations"] == 0 assert theorem4["configurations_with_strictly_more_orientations"] == 88176 assert theorem4["additional_oriented_edges_vs_single_domain"] == 172512 oracles = load("outputs/oracle_gates.json") assert oracles["dseparation_vs_networkx"] == {"disagreements": 0, "queries": 11984} assert oracles["cpdag_vs_causallearn"] == {"disagreements": 0, "models": 600} assert oracles["verma_pearl_invariance"]["compelled_edge_violations"] == 0 assert oracles["verma_pearl_invariance"]["reversibility_violations"] == 0 finite = load("outputs/finite_native_algorithms.json") assert finite["lemma1"]["spurious_with_evolution"] == 46 assert finite["lemma1"]["spurious_without_selection"] == 8 assert finite["theorem4_cdnod"]["mean_correct_oriented_multi"] == 4.65 assert finite["theorem4_cdnod"]["mean_correct_oriented_single"] == 3.55 synthetic = load("outputs/synthetic_native_pipeline.json") assert synthetic["config"]["executed_runs"] == 750 assert synthetic["headline"] == { "GES_cells_oriented_beats_standard": 14, "PC_cells_oriented_beats_standard": 3, "d20_GES_cells_oriented_beats_standard": 4, "d20_PC_cells_oriented_beats_standard": 0, "grid_cells": 15, } real = load("outputs/realdata_native_and_source_audit.json") audit = real["source_reported_seven_dataset_audit"] assert len(audit["per_dataset"]) == 7 assert audit["arithmetic_mismatches_vs_printed_percentages"] == 0 assert audit["claim_i_datasets_with_oriented_gt_unoriented"] == "5 of 6" assert audit["pooled_direction_agrees_with_claim"] is False pan = real["pantheria_native_rerun"] assert (pan["n_samples"], pan["n_variables"]) == (626, 8) assert pan["log_alpha0.05"]["oriented_precision"] == 0.4 assert pan["log_alpha0.05"]["unoriented_precision"] == 1.0 controls = load("outputs/destructive_controls.json") assert controls["claim1_remove_inheritance_edge"]["checker_rejected_mutant"] is True assert controls["claim2_remove_all_inheritance"]["lemma1_converse_failures_without_inheritance"] == 0 assert controls["claim3_delete_selection_clique"]["mismatches"] == 98787 assert controls["claim4_reverse_compelled_orientation"] == { "checker_fired": 328, "corrupted_models_tested": 328, } assert controls["claim5_omit_changed_selection_ancestor_expansion"]["orientation_soundness_violations"] == 583356 assert controls["claim6_mutate_source_and_data"]["source_arithmetic_mutation_detected"] is True assert controls["claim6_mutate_source_and_data"]["data_byte_mutation_detected_by_sha256"] is True source = load("SOURCE_FETCH.json") data_path = ROOT / source["pantheria"]["packaged_data_path"] assert digest(data_path) == source["pantheria"]["packaged_data_sha256"] assert digest(data_path) == "36e64314cae0394a966a63b949504d1975ac5c5629e05e36c1b139b3348f044a" routes = load("LOCAL_ROUTE_AUDIT.json") assert routes["status"] == "pass" and routes["routes_checked"] == 9 assert all(row["exists"] and row["bytes"] > 0 for row in routes["routes"]) # Recursive manifest: exact path set and SHA-256 for every included file. manifest = ROOT / "BUNDLE_SHA256SUMS.txt" assert manifest.is_file() recorded = {} for line in manifest.read_text(encoding="utf-8").splitlines(): value, relative = line.split(" ", 1) recorded[relative] = value excluded_dirs = {"__pycache__", "replay_a", "replay_b", "fresh_replay"} actual = {} for path in ROOT.rglob("*"): relative = path.relative_to(ROOT) if ( path.is_file() and not path.is_symlink() and path != manifest and not any(part in excluded_dirs for part in relative.parts) and path.suffix not in {".log", ".pyc"} and path.name != ".DS_Store" ): actual[relative.as_posix()] = digest(path) assert actual == recorded, "recursive manifest mismatch" print(json.dumps({ "status": "pass", "paper_id": "mOcTXKawFY", "claims": 6, "assessments": EXPECTED_ASSESSMENTS, "paired_replay_files": len(pair_hashes), "manifest_files": len(recorded), }, indent=2))