release/v1/scripts/analyze_claims.py

#!/usr/bin/env python3
"""Turn canonical tables into a numerical claim-to-evidence report."""

from __future__ import annotations

import argparse
import csv
import hashlib
import json
import math
import shutil
import tempfile
from pathlib import Path
from typing import Any, Mapping, Sequence

from make_figures import read_verified_tables


ANALYSIS_VERSION = "1.2.0"


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def finite(value: str, label: str) -> float:
    try:
        result = float(value)
    except (TypeError, ValueError) as exc:
        raise RuntimeError(f"Invalid numeric value for {label}: {value!r}") from exc
    if not math.isfinite(result):
        raise RuntimeError(f"Non-finite value for {label}: {result!r}")
    return result


def summarize(values: Sequence[float]) -> dict[str, float | int]:
    if not values or not all(math.isfinite(value) for value in values):
        raise RuntimeError("Cannot summarize empty or non-finite evidence")
    mean = sum(values) / len(values)
    variance = sum((value - mean) ** 2 for value in values) / len(values)
    return {
        "count": len(values),
        "mean": mean,
        "std_population": math.sqrt(variance),
        "min": min(values),
        "max": max(values),
    }


def parse_optional_int(value: str) -> int | None:
    if value == "":
        return None
    return int(value)


def selected_seed(
    selections: Sequence[Mapping[str, str]],
    method: str,
    *,
    claim_id: str,
    expected_candidate_count: int,
) -> int:
    rows = [
        row
        for row in selections
        if row.get("status") == "success"
        and row.get("claim_id") == claim_id
        and row.get("method") == method
    ]
    if len(rows) != 1:
        raise RuntimeError(
            f"Expected one {claim_id} selection for {method}, found {len(rows)}"
        )
    candidate_count = parse_optional_int(rows[0].get("candidate_count", ""))
    if candidate_count != expected_candidate_count:
        raise RuntimeError(
            f"Selection for {method} has {candidate_count} candidates; "
            f"expected {expected_candidate_count}"
        )
    seed = parse_optional_int(rows[0].get("selected_seed", ""))
    if seed is None:
        raise RuntimeError(f"Selection for {method} has no seed")
    return seed


def episode_summary(
    episodes: Sequence[Mapping[str, str]],
    method: str,
    *,
    claim_id: str,
    panel_id: str,
    expected_count: int,
    seed: int | None = None,
    environment: str,
) -> dict[str, float | int]:
    rows = [
        row
        for row in episodes
        if row.get("status") == "success"
        and row.get("claim_id") == claim_id
        and row.get("panel_id") == panel_id
        and row.get("method") == method
        and (seed is None or parse_optional_int(row.get("seed", "")) == seed)
        and row.get("environment") == environment
    ]
    if len(rows) != expected_count:
        raise RuntimeError(
            f"Expected {expected_count} rows for {claim_id}/{panel_id}/{method}, "
            f"found {len(rows)}"
        )
    return summarize([finite(row["return"], f"{method}.return") for row in rows])


def number_row(
    *,
    component: str,
    paper_value: float,
    reproduced_value: float,
    tolerance: float | None,
    verdict: str | None = None,
    scope: str,
    evidence: str,
    note: str = "",
) -> dict[str, Any]:
    difference = reproduced_value - paper_value
    within_tolerance: bool | str = ""
    if tolerance is not None:
        within_tolerance = abs(difference) <= tolerance
    return {
        "component": component,
        "paper_value": paper_value,
        "reproduced_value": reproduced_value,
        "difference": difference,
        "absolute_difference": abs(difference),
        "tolerance": "" if tolerance is None else tolerance,
        "within_tolerance": within_tolerance,
        "verdict": verdict
        or (
            "numerically_reproduced"
            if tolerance is not None and abs(difference) <= tolerance
            else "not_reproduced"
        ),
        "scope": scope,
        "evidence": evidence,
        "note": note,
    }


def unexecuted_number_row(
    *,
    component: str,
    paper_value: float,
    verdict: str,
    scope: str,
    evidence: str,
    note: str,
) -> dict[str, Any]:
    return {
        "component": component,
        "paper_value": paper_value,
        "reproduced_value": "",
        "difference": "",
        "absolute_difference": "",
        "tolerance": "",
        "within_tolerance": "",
        "verdict": verdict,
        "scope": scope,
        "evidence": evidence,
        "note": note,
    }


def text_row(
    *,
    component: str,
    verdict: str,
    scope: str,
    evidence: str,
    note: str,
) -> dict[str, Any]:
    return {
        "component": component,
        "paper_value": "",
        "reproduced_value": "",
        "difference": "",
        "absolute_difference": "",
        "tolerance": "",
        "within_tolerance": "",
        "verdict": verdict,
        "scope": scope,
        "evidence": evidence,
        "note": note,
    }


def parameter_value(
    rows: Sequence[Mapping[str, str]], key: str
) -> float:
    matches = [
        row
        for row in rows
        if row.get("status") == "success"
        and row.get("claim_id") == "claim3"
        and row.get("task_id") == "claim3_parameter_arithmetic_audit"
    ]
    if len(matches) != 1:
        raise RuntimeError(f"Expected one Claim 3 arithmetic audit, found {len(matches)}")
    return finite(matches[0].get(key, ""), f"parameter_audit.{key}")


def selected_coefficient_count(
    training_runs: Sequence[Mapping[str, str]], *, method: str, seed: int
) -> float:
    matches = [
        row
        for row in training_runs
        if row.get("status") == "success"
        and row.get("claim_id") == "claim3"
        and row.get("method") == method
        and parse_optional_int(row.get("seed", "")) == seed
    ]
    if len(matches) != 1:
        raise RuntimeError(
            f"Expected one Claim 3 training run for {method} seed {seed}, "
            f"found {len(matches)}"
        )
    return finite(matches[0].get("coefficient_count", ""), "training_run.coefficient_count")


def executable_parameter_count(
    rows: Sequence[Mapping[str, str]], *, task_id: str, method: str
) -> float:
    matches = [
        row
        for row in rows
        if row.get("status") == "success"
        and row.get("claim_id") == "claim3"
        and row.get("task_id") == task_id
        and row.get("method") == method
    ]
    if len(matches) != 1:
        raise RuntimeError(
            f"Expected one executable parameter audit for {task_id}/{method}, "
            f"found {len(matches)}"
        )
    return finite(matches[0].get("parameter_count", ""), f"{task_id}.parameter_count")


def positive_ratio(numerator: float, denominator: float, label: str) -> float:
    if numerator <= 0 or denominator <= 0:
        raise RuntimeError(
            f"{label} requires positive regrets; got {numerator=} and {denominator=}"
        )
    return numerator / denominator


def build_claim_rows(tables: Mapping[str, Sequence[Mapping[str, str]]]) -> list[dict[str, Any]]:
    episodes = tables["episodes.csv"]
    selections = tables["selections.csv"]
    parameters = tables["parameter_audit.csv"]
    proofs = tables["proof_certificates.csv"]
    exclusions = tables["exclusions.csv"]
    training_runs = tables["training_runs.csv"]

    mountaincar = "MountainCarContinuous-v0"
    analytic = episode_summary(
        episodes,
        "analytic_two_phase",
        claim_id="claim2",
        panel_id="mountaincar_primary_v1",
        expected_count=100,
        environment=mountaincar,
    )
    ars = episode_summary(
        episodes,
        "ars",
        claim_id="claim2",
        panel_id="mountaincar_primary_v1",
        expected_count=100,
        environment=mountaincar,
    )
    ppo = episode_summary(
        episodes,
        "ppo",
        claim_id="claim2",
        panel_id="mountaincar_primary_v1",
        expected_count=100,
        environment=mountaincar,
    )
    sac = episode_summary(
        episodes,
        "sac",
        claim_id="claim2",
        panel_id="mountaincar_primary_v1",
        expected_count=100,
        environment=mountaincar,
    )
    ch3_seed = selected_seed(
        selections,
        "ch3_ars",
        claim_id="claim3",
        expected_candidate_count=20,
    )
    ch3 = episode_summary(
        episodes,
        "ch3_ars",
        claim_id="claim3",
        panel_id="mountaincar_ch3_ars_seed_grid_v1",
        expected_count=100,
        seed=ch3_seed,
        environment=mountaincar,
    )
    ch3_ppo_seed = selected_seed(
        selections,
        "ch3_ppo",
        claim_id="claim4",
        expected_candidate_count=20,
    )
    ch3_ppo = episode_summary(
        episodes,
        "ch3_ppo",
        claim_id="claim4",
        panel_id="mountaincar_ch3_ppo_seed_grid_v1",
        expected_count=100,
        seed=ch3_ppo_seed,
        environment=mountaincar,
    )
    reinforce = episode_summary(
        episodes,
        "ch3_reinforce_adamw_seeded_selected",
        claim_id="claim4",
        panel_id="mountaincar_ch3_reinforce_primary_v1",
        expected_count=100,
        environment=mountaincar,
    )
    pendulum_seed = selected_seed(
        selections,
        "ch6_ars_pendulum",
        claim_id="claim5",
        expected_candidate_count=30,
    )
    pendulum = episode_summary(
        episodes,
        "ch6_ars_pendulum_selected",
        claim_id="claim5",
        panel_id="pendulum_primary_comparison_v1",
        expected_count=2500,
        seed=pendulum_seed,
        environment="DeterministicPendulum-v1",
    )
    pendulum_ars = episode_summary(
        episodes,
        "ars_baseline_pendulum_released",
        claim_id="claim5",
        panel_id="pendulum_primary_comparison_v1",
        expected_count=2500,
        environment="DeterministicPendulum-v1",
    )
    pendulum_ars_corrected = episode_summary(
        episodes,
        "ars_baseline_pendulum_released_corrected_reset",
        claim_id="claim5",
        panel_id="pendulum_baseline_reset_sensitivity_v1",
        expected_count=2500,
        environment="DeterministicPendulum-v1",
    )

    analytic_mean = float(analytic["mean"])
    ars_regret = analytic_mean - float(ars["mean"])
    ppo_regret = analytic_mean - float(ppo["mean"])
    sac_regret = analytic_mean - float(sac["mean"])
    ch3_regret = analytic_mean - float(ch3["mean"])
    ch3_ppo_regret = analytic_mean - float(ch3_ppo["mean"])
    reinforce_regret = analytic_mean - float(reinforce["mean"])

    proof_rows = [
        row
        for row in proofs
        if row.get("status") == "success"
        and row.get("claim_id") == "claim1"
        and row.get("task_id") == "cauchy_schwarz_discrete_certificate"
    ]
    if len(proof_rows) != 1:
        raise RuntimeError(
            f"Expected one successful Claim 1 proof certificate, found {len(proof_rows)}"
        )
    proof = proof_rows[0]
    minimum_gap = finite(
        proof.get("minimum_perturbed_loss_gap", ""),
        "claim1.minimum_perturbed_loss_gap",
    )
    maximum_constraint_residual = finite(
        proof.get("maximum_constraint_residual", ""),
        "claim1.maximum_constraint_residual",
    )
    maximum_kkt_deviation = finite(
        proof.get("maximum_kkt_ratio_deviation", ""),
        "claim1.maximum_kkt_ratio_deviation",
    )
    theorem_certificate_passed = (
        proof.get("claim_scope") == "continuous_unconstrained_theorem_mechanism"
        and proof.get("historical_priority_tested") == "false"
        and proof.get("discrete_global_optimality_tested") == "false"
        and minimum_gap > 0
        and maximum_constraint_residual <= 1e-12
        and maximum_kkt_deviation <= 1e-12
    )
    ch3_coefficients = selected_coefficient_count(
        training_runs, method="ch3_ars", seed=ch3_seed
    )
    formula_ch3_coefficients = parameter_value(parameters, "chebyshev_parameter_count")
    if ch3_coefficients != formula_ch3_coefficients:
        raise RuntimeError(
            "Selected CH-3-ARS coefficient count disagrees with the basis formula: "
            f"{ch3_coefficients} != {formula_ch3_coefficients}"
        )
    ars_parameter_count = executable_parameter_count(
        parameters,
        task_id="bundled_ars_executable_parameter_count",
        method="ars",
    )
    formula_ars_parameter_count = parameter_value(
        parameters, "released_ars_comparator_parameter_count"
    )
    if ars_parameter_count != formula_ars_parameter_count:
        raise RuntimeError(
            "Released ARS executable count disagrees with the arithmetic audit: "
            f"{ars_parameter_count} != {formula_ars_parameter_count}"
        )

    ars_improvement = positive_ratio(ars_regret, ch3_regret, "Claim 3 improvement")
    ppo_improvement = positive_ratio(
        ppo_regret, ch3_ppo_regret, "Claim 4 PPO improvement"
    )
    reinforce_improvement = positive_ratio(
        ars_regret, reinforce_regret, "Claim 4 REINFORCE improvement"
    )
    c2_headline_means_reproduced = all(
        abs(reproduced - paper) <= 0.01
        for reproduced, paper in (
            (analytic_mean, 99.39),
            (float(ars["mean"]), 96.67),
            (float(ppo["mean"]), 93.91),
            (float(sac["mean"]), 94.61),
        )
    )
    c3_v4_mean_reproduced = abs(float(ch3["mean"]) - 98.95) <= 0.05
    c4_ppo_mean_reproduced = abs(float(ch3_ppo["mean"]) - 98.10) <= 0.05

    rows: list[dict[str, Any]] = [
        text_row(
            component="C1 theorem mechanism: alpha proportional to velocity",
            verdict=(
                "supported_under_continuous_unconstrained_assumptions"
                if theorem_certificate_passed
                else "not_supported_by_certificate"
            ),
            scope="Claim 1 / Theorem 2.4",
            evidence=(
                f"minimum loss gap={minimum_gap:.9g}; maximum constraint residual="
                f"{maximum_constraint_residual:.9g}; maximum KKT-ratio deviation="
                f"{maximum_kkt_deviation:.9g}"
            ),
            note="This certificate tests the Cauchy-Schwarz mechanism, not the discrete Gym global optimum.",
        ),
        text_row(
            component="C1 historical priority and 36-year open-problem claim",
            verdict="firstness_unresolved_chronology_unsupported",
            scope="Claim 1 / historical priority",
            evidence=(
                "Bounded primary-source audit: Moore 1990 defines a materially different "
                "one-dimensional task; Singh and Sutton 1996 attribute the canonical "
                "swing-up task to Moore 1991; ProPS arXiv v1 (2025-11-26) publicly "
                "states force proportional to velocity as an expert heuristic for "
                "MountainCarContinuous-v0."
            ),
            note=(
                "No pre-2026 exact-objective global-optimality proof was found in this "
                "bounded audit, which cannot establish firstness. The 36-year chronology "
                "is unsupported; only the qualitative proportionality idea is shown to "
                "predate this paper."
            ),
        ),
        number_row(
            component="C2 analytic mean return",
            paper_value=99.39,
            reproduced_value=analytic_mean,
            tolerance=0.01,
            scope="Claim 2 / 100 fixed starts",
            evidence=f"n={analytic['count']}; range={analytic['min']:.6f}..{analytic['max']:.6f}",
        ),
        number_row(
            component="C2 analytic minimum return",
            paper_value=99.15,
            reproduced_value=float(analytic["min"]),
            tolerance=0.01,
            scope="Claim 2 / 100 fixed starts",
            evidence="episode-level fixed-grid CSV",
        ),
        number_row(
            component="C2 analytic maximum return",
            paper_value=99.52,
            reproduced_value=float(analytic["max"]),
            tolerance=0.01,
            scope="Claim 2 / 100 fixed starts",
            evidence="episode-level fixed-grid CSV",
        ),
        number_row(
            component="C2 released ARS mean return",
            paper_value=96.67,
            reproduced_value=float(ars["mean"]),
            tolerance=0.01,
            scope="Claim 2 / released checkpoint",
            evidence=f"n={ars['count']}; exact regret={ars_regret:.6f}",
        ),
        number_row(
            component="C2 released PPO mean return",
            paper_value=93.91,
            reproduced_value=float(ppo["mean"]),
            tolerance=0.01,
            scope="Claim 2 / released checkpoint",
            evidence=f"n={ppo['count']}; exact regret={ppo_regret:.6f}",
        ),
        number_row(
            component="C2 released SAC mean return",
            paper_value=94.61,
            reproduced_value=float(sac["mean"]),
            tolerance=0.01,
            scope="Claim 2 / released checkpoint",
            evidence=f"n={sac['count']}; exact regret={sac_regret:.6f}",
        ),
        number_row(
            component="C2 ARS regret 2.72",
            paper_value=2.72,
            reproduced_value=ars_regret,
            tolerance=None,
            verdict=(
                "consistent_at_reported_headline_precision"
                if c2_headline_means_reproduced
                else "not_supported_by_reproduced_headline_means"
            ),
            scope="Claim 2 / derived regret",
            evidence=(
                f"exact episode means give {ars_regret:.6f}; the printed value is "
                "99.39 - 96.67 = 2.72"
            ),
            note="No post-hoc ratio/regret tolerance is applied; both exact and headline-precision arithmetic are shown.",
        ),
        number_row(
            component="C2 PPO regret 5.48",
            paper_value=5.48,
            reproduced_value=ppo_regret,
            tolerance=None,
            verdict=(
                "consistent_at_reported_headline_precision"
                if c2_headline_means_reproduced
                else "not_supported_by_reproduced_headline_means"
            ),
            scope="Claim 2 / derived regret",
            evidence=(
                f"exact episode means give {ppo_regret:.6f}; the printed value is "
                "99.39 - 93.91 = 5.48"
            ),
            note="No post-hoc ratio/regret tolerance is applied; both exact and headline-precision arithmetic are shown.",
        ),
        number_row(
            component="C2 SAC regret 4.78",
            paper_value=4.78,
            reproduced_value=sac_regret,
            tolerance=None,
            verdict=(
                "consistent_at_reported_headline_precision"
                if c2_headline_means_reproduced
                else "not_supported_by_reproduced_headline_means"
            ),
            scope="Claim 2 / derived regret",
            evidence=(
                f"exact episode means give {sac_regret:.6f}; the printed value is "
                "99.39 - 94.61 = 4.78"
            ),
            note="No post-hoc ratio/regret tolerance is applied; both exact and headline-precision arithmetic are shown.",
        ),
        number_row(
            component="C3 current-v4 CH-3-ARS mean return",
            paper_value=98.95,
            reproduced_value=float(ch3["mean"]),
            tolerance=0.05,
            scope="Claim 3 / selected best of 20 seeds",
            evidence=f"selected_seed={ch3_seed}; n={ch3['count']}; range={ch3['min']:.6f}..{ch3['max']:.6f}",
        ),
        number_row(
            component="C3 anchored-v1 regret 0.65",
            paper_value=0.65,
            reproduced_value=ch3_regret,
            tolerance=None,
            verdict="superseded_by_arxiv_v4",
            scope="Claim 3 / challenge wording",
            evidence=f"exact reproduced regret={ch3_regret:.6f}; v4 target is 0.44",
            note="The challenge copied an internally inconsistent v1 table; v4 corrected the mean/range transposition.",
        ),
        number_row(
            component="C3 current-v4 regret 0.44",
            paper_value=0.44,
            reproduced_value=ch3_regret,
            tolerance=None,
            verdict=(
                "consistent_at_reported_headline_precision"
                if c3_v4_mean_reproduced
                else "not_supported_by_reproduced_v4_mean"
            ),
            scope="Claim 3 / arXiv v4",
            evidence=(
                f"analytic mean minus selected CH-3-ARS mean = {ch3_regret:.6f}; "
                "99.39 - 98.95 = 0.44 at headline precision"
            ),
        ),
        number_row(
            component="C3 anchored-v1 regret improvement 4.18x",
            paper_value=4.18,
            reproduced_value=ars_improvement,
            tolerance=None,
            verdict="superseded_by_arxiv_v4",
            scope="Claim 3 / challenge wording",
            evidence=(
                f"exact v4-aligned reproduced ratio={ars_improvement:.6f}; "
                "the challenge value is tied to the stale v1 regret"
            ),
            note="The v1 table transposed the selected mean and minimum; arXiv v4 replaced 4.18x with 6.18x.",
        ),
        number_row(
            component="C3 current-v4 regret improvement 6.18x",
            paper_value=6.18,
            reproduced_value=ars_improvement,
            tolerance=None,
            verdict=(
                "consistent_only_at_reported_headline_precision"
                if c3_v4_mean_reproduced and c2_headline_means_reproduced
                else "not_supported_by_reproduced_headline_means"
            ),
            scope="Claim 3 / arXiv v4",
            evidence=(
                f"exact ratio={ars_improvement:.6f}; printed regrets give "
                "2.72 / 0.44 = 6.181818"
            ),
            note="The exact-ratio value is retained because the spec preregistered no ratio tolerance.",
        ),
        number_row(
            component="C3 Chebyshev actor coefficients",
            paper_value=16,
            reproduced_value=ch3_coefficients,
            tolerance=0,
            scope="Claim 3 / degree-3 bivariate basis",
            evidence=(
                f"selected checkpoint coefficient_count={ch3_coefficients:.0f}; "
                f"formula audit={formula_ch3_coefficients:.0f}; "
                "(3+1)^2=16"
            ),
        ),
        number_row(
            component="C3 stated 2x64x64x1 MLP parameter count",
            paper_value=4355,
            reproduced_value=parameter_value(
                parameters, "correct_dense_2x64x64x1_parameter_count"
            ),
            tolerance=0,
            verdict="arithmetically_false_as_written",
            scope="Claim 3 / architecture audit",
            evidence="64*(2+1)+64*(64+1)+1*(64+1)=4417",
        ),
        number_row(
            component="C3 nominal dense-MLP reduction factor",
            paper_value=277,
            reproduced_value=parameter_value(parameters, "ratio_using_correct_dense_count"),
            tolerance=None,
            verdict="rounding_and_denominator_inconsistent",
            scope="Claim 3 / architecture audit",
            evidence="4417/16=276.0625; the paper's own 4355/16 is 272.1875",
        ),
        number_row(
            component="C3 released neural ARS comparator parameters",
            paper_value=4355,
            reproduced_value=ars_parameter_count,
            tolerance=None,
            verdict="released_comparator_is_not_the_stated_mlp",
            scope="Claim 3 / executable checkpoint audit",
            evidence=(
                f"released ARS checkpoint has {ars_parameter_count:.0f} trainable scalars; "
                "65/16=4.0625"
            ),
            note="The 4,355/4,417 dense architecture is a nominal architecture, not the released ARS comparator state.",
        ),
        number_row(
            component="C4 CH-3-PPO mean return",
            paper_value=98.10,
            reproduced_value=float(ch3_ppo["mean"]),
            tolerance=0.05,
            scope="Claim 4 / selected best of 20 seeds",
            evidence=f"selected_seed={ch3_ppo_seed}; n={ch3_ppo['count']}",
        ),
        number_row(
            component="C4 CH-3-PPO regret",
            paper_value=1.29,
            reproduced_value=ch3_ppo_regret,
            tolerance=None,
            verdict=(
                "consistent_at_reported_headline_precision"
                if c4_ppo_mean_reproduced and c2_headline_means_reproduced
                else "not_supported_by_reproduced_headline_means"
            ),
            scope="Claim 4 / selected policy",
            evidence=(
                f"exact analytic mean minus exact selected-policy mean = {ch3_ppo_regret:.6f}; "
                "99.39 - 98.10 = 1.29 at headline precision"
            ),
        ),
        number_row(
            component="C4 CH-3-PPO regret improvement",
            paper_value=4.24,
            reproduced_value=ppo_improvement,
            tolerance=None,
            verdict=(
                "consistent_only_at_reported_headline_precision"
                if c4_ppo_mean_reproduced and c2_headline_means_reproduced
                else "not_supported_by_reproduced_headline_means"
            ),
            scope="Claim 4 / PPO comparator",
            evidence=(
                f"exact ratio={ppo_improvement:.6f}; printed regrets give "
                "5.48 / 1.29 = 4.248062"
            ),
            note="The exact-ratio value is retained because the spec preregistered no ratio tolerance.",
        ),
        number_row(
            component="C4 CH-3-REINFORCE mean return",
            paper_value=98.62,
            reproduced_value=float(reinforce["mean"]),
            tolerance=0.05,
            verdict=(
                "fresh_seeded_replication_consistent"
                if abs(float(reinforce["mean"]) - 98.62) <= 0.05
                else "fresh_seeded_replication_inconsistent"
            ),
            scope="Claim 4 / fresh seeded AdamW replication",
            evidence=f"n={reinforce['count']}; range={reinforce['min']:.6f}..{reinforce['max']:.6f}",
            note="This is not an exact replay: the paper released neither seeds nor selected coefficients.",
        ),
        number_row(
            component="C4 CH-3-REINFORCE regret",
            paper_value=0.77,
            reproduced_value=reinforce_regret,
            tolerance=None,
            verdict="fresh_seeded_replication_derived",
            scope="Claim 4 / fresh seeded AdamW replication",
            evidence=(
                f"exact analytic mean minus fresh selected-policy mean = {reinforce_regret:.6f}"
            ),
        ),
        number_row(
            component="C4 CH-3-REINFORCE improvement over ARS",
            paper_value=3.53,
            reproduced_value=reinforce_improvement,
            tolerance=None,
            verdict="fresh_seeded_replication_derived",
            scope="Claim 4 / fresh seeded AdamW replication",
            evidence=(
                f"exact ratio={reinforce_improvement:.6f}; printed regrets give "
                "2.72 / 0.77 = 3.532468"
            ),
            note="The paper did not release seeds or selected coefficients, and the spec preregistered no ratio tolerance.",
        ),
        number_row(
            component="C5 Pendulum CH-6-ARS mean return",
            paper_value=-150.8,
            reproduced_value=float(pendulum["mean"]),
            tolerance=0.05,
            scope="Claim 5 / selected best of 30 seeds on 50x50 grid",
            evidence=f"selected_seed={pendulum_seed}; n={pendulum['count']}",
        ),
        number_row(
            component="C5 Pendulum released ARS mean return (author-exact reset)",
            paper_value=-218.3,
            reproduced_value=float(pendulum_ars["mean"]),
            tolerance=0.05,
            scope="Claim 5 / released checkpoint on 50x50 grid",
            evidence=f"n={pendulum_ars['count']}",
            note="The author's protocol bypasses VecNormalize for the first observation.",
        ),
        text_row(
            component="C5 Pendulum ARS corrected-reset sensitivity",
            verdict="secondary_sensitivity_analysis",
            scope="Claim 5 / non-headline protocol audit",
            evidence=(
                f"author-exact mean={pendulum_ars['mean']:.6f}; "
                f"corrected mean={pendulum_ars_corrected['mean']:.6f}; "
                f"delta={float(pendulum_ars_corrected['mean']) - float(pendulum_ars['mean']):.6f}"
            ),
            note="This does not replace the author-exact headline comparison.",
        ),
    ]

    aero_rows = [
        row
        for row in exclusions
        if row.get("status") == "not_executed"
        and row.get("claim_id") == "claim5"
        and row.get("task_id") == "aero2_physical_hardware"
        and row.get("method") == "physical_quanser_aero2"
    ]
    if len(aero_rows) != 1:
        raise RuntimeError(f"Expected one Aero 2 scope limitation, found {len(aero_rows)}")
    aero_reason = aero_rows[0].get("reason", "")
    rows.extend(
        [
            unexecuted_number_row(
                component="C5 Aero 2 reported CH-PPO real-hardware return",
                paper_value=-55.8,
                verdict="inconclusive_physical_hardware_unavailable",
                scope="Claim 5 / Quanser Aero 2",
                evidence=aero_reason,
                note="No physical measurement was produced; simulation is not substituted for transfer evidence.",
            ),
            unexecuted_number_row(
                component="C5 Aero 2 reported PPO real-hardware return",
                paper_value=-182.0,
                verdict="inconclusive_physical_hardware_unavailable",
                scope="Claim 5 / Quanser Aero 2",
                evidence=aero_reason,
                note="No physical measurement was produced; simulation is not substituted for transfer evidence.",
            ),
        ]
    )
    return rows


def csv_value(value: Any) -> str | int | float:
    if isinstance(value, bool):
        return "true" if value else "false"
    return value


def write_outputs(
    output_root: Path,
    rows: Sequence[Mapping[str, Any]],
    tables_manifest: Mapping[str, Any],
    tables_manifest_path: Path,
    command: Sequence[str],
) -> None:
    columns = (
        "component",
        "paper_value",
        "reproduced_value",
        "difference",
        "absolute_difference",
        "tolerance",
        "within_tolerance",
        "verdict",
        "scope",
        "evidence",
        "note",
    )
    csv_path = output_root / "claim_results.csv"
    with csv_path.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=columns, lineterminator="\n")
        writer.writeheader()
        for row in rows:
            writer.writerow({key: csv_value(row[key]) for key in columns})

    lines = [
        "# Claim-to-evidence results",
        "",
        "All empirical values below are recomputed from canonical episode rows; no figure HTML is used as numerical evidence.",
        "",
        "| Component | Paper | Reproduced | Verdict | Evidence |",
        "|---|---:|---:|---|---|",
    ]
    for row in rows:
        paper = "—" if row["paper_value"] == "" else f"{float(row['paper_value']):.6g}"
        reproduced = (
            "—"
            if row["reproduced_value"] == ""
            else f"{float(row['reproduced_value']):.6g}"
        )
        evidence = str(row["evidence"]).replace("|", "\\|").replace("\n", " ")
        lines.append(
            f"| {row['component']} | {paper} | {reproduced} | "
            f"{row['verdict']} | {evidence} |"
        )
    lines.extend(
        [
            "",
            "## Interpretation boundaries",
            "",
            "- Claim 1 is supported only for the continuous, unconstrained theorem mechanism; historical priority and discrete global optimality were not established.",
            "- The challenge's Claim 3 performance numbers come from a stale manuscript version; arXiv v4 corrected the mean/range transposition.",
            "- REINFORCE is a fresh fixed-seed replication because the original seeds and selected coefficients were not released.",
            "- Pendulum and Mountain Car report the best policy selected on the same fixed grid used for reporting; all-seed distributions remain available separately.",
            "- Aero 2 physical transfer remains inconclusive; no simulation-only result is presented as real-hardware evidence.",
            "",
        ]
    )
    markdown_path = output_root / "claim_results.md"
    markdown_path.write_text("\n".join(lines), encoding="utf-8")

    manifest = {
        "schema_version": "1.0.0",
        "builder": "analyze_claims.py",
        "analysis_version": ANALYSIS_VERSION,
        "builder_sha256": sha256_file(Path(__file__).resolve()),
        "command": list(command),
        "status": "success",
        "row_count": len(rows),
        "source_tables_manifest_sha256": sha256_file(tables_manifest_path),
        "source_table_hashes": {
            name: declaration["sha256"]
            for name, declaration in sorted(tables_manifest["tables"].items())
        },
        "outputs": {
            path.name: {
                "sha256": sha256_file(path),
                "bytes": path.stat().st_size,
            }
            for path in (csv_path, markdown_path)
        },
    }
    (output_root / "MANIFEST.json").write_text(
        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
        encoding="utf-8",
    )


def build_command(tables_root: Path, output_root: Path, replace: bool) -> list[str]:
    """Return the canonical, interpreter-independent command for this analysis."""
    command = [
        str(Path(__file__).resolve()),
        "--tables-root",
        str(tables_root.resolve()),
        "--output-root",
        str(output_root.resolve()),
    ]
    if replace:
        command.append("--replace")
    return command


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--tables-root", type=Path, required=True)
    parser.add_argument("--output-root", type=Path, required=True)
    parser.add_argument("--replace", action="store_true")
    return parser.parse_args()


def main() -> int:
    args = parse_args()
    tables_root = args.tables_root.resolve()
    output_root = args.output_root.resolve()
    tables_manifest, tables = read_verified_tables(tables_root)
    rows = build_claim_rows(tables)
    output_root.parent.mkdir(parents=True, exist_ok=True)
    if output_root.exists() and not args.replace:
        raise FileExistsError(f"Refusing to overwrite analysis: {output_root}")
    staging = Path(tempfile.mkdtemp(prefix=f".{output_root.name}.", dir=output_root.parent))
    try:
        write_outputs(
            staging,
            rows,
            tables_manifest,
            tables_root / "MANIFEST.json",
            build_command(tables_root, output_root, args.replace),
        )
        if output_root.exists():
            backup = output_root.with_name(output_root.name + ".old")
            if backup.exists():
                shutil.rmtree(backup)
            output_root.rename(backup)
            staging.rename(output_root)
            shutil.rmtree(backup)
        else:
            staging.rename(output_root)
    except BaseException:
        shutil.rmtree(staging, ignore_errors=True)
        raise
    print(f"Wrote {len(rows)} claim components to {output_root}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())