bundle/v0/tests/test_scaffold.py

from __future__ import annotations

import json
from pathlib import Path

import pytest

from loss_aware_dro_repro.artifacts import validate_result
from loss_aware_dro_repro.batch_control import _task_runtime_contract
from loss_aware_dro_repro.core import (
    CONFIG_ROOT,
    ContractError,
    load_json,
    load_plan,
    plan_hash,
    sha256_value,
)
from loss_aware_dro_repro.datasets import generate_dataset
from loss_aware_dro_repro.empirical_task import normalize_empirical_execution_config
from loss_aware_dro_repro.evaluator import evaluate_bundle
from loss_aware_dro_repro.failure_ledger import append_failure
from loss_aware_dro_repro.matrix import expand_tasks, validate_matrix
from loss_aware_dro_repro.paper_task import normalize_execution_config
from loss_aware_dro_repro.smoke import run_smoke


def test_matrix_count_and_suite_counts():
    summary = validate_matrix()
    assert summary["task_count"] == 14000
    assert summary["suite_counts"] == {
        "portfolio_gaussian_main": 5000,
        "portfolio_gaussian_coverage_ablation": 5000,
        "regression_absolute_main": 500,
        "portfolio_gaussian_highdim": 1000,
        "portfolio_discrete": 1000,
        "portfolio_gmm": 1000,
        "regression_squared": 500,
    }


def test_task_ids_hashes_and_seed_schedule_are_deterministic():
    first = list(expand_tasks())[:2]
    second = list(expand_tasks())[:2]
    assert first == second
    assert first[0]["task_id"].endswith("d001/r00/n010")
    assert first[0]["seeds"]["distribution"] == first[1]["seeds"]["distribution"]
    assert first[0]["seeds"]["dataset"] == first[1]["seeds"]["dataset"]
    assert first[0]["seeds"]["bootstrap"] != first[1]["seeds"]["bootstrap"]


def test_coverage_ablation_is_paired_with_main_suite():
    tasks = {task["task_id"]: task for task in expand_tasks() if task["task_id"].endswith("d001/r00/n010")}
    main = tasks["portfolio_gaussian_main/d001/r00/n010"]["seeds"]
    ablation = tasks["portfolio_gaussian_coverage_ablation/d001/r00/n010"]["seeds"]
    for stream in ("distribution", "dataset", "bootstrap", "oos", "uncertainty"):
        assert main[stream] == ablation[stream]
    assert main["optimizer"] != ablation["optimizer"]


def test_published_parameters_are_complete_and_symmetric():
    rows = load_json(CONFIG_ROOT / "published_gaussians_table2.json")["distributions"]
    models = load_json(CONFIG_ROOT / "regression_models_table3.json")["models"]
    assert [row["id"] for row in rows] == list(range(1, 51))
    assert [row["id"] for row in models] == list(range(1, 11))
    for row in rows:
        covariance = row["covariance"]
        assert len(row["mean"]) == 3
        assert all(covariance[i][j] == covariance[j][i] for i in range(3) for j in range(3))


def test_dataset_generation_is_deterministic_and_nested():
    tasks = list(expand_tasks())[:2]
    small, meta_small = generate_dataset(tasks[0])
    small_again, meta_again = generate_dataset(tasks[0])
    larger, _ = generate_dataset(tasks[1])
    assert meta_small["fingerprint"] == meta_again["fingerprint"]
    assert (small == small_again).all()
    assert (small == larger[: len(small)]).all()


def test_plan_tracks_paper_schedule_not_author_adam_path():
    plan = load_plan()
    assert plan["paper_hyperparameters"]["optimizer"] == "paper_algorithm_plain_gradient_descent"
    assert plan["paper_hyperparameters"]["adam_allowed"] is False
    assert plan["paper_hyperparameters"]["bootstrap_count"] == 20
    assert plan["paper_hyperparameters"]["stopping_contract"]["primary_gate"] == "paper_total_phi_literal"


def test_stopping_storage_batch_identity_matches_production_runner_identity_formula():
    plan = load_plan()
    requested = {
        "schema_version": 2,
        "execution_scale": "stopping_storage_sample",
        "max_outer_iterations": 5000,
    }
    tasks_by_id = {task["task_id"]: task for task in expand_tasks(plan)}
    tasks = [
        tasks_by_id["portfolio_gaussian_main/d001/r00/n010"],
        tasks_by_id["regression_squared/d001/r00/n010"],
    ]
    snapshot = {"commit": "a" * 40, "clean": True}
    for task in tasks:
        runtime = _task_runtime_contract(task, plan, requested, snapshot)
        if task["suite"] == "portfolio_gaussian_main":
            runner_config = normalize_execution_config(plan, requested)
        else:
            runner_config = normalize_empirical_execution_config(
                plan,
                requested,
                suite=task["suite"],
            )
        assert runtime["normalized_execution_config"] == runner_config
        assert runner_config["optimizer"] == "paper_algorithm_plain_gradient_descent"
        assert runtime["execution_config_hash"] == sha256_value(runner_config)
        expected_identity = sha256_value(
            {
                "runner_schema_version": runtime["runner_schema_version"],
                "plan_hash": plan_hash(plan),
                "task_hash": task["task_hash"],
                "execution_config_hash": sha256_value(runner_config),
                "source_tree_hash": runtime["scientific_source_tree_hash"],
                "implementation_commit": snapshot["commit"],
                "working_tree_clean": snapshot["clean"],
                "trace_stream_contract": runtime["trace_stream_contract"],
            }
        )
        assert runtime["scientific_run_identity"] == expected_identity


def _valid_result(task):
    zeros = "0" * 64
    stopping = {
        "contract_version": 2,
        "reason": "maximum_outer_iterations_reached",
        "tolerance": 1e-6,
        "primary_rule": "paper_algorithm2_total_penalized_phi_literal_previous",
        "paper_total_phi_literal": {
            "objective": "total_penalized_phi", "previous_objective": None,
            "current_objective": 1.0, "literal_denominator": "phi_previous",
            "signed_relative_improvement": None, "denominator_defined": False,
            "denominator_sign_inversion_risk": False, "stop_triggered": False,
            "stop_trigger_caused_by_worsening": False, "objective_worsened": False,
        },
        "paper_total_phi_abs_denominator": {
            "role": "safety_diagnostic_only", "objective": "total_penalized_phi",
            "previous_objective": None, "current_objective": 1.0,
            "denominator": "abs_phi_previous", "signed_relative_improvement": None,
            "denominator_defined": False, "stop_trigger_observed": False,
            "stop_trigger_caused_by_worsening": False, "objective_worsened": False,
        },
        "released_lower_objective_abs_denominator": {
            "role": "diagnostic_only", "objective": "nonpenalized_lower_objective",
            "previous_objective": None, "current_objective": 1.0,
            "denominator": "abs_previous_lower_objective", "signed_relative_improvement": None,
            "denominator_defined": False, "stop_trigger_observed": False,
            "stop_trigger_caused_by_worsening": False, "objective_worsened": False,
        },
    }
    return {
        "schema_version": 2,
        "task_id": task["task_id"],
        "run_identity": "sha256:" + zeros,
        "plan_hash": plan_hash(load_plan()),
        "task_hash": task["task_hash"],
        "implementation": {"kind": "independent", "commit": "0" * 40, "reference_commit": "6e8d18f023f4f2e920af22dc1ef32b292dbb9631"},
        "status": "success",
        "seeds": task["seeds"],
        "dataset": {"suite": task["suite"], "distribution_id": task["distribution_id"], "replicate": task["replicate"], "sample_size": task["sample_size"], "fingerprint": "sha256:" + zeros, "covariance_adjustment_fro": 0.0},
        "scientific_contract": {"route": "gaussian_gelbrich_cvar_socp", "transport": task["transport"], "loss": task["loss"], "risk_estimand": "gaussian_exact", "wasserstein_order": 2, "epsilon": 0.1, "bootstrap_count": task["hyperparameters"]["bootstrap_count"], "coverage_beta": task["hyperparameters"]["coverage_beta"], "coverage_penalty_lambda": task["hyperparameters"]["coverage_penalty_lambda"], "initial_metric_factor": [[1.0]], "final_metric_factor": [[1.0]], "initial_decision": [1.0], "final_decision": [1.0], "true_parameters_hash": "sha256:" + zeros, "paired_task_id": None, "oos_evaluation": {"kind": "test"}, "stopping_rule": "paper_total_phi_literal", "stopping_interpretation": "iteration_cap_reached"},
        "runtime": {"started_at": "2026-07-19T00:00:00Z", "finished_at": "2026-07-19T00:00:01Z", "duration_seconds": 1.0, "hostname_hash": "sha256:" + zeros, "python": "3.11.15", "packages": {}, "cpu_model": "test", "blas_threads": 1, "cost_usd": 0.0},
        "solver": {
            "name": "CLARABEL",
            "status": "optimal",
            "residual_contract_version": 2,
            "residuals": {
                "contract_version": 2,
                "primal": 0.0,
                "dual": 0.0,
                "equality": 0.0,
                "cone": 0.0,
                "dual_cone": 0.0,
                "complementarity": 0.0,
                "duality_gap": 0.0,
                "solver_native_primal": 0.0,
                "solver_native_dual": 0.0,
                "primal_relative": 0.0,
                "dual_relative": 0.0,
                "equality_relative": 0.0,
                "cone_relative": 0.0,
                "dual_cone_relative": 0.0,
                "duality_gap_relative": 0.0,
                "complementarity_relative": 0.0,
            },
            "primal_residual": 0.0,
            "dual_residual": 0.0,
            "duality_gap": 0.0,
        },
        "optimization": {
            "iterations": 1,
            "stop_reason": "maximum_outer_iterations_reached",
            "stopping_contract_version": 2,
            "stopping": stopping,
            "initial_stationarity": 1.0,
            "final_stationarity": 0.01,
            "trace_stream_binding": "sha256:" + zeros,
            "checkpoint_store_every": 100,
            "iteration_trace_path": "iteration_trace.jsonl",
            "iteration_trace_sha256": zeros,
            "iteration_trace_records": 1,
            "checkpoint_state_path": "checkpoint_states.jsonl",
            "checkpoint_state_sha256": zeros,
            "checkpoint_state_records": 2,
        },
        "metrics": {"worst_case_initial": 1.0, "worst_case_final": 0.8, "oos_initial": 1.0, "oos_final": 0.9, "relative_worst_case_improvement": 0.2, "relative_oos_improvement": 0.1, "coverage_initial": True, "coverage_final": True},
        "artifact_hashes": {
            "iteration_trace": zeros,
            "checkpoint_states": zeros,
            "dataset": zeros,
        },
    }


def test_artifact_validator_accepts_independent_lineage():
    task = next(expand_tasks())
    validate_result(_valid_result(task), task, plan_hash(load_plan()))


def test_artifact_validator_rejects_author_or_cached_output():
    task = next(expand_tasks())
    record = _valid_result(task)
    record["implementation"]["kind"] = "author"
    with pytest.raises(ContractError, match="independent"):
        validate_result(record, task, plan_hash(load_plan()))


def test_artifact_validator_rejects_trace_free_result():
    task = next(expand_tasks())
    record = _valid_result(task)
    record["optimization"]["iteration_trace_path"] = ""
    with pytest.raises(ContractError, match="schema|trace"):
        validate_result(record, task, plan_hash(load_plan()))


def test_evaluator_fails_closed_on_incomplete_results(tmp_path: Path):
    results = tmp_path / "results.jsonl"
    gradient = tmp_path / "gradient.json"
    proof = tmp_path / "proof.json"
    results.write_text("", encoding="utf-8")
    gradient.write_text("{}", encoding="utf-8")
    proof.write_text("{}", encoding="utf-8")
    verdict = evaluate_bundle(results, gradient, proof)
    assert verdict["gate"] == "FAIL_MISSING_PAPER_SCALE_RESULTS"
    assert verdict["claim_verdicts"] == {
        claim_id: "inconclusive" for claim_id in ("A1", "A2", "A3", "A4", "A5", "A6")
    }
    assert verdict["missing_task_count"] == 14000


def test_failure_ledger_is_append_only_and_denies_identity_reuse(tmp_path: Path):
    ledger = tmp_path / "failures.jsonl"
    entry = {"task_id": "x", "run_identity": "sha256:" + "1" * 64, "attempt": 1, "timestamp": "2026-07-19T00:00:00Z", "phase": "solver", "exception_type": "SolverError", "message": "diagnostic"}
    failure_id = append_failure(ledger, entry)
    assert failure_id.startswith("failure-")
    row = json.loads(ledger.read_text(encoding="utf-8"))
    assert row["retry_allowed"] is False
    with pytest.raises(ContractError, match="may not be reused"):
        append_failure(ledger, entry)


def test_smoke_is_explicitly_ineligible_for_claims():
    receipt = run_smoke()
    assert receipt["evidence_scale"] == "TOY_DIAGNOSTIC"
    assert receipt["claim_eligible"] is False
    assert receipt["derivative_plumbing"]["passed"] is True
    assert receipt["scientific_verdicts"] == {"C1": "HOLD", "C2": "HOLD", "C3": "HOLD"}


def test_control_authority_and_scientific_state_remain_closed():
    paper = load_json(Path(__file__).parents[1] / "paper.json")
    assert paper["state"] == "PREPARATION_HOLD"
    assert paper["current_evidence_scale"] == "scientific_canary"
    assert paper["scientific_claims"] == {
        claim_id: "HOLD" for claim_id in ("A1", "A2", "A3", "A4", "A5", "A6")
    }
    assert not any(paper["authority"].values())


def test_openresearch_manifest_hash_matches_canonical_plan():
    manifest = load_json(Path(__file__).parents[1] / ".openresearch" / "control" / "run-manifest.json")
    expected = plan_hash(load_plan())
    assert manifest["science"]["analysis_plan_hash"] == expected
    assert manifest["run"]["config_hash"] == expected
    assert manifest["launch_authorized"] is False
    assert not any(manifest["authority"].values())


def test_openresearch_launch_ledger_is_empty():
    ledger = load_json(Path(__file__).parents[1] / ".openresearch" / "control" / "launch-ledger.json")
    assert ledger == {"schema_version": 1, "launches": []}


@pytest.mark.parametrize("name", ["paper_scale.json", "published_gaussians_table2.json", "regression_models_table3.json", "artifact.schema.json", "failure-ledger.schema.json", "claim-verdicts.json"])
def test_all_contract_json_parses(name):
    data = load_json(CONFIG_ROOT / name)
    assert data.get("schema_version") in {1, 2} or data.get("$schema") == "https://json-schema.org/draft/2020-12/schema"