tests/test_paper_artifacts.py

from __future__ import annotations

import subprocess
import sys
from pathlib import Path

from dovla_cil.utils.io import read_json, write_json


def test_make_paper_artifacts_with_fake_metrics(tmp_path: Path) -> None:
    runs_dir = tmp_path / "runs"
    out_dir = tmp_path / "paper_artifacts"
    _write_fake_paper_runs(runs_dir)

    result = subprocess.run(
        [
            sys.executable,
            "scripts/make_paper_artifacts.py",
            "--runs",
            str(runs_dir),
            "--out",
            str(out_dir),
        ],
        check=True,
        text=True,
        capture_output=True,
    )

    assert "paper artifacts:" in result.stdout
    for filename in (
        "main_scaling_table.csv",
        "main_scaling_table.md",
        "baseline_comparison_table.csv",
        "baseline_comparison_table.md",
        "ablation_table.csv",
        "ablation_table.md",
        "causalstress_per_category_table.csv",
        "causalstress_per_category_table.md",
        "result_summary.md",
        "artifact_manifest.json",
    ):
        assert (out_dir / filename).exists()

    for filename in (
        "performance_vs_k.png",
        "same_state_vs_cross_state_ranking.png",
        "physical_outcome_vs_label_only.png",
        "success_by_failure_category.png",
        "regret_calibration.png",
    ):
        path = out_dir / "figures" / filename
        assert path.exists()
        assert path.stat().st_size > 0

    summary = (out_dir / "result_summary.md").read_text(encoding="utf-8")
    manifest = read_json(out_dir / "artifact_manifest.json")
    assert "Best detected model" in summary
    assert "Expected Claim Checks" in summary
    assert manifest["num_scaling_rows"] == 2
    assert manifest["num_baseline_rows"] >= 4
    assert manifest["num_category_rows"] == 2


def test_paper_artifacts_loads_measured_lattice_eval(tmp_path: Path) -> None:
    from scripts.make_paper_artifacts import collect_metric_rows, load_result_payloads

    runs = tmp_path / "runs"
    write_json(
        {
            "k": 8,
            "selected_success_rate": 0.75,
            "pairwise_ranking_accuracy": 0.8,
            "top1_action_selection": 0.7,
            "effect_prediction_mae": 0.2,
        },
        runs / "scaling" / "k_8" / "seed_0" / "lattice_eval.json",
    )

    rows = collect_metric_rows(load_result_payloads(runs), runs)

    assert len(rows) == 1
    assert rows[0]["k"] == 8
    assert rows[0]["success_rate"] == 0.75


def _write_fake_paper_runs(root: Path) -> None:
    scaling_payloads = [
        {
            "run_name": "k1",
            "k": 1,
            "num_states": 16,
            "effective_total_records": 16,
            "task_success_rate": 0.35,
            "pairwise_ranking_accuracy": 0.50,
            "top1_action_selection": 0.45,
            "instruction_switch_accuracy": 0.30,
            "effect_prediction_mae": 0.80,
            "regret_calibration_error": 0.25,
        },
        {
            "run_name": "k4",
            "k": 4,
            "num_states": 4,
            "effective_total_records": 16,
            "task_success_rate": 0.65,
            "pairwise_ranking_accuracy": 0.78,
            "top1_action_selection": 0.70,
            "instruction_switch_accuracy": 0.55,
            "effect_prediction_mae": 0.50,
            "regret_calibration_error": 0.12,
        },
    ]
    for payload in scaling_payloads:
        write_json(payload, root / "scaling_toy" / f"k_{payload['k']:04d}" / "metrics.json")

    baselines = {
        "expert_only_bc": (0.45, 0.55),
        "cross_state_negatives": (0.40, 0.45),
        "label_only_counterfactual": (0.42, 0.48),
        "no_rank_regret": (0.50, 0.52),
        "world_model_auxiliary": (0.54, 0.56),
    }
    for baseline, (success, ranking) in baselines.items():
        write_json(
            {
                "baseline": baseline,
                "eval": {
                    "task_success_rate": success,
                    "pairwise_ranking_accuracy": ranking,
                    "top1_action_selection": success,
                    "instruction_switch_accuracy": success - 0.05,
                    "effect_prediction_mae": 1.0 - success,
                    "regret_calibration_error": 1.0 - ranking,
                },
            },
            root / "baselines" / baseline / "metrics.json",
        )

    write_json(
        {
            "run_name": "causalstress_best",
            "task_success_rate": 0.66,
            "pairwise_ranking_accuracy": 0.79,
            "per_category": {
                "wrong_target_distractor": {
                    "success": 0.60,
                    "selected_success": 0.70,
                    "failure_rate": 0.40,
                    "instruction_switch": 0.75,
                    "top1": 0.80,
                    "pair_correct": 0.79,
                },
                "near_miss_boundary": {
                    "success": 0.55,
                    "selected_success": 0.65,
                    "failure_rate": 0.45,
                    "instruction_switch": 0.70,
                    "top1": 0.76,
                    "pair_correct": 0.74,
                },
            },
        },
        root / "dovla_toy" / "causalstress.json",
    )