tests/test_bootstrap_ci.py

"""Unit tests for bootstrap CI helpers in src/eval/metrics.py.

Locks the contract of bootstrap_macro_ci and aggregate_with_ci per
spec docs/superpowers/specs/2026-06-07-v07-bootstrap-ci-design.md §6.1.
"""

from __future__ import annotations

from datetime import date

import pytest

from src.eval.metrics import aggregate_with_ci, bootstrap_macro_ci
from src.models.schemas import GoldEvaluation


# -----------------------------------------------------------
# bootstrap_macro_ci — 7 unit cases
# -----------------------------------------------------------


def test_basic_smoke():
    """Fixed inputs produce mean = 0.55 with a CI that brackets the mean."""
    out = bootstrap_macro_ci(
        [0.3, 0.4, 0.5, 0.6, 0.7, 0.8],
        n_resamples=1000,
        confidence=0.95,
        seed=42,
    )
    assert out["mean"] == pytest.approx(0.55, abs=1e-9)
    assert out["n"] == 6
    assert out["n_resamples"] == 1000
    assert out["confidence"] == 0.95
    assert out["ci_low"] <= out["mean"] <= out["ci_high"]
    assert 0.3 <= out["ci_low"] <= out["ci_high"] <= 0.8


def test_determinism():
    """Same input + same seed → bit-identical output across calls."""
    vals = [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]
    a = bootstrap_macro_ci(vals, seed=42)
    b = bootstrap_macro_ci(vals, seed=42)
    assert a == b


def test_degenerate_n1():
    """Single value collapses CI onto the point estimate."""
    out = bootstrap_macro_ci([0.5], seed=42)
    assert out["mean"] == 0.5
    assert out["ci_low"] == 0.5
    assert out["ci_high"] == 0.5
    assert out["n"] == 1


def test_degenerate_all_same():
    """Zero variance → CI collapses to the mean."""
    out = bootstrap_macro_ci([0.5] * 6, seed=42)
    assert out["mean"] == 0.5
    assert out["ci_low"] == 0.5
    assert out["ci_high"] == 0.5
    assert out["n"] == 6


def test_empty_raises():
    """Empty input is a programmer error — raise, do not silently NaN."""
    with pytest.raises(ValueError):
        bootstrap_macro_ci([])


def test_all_none_raises():
    """All-None input is a programmer error too."""
    with pytest.raises(ValueError):
        bootstrap_macro_ci([None, None, None])


def test_partial_none_skipped():
    """Mixed None / float — None entries are dropped; n reflects survivors."""
    out = bootstrap_macro_ci(
        [0.5, None, 0.5, 0.5, None, 0.5], seed=42
    )
    assert out["mean"] == 0.5
    assert out["n"] == 4
    assert out["ci_low"] == 0.5
    assert out["ci_high"] == 0.5


# -----------------------------------------------------------
# aggregate_with_ci — 2 unit cases
# -----------------------------------------------------------


def _mk_eval(event_id: str, seed: int, *, cat_r: float, cat_sev: float,
             cos_f1: float, cat_f1: float) -> GoldEvaluation:
    """Minimal GoldEvaluation builder for tests — only fields touched by
    aggregate_with_ci need realistic values. Other required fields get
    placeholder content.
    """
    return GoldEvaluation(
        event_id=event_id,
        predicted_chain={
            "event_id": event_id,
            "trigger_summary": "test",
            "trigger_country": "Testland",
            "trigger_iso": "TST",
            "trigger_date": date(2026, 6, 7),
            "cascade_events": [],
        },
        gold_event_id=event_id,
        gold_node_count=0,
        predicted_node_count=0,
        matches=[],
        precision=0.0,
        recall=0.0,
        f1=cos_f1,
        domain_jaccard=0.0,
        severity_match_rate=None,
        threshold=0.35,
        embedder_model="all-MiniLM-L6-v2",
        algorithm_version="v10",
        gold_eval_fingerprint=f"fp_{event_id}_{seed}",
        retrieval_info=[],
        evaluated_at=date(2026, 6, 7),
        category_precision=None,
        category_recall=cat_r,
        category_f1=cat_f1,
        category_severity_match_rate=cat_sev,
        category_matches=[],
    )


def test_aggregate_with_ci_groups_by_event():
    """3 events × 3 seeds → seed-mean per event, then bootstrap over events."""
    evals = []
    for eid, recalls in [
        ("E1", [0.3, 0.4, 0.5]),
        ("E2", [0.6, 0.6, 0.6]),
        ("E3", [0.7, 0.8, 0.9]),
    ]:
        for seed, r in zip([42, 1337, 2026], recalls):
            evals.append(_mk_eval(eid, seed,
                                  cat_r=r, cat_sev=0.5,
                                  cos_f1=r, cat_f1=r))

    out = aggregate_with_ci(evals, bootstrap_seed=42)

    assert set(out.keys()) == {
        "category_recall", "category_severity_match_rate",
        "cosine_f1", "category_f1",
    }
    # Per-event seed-means: E1=0.4, E2=0.6, E3=0.8 → grand mean = 0.6
    cat_r = out["category_recall"]
    assert cat_r["mean"] == pytest.approx(0.6, abs=1e-9)
    assert cat_r["n"] == 3
    assert cat_r["per_event_values"]["E1"] == pytest.approx(0.4, abs=1e-9)
    assert cat_r["per_event_values"]["E2"] == pytest.approx(0.6, abs=1e-9)
    assert cat_r["per_event_values"]["E3"] == pytest.approx(0.8, abs=1e-9)
    assert cat_r["per_event_seed_values"]["E1"] == [0.3, 0.4, 0.5]


def test_aggregate_skips_outlier_events():
    """outlier_event_ids removes events from CI computation."""
    evals = [
        _mk_eval("E1", 42, cat_r=0.4, cat_sev=0.5, cos_f1=0.4, cat_f1=0.4),
        _mk_eval("E2", 42, cat_r=0.6, cat_sev=0.5, cos_f1=0.6, cat_f1=0.6),
        _mk_eval("OUTLIER", 42, cat_r=0.0, cat_sev=0.0,
                 cos_f1=0.0, cat_f1=0.0),
    ]
    out = aggregate_with_ci(evals, outlier_event_ids={"OUTLIER"})
    assert "OUTLIER" not in out["category_recall"]["per_event_values"]
    assert out["category_recall"]["n"] == 2
    assert out["category_recall"]["mean"] == pytest.approx(0.5, abs=1e-9)