File size: 3,488 Bytes
3c8ce9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import json

import numpy as np
import pytest
from sklearn.metrics import average_precision_score, f1_score, recall_score

from otfusion import (
    binary_metrics,
    bootstrap_metrics,
    cross_fitted_calibration,
    net_benefit,
    net_benefit_treat_all,
    paired_comparison,
    run_evaluation,
    simulate_dropout,
)


@pytest.fixture
def data():
    rng = np.random.default_rng(0)
    y = (rng.random(300) < 0.7).astype(int)
    hist = np.clip(0.6 * y + rng.normal(0.2, 0.2, 300), 0.01, 0.99)
    risk = np.clip(0.2 * y + rng.normal(0.4, 0.2, 300), 0.01, 0.99)
    return hist, risk, y


# ---------------------------------------------------------------------------
# Dropout
# ---------------------------------------------------------------------------


def test_dropout_rates_and_modes():
    rng = np.random.default_rng(0)
    m_p, m_q = simulate_dropout(20000, 0.2, "either", rng)
    assert (~m_p | ~m_q).mean() == pytest.approx(0.2, abs=0.01)
    assert not (~m_p & ~m_q).any()  # "either" never drops both

    m_p, m_q = simulate_dropout(1000, 0.5, "hist", rng)
    assert m_q.all() and (~m_p).any()

    m_p, m_q = simulate_dropout(1000, 0.0, "either", rng)
    assert m_p.all() and m_q.all()

    with pytest.raises(ValueError):
        simulate_dropout(10, 1.5)


# ---------------------------------------------------------------------------
# Metrics
# ---------------------------------------------------------------------------


def test_binary_metrics_match_sklearn(data):
    hist, _, y = data
    m = binary_metrics(y, hist)
    assert m["pr_auc"] == pytest.approx(average_precision_score(y, hist))
    assert m["recall"] == pytest.approx(recall_score(y, hist >= 0.5))
    assert m["f1"] == pytest.approx(f1_score(y, hist >= 0.5))


def test_bootstrap_ci_contains_point_estimate(data):
    hist, _, y = data
    out = bootstrap_metrics(y, hist, n_boot=200, seed=0)
    for v in out.values():
        assert v["ci_low"] <= v["value"] <= v["ci_high"] + 1e-9


def test_paired_comparison_of_identical_models_is_null(data):
    hist, _, y = data
    c = paired_comparison(y, hist, hist, n_boot=100)
    assert c["diff"] == 0 and c["ci_low"] == 0 and c["ci_high"] == 0
    assert c["p_bootstrap"] == 1.0


def test_net_benefit_treat_all_matches_formula(data):
    _, _, y = data
    t = np.linspace(0.05, 0.9, 10)
    np.testing.assert_allclose(net_benefit(y, np.ones_like(y, dtype=float), t), net_benefit_treat_all(y, t))
    np.testing.assert_allclose(net_benefit(y, np.zeros_like(y, dtype=float), t), 0.0)


# ---------------------------------------------------------------------------
# Calibration and pipeline
# ---------------------------------------------------------------------------


def test_cross_fitted_calibration_range(data):
    hist, _, y = data
    cal = cross_fitted_calibration(hist, y)
    assert cal.shape == hist.shape
    assert ((cal >= 0) & (cal <= 1)).all()


def test_pipeline_runs_end_to_end(data, tmp_path):
    hist, risk, y = data
    results = run_evaluation(hist, risk, y, n_boot=50, out_dir=str(tmp_path), calibration=(hist, risk, y))
    assert results["calibration_method"] == "separate calibration split"
    assert set(results["dropout"]["results"]) == {"0", "0.2", "0.5"}
    saved = json.loads((tmp_path / "results.json").read_text())
    assert saved["n_samples"] == 300
    for name in ("calibration_curve", "pr_curve", "decision_curve", "average_transport_plan"):
        assert (tmp_path / f"{name}.png").exists()