import json import numpy as np import pytest from sklearn.metrics import average_precision_score, f1_score, recall_score from otfusion import ( binary_metrics, bootstrap_metrics, cross_fitted_calibration, net_benefit, net_benefit_treat_all, paired_comparison, run_evaluation, simulate_dropout, ) @pytest.fixture def data(): rng = np.random.default_rng(0) y = (rng.random(300) < 0.7).astype(int) hist = np.clip(0.6 * y + rng.normal(0.2, 0.2, 300), 0.01, 0.99) risk = np.clip(0.2 * y + rng.normal(0.4, 0.2, 300), 0.01, 0.99) return hist, risk, y # --------------------------------------------------------------------------- # Dropout # --------------------------------------------------------------------------- def test_dropout_rates_and_modes(): rng = np.random.default_rng(0) m_p, m_q = simulate_dropout(20000, 0.2, "either", rng) assert (~m_p | ~m_q).mean() == pytest.approx(0.2, abs=0.01) assert not (~m_p & ~m_q).any() # "either" never drops both m_p, m_q = simulate_dropout(1000, 0.5, "hist", rng) assert m_q.all() and (~m_p).any() m_p, m_q = simulate_dropout(1000, 0.0, "either", rng) assert m_p.all() and m_q.all() with pytest.raises(ValueError): simulate_dropout(10, 1.5) # --------------------------------------------------------------------------- # Metrics # --------------------------------------------------------------------------- def test_binary_metrics_match_sklearn(data): hist, _, y = data m = binary_metrics(y, hist) assert m["pr_auc"] == pytest.approx(average_precision_score(y, hist)) assert m["recall"] == pytest.approx(recall_score(y, hist >= 0.5)) assert m["f1"] == pytest.approx(f1_score(y, hist >= 0.5)) def test_bootstrap_ci_contains_point_estimate(data): hist, _, y = data out = bootstrap_metrics(y, hist, n_boot=200, seed=0) for v in out.values(): assert v["ci_low"] <= v["value"] <= v["ci_high"] + 1e-9 def test_paired_comparison_of_identical_models_is_null(data): hist, _, y = data c = paired_comparison(y, hist, hist, n_boot=100) assert c["diff"] == 0 and c["ci_low"] == 0 and c["ci_high"] == 0 assert c["p_bootstrap"] == 1.0 def test_net_benefit_treat_all_matches_formula(data): _, _, y = data t = np.linspace(0.05, 0.9, 10) np.testing.assert_allclose(net_benefit(y, np.ones_like(y, dtype=float), t), net_benefit_treat_all(y, t)) np.testing.assert_allclose(net_benefit(y, np.zeros_like(y, dtype=float), t), 0.0) # --------------------------------------------------------------------------- # Calibration and pipeline # --------------------------------------------------------------------------- def test_cross_fitted_calibration_range(data): hist, _, y = data cal = cross_fitted_calibration(hist, y) assert cal.shape == hist.shape assert ((cal >= 0) & (cal <= 1)).all() def test_pipeline_runs_end_to_end(data, tmp_path): hist, risk, y = data results = run_evaluation(hist, risk, y, n_boot=50, out_dir=str(tmp_path), calibration=(hist, risk, y)) assert results["calibration_method"] == "separate calibration split" assert set(results["dropout"]["results"]) == {"0", "0.2", "0.5"} saved = json.loads((tmp_path / "results.json").read_text()) assert saved["n_samples"] == 300 for name in ("calibration_curve", "pr_curve", "decision_curve", "average_transport_plan"): assert (tmp_path / f"{name}.png").exists()