bnet-ot-fusion / tests /test_evaluation.py
BioinstLab's picture
Initial commit: BNet-OT multimodal late fusion implementation
3c8ce9d
Raw History Blame Contribute Delete
3.49 kB
import json
import numpy as np
import pytest
from sklearn.metrics import average_precision_score, f1_score, recall_score
from otfusion import (
binary_metrics,
bootstrap_metrics,
cross_fitted_calibration,
net_benefit,
net_benefit_treat_all,
paired_comparison,
run_evaluation,
simulate_dropout,
)
@pytest.fixture
def data():
rng = np.random.default_rng(0)
y = (rng.random(300) < 0.7).astype(int)
hist = np.clip(0.6 * y + rng.normal(0.2, 0.2, 300), 0.01, 0.99)
risk = np.clip(0.2 * y + rng.normal(0.4, 0.2, 300), 0.01, 0.99)
return hist, risk, y
# ---------------------------------------------------------------------------
# Dropout
# ---------------------------------------------------------------------------
def test_dropout_rates_and_modes():
rng = np.random.default_rng(0)
m_p, m_q = simulate_dropout(20000, 0.2, "either", rng)
assert (~m_p | ~m_q).mean() == pytest.approx(0.2, abs=0.01)
assert not (~m_p & ~m_q).any() # "either" never drops both
m_p, m_q = simulate_dropout(1000, 0.5, "hist", rng)
assert m_q.all() and (~m_p).any()
m_p, m_q = simulate_dropout(1000, 0.0, "either", rng)
assert m_p.all() and m_q.all()
with pytest.raises(ValueError):
simulate_dropout(10, 1.5)
# ---------------------------------------------------------------------------
# Metrics
# ---------------------------------------------------------------------------
def test_binary_metrics_match_sklearn(data):
hist, _, y = data
m = binary_metrics(y, hist)
assert m["pr_auc"] == pytest.approx(average_precision_score(y, hist))
assert m["recall"] == pytest.approx(recall_score(y, hist >= 0.5))
assert m["f1"] == pytest.approx(f1_score(y, hist >= 0.5))
def test_bootstrap_ci_contains_point_estimate(data):
hist, _, y = data
out = bootstrap_metrics(y, hist, n_boot=200, seed=0)
for v in out.values():
assert v["ci_low"] <= v["value"] <= v["ci_high"] + 1e-9
def test_paired_comparison_of_identical_models_is_null(data):
hist, _, y = data
c = paired_comparison(y, hist, hist, n_boot=100)
assert c["diff"] == 0 and c["ci_low"] == 0 and c["ci_high"] == 0
assert c["p_bootstrap"] == 1.0
def test_net_benefit_treat_all_matches_formula(data):
_, _, y = data
t = np.linspace(0.05, 0.9, 10)
np.testing.assert_allclose(net_benefit(y, np.ones_like(y, dtype=float), t), net_benefit_treat_all(y, t))
np.testing.assert_allclose(net_benefit(y, np.zeros_like(y, dtype=float), t), 0.0)
# ---------------------------------------------------------------------------
# Calibration and pipeline
# ---------------------------------------------------------------------------
def test_cross_fitted_calibration_range(data):
hist, _, y = data
cal = cross_fitted_calibration(hist, y)
assert cal.shape == hist.shape
assert ((cal >= 0) & (cal <= 1)).all()
def test_pipeline_runs_end_to_end(data, tmp_path):
hist, risk, y = data
results = run_evaluation(hist, risk, y, n_boot=50, out_dir=str(tmp_path), calibration=(hist, risk, y))
assert results["calibration_method"] == "separate calibration split"
assert set(results["dropout"]["results"]) == {"0", "0.2", "0.5"}
saved = json.loads((tmp_path / "results.json").read_text())
assert saved["n_samples"] == 300
for name in ("calibration_curve", "pr_curve", "decision_curve", "average_transport_plan"):
assert (tmp_path / f"{name}.png").exists()