tests/test_evaluate.py

"""Correctness tests for the evaluation stack.

Two levels of assurance:

* :func:`test_reference_output` is the important one - it runs the port on the
  sample submission bundled with the official LUNA16 archive and requires every
  published counter to match. If this passes, the matching logic *is* the
  official one.
* the synthetic tests pin down the degenerate cases that the reference
  submission never exercises (a detector with no false positives, a detector
  with no detections, self-paired differences).

Run with ``python tests/test_evaluate.py`` or ``pytest tests/``.
"""

from __future__ import annotations

import sys
from pathlib import Path

import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from luna_rev import evaluate as ev
from luna_rev import stats
from luna_rev.io_luna import all_uids, group_by_uid, load_annotations, load_excluded

UIDS = list(all_uids())
INCLUDED = group_by_uid(load_annotations())
EXCLUDED = group_by_uid(load_excluded())
N_NODULES = sum(len(v) for v in INCLUDED.values())


def _perfect() -> dict[str, np.ndarray]:
    """One candidate exactly on every reference nodule, at maximum confidence."""
    return {u: np.hstack([g[:, :3], np.ones((len(g), 1))]) for u, g in INCLUDED.items()}


def test_reference_output():
    """The port must reproduce the official CADAnalysis.txt counters exactly."""
    r = ev.validate_against_reference("legacy_abs")
    assert r["matches_reference"], f"counter mismatch: {r['mismatches']}"


def test_perfect_detector():
    """No false positives anywhere: CPM is 1 and every nodule is a true positive."""
    v = ev.match(_perfect(), INCLUDED, EXCLUDED, UIDS)
    r = ev.evaluate(v)
    assert r["true_positives"] == N_NODULES
    assert r["false_positives"] == 0
    assert abs(r["cpm"] - 1.0) < 1e-9, r["cpm"]


def test_empty_detector():
    """No candidates at all: sensitivity is 0 and every nodule is a false negative."""
    r = ev.evaluate(ev.match({}, INCLUDED, EXCLUDED, UIDS))
    assert r["false_negatives"] == N_NODULES
    assert r["cpm"] == 0.0


def test_excluded_findings_are_ignored_not_scored():
    """A candidate on an irrelevant finding must not be counted as a false positive."""
    uid = next(u for u in UIDS if u in EXCLUDED)
    x, y, z, _ = EXCLUDED[uid][0]
    cands = {uid: np.array([[x, y, z, 0.9]])}

    with_list = ev.match(cands, INCLUDED, EXCLUDED, [uid]).counters
    without = ev.match(cands, INCLUDED, None, [uid]).counters
    assert with_list["ignored_on_excluded"] == 1
    assert with_list["false_positives"] == 0
    assert without["false_positives"] == 1, "without the list it must count as an FP"


def test_double_detections_are_ignored():
    """Extra candidates inside a detected nodule are ignored, not false positives."""
    uid, g = next(iter(INCLUDED.items()))
    x, y, z, d = g[0]
    eps = d / 8.0
    cands = {uid: np.array([[x, y, z, 0.9], [x + eps, y, z, 0.5]])}
    c = ev.match(cands, INCLUDED, EXCLUDED, [uid]).counters
    assert c["true_positives"] == 1
    assert c["ignored_double_detections"] == 1
    assert c["false_positives"] == 0


def test_highest_confidence_candidate_wins():
    """A nodule is scored with its most confident hit, not its first."""
    uid, g = next(iter(INCLUDED.items()))
    x, y, z, d = g[0]
    cands = {uid: np.array([[x, y, z, 0.2], [x + d / 8.0, y, z, 0.8]])}
    v = ev.match(cands, INCLUDED, EXCLUDED, [uid])
    assert float(v.prob[v.gt == 1.0].max()) == 0.8


def test_self_paired_difference_is_zero():
    """Sharing bootstrap resamples must make a model's difference with itself exactly 0."""
    v = ev.match(_perfect(), INCLUDED, EXCLUDED, UIDS)
    b = ev.bootstrap_cpm({"a": v, "b": v}, UIDS, n_iter=25)
    d = ev.paired_difference(b["a"], b["b"])
    assert d["delta_bootstrap_mean"] == 0.0
    assert (d["ci_low"], d["ci_high"]) == (0.0, 0.0)


def test_size_stratified_recall_perfect():
    """A perfect detector recalls every size band, and the bands cover all nodules."""
    df = stats.size_stratified_recall(_perfect(), load_annotations(), EXCLUDED, UIDS)
    assert df["n_nodules"].sum() == N_NODULES
    assert (df["recall"] == 1.0).all()


def test_monotone_in_false_positives():
    """Adding pure noise can never increase CPM."""
    rng = np.random.default_rng(0)
    perfect = _perfect()
    noisy = {u: np.vstack([perfect.get(u, np.zeros((0, 4))),
                           np.hstack([rng.normal(0, 300, (20, 3)), rng.uniform(0, 1, (20, 1))])])
             for u in UIDS}
    a = ev.evaluate(ev.match(perfect, INCLUDED, EXCLUDED, UIDS))["cpm"]
    b = ev.evaluate(ev.match(noisy, INCLUDED, EXCLUDED, UIDS))["cpm"]
    assert b <= a + 1e-9, (a, b)


def main() -> int:
    tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
    failed = 0
    for t in tests:
        try:
            t()
            print(f"PASS  {t.__name__}")
        except AssertionError as e:
            failed += 1
            print(f"FAIL  {t.__name__}: {e}")
    print(f"\n{len(tests) - failed}/{len(tests)} passed")
    return 1 if failed else 0


if __name__ == "__main__":
    raise SystemExit(main())