File size: 3,494 Bytes
3c8ce9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Evaluate the fusion layer on saved model predictions.

Each input is a .npy file holding either positive-class probabilities, shape (N,),
or two-class probability vectors, shape (N, 2).

Example
-------
python scripts/evaluate.py \
    --hist data/p_hist_test.npy --risk data/p_risk_test.npy --labels data/y_test.npy \
    --calib-hist data/p_hist_val.npy --calib-risk data/p_risk_val.npy --calib-labels data/y_val.npy \
    --out results/
"""

from __future__ import annotations

import argparse
import os
import sys

import numpy as np

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src"))

from otfusion import FusionConfig, format_summary, run_evaluation  # noqa: E402


def _alpha(value: str):
    return value if value == "dynamic" else float(value)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--hist", required=True, help="Histopathology model probabilities (.npy)")
    parser.add_argument("--risk", required=True, help="Clinical risk model probabilities (.npy)")
    parser.add_argument("--labels", required=True, help="Reference labels, 0/1 (.npy)")
    parser.add_argument("--calib-hist", help="Calibration split: histopathology probabilities")
    parser.add_argument("--calib-risk", help="Calibration split: risk probabilities")
    parser.add_argument("--calib-labels", help="Calibration split: labels")
    parser.add_argument("--out", default="results", help="Output directory")

    defaults = FusionConfig()
    parser.add_argument("--fp-weight", type=float, default=defaults.fp_weight)
    parser.add_argument("--fn-weight", type=float, default=defaults.fn_weight)
    parser.add_argument("--alpha", type=_alpha, default=defaults.alpha, help='float in [0, 1] or "dynamic"')
    parser.add_argument("--reg", type=float, default=defaults.reg)
    parser.add_argument("--max-iter", type=int, default=defaults.max_iter)

    parser.add_argument("--dropout-rates", type=float, nargs="+", default=[0.0, 0.2, 0.5])
    parser.add_argument("--dropout-mode", default="either", choices=["either", "hist", "risk", "both"])
    parser.add_argument("--threshold", type=float, default=0.5)
    parser.add_argument("--n-boot", type=int, default=1000)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--no-figures", action="store_true")
    args = parser.parse_args()

    calib_args = (args.calib_hist, args.calib_risk, args.calib_labels)
    if any(calib_args) and not all(calib_args):
        parser.error("--calib-hist, --calib-risk and --calib-labels must be given together")
    calibration = tuple(np.load(path) for path in calib_args) if all(calib_args) else None

    config = FusionConfig(
        fp_weight=args.fp_weight,
        fn_weight=args.fn_weight,
        alpha=args.alpha,
        reg=args.reg,
        max_iter=args.max_iter,
    )
    results = run_evaluation(
        np.load(args.hist),
        np.load(args.risk),
        np.load(args.labels),
        config=config,
        calibration=calibration,
        dropout_rates=args.dropout_rates,
        dropout_mode=args.dropout_mode,
        threshold=args.threshold,
        n_boot=args.n_boot,
        seed=args.seed,
        out_dir=args.out,
        make_figures=not args.no_figures,
    )
    print(format_summary(results))
    print(f"\nSaved results to {os.path.abspath(args.out)}")


if __name__ == "__main__":
    main()