scripts/issue11_threshold_sweep.py

"""Offline threshold sweep for v0.2 issue #11 — replays greedy matching at
arbitrary cosine thresholds against the .diag.json files written by
``scripts/05_evaluate.py --dump-match-debug``. No LLM, no embedder.

Usage:
    PYTHONPATH=. python scripts/issue11_threshold_sweep.py
    PYTHONPATH=. python scripts/issue11_threshold_sweep.py --thresholds 0.30 0.35 0.40 0.45 0.50

Reads:
    data/evaluation/gold/{event_id}.json       # gold cache (for pred_count, gold_count)
    data/evaluation/gold/{event_id}.diag.json  # all same-domain pair cosines

Prints a per-threshold table (macro_P / macro_R / macro_F1 / micro_F1).
Does NOT mutate caches or the codebase — purely analytic.
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

GOLD_DIR = Path("data/evaluation/gold")


def replay_greedy(candidates: list[dict], threshold: float) -> int:
    """Re-run greedy match selection on a sorted-desc candidate list.

    candidates: list of {p_id, g_id, cosine, severity_match, accepted}
    Returns the number of accepted matches at the given threshold.
    """
    matched_p: set[str] = set()
    matched_g: set[str] = set()
    matched = 0
    for c in candidates:
        if c["cosine"] < threshold:
            break  # sorted desc
        if c["p_id"] in matched_p or c["g_id"] in matched_g:
            continue
        matched_p.add(c["p_id"])
        matched_g.add(c["g_id"])
        matched += 1
    return matched


def per_event_metrics(diag_path: Path, gold_cache_path: Path, threshold: float) -> dict:
    diag = json.loads(diag_path.read_text())
    gold = json.loads(gold_cache_path.read_text())

    matched = replay_greedy(diag["candidates"], threshold)
    pred_count = gold["predicted_node_count"]
    gold_count = gold["gold_node_count"]

    precision = matched / pred_count if pred_count else 0.0
    recall = matched / gold_count if gold_count else 0.0
    f1 = (
        2 * precision * recall / (precision + recall)
        if (precision + recall) else 0.0
    )
    return {
        "event_id": diag["event_id"],
        "matched": matched,
        "pred_count": pred_count,
        "gold_count": gold_count,
        "precision": precision,
        "recall": recall,
        "f1": f1,
    }


def sweep(thresholds: list[float]) -> None:
    diag_files = sorted(GOLD_DIR.glob("*.diag.json"))
    if not diag_files:
        raise SystemExit(
            f"No .diag.json files in {GOLD_DIR}. Run "
            "`scripts/05_evaluate.py --force --dump-match-debug` first."
        )

    print(f"Found {len(diag_files)} events with .diag.json:")
    for d in diag_files:
        print(f"  {d.stem.replace('.diag', '')}")
    print()

    print(f"{'threshold':>9}  {'macro_P':>8}  {'macro_R':>8}  {'macro_F1':>9}  "
          f"{'micro_F1':>9}  {'matched/pred/gold':>22}")
    print("-" * 80)

    rows: list[dict] = []
    for t in thresholds:
        per_event = []
        for diag_path in diag_files:
            event_id = diag_path.stem.replace(".diag", "")
            gold_cache = GOLD_DIR / f"{event_id}.json"
            if not gold_cache.exists():
                continue
            per_event.append(per_event_metrics(diag_path, gold_cache, t))

        n = len(per_event)
        if not n:
            continue
        macro_p = sum(e["precision"] for e in per_event) / n
        macro_r = sum(e["recall"] for e in per_event) / n
        macro_f1 = sum(e["f1"] for e in per_event) / n

        m = sum(e["matched"] for e in per_event)
        p_total = sum(e["pred_count"] for e in per_event)
        g_total = sum(e["gold_count"] for e in per_event)
        micro_p = m / p_total if p_total else 0.0
        micro_r = m / g_total if g_total else 0.0
        micro_f1 = (
            2 * micro_p * micro_r / (micro_p + micro_r)
            if (micro_p + micro_r) else 0.0
        )

        print(
            f"{t:>9.2f}  {macro_p:>8.3f}  {macro_r:>8.3f}  {macro_f1:>9.3f}  "
            f"{micro_f1:>9.3f}  {f'{m}/{p_total}/{g_total}':>22}"
        )
        rows.append({
            "threshold": t, "macro_p": macro_p, "macro_r": macro_r,
            "macro_f1": macro_f1, "micro_f1": micro_f1,
            "per_event": per_event,
        })

    # Best threshold by macro_F1, tiebreak by precision
    best = max(rows, key=lambda r: (r["macro_f1"], r["macro_p"]))
    print()
    print(f"Best threshold (by macro_F1, tiebreak macro_P): {best['threshold']:.2f}")
    print(f"  macro_F1={best['macro_f1']:.4f}  macro_P={best['macro_p']:.4f}  "
          f"macro_R={best['macro_r']:.4f}  micro_F1={best['micro_f1']:.4f}")
    print()
    print("Per-event detail at best threshold:")
    for e in best["per_event"]:
        print(
            f"  {e['event_id']:<18} matched={e['matched']:>2}/"
            f"pred={e['pred_count']:>2}/gold={e['gold_count']:>2}  "
            f"P={e['precision']:.3f}  R={e['recall']:.3f}  F1={e['f1']:.3f}"
        )


def main() -> None:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument(
        "--thresholds", type=float, nargs="+",
        default=[0.30, 0.35, 0.40, 0.45, 0.50],
        help="Cosine thresholds to sweep (default: 0.30 0.35 0.40 0.45 0.50).",
    )
    args = p.parse_args()
    sweep(args.thresholds)


if __name__ == "__main__":
    main()