feat(eval): field-level extraction eval harness + tests

Co-authored-by: Codex <chatgpt-codex-connector[bot]@users.noreply.github.com>

eval/run_eval.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Run extraction evaluation: gold labels vs predictions.
+
+Two modes:
+  1. Score precomputed predictions:
+        python eval/run_eval.py --labels eval/data/synth_eval/labels.jsonl \
+                                --predictions runs/pred.jsonl
+  2. Run the configured extractor over the images and score live (needs the model):
+        EXTRACTOR_BACKEND=local LOCAL_MODEL_PATH=... LOCAL_MMPROJ_PATH=... \
+        python eval/run_eval.py --labels eval/data/synth_eval/labels.jsonl --run
+
+Use mode 2 twice (base vs fine-tuned GGUF) to produce the OpenBMB before/after numbers.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from src.eval_scoring import format_metrics, score  # noqa: E402
+
+
+def _load_jsonl(path: Path) -> list[dict]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _predict_live(labels: list[dict], labels_path: Path) -> list[dict]:
+    from src.extraction import build_extractor
+
+    extractor = build_extractor()
+    base = labels_path.parent
+    preds = []
+    for i, row in enumerate(labels):
+        image_path = str((base / row["image"]).resolve())
+        try:
+            result = extractor.extract(image_path, max_pages=3)
+            preds.append({"tests": result.tests})
+        except Exception as error:  # keep going; a failed page is a miss
+            print(f"  [{i}] extraction failed: {error}", file=sys.stderr)
+            preds.append({"tests": []})
+    return preds
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--labels", type=Path, required=True)
+    ap.add_argument("--predictions", type=Path, help="precomputed predictions JSONL")
+    ap.add_argument("--run", action="store_true", help="run the configured extractor live")
+    args = ap.parse_args()
+
+    gold = _load_jsonl(args.labels)
+    if args.predictions:
+        pred = _load_jsonl(args.predictions)
+    elif args.run:
+        pred = _predict_live(gold, args.labels)
+    else:
+        ap.error("provide --predictions or --run")
+
+    if len(pred) != len(gold):
+        ap.error(f"predictions ({len(pred)}) and labels ({len(gold)}) length mismatch")
+
+    m = score(gold, pred)
+    print(f"\n  Extraction eval — {args.labels.name} ({len(gold)} reports)\n")
+    print(format_metrics(m))
+    worst = sorted(m.by_marker_fn.items(), key=lambda kv: -kv[1])[:5]
+    if worst:
+        print("\n  most-missed markers:", ", ".join(f"{k}×{n}" for k, n in worst))
+    print()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/eval_scoring.py

@@ -0,0 +1,115 @@
+"""Field-level scoring for extraction quality.
+
+Compares predicted lab values against gold labels and reports the metrics that matter for the
+OpenBMB before/after story:
+  - **marker P / R / F1** — did we find the right markers (matched by canonical name/alias)?
+  - **value / unit / status accuracy** — for matched markers, are the fields right?
+
+Pure functions, no model or I/O, so they are unit-tested directly.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from src.markers import resolve
+
+
+def _canon(name: str) -> str:
+    m = resolve(name)
+    return m.name.casefold() if m else (name or "").strip().casefold()
+
+
+def _num(s) -> float | None:
+    try:
+        return float(str(s).replace(",", "").strip())
+    except (TypeError, ValueError):
+        return None
+
+
+def _value_match(a, b, rel_tol: float = 0.001) -> bool:
+    na, nb = _num(a), _num(b)
+    if na is not None and nb is not None:
+        return abs(na - nb) <= rel_tol * max(1.0, abs(nb))
+    return str(a).strip().casefold() == str(b).strip().casefold()
+
+
+def _unit_match(a, b) -> bool:
+    norm = lambda s: (str(s or "").strip().casefold().replace(" ", ""))
+    return norm(a) == norm(b)
+
+
+@dataclass
+class Metrics:
+    tp: int = 0
+    fp: int = 0
+    fn: int = 0
+    value_ok: int = 0
+    unit_ok: int = 0
+    status_ok: int = 0
+    matched: int = 0
+    by_marker_fn: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def precision(self) -> float:
+        return self.tp / (self.tp + self.fp) if (self.tp + self.fp) else 0.0
+
+    @property
+    def recall(self) -> float:
+        return self.tp / (self.tp + self.fn) if (self.tp + self.fn) else 0.0
+
+    @property
+    def f1(self) -> float:
+        p, r = self.precision, self.recall
+        return 2 * p * r / (p + r) if (p + r) else 0.0
+
+    @property
+    def value_acc(self) -> float:
+        return self.value_ok / self.matched if self.matched else 0.0
+
+    @property
+    def unit_acc(self) -> float:
+        return self.unit_ok / self.matched if self.matched else 0.0
+
+    @property
+    def status_acc(self) -> float:
+        return self.status_ok / self.matched if self.matched else 0.0
+
+
+def score_report(gold_tests: list[dict], pred_tests: list[dict], m: Metrics) -> None:
+    """Accumulate one report's gold-vs-pred comparison into `m`."""
+    gold_by = {_canon(t.get("marker", "")): t for t in gold_tests}
+    pred_by = {_canon(t.get("marker", "")): t for t in pred_tests}
+
+    for key, g in gold_by.items():
+        p = pred_by.get(key)
+        if p is None:
+            m.fn += 1
+            m.by_marker_fn[key] = m.by_marker_fn.get(key, 0) + 1
+            continue
+        m.tp += 1
+        m.matched += 1
+        m.value_ok += _value_match(p.get("value"), g.get("value"))
+        m.unit_ok += _unit_match(p.get("unit"), g.get("unit"))
+        m.status_ok += str(p.get("status", "")).strip().casefold() == str(g.get("status", "")).strip().casefold()
+
+    for key in pred_by:
+        if key not in gold_by:
+            m.fp += 1
+
+
+def score(gold_rows: list[dict], pred_rows: list[dict]) -> Metrics:
+    """Score aligned lists of {tests:[...]} rows (same order/length)."""
+    m = Metrics()
+    for g, p in zip(gold_rows, pred_rows):
+        score_report(g.get("tests", []), p.get("tests", []), m)
+    return m
+
+
+def format_metrics(m: Metrics) -> str:
+    return (
+        f"  markers   P={m.precision:.3f}  R={m.recall:.3f}  F1={m.f1:.3f}  "
+        f"(tp={m.tp} fp={m.fp} fn={m.fn})\n"
+        f"  fields    value={m.value_acc:.3f}  unit={m.unit_acc:.3f}  status={m.status_acc:.3f}  "
+        f"(matched={m.matched})"
+    )

tests/test_data_pipeline.py

@@ -0,0 +1,43 @@
+"""End-to-end check of the data pipeline: generate → SFT convert → self-score."""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from src.eval_scoring import score  # noqa: E402
+from train.synth_reports import generate  # noqa: E402
+from train.to_sft_dataset import convert  # noqa: E402
+
+
+def test_generate_produces_valid_labels(tmp_path):
+    labels = generate(5, tmp_path, seed=1)
+    rows = [json.loads(l) for l in labels.read_text().splitlines() if l.strip()]
+    assert len(rows) == 5
+    for r in rows:
+        assert (tmp_path / r["image"]).exists()
+        assert r["tests"], "every report should have at least one marker"
+        for t in r["tests"]:
+            assert t["status"] in {"low", "normal", "high"}
+            assert set(t) >= {"marker", "value", "unit", "reference_range", "status"}
+
+
+def test_gold_scores_perfectly_against_itself(tmp_path):
+    labels = generate(8, tmp_path, seed=2)
+    rows = [json.loads(l) for l in labels.read_text().splitlines() if l.strip()]
+    m = score(rows, rows)
+    assert m.recall == 1.0 and m.precision == 1.0
+    assert m.value_acc == 1.0 and m.status_acc == 1.0
+
+
+def test_sft_conversion_targets_are_valid_json(tmp_path):
+    labels = generate(4, tmp_path, seed=3)
+    out = tmp_path / "sft.jsonl"
+    n = convert(labels, out)
+    assert n == 4
+    for line in out.read_text().splitlines():
+        rec = json.loads(line)
+        assert [m["role"] for m in rec["messages"]] == ["user", "assistant"]
+        json.loads(rec["messages"][1]["content"])  # assistant target parses as JSON
+        assert rec["images"] and Path(rec["images"][0]).exists()

tests/test_eval_scoring.py

@@ -0,0 +1,51 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from src.eval_scoring import score  # noqa: E402
+
+
+def _row(tests):
+    return {"tests": tests}
+
+
+def _t(marker, value, unit="mg/dL", status="normal"):
+    return {"marker": marker, "value": value, "unit": unit, "status": status}
+
+
+def test_perfect_match():
+    gold = [_row([_t("Glucose", "95"), _t("ALT", "30", "U/L")])]
+    m = score(gold, gold)
+    assert m.precision == 1.0 and m.recall == 1.0 and m.f1 == 1.0
+    assert m.value_acc == 1.0 and m.unit_acc == 1.0 and m.status_acc == 1.0
+
+
+def test_alias_is_matched_to_canonical():
+    gold = [_row([_t("Creatinine", "1.0")])]
+    pred = [_row([_t("Cr", "1.0")])]  # alias
+    m = score(gold, pred)
+    assert m.tp == 1 and m.fp == 0 and m.fn == 0
+
+
+def test_missing_and_extra_markers():
+    gold = [_row([_t("Glucose", "95"), _t("ALT", "30")])]
+    pred = [_row([_t("Glucose", "95"), _t("HDL", "55")])]  # missed ALT, hallucinated HDL
+    m = score(gold, pred)
+    assert m.tp == 1 and m.fn == 1 and m.fp == 1
+    assert "alt" in m.by_marker_fn
+
+
+def test_value_numeric_tolerance_and_status():
+    gold = [_row([_t("Glucose", "95", status="normal")])]
+    pred = [_row([_t("Glucose", "95.0", status="high")])]  # value ok (numeric), status wrong
+    m = score(gold, pred)
+    assert m.value_ok == 1
+    assert m.status_ok == 0
+
+
+def test_unit_mismatch():
+    gold = [_row([_t("ALT", "30", "U/L")])]
+    pred = [_row([_t("ALT", "30", "IU/L")])]
+    m = score(gold, pred)
+    assert m.unit_ok == 0 and m.value_ok == 1