"""Measure localization quality against the hand-checked ground-truth set. python tools/evaluate.py # summary python tools/evaluate.py --detail # every gold item, failures marked Two metrics, because they answer different questions: * `IoU >= 0.5` -- does the predicted box match the component's extent? * `on target` -- does the box point at the *right part* at all? True when the prediction and the gold box each contain the other's centre-point and overlap at all. A conservative Tier B box sitting on the correct part scores here but not on IoU, which is exactly the distinction worth reporting. """ from __future__ import annotations import argparse import json import sys from collections import defaultdict from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from spareparts import config, index_db # noqa: E402 GOLD_PATH = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "gold_boxes.json" def iou(a, b) -> float: ax0, ay0, ax1, ay1 = a bx0, by0, bx1, by1 = b ix0, iy0 = max(ax0, bx0), max(ay0, by0) ix1, iy1 = min(ax1, bx1), min(ay1, by1) if ix1 <= ix0 or iy1 <= iy0: return 0.0 inter = (ix1 - ix0) * (iy1 - iy0) union = (ax1 - ax0) * (ay1 - ay0) + (bx1 - bx0) * (by1 - by0) - inter return inter / union if union else 0.0 def _contains_centre(outer, inner) -> bool: cx, cy = (inner[0] + inner[2]) / 2, (inner[1] + inner[3]) / 2 return outer[0] <= cx <= outer[2] and outer[1] <= cy <= outer[3] def on_target(pred, gold) -> bool: """Does the prediction indicate the right part, regardless of tightness?""" if iou(pred, gold) <= 0: return False return _contains_centre(gold, pred) and _contains_centre(pred, gold) def evaluate(db_path: Path = config.DB_PATH, gold_path: Path = GOLD_PATH) -> dict: gold = json.loads(gold_path.read_text(encoding="utf-8"))["items"] conn = index_db.connect(db_path) try: rows = {} for row in conn.execute( """ SELECT l.pdf_page, l.part_no, l.instance, c.cx0, c.cy0, c.cx1, c.cy1, c.method, c.confidence FROM labels l LEFT JOIN components c ON c.label_id = l.id """ ): rows[(row["pdf_page"], row["part_no"], row["instance"])] = dict(row) finally: conn.close() results = [] for item in gold: key = (item["page"], item["part_no"], item["instance"]) row = rows.get(key) pred = None if row and row["cx0"] is not None: pred = (row["cx0"], row["cy0"], row["cx1"], row["cy1"]) results.append( { **item, "found": row is not None, "method": row["method"] if row else "MISSING", "confidence": row["confidence"] if row else 0.0, "pred": pred, "iou": iou(pred, item["box"]) if pred else 0.0, "on_target": on_target(pred, item["box"]) if pred else False, } ) return {"results": results} def _summarise(results, key=None): buckets = defaultdict(list) for r in results: buckets[key(r) if key else "all"].append(r) out = {} for name, items in sorted(buckets.items(), key=lambda kv: str(kv[0])): n = len(items) out[name] = { "n": n, "on_target": sum(r["on_target"] for r in items), "iou50": sum(r["iou"] >= 0.5 for r in items), "mean_iou": sum(r["iou"] for r in items) / n if n else 0.0, } return out def main(argv=None) -> int: parser = argparse.ArgumentParser(description="Evaluate localization accuracy.") parser.add_argument("--db", type=Path, default=config.DB_PATH) parser.add_argument("--gold", type=Path, default=GOLD_PATH) parser.add_argument("--detail", action="store_true") args = parser.parse_args(argv) results = evaluate(args.db, args.gold)["results"] total = len(results) def line(name, stats): return ( f" {str(name):<12s} n={stats['n']:<4d}" f" on-target {stats['on_target']:>3d} ({100 * stats['on_target'] / stats['n']:5.1f}%)" f" IoU>=0.5 {stats['iou50']:>3d} ({100 * stats['iou50'] / stats['n']:5.1f}%)" f" mean IoU {stats['mean_iou']:.2f}" ) overall = _summarise(results)["all"] verified = sum(1 for r in results if r.get("source") == "verified") print(f"Localization evaluation ({total} hand-checked callouts)") print( f" NOTE: for the {verified} items inspection judged correct, the gold box IS the\n" " prediction, so their IoU is 1.0 by construction. The honest headline number\n" " is 'on-target', which reflects the independent visual verdict of whether the\n" " box lands on the right component. Read IoU as a description of how far the\n" f" {total - verified} judged-wrong cases miss by, not as an independent measure." ) print(line("OVERALL", overall)) print("\nBy page:") for name, stats in _summarise(results, lambda r: r["page"]).items(): print(line(f"page {name}", stats)) print("\nBy tier:") for name, stats in _summarise(results, lambda r: r["method"]).items(): print(line(name, stats)) failures = [r for r in results if not r["on_target"]] print(f"\nOff-target: {len(failures)}") for r in failures: print( f" p{r['page']} {r['part_no']}#{r['instance']} [{r['method']}]" f" IoU={r['iou']:.2f} pred={r['pred']} gold={tuple(r['box'])}" f" {r.get('note', '')}" ) if args.detail: print("\nAll items:") for r in sorted(results, key=lambda r: (r["page"], r["part_no"], r["instance"])): mark = "ok " if r["on_target"] else "MISS" print( f" {mark} p{r['page']} {r['part_no']}#{r['instance']:<2d}" f" {r['method']:<9s} IoU={r['iou']:.2f} conf={r['confidence']:.2f}" ) return 0 if __name__ == "__main__": sys.exit(main())