Spaces:
Configuration error
Configuration error
| """Measure localization quality against the hand-checked ground-truth set. | |
| python tools/evaluate.py # summary | |
| python tools/evaluate.py --detail # every gold item, failures marked | |
| Two metrics, because they answer different questions: | |
| * `IoU >= 0.5` -- does the predicted box match the component's extent? | |
| * `on target` -- does the box point at the *right part* at all? True when the | |
| prediction and the gold box each contain the other's centre-point and overlap | |
| at all. A conservative Tier B box sitting on the correct part scores here but | |
| not on IoU, which is exactly the distinction worth reporting. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from collections import defaultdict | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from spareparts import config, index_db # noqa: E402 | |
| GOLD_PATH = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "gold_boxes.json" | |
| def iou(a, b) -> float: | |
| ax0, ay0, ax1, ay1 = a | |
| bx0, by0, bx1, by1 = b | |
| ix0, iy0 = max(ax0, bx0), max(ay0, by0) | |
| ix1, iy1 = min(ax1, bx1), min(ay1, by1) | |
| if ix1 <= ix0 or iy1 <= iy0: | |
| return 0.0 | |
| inter = (ix1 - ix0) * (iy1 - iy0) | |
| union = (ax1 - ax0) * (ay1 - ay0) + (bx1 - bx0) * (by1 - by0) - inter | |
| return inter / union if union else 0.0 | |
| def _contains_centre(outer, inner) -> bool: | |
| cx, cy = (inner[0] + inner[2]) / 2, (inner[1] + inner[3]) / 2 | |
| return outer[0] <= cx <= outer[2] and outer[1] <= cy <= outer[3] | |
| def on_target(pred, gold) -> bool: | |
| """Does the prediction indicate the right part, regardless of tightness?""" | |
| if iou(pred, gold) <= 0: | |
| return False | |
| return _contains_centre(gold, pred) and _contains_centre(pred, gold) | |
| def evaluate(db_path: Path = config.DB_PATH, gold_path: Path = GOLD_PATH) -> dict: | |
| gold = json.loads(gold_path.read_text(encoding="utf-8"))["items"] | |
| conn = index_db.connect(db_path) | |
| try: | |
| rows = {} | |
| for row in conn.execute( | |
| """ | |
| SELECT l.pdf_page, l.part_no, l.instance, | |
| c.cx0, c.cy0, c.cx1, c.cy1, c.method, c.confidence | |
| FROM labels l LEFT JOIN components c ON c.label_id = l.id | |
| """ | |
| ): | |
| rows[(row["pdf_page"], row["part_no"], row["instance"])] = dict(row) | |
| finally: | |
| conn.close() | |
| results = [] | |
| for item in gold: | |
| key = (item["page"], item["part_no"], item["instance"]) | |
| row = rows.get(key) | |
| pred = None | |
| if row and row["cx0"] is not None: | |
| pred = (row["cx0"], row["cy0"], row["cx1"], row["cy1"]) | |
| results.append( | |
| { | |
| **item, | |
| "found": row is not None, | |
| "method": row["method"] if row else "MISSING", | |
| "confidence": row["confidence"] if row else 0.0, | |
| "pred": pred, | |
| "iou": iou(pred, item["box"]) if pred else 0.0, | |
| "on_target": on_target(pred, item["box"]) if pred else False, | |
| } | |
| ) | |
| return {"results": results} | |
| def _summarise(results, key=None): | |
| buckets = defaultdict(list) | |
| for r in results: | |
| buckets[key(r) if key else "all"].append(r) | |
| out = {} | |
| for name, items in sorted(buckets.items(), key=lambda kv: str(kv[0])): | |
| n = len(items) | |
| out[name] = { | |
| "n": n, | |
| "on_target": sum(r["on_target"] for r in items), | |
| "iou50": sum(r["iou"] >= 0.5 for r in items), | |
| "mean_iou": sum(r["iou"] for r in items) / n if n else 0.0, | |
| } | |
| return out | |
| def main(argv=None) -> int: | |
| parser = argparse.ArgumentParser(description="Evaluate localization accuracy.") | |
| parser.add_argument("--db", type=Path, default=config.DB_PATH) | |
| parser.add_argument("--gold", type=Path, default=GOLD_PATH) | |
| parser.add_argument("--detail", action="store_true") | |
| args = parser.parse_args(argv) | |
| results = evaluate(args.db, args.gold)["results"] | |
| total = len(results) | |
| def line(name, stats): | |
| return ( | |
| f" {str(name):<12s} n={stats['n']:<4d}" | |
| f" on-target {stats['on_target']:>3d} ({100 * stats['on_target'] / stats['n']:5.1f}%)" | |
| f" IoU>=0.5 {stats['iou50']:>3d} ({100 * stats['iou50'] / stats['n']:5.1f}%)" | |
| f" mean IoU {stats['mean_iou']:.2f}" | |
| ) | |
| overall = _summarise(results)["all"] | |
| verified = sum(1 for r in results if r.get("source") == "verified") | |
| print(f"Localization evaluation ({total} hand-checked callouts)") | |
| print( | |
| f" NOTE: for the {verified} items inspection judged correct, the gold box IS the\n" | |
| " prediction, so their IoU is 1.0 by construction. The honest headline number\n" | |
| " is 'on-target', which reflects the independent visual verdict of whether the\n" | |
| " box lands on the right component. Read IoU as a description of how far the\n" | |
| f" {total - verified} judged-wrong cases miss by, not as an independent measure." | |
| ) | |
| print(line("OVERALL", overall)) | |
| print("\nBy page:") | |
| for name, stats in _summarise(results, lambda r: r["page"]).items(): | |
| print(line(f"page {name}", stats)) | |
| print("\nBy tier:") | |
| for name, stats in _summarise(results, lambda r: r["method"]).items(): | |
| print(line(name, stats)) | |
| failures = [r for r in results if not r["on_target"]] | |
| print(f"\nOff-target: {len(failures)}") | |
| for r in failures: | |
| print( | |
| f" p{r['page']} {r['part_no']}#{r['instance']} [{r['method']}]" | |
| f" IoU={r['iou']:.2f} pred={r['pred']} gold={tuple(r['box'])}" | |
| f" {r.get('note', '')}" | |
| ) | |
| if args.detail: | |
| print("\nAll items:") | |
| for r in sorted(results, key=lambda r: (r["page"], r["part_no"], r["instance"])): | |
| mark = "ok " if r["on_target"] else "MISS" | |
| print( | |
| f" {mark} p{r['page']} {r['part_no']}#{r['instance']:<2d}" | |
| f" {r['method']:<9s} IoU={r['iou']:.2f} conf={r['confidence']:.2f}" | |
| ) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |