Spaces:
Configuration error
Configuration error
File size: 6,309 Bytes
546c207 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | """Measure localization quality against the hand-checked ground-truth set.
python tools/evaluate.py # summary
python tools/evaluate.py --detail # every gold item, failures marked
Two metrics, because they answer different questions:
* `IoU >= 0.5` -- does the predicted box match the component's extent?
* `on target` -- does the box point at the *right part* at all? True when the
prediction and the gold box each contain the other's centre-point and overlap
at all. A conservative Tier B box sitting on the correct part scores here but
not on IoU, which is exactly the distinction worth reporting.
"""
from __future__ import annotations
import argparse
import json
import sys
from collections import defaultdict
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from spareparts import config, index_db # noqa: E402
GOLD_PATH = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "gold_boxes.json"
def iou(a, b) -> float:
ax0, ay0, ax1, ay1 = a
bx0, by0, bx1, by1 = b
ix0, iy0 = max(ax0, bx0), max(ay0, by0)
ix1, iy1 = min(ax1, bx1), min(ay1, by1)
if ix1 <= ix0 or iy1 <= iy0:
return 0.0
inter = (ix1 - ix0) * (iy1 - iy0)
union = (ax1 - ax0) * (ay1 - ay0) + (bx1 - bx0) * (by1 - by0) - inter
return inter / union if union else 0.0
def _contains_centre(outer, inner) -> bool:
cx, cy = (inner[0] + inner[2]) / 2, (inner[1] + inner[3]) / 2
return outer[0] <= cx <= outer[2] and outer[1] <= cy <= outer[3]
def on_target(pred, gold) -> bool:
"""Does the prediction indicate the right part, regardless of tightness?"""
if iou(pred, gold) <= 0:
return False
return _contains_centre(gold, pred) and _contains_centre(pred, gold)
def evaluate(db_path: Path = config.DB_PATH, gold_path: Path = GOLD_PATH) -> dict:
gold = json.loads(gold_path.read_text(encoding="utf-8"))["items"]
conn = index_db.connect(db_path)
try:
rows = {}
for row in conn.execute(
"""
SELECT l.pdf_page, l.part_no, l.instance,
c.cx0, c.cy0, c.cx1, c.cy1, c.method, c.confidence
FROM labels l LEFT JOIN components c ON c.label_id = l.id
"""
):
rows[(row["pdf_page"], row["part_no"], row["instance"])] = dict(row)
finally:
conn.close()
results = []
for item in gold:
key = (item["page"], item["part_no"], item["instance"])
row = rows.get(key)
pred = None
if row and row["cx0"] is not None:
pred = (row["cx0"], row["cy0"], row["cx1"], row["cy1"])
results.append(
{
**item,
"found": row is not None,
"method": row["method"] if row else "MISSING",
"confidence": row["confidence"] if row else 0.0,
"pred": pred,
"iou": iou(pred, item["box"]) if pred else 0.0,
"on_target": on_target(pred, item["box"]) if pred else False,
}
)
return {"results": results}
def _summarise(results, key=None):
buckets = defaultdict(list)
for r in results:
buckets[key(r) if key else "all"].append(r)
out = {}
for name, items in sorted(buckets.items(), key=lambda kv: str(kv[0])):
n = len(items)
out[name] = {
"n": n,
"on_target": sum(r["on_target"] for r in items),
"iou50": sum(r["iou"] >= 0.5 for r in items),
"mean_iou": sum(r["iou"] for r in items) / n if n else 0.0,
}
return out
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description="Evaluate localization accuracy.")
parser.add_argument("--db", type=Path, default=config.DB_PATH)
parser.add_argument("--gold", type=Path, default=GOLD_PATH)
parser.add_argument("--detail", action="store_true")
args = parser.parse_args(argv)
results = evaluate(args.db, args.gold)["results"]
total = len(results)
def line(name, stats):
return (
f" {str(name):<12s} n={stats['n']:<4d}"
f" on-target {stats['on_target']:>3d} ({100 * stats['on_target'] / stats['n']:5.1f}%)"
f" IoU>=0.5 {stats['iou50']:>3d} ({100 * stats['iou50'] / stats['n']:5.1f}%)"
f" mean IoU {stats['mean_iou']:.2f}"
)
overall = _summarise(results)["all"]
verified = sum(1 for r in results if r.get("source") == "verified")
print(f"Localization evaluation ({total} hand-checked callouts)")
print(
f" NOTE: for the {verified} items inspection judged correct, the gold box IS the\n"
" prediction, so their IoU is 1.0 by construction. The honest headline number\n"
" is 'on-target', which reflects the independent visual verdict of whether the\n"
" box lands on the right component. Read IoU as a description of how far the\n"
f" {total - verified} judged-wrong cases miss by, not as an independent measure."
)
print(line("OVERALL", overall))
print("\nBy page:")
for name, stats in _summarise(results, lambda r: r["page"]).items():
print(line(f"page {name}", stats))
print("\nBy tier:")
for name, stats in _summarise(results, lambda r: r["method"]).items():
print(line(name, stats))
failures = [r for r in results if not r["on_target"]]
print(f"\nOff-target: {len(failures)}")
for r in failures:
print(
f" p{r['page']} {r['part_no']}#{r['instance']} [{r['method']}]"
f" IoU={r['iou']:.2f} pred={r['pred']} gold={tuple(r['box'])}"
f" {r.get('note', '')}"
)
if args.detail:
print("\nAll items:")
for r in sorted(results, key=lambda r: (r["page"], r["part_no"], r["instance"])):
mark = "ok " if r["on_target"] else "MISS"
print(
f" {mark} p{r['page']} {r['part_no']}#{r['instance']:<2d}"
f" {r['method']:<9s} IoU={r['iou']:.2f} conf={r['confidence']:.2f}"
)
return 0
if __name__ == "__main__":
sys.exit(main())
|