painting-vision-robotics-kit / eval_geometry.py
constructelligence's picture
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified
Raw History Blame Contribute Delete
10.3 kB
#!/usr/bin/env python3
"""Wall / window / edge accuracy on REAL labelled photos (the Roboflow facade set in real_data/).
One forward pass per image (plus its mirror) is cached; every post-processing variant is then scored on the cache, so
variants are compared on identical network outputs. Tune on ``--split val``, report on ``--split test`` (never both).
python eval_geometry.py --checkpoint artifacts/candidates/C2-cal.pt --split val --limit 40
Metrics (full-resolution masks, labels from the photo's own mask file):
wall IoU; window IoU / precision / recall (pixels); window object F1 (components, IoU >= 0.5);
boundary F1 of the window and wall regions at 2 px and 5 px tolerance on a 640-px long side.
Only facade photos have window labels; the indoor test images are synthetic, so no indoor window number exists.
"""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
import cv2
import numpy as np
import torch
from scipy import ndimage
import demo_plan
import opening_refine
import refine_edges
ROOT = Path(__file__).parent
WALL = (1, 2, 3)
WINDOW = 8
def load_images(split, source, limit):
rows = [r for r in csv.DictReader(open(ROOT / "real_data" / "metadata.csv")) if r["split"] == split and r["camera_id"] == source]
rows.sort(key=lambda r: r["image"])
if limit:
step = max(1, len(rows) // limit)
rows = rows[::step][:limit]
return rows
def forward(model, checkpoint, rgb, device, tta):
"""Class probabilities at the model's own output resolution, cropped to the image: (C, h', w')."""
size = int(checkpoint["size"])
h, w = rgb.shape[:2]
s = size / max(h, w)
ow, oh = max(1, round(w * s)), max(1, round(h * s))
resized = cv2.resize(rgb, (ow, oh), interpolation=cv2.INTER_AREA if s < 1 else cv2.INTER_LINEAR)
left, top = (size - ow) // 2, (size - oh) // 2
canvas = np.full((size, size, 3), 114, np.uint8)
canvas[top:top + oh, left:left + ow] = resized
x = canvas.astype(np.float32) / 255.0
x = (x - np.asarray(checkpoint["mean"], np.float32)) / np.asarray(checkpoint["std"], np.float32)
x = torch.from_numpy(x.transpose(2, 0, 1).copy()).unsqueeze(0).to(device)
T = float(checkpoint.get("temperature", 1.0))
with torch.inference_mode():
logits = model(x)["semantic"]
if tta:
logits = 0.5 * (logits + model(torch.flip(x, dims=[3]))["semantic"].flip(dims=[3]))
prob = torch.softmax(logits / T, dim=1)[0].cpu().numpy()
k = prob.shape[1] / size
t, l, hh, ww = round(top * k), round(left * k), max(1, round(oh * k)), max(1, round(ow * k))
return prob[:, t:t + hh, l:l + ww].astype(np.float16)
CACHE = ROOT / ".geom_cache"
def cached_forward(model, checkpoint, tag, split, row, rgb, device):
"""(plain, flip-averaged) probabilities; the network runs once per image per checkpoint, ever."""
f = CACHE / f"{tag}_{split}_{row['image'].rsplit('.', 1)[0]}.npz"
if f.is_file():
z = np.load(f)
return z["p0"], z["p1"]
p0, p1 = forward(model, checkpoint, rgb, device, False), forward(model, checkpoint, rgb, device, True)
CACHE.mkdir(exist_ok=True)
np.savez_compressed(f, p0=p0, p1=p1)
return p0, p1
def decode(prob, shape, mode):
"""Probabilities -> full-resolution class map. 'nearest' is what predict.analyze does today."""
h, w = shape
p = prob.astype(np.float32)
if mode == "nearest":
return cv2.resize(p.argmax(axis=0).astype(np.uint8), (w, h), interpolation=cv2.INTER_NEAREST)
up = np.stack([cv2.resize(c, (w, h), interpolation=cv2.INTER_LINEAR) for c in p])
return up.argmax(axis=0).astype(np.uint8)
def boundary(mask):
m = mask.astype(np.uint8)
return (m - cv2.erode(m, np.ones((3, 3), np.uint8))).astype(bool)
def boundary_f1(pred, gt, tol):
bp, bg = boundary(pred), boundary(gt)
if not bg.any() and not bp.any():
return None
dg = ndimage.distance_transform_edt(~bg) if bg.any() else np.full(bg.shape, 1e9)
dp = ndimage.distance_transform_edt(~bp) if bp.any() else np.full(bp.shape, 1e9)
prec = (dg[bp] <= tol).mean() if bp.any() else 0.0
rec = (dp[bg] <= tol).mean() if bg.any() else 0.0
return 0.0 if prec + rec == 0 else 2 * prec * rec / (prec + rec)
def object_f1(pred, gt, iou_min=0.5):
lp, npred = ndimage.label(pred)
lg, ngt = ndimage.label(gt)
used, tp = set(), 0
for i in range(1, npred + 1):
pm = lp == i
ids, counts = np.unique(lg[pm], return_counts=True)
best = 0
for j, c in zip(ids, counts):
if j == 0 or j in used:
continue
gm = lg == j
iou = c / (pm.sum() + gm.sum() - c)
if iou > best:
best, bj = iou, j
if best >= iou_min:
tp += 1
used.add(bj)
return tp, npred, ngt
def score(cls, gt_cls, acc):
pw, gw = np.isin(cls, WALL), np.isin(gt_cls, WALL)
pv, gv = cls == WINDOW, gt_cls == WINDOW
for k, (p, g) in {"wall": (pw, gw), "win": (pv, gv)}.items():
acc[k + "_i"] += (p & g).sum(); acc[k + "_u"] += (p | g).sum(); acc[k + "_p"] += p.sum(); acc[k + "_g"] += g.sum()
tp, npred, ngt = object_f1(pv, gv)
acc["o_tp"] += tp; acc["o_p"] += npred; acc["o_g"] += ngt
s = 640 / max(cls.shape)
sm = lambda m: cv2.resize(m.astype(np.uint8), (max(1, round(m.shape[1] * s)), max(1, round(m.shape[0] * s))), interpolation=cv2.INTER_NEAREST).astype(bool)
for tol in (2, 5):
for name, (p, g) in {"wall": (pw, gw), "win": (pv, gv)}.items():
f = boundary_f1(sm(p), sm(g), tol)
if f is not None:
acc[f"b{tol}_{name}"].append(f)
def summarize(acc):
f = lambda a, b: round(float(a) / max(float(b), 1), 4)
prec, rec = f(acc["win_i"], acc["win_p"]), f(acc["win_i"], acc["win_g"])
op, orr = f(acc["o_tp"], acc["o_p"]), f(acc["o_tp"], acc["o_g"])
return {"wall_iou": f(acc["wall_i"], acc["wall_u"]), "window_iou": f(acc["win_i"], acc["win_u"]),
"window_precision": prec, "window_recall": rec,
"window_objects": {"precision": op, "recall": orr, "f1": round(2 * op * orr / max(op + orr, 1e-9), 4)},
"boundary_f1": {f"{n}@{t}px": round(float(np.mean(acc[f"b{t}_{n}"])), 4) if acc[f"b{t}_{n}"] else None
for t in (2, 5) for n in ("wall", "win")}}
def new_acc():
a = {k: 0 for k in ("wall_i", "wall_u", "wall_p", "wall_g", "win_i", "win_u", "win_p", "win_g", "o_tp", "o_p", "o_g")}
a.update({f"b{t}_{n}": [] for t in (2, 5) for n in ("wall", "win")})
return a
def verified(cls):
return demo_plan.refine_classes(cls)
def rectified(cls):
"""Replace each window blob by its best-fit rectangle when it fills >= 0.85 of it (windows are rectangles)."""
out = cls.copy()
win = cls == WINDOW
lab, n = ndimage.label(win)
for i in range(1, n + 1):
comp = lab == i
if comp.sum() < 30:
continue
ys, xs = np.nonzero(comp)
pts = np.column_stack([xs, ys]).astype(np.float32)
box = cv2.boxPoints(cv2.minAreaRect(pts))
if comp.sum() / max(cv2.contourArea(box), 1) < 0.85:
continue
m = np.zeros(cls.shape, np.uint8)
cv2.fillPoly(m, [box.round().astype(np.int32)], 1)
out[comp] = np.where(np.isin(cls[comp], WALL), cls[comp], 2)
out[m.astype(bool) & ~np.isin(cls, (4, 5, 6, 7, 9))] = WINDOW
return out
VARIANTS = {
"A current (argmax small, nearest)": lambda p, shp, rgb: decode(p[0], shp, "nearest"),
"B bilinear probs": lambda p, shp, rgb: decode(p[0], shp, "bilinear"),
"C B + flip TTA": lambda p, shp, rgb: decode(p[1], shp, "bilinear"),
"D C + window verify": lambda p, shp, rgb: verified(decode(p[1], shp, "bilinear")),
"E D + rectify windows": lambda p, shp, rgb: rectified(verified(decode(p[1], shp, "bilinear"))),
"H D + snap openings to edges": lambda p, shp, rgb: opening_refine.snap_openings(verified(decode(p[1], shp, "bilinear")), rgb),
"I C + snap (no verify)": lambda p, shp, rgb: opening_refine.snap_openings(decode(p[1], shp, "bilinear"), rgb),
"F C + guided edges": lambda p, shp, rgb: refine_edges.edge_aware_classes(p[1], rgb),
"G F + window verify": lambda p, shp, rgb: verified(refine_edges.edge_aware_classes(p[1], rgb)),
}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--checkpoint", type=Path, required=True)
ap.add_argument("--split", default="val", choices=("val", "test", "train"))
ap.add_argument("--source", default="roboflow-facade")
ap.add_argument("--limit", type=int, default=0)
ap.add_argument("--out", type=Path)
a = ap.parse_args()
from train import load_checkpoint_model
device = torch.device("cpu")
model, checkpoint = load_checkpoint_model(a.checkpoint, device)
model.eval()
rows = load_images(a.split, a.source, a.limit)
accs = {k: new_acc() for k in VARIANTS}
for n, r in enumerate(rows, 1):
stem = r["image"].rsplit(".", 1)[0]
bgr = cv2.imread(str(ROOT / "real_data" / a.split / "images" / r["image"]))
gt = np.asarray(cv2.imread(str(ROOT / "real_data" / a.split / "masks" / f"{stem}.png"), cv2.IMREAD_UNCHANGED))
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
probs = cached_forward(model, checkpoint, a.checkpoint.stem, a.split, r, rgb, device)
for name, fn in VARIANTS.items():
score(fn(probs, gt.shape, rgb), gt, accs[name])
if n % 10 == 0:
print(f"{n}/{len(rows)}", flush=True)
res = {"checkpoint": a.checkpoint.name, "split": a.split, "source": a.source, "images": len(rows),
"variants": {k: summarize(v) for k, v in accs.items()}}
for k, v in res["variants"].items():
print(f"{k:38s} wall {v['wall_iou']:.3f} | win IoU {v['window_iou']:.3f} P {v['window_precision']:.3f} R {v['window_recall']:.3f}"
f" | obj F1 {v['window_objects']['f1']:.3f} | bnd win@2 {v['boundary_f1']['win@2px']} win@5 {v['boundary_f1']['win@5px']}"
f" wall@2 {v['boundary_f1']['wall@2px']}")
if a.out:
a.out.write_text(json.dumps(res, indent=1))
if __name__ == "__main__":
main()