#!/usr/bin/env python3 """Wall / window / edge accuracy on REAL labelled photos (the Roboflow facade set in real_data/). One forward pass per image (plus its mirror) is cached; every post-processing variant is then scored on the cache, so variants are compared on identical network outputs. Tune on ``--split val``, report on ``--split test`` (never both). python eval_geometry.py --checkpoint artifacts/candidates/C2-cal.pt --split val --limit 40 Metrics (full-resolution masks, labels from the photo's own mask file): wall IoU; window IoU / precision / recall (pixels); window object F1 (components, IoU >= 0.5); boundary F1 of the window and wall regions at 2 px and 5 px tolerance on a 640-px long side. Only facade photos have window labels; the indoor test images are synthetic, so no indoor window number exists. """ from __future__ import annotations import argparse import csv import json from pathlib import Path import cv2 import numpy as np import torch from scipy import ndimage import demo_plan import opening_refine import refine_edges ROOT = Path(__file__).parent WALL = (1, 2, 3) WINDOW = 8 def load_images(split, source, limit): rows = [r for r in csv.DictReader(open(ROOT / "real_data" / "metadata.csv")) if r["split"] == split and r["camera_id"] == source] rows.sort(key=lambda r: r["image"]) if limit: step = max(1, len(rows) // limit) rows = rows[::step][:limit] return rows def forward(model, checkpoint, rgb, device, tta): """Class probabilities at the model's own output resolution, cropped to the image: (C, h', w').""" size = int(checkpoint["size"]) h, w = rgb.shape[:2] s = size / max(h, w) ow, oh = max(1, round(w * s)), max(1, round(h * s)) resized = cv2.resize(rgb, (ow, oh), interpolation=cv2.INTER_AREA if s < 1 else cv2.INTER_LINEAR) left, top = (size - ow) // 2, (size - oh) // 2 canvas = np.full((size, size, 3), 114, np.uint8) canvas[top:top + oh, left:left + ow] = resized x = canvas.astype(np.float32) / 255.0 x = (x - np.asarray(checkpoint["mean"], np.float32)) / np.asarray(checkpoint["std"], np.float32) x = torch.from_numpy(x.transpose(2, 0, 1).copy()).unsqueeze(0).to(device) T = float(checkpoint.get("temperature", 1.0)) with torch.inference_mode(): logits = model(x)["semantic"] if tta: logits = 0.5 * (logits + model(torch.flip(x, dims=[3]))["semantic"].flip(dims=[3])) prob = torch.softmax(logits / T, dim=1)[0].cpu().numpy() k = prob.shape[1] / size t, l, hh, ww = round(top * k), round(left * k), max(1, round(oh * k)), max(1, round(ow * k)) return prob[:, t:t + hh, l:l + ww].astype(np.float16) CACHE = ROOT / ".geom_cache" def cached_forward(model, checkpoint, tag, split, row, rgb, device): """(plain, flip-averaged) probabilities; the network runs once per image per checkpoint, ever.""" f = CACHE / f"{tag}_{split}_{row['image'].rsplit('.', 1)[0]}.npz" if f.is_file(): z = np.load(f) return z["p0"], z["p1"] p0, p1 = forward(model, checkpoint, rgb, device, False), forward(model, checkpoint, rgb, device, True) CACHE.mkdir(exist_ok=True) np.savez_compressed(f, p0=p0, p1=p1) return p0, p1 def decode(prob, shape, mode): """Probabilities -> full-resolution class map. 'nearest' is what predict.analyze does today.""" h, w = shape p = prob.astype(np.float32) if mode == "nearest": return cv2.resize(p.argmax(axis=0).astype(np.uint8), (w, h), interpolation=cv2.INTER_NEAREST) up = np.stack([cv2.resize(c, (w, h), interpolation=cv2.INTER_LINEAR) for c in p]) return up.argmax(axis=0).astype(np.uint8) def boundary(mask): m = mask.astype(np.uint8) return (m - cv2.erode(m, np.ones((3, 3), np.uint8))).astype(bool) def boundary_f1(pred, gt, tol): bp, bg = boundary(pred), boundary(gt) if not bg.any() and not bp.any(): return None dg = ndimage.distance_transform_edt(~bg) if bg.any() else np.full(bg.shape, 1e9) dp = ndimage.distance_transform_edt(~bp) if bp.any() else np.full(bp.shape, 1e9) prec = (dg[bp] <= tol).mean() if bp.any() else 0.0 rec = (dp[bg] <= tol).mean() if bg.any() else 0.0 return 0.0 if prec + rec == 0 else 2 * prec * rec / (prec + rec) def object_f1(pred, gt, iou_min=0.5): lp, npred = ndimage.label(pred) lg, ngt = ndimage.label(gt) used, tp = set(), 0 for i in range(1, npred + 1): pm = lp == i ids, counts = np.unique(lg[pm], return_counts=True) best = 0 for j, c in zip(ids, counts): if j == 0 or j in used: continue gm = lg == j iou = c / (pm.sum() + gm.sum() - c) if iou > best: best, bj = iou, j if best >= iou_min: tp += 1 used.add(bj) return tp, npred, ngt def score(cls, gt_cls, acc): pw, gw = np.isin(cls, WALL), np.isin(gt_cls, WALL) pv, gv = cls == WINDOW, gt_cls == WINDOW for k, (p, g) in {"wall": (pw, gw), "win": (pv, gv)}.items(): acc[k + "_i"] += (p & g).sum(); acc[k + "_u"] += (p | g).sum(); acc[k + "_p"] += p.sum(); acc[k + "_g"] += g.sum() tp, npred, ngt = object_f1(pv, gv) acc["o_tp"] += tp; acc["o_p"] += npred; acc["o_g"] += ngt s = 640 / max(cls.shape) sm = lambda m: cv2.resize(m.astype(np.uint8), (max(1, round(m.shape[1] * s)), max(1, round(m.shape[0] * s))), interpolation=cv2.INTER_NEAREST).astype(bool) for tol in (2, 5): for name, (p, g) in {"wall": (pw, gw), "win": (pv, gv)}.items(): f = boundary_f1(sm(p), sm(g), tol) if f is not None: acc[f"b{tol}_{name}"].append(f) def summarize(acc): f = lambda a, b: round(float(a) / max(float(b), 1), 4) prec, rec = f(acc["win_i"], acc["win_p"]), f(acc["win_i"], acc["win_g"]) op, orr = f(acc["o_tp"], acc["o_p"]), f(acc["o_tp"], acc["o_g"]) return {"wall_iou": f(acc["wall_i"], acc["wall_u"]), "window_iou": f(acc["win_i"], acc["win_u"]), "window_precision": prec, "window_recall": rec, "window_objects": {"precision": op, "recall": orr, "f1": round(2 * op * orr / max(op + orr, 1e-9), 4)}, "boundary_f1": {f"{n}@{t}px": round(float(np.mean(acc[f"b{t}_{n}"])), 4) if acc[f"b{t}_{n}"] else None for t in (2, 5) for n in ("wall", "win")}} def new_acc(): a = {k: 0 for k in ("wall_i", "wall_u", "wall_p", "wall_g", "win_i", "win_u", "win_p", "win_g", "o_tp", "o_p", "o_g")} a.update({f"b{t}_{n}": [] for t in (2, 5) for n in ("wall", "win")}) return a def verified(cls): return demo_plan.refine_classes(cls) def rectified(cls): """Replace each window blob by its best-fit rectangle when it fills >= 0.85 of it (windows are rectangles).""" out = cls.copy() win = cls == WINDOW lab, n = ndimage.label(win) for i in range(1, n + 1): comp = lab == i if comp.sum() < 30: continue ys, xs = np.nonzero(comp) pts = np.column_stack([xs, ys]).astype(np.float32) box = cv2.boxPoints(cv2.minAreaRect(pts)) if comp.sum() / max(cv2.contourArea(box), 1) < 0.85: continue m = np.zeros(cls.shape, np.uint8) cv2.fillPoly(m, [box.round().astype(np.int32)], 1) out[comp] = np.where(np.isin(cls[comp], WALL), cls[comp], 2) out[m.astype(bool) & ~np.isin(cls, (4, 5, 6, 7, 9))] = WINDOW return out VARIANTS = { "A current (argmax small, nearest)": lambda p, shp, rgb: decode(p[0], shp, "nearest"), "B bilinear probs": lambda p, shp, rgb: decode(p[0], shp, "bilinear"), "C B + flip TTA": lambda p, shp, rgb: decode(p[1], shp, "bilinear"), "D C + window verify": lambda p, shp, rgb: verified(decode(p[1], shp, "bilinear")), "E D + rectify windows": lambda p, shp, rgb: rectified(verified(decode(p[1], shp, "bilinear"))), "H D + snap openings to edges": lambda p, shp, rgb: opening_refine.snap_openings(verified(decode(p[1], shp, "bilinear")), rgb), "I C + snap (no verify)": lambda p, shp, rgb: opening_refine.snap_openings(decode(p[1], shp, "bilinear"), rgb), "F C + guided edges": lambda p, shp, rgb: refine_edges.edge_aware_classes(p[1], rgb), "G F + window verify": lambda p, shp, rgb: verified(refine_edges.edge_aware_classes(p[1], rgb)), } def main(): ap = argparse.ArgumentParser() ap.add_argument("--checkpoint", type=Path, required=True) ap.add_argument("--split", default="val", choices=("val", "test", "train")) ap.add_argument("--source", default="roboflow-facade") ap.add_argument("--limit", type=int, default=0) ap.add_argument("--out", type=Path) a = ap.parse_args() from train import load_checkpoint_model device = torch.device("cpu") model, checkpoint = load_checkpoint_model(a.checkpoint, device) model.eval() rows = load_images(a.split, a.source, a.limit) accs = {k: new_acc() for k in VARIANTS} for n, r in enumerate(rows, 1): stem = r["image"].rsplit(".", 1)[0] bgr = cv2.imread(str(ROOT / "real_data" / a.split / "images" / r["image"])) gt = np.asarray(cv2.imread(str(ROOT / "real_data" / a.split / "masks" / f"{stem}.png"), cv2.IMREAD_UNCHANGED)) rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) probs = cached_forward(model, checkpoint, a.checkpoint.stem, a.split, r, rgb, device) for name, fn in VARIANTS.items(): score(fn(probs, gt.shape, rgb), gt, accs[name]) if n % 10 == 0: print(f"{n}/{len(rows)}", flush=True) res = {"checkpoint": a.checkpoint.name, "split": a.split, "source": a.source, "images": len(rows), "variants": {k: summarize(v) for k, v in accs.items()}} for k, v in res["variants"].items(): print(f"{k:38s} wall {v['wall_iou']:.3f} | win IoU {v['window_iou']:.3f} P {v['window_precision']:.3f} R {v['window_recall']:.3f}" f" | obj F1 {v['window_objects']['f1']:.3f} | bnd win@2 {v['boundary_f1']['win@2px']} win@5 {v['boundary_f1']['win@5px']}" f" wall@2 {v['boundary_f1']['wall@2px']}") if a.out: a.out.write_text(json.dumps(res, indent=1)) if __name__ == "__main__": main()