painting-vision-robotics-kit / eval_align_real.py
constructelligence's picture
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified
Raw History Blame Contribute Delete
7.73 kB
#!/usr/bin/env python3
"""How far off is a re-scan's alignment on REAL facade photos? The number the drone twin's gating study needs.
``drone_align.homography`` lines a re-scan photo up with the job photo. Its error is what puts a camera read of the
paint on the wrong wall pixels -- the misregistration that observation gating near paint edges is meant to absorb.
No real re-scan pairs with ground truth exist yet, so each real facade test photo (real texture, real repetition,
real windows) is re-shot synthetically: a random perspective change (corners moved up to ``--jitter`` of the image),
brightness/contrast change, blur and JPEG. The true warp is known, so the error is exact:
error = distance between where the estimated and the true warp put each wall pixel, in job-photo pixels,
reported also as a share of the image width (so it transfers to the twin's grid).
Refusals (drone_align says the photos cannot be trusted) are counted, not scored: a refusal is safe.
A synthetic move has no parallax, moving shadows or lens change, so that number is a floor. The other real source of a
read landing on the wrong pixels is the segmentation's own edge error, measured with ``--edge-error``: for every true
wall edge pixel on the real facade frames, the distance to the nearest predicted wall edge (from eval_geometry's cached
network outputs, so nothing is re-run), as a share of the image width.
python eval_align_real.py [--split test] [--pairs-per-image 3] [--out benchmark/records/align_real.json]
python eval_align_real.py --edge-error C2-cal --split val [--out ...]
"""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
import cv2
import numpy as np
import drone_align
ROOT = Path(__file__).resolve().parent
WALL = 2
def reshoot(rgb, rng, jitter):
"""A synthetic second visit: (new photo, true H mapping new -> job pixels)."""
h, w = rgb.shape[:2]
src = np.float32([[0, 0], [w, 0], [w, h], [0, h]])
dst = src + rng.uniform(-jitter, jitter, (4, 2)).astype(np.float32) * np.float32([w, h])
H_job_to_new = cv2.getPerspectiveTransform(src, dst)
new = cv2.warpPerspective(rgb, H_job_to_new, (w, h), borderMode=cv2.BORDER_REFLECT)
new = np.clip(new.astype(np.float32) * rng.uniform(0.75, 1.25) + rng.uniform(-20, 20), 0, 255).astype(np.uint8)
if rng.random() < 0.5:
new = cv2.GaussianBlur(new, (0, 0), rng.uniform(0.5, 1.5))
ok, enc = cv2.imencode(".jpg", new, [cv2.IMWRITE_JPEG_QUALITY, int(rng.integers(55, 90))])
new = cv2.imdecode(enc, cv2.IMREAD_UNCHANGED)
return new, np.linalg.inv(H_job_to_new)
def warp_points(H, pts):
p = np.c_[pts, np.ones(len(pts))] @ H.T
return p[:, :2] / p[:, 2:3]
def edge_error(tag, split, source):
"""Per-image median / p90 distance (share of width) from true wall edges to predicted ones, from .geom_cache."""
cache = ROOT / ".geom_cache"
rows = [r for r in csv.DictReader(open(ROOT / "real_data" / "metadata.csv")) if r["split"] == split and r["camera_id"] == source]
med, p90 = [], []
for r in rows:
stem = r["image"].rsplit(".", 1)[0]
f = cache / f"{tag}_{split}_{stem}.npz"
gt = cv2.imread(str(ROOT / "real_data" / split / "masks" / f"{stem}.png"), cv2.IMREAD_UNCHANGED)
if not f.is_file() or gt is None:
continue
prob = np.load(f)["p1"].astype(np.float32)
h, w = gt.shape
pred = np.stack([cv2.resize(c, (w, h), interpolation=cv2.INTER_LINEAR) for c in prob]).argmax(0) == WALL
truth = gt == WALL
edge = lambda m: cv2.morphologyEx(m.astype(np.uint8), cv2.MORPH_GRADIENT, np.ones((3, 3), np.uint8)) > 0
te, pe = edge(truth) & (gt != 255), edge(pred)
if te.sum() < 50 or not pe.any():
continue
dist = cv2.distanceTransform((~pe).astype(np.uint8), cv2.DIST_L2, 5)[te] / w
med.append(float(np.median(dist)))
p90.append(float(np.percentile(dist, 90)))
return {"tag": tag, "split": split, "images": len(med), "what": "true wall edge -> nearest predicted wall edge",
"median_share_of_width": round(float(np.median(med)), 5), "p90_of_image_medians": round(float(np.percentile(med, 90)), 5),
"median_of_image_p90s": round(float(np.median(p90)), 5)}
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--split", default="test")
ap.add_argument("--source", default="roboflow-facade")
ap.add_argument("--pairs-per-image", type=int, default=3)
ap.add_argument("--jitter", type=float, default=0.06, help="max corner move as a share of the image side")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--edge-error", metavar="TAG", help="measure segmentation edge error from .geom_cache/TAG_* instead")
ap.add_argument("--out", type=Path)
a = ap.parse_args()
if a.edge_error:
res = edge_error(a.edge_error, a.split, a.source)
print(json.dumps(res, indent=1))
if a.out:
a.out.write_text(json.dumps(res, indent=1))
return
rng = np.random.default_rng(a.seed)
rows = [r for r in csv.DictReader(open(ROOT / "real_data" / "metadata.csv"))
if r["split"] == a.split and r["camera_id"] == a.source]
errors_px, errors_share, refused, reasons = [], [], 0, {}
for r in sorted(rows, key=lambda r: r["image"]):
stem = r["image"].rsplit(".", 1)[0]
bgr = cv2.imread(str(ROOT / "real_data" / a.split / "images" / r["image"]))
mask = cv2.imread(str(ROOT / "real_data" / a.split / "masks" / f"{stem}.png"), cv2.IMREAD_UNCHANGED)
if bgr is None or mask is None:
continue
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
h, w = rgb.shape[:2]
ys, xs = np.nonzero(cv2.resize(mask, (w, h), interpolation=cv2.INTER_NEAREST) == WALL)
if len(xs) < 100:
continue
pick = rng.choice(len(xs), min(2000, len(xs)), replace=False)
job_pts = np.c_[xs[pick], ys[pick]].astype(np.float64)
for _ in range(a.pairs_per_image):
new, H_true = reshoot(rgb, rng, a.jitter)
try:
H_est, _ = drone_align.homography(rgb, new)
except ValueError as exc:
refused += 1
key = str(exc).split(":")[0][:60]
reasons[key] = reasons.get(key, 0) + 1
continue
new_pts = warp_points(np.linalg.inv(H_true), job_pts) # where each wall pixel is in the new photo
err = np.linalg.norm(warp_points(H_est, new_pts) - job_pts, axis=1)
errors_px.append(float(np.median(err)))
errors_share.append(float(np.median(err)) / w)
px, share = np.array(errors_px), np.array(errors_share)
res = {"split": a.split, "source": a.source, "pairs_scored": int(len(px)), "refused": refused, "refusal_reasons": reasons,
"jitter": a.jitter, "note": "real facade photos, synthetic camera move with known truth",
"median_error_px": round(float(np.median(px)), 2), "p90_error_px": round(float(np.percentile(px, 90)), 2),
"p99_error_px": round(float(np.percentile(px, 99)), 2),
"median_error_share_of_width": round(float(np.median(share)), 5),
"p90_error_share_of_width": round(float(np.percentile(share, 90)), 5),
"p99_error_share_of_width": round(float(np.percentile(share, 99)), 5),
"share_over_1pct_width": round(float((share > 0.01).mean()), 3)}
print(json.dumps(res, indent=1))
if a.out:
a.out.write_text(json.dumps(res, indent=1))
if __name__ == "__main__":
main()