File size: 7,728 Bytes
4eecd03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#!/usr/bin/env python3
"""How far off is a re-scan's alignment on REAL facade photos? The number the drone twin's gating study needs.

``drone_align.homography`` lines a re-scan photo up with the job photo. Its error is what puts a camera read of the
paint on the wrong wall pixels -- the misregistration that observation gating near paint edges is meant to absorb.
No real re-scan pairs with ground truth exist yet, so each real facade test photo (real texture, real repetition,
real windows) is re-shot synthetically: a random perspective change (corners moved up to ``--jitter`` of the image),
brightness/contrast change, blur and JPEG. The true warp is known, so the error is exact:

  error = distance between where the estimated and the true warp put each wall pixel, in job-photo pixels,
          reported also as a share of the image width (so it transfers to the twin's grid).

Refusals (drone_align says the photos cannot be trusted) are counted, not scored: a refusal is safe.

A synthetic move has no parallax, moving shadows or lens change, so that number is a floor. The other real source of a
read landing on the wrong pixels is the segmentation's own edge error, measured with ``--edge-error``: for every true
wall edge pixel on the real facade frames, the distance to the nearest predicted wall edge (from eval_geometry's cached
network outputs, so nothing is re-run), as a share of the image width.

    python eval_align_real.py [--split test] [--pairs-per-image 3] [--out benchmark/records/align_real.json]
    python eval_align_real.py --edge-error C2-cal --split val [--out ...]
"""
from __future__ import annotations

import argparse
import csv
import json
from pathlib import Path

import cv2
import numpy as np

import drone_align

ROOT = Path(__file__).resolve().parent
WALL = 2


def reshoot(rgb, rng, jitter):
    """A synthetic second visit: (new photo, true H mapping new -> job pixels)."""
    h, w = rgb.shape[:2]
    src = np.float32([[0, 0], [w, 0], [w, h], [0, h]])
    dst = src + rng.uniform(-jitter, jitter, (4, 2)).astype(np.float32) * np.float32([w, h])
    H_job_to_new = cv2.getPerspectiveTransform(src, dst)
    new = cv2.warpPerspective(rgb, H_job_to_new, (w, h), borderMode=cv2.BORDER_REFLECT)
    new = np.clip(new.astype(np.float32) * rng.uniform(0.75, 1.25) + rng.uniform(-20, 20), 0, 255).astype(np.uint8)
    if rng.random() < 0.5:
        new = cv2.GaussianBlur(new, (0, 0), rng.uniform(0.5, 1.5))
    ok, enc = cv2.imencode(".jpg", new, [cv2.IMWRITE_JPEG_QUALITY, int(rng.integers(55, 90))])
    new = cv2.imdecode(enc, cv2.IMREAD_UNCHANGED)
    return new, np.linalg.inv(H_job_to_new)


def warp_points(H, pts):
    p = np.c_[pts, np.ones(len(pts))] @ H.T
    return p[:, :2] / p[:, 2:3]


def edge_error(tag, split, source):
    """Per-image median / p90 distance (share of width) from true wall edges to predicted ones, from .geom_cache."""
    cache = ROOT / ".geom_cache"
    rows = [r for r in csv.DictReader(open(ROOT / "real_data" / "metadata.csv")) if r["split"] == split and r["camera_id"] == source]
    med, p90 = [], []
    for r in rows:
        stem = r["image"].rsplit(".", 1)[0]
        f = cache / f"{tag}_{split}_{stem}.npz"
        gt = cv2.imread(str(ROOT / "real_data" / split / "masks" / f"{stem}.png"), cv2.IMREAD_UNCHANGED)
        if not f.is_file() or gt is None:
            continue
        prob = np.load(f)["p1"].astype(np.float32)
        h, w = gt.shape
        pred = np.stack([cv2.resize(c, (w, h), interpolation=cv2.INTER_LINEAR) for c in prob]).argmax(0) == WALL
        truth = gt == WALL
        edge = lambda m: cv2.morphologyEx(m.astype(np.uint8), cv2.MORPH_GRADIENT, np.ones((3, 3), np.uint8)) > 0
        te, pe = edge(truth) & (gt != 255), edge(pred)
        if te.sum() < 50 or not pe.any():
            continue
        dist = cv2.distanceTransform((~pe).astype(np.uint8), cv2.DIST_L2, 5)[te] / w
        med.append(float(np.median(dist)))
        p90.append(float(np.percentile(dist, 90)))
    return {"tag": tag, "split": split, "images": len(med), "what": "true wall edge -> nearest predicted wall edge",
            "median_share_of_width": round(float(np.median(med)), 5), "p90_of_image_medians": round(float(np.percentile(med, 90)), 5),
            "median_of_image_p90s": round(float(np.median(p90)), 5)}


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--split", default="test")
    ap.add_argument("--source", default="roboflow-facade")
    ap.add_argument("--pairs-per-image", type=int, default=3)
    ap.add_argument("--jitter", type=float, default=0.06, help="max corner move as a share of the image side")
    ap.add_argument("--seed", type=int, default=0)
    ap.add_argument("--edge-error", metavar="TAG", help="measure segmentation edge error from .geom_cache/TAG_* instead")
    ap.add_argument("--out", type=Path)
    a = ap.parse_args()
    if a.edge_error:
        res = edge_error(a.edge_error, a.split, a.source)
        print(json.dumps(res, indent=1))
        if a.out:
            a.out.write_text(json.dumps(res, indent=1))
        return
    rng = np.random.default_rng(a.seed)
    rows = [r for r in csv.DictReader(open(ROOT / "real_data" / "metadata.csv"))
            if r["split"] == a.split and r["camera_id"] == a.source]
    errors_px, errors_share, refused, reasons = [], [], 0, {}
    for r in sorted(rows, key=lambda r: r["image"]):
        stem = r["image"].rsplit(".", 1)[0]
        bgr = cv2.imread(str(ROOT / "real_data" / a.split / "images" / r["image"]))
        mask = cv2.imread(str(ROOT / "real_data" / a.split / "masks" / f"{stem}.png"), cv2.IMREAD_UNCHANGED)
        if bgr is None or mask is None:
            continue
        rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
        h, w = rgb.shape[:2]
        ys, xs = np.nonzero(cv2.resize(mask, (w, h), interpolation=cv2.INTER_NEAREST) == WALL)
        if len(xs) < 100:
            continue
        pick = rng.choice(len(xs), min(2000, len(xs)), replace=False)
        job_pts = np.c_[xs[pick], ys[pick]].astype(np.float64)
        for _ in range(a.pairs_per_image):
            new, H_true = reshoot(rgb, rng, a.jitter)
            try:
                H_est, _ = drone_align.homography(rgb, new)
            except ValueError as exc:
                refused += 1
                key = str(exc).split(":")[0][:60]
                reasons[key] = reasons.get(key, 0) + 1
                continue
            new_pts = warp_points(np.linalg.inv(H_true), job_pts)        # where each wall pixel is in the new photo
            err = np.linalg.norm(warp_points(H_est, new_pts) - job_pts, axis=1)
            errors_px.append(float(np.median(err)))
            errors_share.append(float(np.median(err)) / w)
    px, share = np.array(errors_px), np.array(errors_share)
    res = {"split": a.split, "source": a.source, "pairs_scored": int(len(px)), "refused": refused, "refusal_reasons": reasons,
           "jitter": a.jitter, "note": "real facade photos, synthetic camera move with known truth",
           "median_error_px": round(float(np.median(px)), 2), "p90_error_px": round(float(np.percentile(px, 90)), 2),
           "p99_error_px": round(float(np.percentile(px, 99)), 2),
           "median_error_share_of_width": round(float(np.median(share)), 5),
           "p90_error_share_of_width": round(float(np.percentile(share, 90)), 5),
           "p99_error_share_of_width": round(float(np.percentile(share, 99)), 5),
           "share_over_1pct_width": round(float((share > 0.01).mean()), 3)}
    print(json.dumps(res, indent=1))
    if a.out:
        a.out.write_text(json.dumps(res, indent=1))


if __name__ == "__main__":
    main()