Vertex view-projection refinement: snap 3D corners to 2D gestalt evidence

For each predicted 3D vertex V:
1. Project V into every registered COLMAP view via its P matrix.
2. Find the nearest gestalt corner-class pixel (apex / eave_end_point /
flashing_end_point) within 15 px in each view.
3. If at least 2 views have a match, DLT-triangulate a new 3D position
from the matched 2D corner pixels.
4. Sanity-check: refined position must lie within 0.5 m of V and
reproject with mean error <= 10 px.
5. If both checks pass, replace V with the refined position.

This is precision refinement of vertex *positions* — topology (edges)
is unchanged, no new geometry is introduced. Directly targets corner_f1
(currently 0.517 vs leader ~0.65+) by snapping model approximations
to exact gestalt corner detections that already drove training.

Reuses validated machinery: mvs_utils.collect_views + project_world_to_image
+ triangulate_dlt + mean_reprojection_error. Falls back to the input on
any error so the pipeline cannot regress structurally.

Local 100-sample A/B (paired, fixed seed):
baseline: 0.3770
+ orphan only: 0.3800 (+0.003, t=0.98)
+ refine only: 0.3839 (+0.007, t=1.42, 22 big wins vs 13 big losses)
+ both: 0.3856 (+0.009, t=1.69) <-- this commit

Refine runs BEFORE orphan cleanup so vertex movements complete first,
then orphan pass removes any vertices whose edges were dropped earlier
by hybrid_merge. Local A/B showed combined effect is partially additive
(refine +0.007, orphan +0.003, combined +0.009).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

local_eval.py

@@ -50,12 +50,21 @@ def parse_args():
                    help="override CONF_THRESH in script.py for this run")
     p.add_argument("--snap-apex", action="store_true",
                    help="extend snap_to_point_cloud target_classes to include apex (class 0)")
+    p.add_argument("--vertex-refine", action="store_true",
+                   help="apply vertex view-projection refinement after all post-process")
+    p.add_argument("--refine-max-pixel-dist", type=float, default=15.0,
+                   help="vertex refine: max 2D pixel distance for corner matching")
+    p.add_argument("--refine-min-views", type=int, default=2,
+                   help="vertex refine: min views with 2D match")
+    p.add_argument("--refine-max-move", type=float, default=0.5,
+                   help="vertex refine: max 3D displacement in meters")
     return p.parse_args()
 
 
 def predict_one(sample, model, device, cfg, rng,
                 use_tracks=True, use_2d_filter=True, orphan_only=False,
-                strict_no_support=False):
+                strict_no_support=False, vertex_refine=False,
+                refine_kwargs=None):
     """Run the full inference pipeline on one sample. Returns (pv, pe, diag)."""
     diag = {"colmap": -1, "fused": 0, "track_v": 0, "track_e": 0,
             "pred_v": 0, "pred_e": 0, "2dfilt_in": 0, "2dfilt_out": 0,
@@ -94,6 +103,22 @@ def predict_one(sample, model, device, cfg, rng,
             diag["status"] = f"track_failed:{type(e).__name__}"
 
     diag["2dfilt_in"] = len(pred_e) if hasattr(pred_e, '__len__') else 0
+    # Vertex refinement runs FIRST (refines vertex positions while orphan is still present;
+    # orphan/2d-filter then cleans up afterwards).
+    if vertex_refine:
+        try:
+            from vertex_refine import refine_vertices_view_projection
+            pv_before = pred_v
+            pred_v, pred_e = refine_vertices_view_projection(
+                pred_v, pred_e, sample,
+                **(refine_kwargs or {}))
+            if hasattr(pred_v, '__len__') and len(pred_v) == len(pv_before):
+                moved = int(np.sum(np.linalg.norm(
+                    np.asarray(pred_v) - np.asarray(pv_before), axis=1) > 1e-6))
+                diag["refined"] = moved
+        except Exception as e:
+            diag["status"] = f"refine_failed:{type(e).__name__}"
+
     if orphan_only:
         try:
             from edge_2d_filter import drop_orphan_vertices
@@ -182,12 +207,19 @@ def main():
             continue
 
         try:
+            refine_kwargs = {
+                "max_pixel_dist": args.refine_max_pixel_dist,
+                "min_views": args.refine_min_views,
+                "max_move_meters": args.refine_max_move,
+            }
             pred_v, pred_e, diag = predict_one(
                 sample, model, device, cfg, rng,
                 use_tracks=not args.no_tracks,
                 use_2d_filter=not args.no_filter,
                 orphan_only=args.orphan_only,
-                strict_no_support=args.strict_no_support)
+                strict_no_support=args.strict_no_support,
+                vertex_refine=args.vertex_refine,
+                refine_kwargs=refine_kwargs)
             if torch.backends.mps.is_available():
                 torch.mps.empty_cache()
 

script.py

@@ -426,11 +426,25 @@ if __name__ == "__main__":
                         print(f"  Track ensemble failed for {order_id}: {track_e_err}")
                         pred_status = "track_failed"
 
+                    # Vertex view-projection refinement. For each predicted 3D
+                    # vertex, find the nearest gestalt-corner pixel in each
+                    # view, re-triangulate via DLT, and replace the vertex if
+                    # the refined position is close (<=0.5m) and reprojects
+                    # well (<=10px). Local 100-sample A/B: +0.007 hss_mean,
+                    # 56 wins / 41 losses vs baseline.
+                    try:
+                        from vertex_refine import refine_vertices_view_projection
+                        pred_v, pred_e = refine_vertices_view_projection(
+                            pred_v, pred_e, sample,
+                            max_pixel_dist=15.0, min_views=2,
+                            max_move_meters=0.5, max_reproj_px=10.0,
+                        )
+                    except Exception as ref_err:
+                        print(f"  vertex refine failed for {order_id}: {ref_err}")
+
                     # Drop orphan vertices (vertices with no incident edges).
-                    # Pure precision pass: orphans hurt corner_f1 without
-                    # contributing to edge_iou. Local 100-sample A/B vs the
-                    # earlier 2D edge-content filter showed orphan-only is
-                    # +0.018 hss_mean (the 2D edge filter regressed q5 by 28%).
+                    # Local 100-sample A/B: combined refine + orphan = +0.009
+                    # hss_mean over baseline (t=1.69).
                     edges_before_2d = len(pred_e) if hasattr(pred_e, '__len__') else 0
                     try:
                         from edge_2d_filter import drop_orphan_vertices

vertex_refine.py

@@ -0,0 +1,172 @@
+"""Vertex view-projection refinement.
+
+For each predicted 3D vertex V:
+
+  1. Project V to 2D in every registered COLMAP view.
+  2. In each view, find the nearest gestalt corner-class pixel
+     (apex / eave_end_point / flashing_end_point) within ``max_pixel_dist``.
+  3. If at least ``min_views`` views have a 2D match, DLT-triangulate a new
+     3D position from those 2D corner detections.
+  4. Sanity-check: the refined position must (a) lie within
+     ``max_move_meters`` of the original V, and (b) have mean reprojection
+     error below ``max_reproj_px`` across the supporting views.
+  5. If both checks pass, replace V with the refined position.
+
+This is pure precision refinement of corner positions. Topology (edges)
+is preserved. Falls back to the input on any error — the function is
+guaranteed to never return fewer vertices than it received.
+
+Targets corner_f1: the learned model's 3D vertices are approximate; the
+gestalt segmentation gives us *exact* pixel-level corner detections per
+view; triangulating from those gives a much tighter 3D position whenever
+multiple views agree.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import cv2
+
+
+POINT_CLASSES = ("apex", "eave_end_point", "flashing_end_point")
+
+
+def _build_corner_pixels_per_view(good, views):
+    """Build per-view corner-pixel catalog.
+
+    Returns dict ``img_id -> {P, corners (N,2), tree, H, W}``.
+    Only includes views that have at least one corner pixel.
+    """
+    from hoho2025.color_mappings import gestalt_color_mapping
+    from scipy.spatial import cKDTree
+
+    out = {}
+    for gest_pil, depth_pil, img_id in zip(
+        good["gestalt"], good["depth"], good["image_ids"]
+    ):
+        if img_id not in views:
+            continue
+        depth_np = np.array(depth_pil)
+        H, W = depth_np.shape[:2]
+        gest_np = np.array(gest_pil.resize((W, H))).astype(np.uint8)
+
+        corners = []
+        for cls in POINT_CLASSES:
+            color = np.array(gestalt_color_mapping[cls])
+            mask = cv2.inRange(gest_np, color - 0.5, color + 0.5)
+            if mask.sum() == 0:
+                continue
+            n_cc, _, _, centroids = cv2.connectedComponentsWithStats(
+                mask, 8, cv2.CV_32S
+            )
+            if n_cc > 1:
+                # Skip the background centroid at index 0; rest are blob centers.
+                corners.extend(centroids[1:].tolist())
+
+        if not corners:
+            continue
+        corners_arr = np.asarray(corners, dtype=np.float64)
+        out[img_id] = {
+            "P": views[img_id]["P"],
+            "corners": corners_arr,
+            "tree": cKDTree(corners_arr),
+            "H": H,
+            "W": W,
+        }
+    return out
+
+
+def refine_vertices_view_projection(
+    pv,
+    pe,
+    sample,
+    max_pixel_dist: float = 15.0,
+    min_views: int = 2,
+    max_move_meters: float = 0.5,
+    max_reproj_px: float = 10.0,
+):
+    """Refine predicted vertex positions by re-triangulating from gestalt corners.
+
+    Args:
+        pv: (N, 3) predicted vertices in world coordinates.
+        pe: edge list (unchanged on return).
+        sample: raw dataset entry.
+        max_pixel_dist: max 2D distance from projected vertex to a gestalt
+            corner pixel to count as a match (15px ~= 2% of 768px width).
+        min_views: minimum views with a 2D match to attempt re-triangulation.
+        max_move_meters: refined vertex must lie within this distance of
+            the original — guards against spurious cross-corner matches.
+        max_reproj_px: refined vertex must have mean reprojection error
+            below this across the supporting views.
+
+    Returns:
+        (pv_refined, pe). Vertex count unchanged; pe is the same list.
+        Falls back to (pv, pe) on any error.
+    """
+    try:
+        from hoho2025.example_solutions import convert_entry_to_human_readable
+        from mvs_utils import (
+            collect_views, project_world_to_image,
+            triangulate_dlt, mean_reprojection_error,
+        )
+
+        pv_arr = np.asarray(pv, dtype=np.float64)
+        if pv_arr.ndim != 2 or pv_arr.shape[0] < 1:
+            return pv, pe
+
+        good = convert_entry_to_human_readable(sample)
+        colmap_rec = good.get("colmap") or good.get("colmap_binary")
+        if colmap_rec is None:
+            return pv, pe
+
+        views = collect_views(colmap_rec, good["image_ids"])
+        if len(views) < min_views:
+            return pv, pe
+
+        view_data = _build_corner_pixels_per_view(good, views)
+        if len(view_data) < min_views:
+            return pv, pe
+
+        refined = pv_arr.copy()
+        n_refined = 0
+
+        for i, v in enumerate(pv_arr):
+            Ps_match = []
+            pts2d_match = []
+
+            for img_id, vd in view_data.items():
+                uv, z = project_world_to_image(vd["P"], v.reshape(1, 3))
+                if z[0] <= 0:
+                    continue
+                u, vp = float(uv[0, 0]), float(uv[0, 1])
+                if not (0 <= u < vd["W"] and 0 <= vp < vd["H"]):
+                    continue
+
+                dist, idx = vd["tree"].query([u, vp], k=1)
+                if dist <= max_pixel_dist:
+                    Ps_match.append(vd["P"])
+                    pts2d_match.append(vd["corners"][idx])
+
+            if len(Ps_match) < min_views:
+                continue  # not enough 2D evidence; keep original V
+
+            v_new = triangulate_dlt(Ps_match, pts2d_match)
+            if not np.all(np.isfinite(v_new)):
+                continue
+
+            # Sanity 1: refined position shouldn't have moved too far.
+            if float(np.linalg.norm(v_new - v)) > max_move_meters:
+                continue
+
+            # Sanity 2: refined position must reproject well to its 2D matches.
+            err = mean_reprojection_error(v_new, Ps_match, pts2d_match)
+            if not np.isfinite(err) or err > max_reproj_px:
+                continue
+
+            refined[i] = v_new
+            n_refined += 1
+
+        return refined.astype(np.asarray(pv).dtype), pe
+
+    except Exception:
+        return pv, pe