xsponenta Claude Opus 4.7 commited on
Commit
9bb88ff
·
1 Parent(s): 55ba9bc

Add vertex POSITION REGRESSOR (DINOv2) on top of edge classifier

Browse files

Per-vertex DINOv2 patch features + geometric features -> 3D offset
predicted by MLP (60K head params). Trained on 800 samples to regress
toward the nearest GT vertex within 0.5m. Applied AFTER orphan-drop,
BEFORE the edge classifier in the production pipeline.

Offset clamped at 0.3m magnitude for safety; vertices visible in fewer
than 2 views are not moved (avoid moving low-confidence verts).

Local 200-sample A/B results:
vr + edge_v4 vs baseline: +0.0095 hss_mean (t=+3.84, 116w/70l)
vr + edge_v4 vs edge_v4: +0.0065 hss_mean (t=+2.39, 109w/79l)

Strongly significant signal. First positive ON TOP of edge classifier.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Files changed (4) hide show
  1. local_eval.py +82 -0
  2. script.py +50 -0
  3. vertex_classifier_v4.py +281 -0
  4. vertex_regressor_v4.py +119 -0
local_eval.py CHANGED
@@ -110,6 +110,16 @@ def parse_args():
110
  help="path to edge_classifier_v3.pt; v3 = CNN patches + v2 features")
111
  p.add_argument("--edge-classifier-v4", type=str, default="",
112
  help="path to edge_classifier_v4.pt; v4 = DINOv2 features + v2 features")
 
 
 
 
 
 
 
 
 
 
113
  p.add_argument("--edge-class-thresh", type=float, default=0.5,
114
  help="edge classifier: keep edges with P(keep) >= threshold")
115
  p.add_argument("--edge-class-min-keep", type=float, default=0.5,
@@ -266,6 +276,40 @@ def predict_one(sample, model, device, cfg, rng,
266
  diag["status"] = f"2dfilt_failed:{type(e).__name__}"
267
  diag["2dfilt_out"] = len(pred_e) if hasattr(pred_e, '__len__') else 0
268
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
269
  # Edge classifier: learned keep/drop on top of post-filter edges.
270
  ec = getattr(predict_one, "_edge_classifier", None)
271
  if ec is not None:
@@ -436,6 +480,42 @@ def main():
436
  "version": 3,
437
  }
438
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
439
  if args.edge_classifier_v4:
440
  from edge_classifier_v4 import load_classifier_v4, get_dino_model
441
  ec_model, g_mean, g_std, e_mean, e_std = load_classifier_v4(args.edge_classifier_v4, device="cpu")
@@ -523,6 +603,8 @@ def main():
523
  predict_one._tri_sparse_threshold = args.tri_sparse_threshold
524
  predict_one._tri_merge_radius = args.tri_merge_radius
525
  predict_one._edge_classifier = getattr(predict_one, "_edge_classifier_loaded", None)
 
 
526
  fallback_tracks = None
527
  if args.fallback_to_tracks_when:
528
  pv_thr, tv_thr = args.fallback_to_tracks_when.split(",")
 
110
  help="path to edge_classifier_v3.pt; v3 = CNN patches + v2 features")
111
  p.add_argument("--edge-classifier-v4", type=str, default="",
112
  help="path to edge_classifier_v4.pt; v4 = DINOv2 features + v2 features")
113
+ p.add_argument("--vertex-regressor-v4", type=str, default="",
114
+ help="path to vertex_regressor_v4.pt; learned 3D position refinement")
115
+ p.add_argument("--vertex-reg-max-move", type=float, default=0.4,
116
+ help="vertex regressor: clamp predicted offset to this magnitude")
117
+ p.add_argument("--vertex-classifier-v4", type=str, default="",
118
+ help="path to vertex_classifier_v4.pt; drops low-conf vertices")
119
+ p.add_argument("--vertex-class-thresh", type=float, default=0.3,
120
+ help="vertex classifier: keep if P(keep) >= threshold")
121
+ p.add_argument("--vertex-class-min-keep", type=float, default=0.85,
122
+ help="vertex classifier: never drop more than (1 - this) of vertices")
123
  p.add_argument("--edge-class-thresh", type=float, default=0.5,
124
  help="edge classifier: keep edges with P(keep) >= threshold")
125
  p.add_argument("--edge-class-min-keep", type=float, default=0.5,
 
276
  diag["status"] = f"2dfilt_failed:{type(e).__name__}"
277
  diag["2dfilt_out"] = len(pred_e) if hasattr(pred_e, '__len__') else 0
278
 
279
+ # Vertex regressor (moves vertices toward learned position before classifier)
280
+ vr = getattr(predict_one, "_vertex_regressor", None)
281
+ if vr is not None:
282
+ try:
283
+ from vertex_regressor_v4 import refine_vertices_with_regressor
284
+ pred_v, pred_e = refine_vertices_with_regressor(
285
+ pred_v, pred_e, sample,
286
+ vr["model"], vr["dino"], device=vr["dino_device"],
287
+ feature_mean=vr["mean"], feature_std=vr["std"],
288
+ edge_feat_mean=vr["edge_feat_mean"], edge_feat_std=vr["edge_feat_std"],
289
+ max_move_meters=vr["max_move_meters"],
290
+ )
291
+ except Exception as e:
292
+ diag["status"] = f"vr_failed:{type(e).__name__}"
293
+
294
+ # Vertex classifier (drops low-conf vertices before edge classifier)
295
+ vc = getattr(predict_one, "_vertex_classifier", None)
296
+ if vc is not None:
297
+ try:
298
+ from vertex_classifier_v4 import classify_vertices_v4
299
+ v_before = len(pred_v) if hasattr(pred_v, '__len__') else 0
300
+ pred_v, pred_e = classify_vertices_v4(
301
+ pred_v, pred_e, sample,
302
+ vc["model"], vc["dino"], device=vc["dino_device"],
303
+ threshold=vc["threshold"],
304
+ feature_mean=vc["mean"], feature_std=vc["std"],
305
+ edge_feat_mean=vc["edge_feat_mean"], edge_feat_std=vc["edge_feat_std"],
306
+ min_keep_frac=vc["min_keep_frac"],
307
+ )
308
+ diag["vc_kept"] = (len(pred_v) if hasattr(pred_v, '__len__') else 0)
309
+ diag["vc_dropped"] = v_before - diag["vc_kept"]
310
+ except Exception as e:
311
+ diag["status"] = f"vc_failed:{type(e).__name__}"
312
+
313
  # Edge classifier: learned keep/drop on top of post-filter edges.
314
  ec = getattr(predict_one, "_edge_classifier", None)
315
  if ec is not None:
 
480
  "version": 3,
481
  }
482
 
483
+ if args.vertex_regressor_v4:
484
+ from vertex_regressor_v4 import load_regressor_v4
485
+ from edge_classifier_v4 import get_dino_model
486
+ vr_model, vrg_mean, vrg_std, vre_mean, vre_std = load_regressor_v4(
487
+ args.vertex_regressor_v4, device="cpu")
488
+ dino_vr = get_dino_model(device=device)
489
+ print(f"Vertex regressor V4 loaded from {args.vertex_regressor_v4}")
490
+ predict_one._vertex_regressor_loaded = {
491
+ "model": vr_model,
492
+ "dino": dino_vr,
493
+ "dino_device": device,
494
+ "mean": vrg_mean.cpu().numpy() if hasattr(vrg_mean, "cpu") else vrg_mean,
495
+ "std": vrg_std.cpu().numpy() if hasattr(vrg_std, "cpu") else vrg_std,
496
+ "edge_feat_mean": vre_mean.cpu().numpy() if hasattr(vre_mean, "cpu") else vre_mean,
497
+ "edge_feat_std": vre_std.cpu().numpy() if hasattr(vre_std, "cpu") else vre_std,
498
+ "max_move_meters": args.vertex_reg_max_move,
499
+ }
500
+
501
+ if args.vertex_classifier_v4:
502
+ from vertex_classifier_v4 import load_classifier_v4 as load_vc4
503
+ from edge_classifier_v4 import get_dino_model
504
+ vc_model, vg_mean, vg_std, ve_mean, ve_std = load_vc4(args.vertex_classifier_v4, device="cpu")
505
+ dino_v = get_dino_model(device=device)
506
+ print(f"Vertex classifier V4 loaded from {args.vertex_classifier_v4}")
507
+ predict_one._vertex_classifier_loaded = {
508
+ "model": vc_model,
509
+ "dino": dino_v,
510
+ "dino_device": device,
511
+ "mean": vg_mean.cpu().numpy() if hasattr(vg_mean, "cpu") else vg_mean,
512
+ "std": vg_std.cpu().numpy() if hasattr(vg_std, "cpu") else vg_std,
513
+ "edge_feat_mean": ve_mean.cpu().numpy() if hasattr(ve_mean, "cpu") else ve_mean,
514
+ "edge_feat_std": ve_std.cpu().numpy() if hasattr(ve_std, "cpu") else ve_std,
515
+ "threshold": args.vertex_class_thresh,
516
+ "min_keep_frac": args.vertex_class_min_keep,
517
+ }
518
+
519
  if args.edge_classifier_v4:
520
  from edge_classifier_v4 import load_classifier_v4, get_dino_model
521
  ec_model, g_mean, g_std, e_mean, e_std = load_classifier_v4(args.edge_classifier_v4, device="cpu")
 
603
  predict_one._tri_sparse_threshold = args.tri_sparse_threshold
604
  predict_one._tri_merge_radius = args.tri_merge_radius
605
  predict_one._edge_classifier = getattr(predict_one, "_edge_classifier_loaded", None)
606
+ predict_one._vertex_classifier = getattr(predict_one, "_vertex_classifier_loaded", None)
607
+ predict_one._vertex_regressor = getattr(predict_one, "_vertex_regressor_loaded", None)
608
  fallback_tracks = None
609
  if args.fallback_to_tracks_when:
610
  pv_thr, tv_thr = args.fallback_to_tracks_when.split(",")
script.py CHANGED
@@ -391,6 +391,38 @@ if __name__ == "__main__":
391
  model = load_model(checkpoint_path, device)
392
  print(f"Model loaded: {sum(p.numel() for p in model.parameters()):,} params")
393
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
394
  # Edge classifier (DINOv2 patch features + geometric features → P(keep edge)).
395
  # Trained on 400 samples with match_radius=0.4. Best val acc 82.1%.
396
  # Operating point: thresh=0.15, min_keep=0.85 — drop only the bottom ~15%
@@ -581,6 +613,24 @@ if __name__ == "__main__":
581
  print(f" orphan drop failed for {order_id}: {filt_err}")
582
  edges_after_2d = len(pred_e) if hasattr(pred_e, '__len__') else 0
583
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
584
  # Edge classifier v4 (DINOv2 + geom features): drop the bottom
585
  # ~15% of edges whose learned P(keep) is lowest. 200-sample
586
  # local A/B: +0.0030 hss_mean (t=+1.26).
 
391
  model = load_model(checkpoint_path, device)
392
  print(f"Model loaded: {sum(p.numel() for p in model.parameters()):,} params")
393
 
394
+ # Vertex position regressor (DINOv2 patch features per vertex → 3D offset).
395
+ # Trained on 800 samples. Predicts a small move toward the nearest GT vertex
396
+ # using bilinear-sampled DINOv2 features and per-vertex geometric features.
397
+ # Local 200-sample A/B vs edge-classifier-only: +0.0065 hss_mean (t=+2.39,
398
+ # 109 wins / 79 losses). Combined vs baseline: +0.0095 (t=+3.84).
399
+ # Offset clamped at 0.3m magnitude for safety.
400
+ vertex_regressor_bundle = None
401
+ vertex_regressor_path = SCRIPT_DIR / "vertex_regressor_v4_800.pt"
402
+ if vertex_regressor_path.exists():
403
+ try:
404
+ from vertex_regressor_v4 import load_regressor_v4
405
+ from edge_classifier_v4 import get_dino_model as _get_dino
406
+ vr_model, vrg_mean, vrg_std, vre_mean, vre_std = load_regressor_v4(
407
+ str(vertex_regressor_path), device="cpu")
408
+ dino_vr = _get_dino(device=device)
409
+ vertex_regressor_bundle = {
410
+ "model": vr_model,
411
+ "dino": dino_vr,
412
+ "dino_device": device,
413
+ "mean": vrg_mean.cpu().numpy() if hasattr(vrg_mean, "cpu") else vrg_mean,
414
+ "std": vrg_std.cpu().numpy() if hasattr(vrg_std, "cpu") else vrg_std,
415
+ "edge_feat_mean": vre_mean.cpu().numpy() if hasattr(vre_mean, "cpu") else vre_mean,
416
+ "edge_feat_std": vre_std.cpu().numpy() if hasattr(vre_std, "cpu") else vre_std,
417
+ "max_move_meters": 0.3,
418
+ }
419
+ print(f"Vertex regressor v4 loaded ({sum(p.numel() for p in vr_model.parameters()):,} head params)")
420
+ except Exception as vr_err:
421
+ print(f"Vertex regressor load failed: {vr_err}; running without")
422
+ vertex_regressor_bundle = None
423
+ else:
424
+ print(f"No vertex_regressor_v4_800.pt at {vertex_regressor_path}; running without")
425
+
426
  # Edge classifier (DINOv2 patch features + geometric features → P(keep edge)).
427
  # Trained on 400 samples with match_radius=0.4. Best val acc 82.1%.
428
  # Operating point: thresh=0.15, min_keep=0.85 — drop only the bottom ~15%
 
613
  print(f" orphan drop failed for {order_id}: {filt_err}")
614
  edges_after_2d = len(pred_e) if hasattr(pred_e, '__len__') else 0
615
 
616
+ # Vertex position regressor (DINOv2): predict per-vertex 3D
617
+ # offset toward nearest learned position. Local 200-sample
618
+ # A/B: +0.0065 hss_mean on top of edge classifier (t=+2.39).
619
+ if vertex_regressor_bundle is not None:
620
+ try:
621
+ from vertex_regressor_v4 import refine_vertices_with_regressor
622
+ vr = vertex_regressor_bundle
623
+ pred_v, pred_e = refine_vertices_with_regressor(
624
+ pred_v, pred_e, sample,
625
+ vr["model"], vr["dino"], device=vr["dino_device"],
626
+ feature_mean=vr["mean"], feature_std=vr["std"],
627
+ edge_feat_mean=vr["edge_feat_mean"],
628
+ edge_feat_std=vr["edge_feat_std"],
629
+ max_move_meters=vr["max_move_meters"],
630
+ )
631
+ except Exception as vr_err:
632
+ print(f" vertex regressor failed for {order_id}: {vr_err}")
633
+
634
  # Edge classifier v4 (DINOv2 + geom features): drop the bottom
635
  # ~15% of edges whose learned P(keep) is lowest. 200-sample
636
  # local A/B: +0.0030 hss_mean (t=+1.26).
vertex_classifier_v4.py ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """V4 vertex classifier with DINOv2 patch features.
2
+
3
+ Per-vertex analog of edge_classifier_v4. For each predicted 3D vertex,
4
+ project to each view, bilinearly sample DINOv2 patch features at the
5
+ projected pixel location. Mean+max pool across views -> 768-dim. Concat
6
+ with simple geometric features (10-dim) -> 778-dim. MLP head -> P(keep).
7
+
8
+ Decisions:
9
+ - drop a vertex if classifier P(keep) < threshold
10
+ - re-run drop_orphan after to clean dangling edges
11
+
12
+ Label for training: vertex is "true" if there's a GT vertex within
13
+ `match_radius` meters.
14
+
15
+ Vertex geometric features (10):
16
+ 0 z (height)
17
+ 1 degree (number of incident edges)
18
+ 2 dist to nearest other vertex
19
+ 3 dist to nearest colmap point
20
+ 4 num views vertex is in-frame
21
+ 5 min projected dist to nearest gestalt corner pixel (clipped 30)
22
+ 6 mean projected dist to nearest gestalt corner pixel
23
+ 7 num views with corner pixel within 5px
24
+ 8 num views with corner pixel within 15px
25
+ 9 vertex_count / median scene vertex count (graph density hint)
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import numpy as np
31
+ import torch
32
+ import torch.nn as nn
33
+
34
+ from edge_classifier_v4 import (
35
+ DINO_FEAT_DIM, get_dino_model, _encode_views_with_dino, _bilinear_sample_grid,
36
+ )
37
+
38
+ POINT_CLASSES = ("apex", "eave_end_point", "flashing_end_point")
39
+ V_GEOM_DIM = 10
40
+ V_EDGE_FEAT_DIM = DINO_FEAT_DIM * 2 # mean + max pool
41
+
42
+
43
+ class VertexClassifierV4(nn.Module):
44
+ def __init__(self, geom_dim: int = V_GEOM_DIM, edge_feat_dim: int = V_EDGE_FEAT_DIM,
45
+ hidden: int = 128):
46
+ super().__init__()
47
+ in_dim = geom_dim + edge_feat_dim
48
+ self.net = nn.Sequential(
49
+ nn.Linear(in_dim, hidden),
50
+ nn.GELU(),
51
+ nn.Dropout(0.2),
52
+ nn.Linear(hidden, hidden),
53
+ nn.GELU(),
54
+ nn.Dropout(0.2),
55
+ nn.Linear(hidden, hidden // 2),
56
+ nn.GELU(),
57
+ nn.Linear(hidden // 2, 1),
58
+ )
59
+
60
+ def forward(self, geom_feats, dino_feats):
61
+ x = torch.cat([geom_feats, dino_feats], dim=1)
62
+ return self.net(x).squeeze(-1)
63
+
64
+
65
+ def _build_corner_dt(good, views, dilate_px=0):
66
+ """Per-view corner DT (distance to nearest gestalt point-class pixel)."""
67
+ import cv2
68
+ from hoho2025.color_mappings import gestalt_color_mapping
69
+ out = {}
70
+ for gest_pil, depth_pil, img_id in zip(
71
+ good["gestalt"], good["depth"], good["image_ids"]
72
+ ):
73
+ if img_id not in views:
74
+ continue
75
+ depth_np = np.array(depth_pil)
76
+ H, W = depth_np.shape[:2]
77
+ gest_np = np.array(gest_pil.resize((W, H))).astype(np.uint8)
78
+ mask = np.zeros((H, W), dtype=np.uint8)
79
+ for cls in POINT_CLASSES:
80
+ color = np.array(gestalt_color_mapping[cls])
81
+ mask |= cv2.inRange(gest_np, color - 0.5, color + 0.5)
82
+ if mask.sum() > 0:
83
+ dt = cv2.distanceTransform(255 - mask, cv2.DIST_L2, 5)
84
+ dt = np.minimum(dt, 30.0).astype(np.float32)
85
+ else:
86
+ dt = np.full((H, W), 30.0, dtype=np.float32)
87
+ out[img_id] = (dt, H, W)
88
+ return out
89
+
90
+
91
+ def _vertex_geom_features(pv, pe, sample):
92
+ """Per-vertex geometric features (V, 10)."""
93
+ pv_arr = np.asarray(pv, dtype=np.float32)
94
+ V = len(pv_arr)
95
+ feats = np.zeros((V, V_GEOM_DIM), dtype=np.float32)
96
+ if V == 0:
97
+ return feats
98
+
99
+ deg = np.zeros(V, dtype=np.int32)
100
+ for a, b in pe:
101
+ if 0 <= int(a) < V: deg[int(a)] += 1
102
+ if 0 <= int(b) < V: deg[int(b)] += 1
103
+
104
+ try:
105
+ from hoho2025.example_solutions import convert_entry_to_human_readable
106
+ from mvs_utils import collect_views, project_world_to_image
107
+ from scipy.spatial import cKDTree
108
+
109
+ good = convert_entry_to_human_readable(sample)
110
+ colmap_rec = good.get("colmap") or good.get("colmap_binary")
111
+ if colmap_rec is None:
112
+ return feats
113
+ views = collect_views(colmap_rec, good["image_ids"])
114
+ if not views:
115
+ return feats
116
+ corner_dt = _build_corner_dt(good, views)
117
+
118
+ # KDTree over colmap points
119
+ c_pts = []
120
+ if hasattr(colmap_rec, "points3D"):
121
+ for p in colmap_rec.points3D.values():
122
+ c_pts.append(p.xyz)
123
+ c_tree = cKDTree(np.asarray(c_pts, dtype=np.float32)) if c_pts else None
124
+
125
+ # KDTree over pred vertices (for nearest-neighbor)
126
+ pv_tree = cKDTree(pv_arr) if V >= 2 else None
127
+
128
+ for i in range(V):
129
+ v = pv_arr[i]
130
+ feats[i, 0] = float(v[2])
131
+ feats[i, 1] = float(deg[i])
132
+ if pv_tree is not None:
133
+ dists, _ = pv_tree.query(v, k=min(2, V))
134
+ feats[i, 2] = float(dists[1] if len(dists) > 1 else 0.0)
135
+ if c_tree is not None:
136
+ d, _ = c_tree.query(v)
137
+ feats[i, 3] = float(d)
138
+
139
+ # Per-view corner distances
140
+ view_dists = []
141
+ in_frame = 0
142
+ close_5 = 0
143
+ close_15 = 0
144
+ for img_id, view in views.items():
145
+ if img_id not in corner_dt:
146
+ continue
147
+ dt, H, W = corner_dt[img_id]
148
+ uv, z = project_world_to_image(view["P"], v.reshape(1, 3))
149
+ if z[0] <= 0:
150
+ continue
151
+ if not (0 <= uv[0, 0] < W and 0 <= uv[0, 1] < H):
152
+ continue
153
+ in_frame += 1
154
+ cd = float(dt[int(uv[0, 1]), int(uv[0, 0])])
155
+ view_dists.append(cd)
156
+ if cd < 5: close_5 += 1
157
+ if cd < 15: close_15 += 1
158
+ feats[i, 4] = float(in_frame)
159
+ if view_dists:
160
+ arr = np.asarray(view_dists)
161
+ feats[i, 5] = float(arr.min())
162
+ feats[i, 6] = float(arr.mean())
163
+ else:
164
+ feats[i, 5] = 30.0
165
+ feats[i, 6] = 30.0
166
+ feats[i, 7] = float(close_5)
167
+ feats[i, 8] = float(close_15)
168
+ feats[i, 9] = float(V)
169
+ except Exception:
170
+ pass
171
+ return feats
172
+
173
+
174
+ def extract_vertex_features_v4(pv, pe, sample, dino, device="cpu"):
175
+ """Return (V, 10) geom + (V, 768) dino features per vertex."""
176
+ pv_arr = np.asarray(pv)
177
+ V = len(pv_arr)
178
+ geom = _vertex_geom_features(pv, pe, sample)
179
+ dino_feats = np.zeros((V, V_EDGE_FEAT_DIM), dtype=np.float32)
180
+ if V == 0:
181
+ return geom, dino_feats
182
+
183
+ try:
184
+ from hoho2025.example_solutions import convert_entry_to_human_readable
185
+ from mvs_utils import collect_views, project_world_to_image
186
+ good = convert_entry_to_human_readable(sample)
187
+ colmap_rec = good.get("colmap") or good.get("colmap_binary")
188
+ if colmap_rec is None:
189
+ return geom, dino_feats
190
+ views = collect_views(colmap_rec, good["image_ids"])
191
+ if not views:
192
+ return geom, dino_feats
193
+ per_view, Hs, Ws = _encode_views_with_dino(good, views, dino, device)
194
+ if not per_view:
195
+ return geom, dino_feats
196
+
197
+ for i in range(V):
198
+ v = pv_arr[i].reshape(1, 3)
199
+ per_view_feats = []
200
+ for img_id, view in views.items():
201
+ if img_id not in per_view:
202
+ continue
203
+ H, W = Hs[img_id], Ws[img_id]
204
+ uv, z = project_world_to_image(view["P"], v)
205
+ if z[0] <= 0:
206
+ continue
207
+ u, vu = uv[0, 0], uv[0, 1]
208
+ if not (0 <= u < W and 0 <= vu < H):
209
+ continue
210
+ feat = _bilinear_sample_grid(per_view[img_id], u / max(W - 1, 1), vu / max(H - 1, 1))
211
+ per_view_feats.append(feat)
212
+ if per_view_feats:
213
+ arr = np.asarray(per_view_feats)
214
+ dino_feats[i] = np.concatenate([arr.mean(axis=0), arr.max(axis=0)])
215
+ except Exception:
216
+ pass
217
+ return geom, dino_feats
218
+
219
+
220
+ def label_vertices_vs_gt(pv, gt_v, match_radius: float = 0.5):
221
+ """Vertex is positive if a GT vertex is within match_radius meters."""
222
+ pv_arr = np.asarray(pv, dtype=np.float32)
223
+ gt_v_arr = np.asarray(gt_v, dtype=np.float32)
224
+ if pv_arr.shape[0] == 0 or gt_v_arr.shape[0] == 0:
225
+ return np.zeros(pv_arr.shape[0], dtype=np.float32)
226
+ from scipy.spatial import cKDTree
227
+ tree = cKDTree(gt_v_arr)
228
+ dists, _ = tree.query(pv_arr)
229
+ return (dists <= match_radius).astype(np.float32)
230
+
231
+
232
+ def classify_vertices_v4(pv, pe, sample, classifier, dino, device="cpu",
233
+ threshold: float = 0.5,
234
+ feature_mean=None, feature_std=None,
235
+ edge_feat_mean=None, edge_feat_std=None,
236
+ min_keep_frac: float = 0.85):
237
+ """Drop low-confidence vertices, rebuild edge indexing."""
238
+ try:
239
+ pv_arr = np.asarray(pv)
240
+ if len(pv_arr) == 0:
241
+ return pv, pe
242
+ geom, dino_feats = extract_vertex_features_v4(pv, pe, sample, dino, device=device)
243
+ if feature_mean is not None and feature_std is not None:
244
+ geom = (geom - feature_mean) / (feature_std + 1e-6)
245
+ if edge_feat_mean is not None and edge_feat_std is not None:
246
+ dino_feats = (dino_feats - edge_feat_mean) / (edge_feat_std + 1e-6)
247
+ with torch.no_grad():
248
+ g = torch.tensor(geom, dtype=torch.float32)
249
+ d = torch.tensor(dino_feats, dtype=torch.float32)
250
+ scores = torch.sigmoid(classifier(g, d)).numpy()
251
+
252
+ keep_mask = scores >= threshold
253
+ min_keep = max(2, int(np.ceil(min_keep_frac * len(pv_arr))))
254
+ if keep_mask.sum() < min_keep:
255
+ top_idx = np.argsort(-scores)[:min_keep]
256
+ keep_mask = np.zeros_like(keep_mask)
257
+ keep_mask[top_idx] = True
258
+
259
+ if keep_mask.all():
260
+ return pv, pe
261
+
262
+ keep_idx = np.where(keep_mask)[0]
263
+ old_to_new = {int(o): int(n) for n, o in enumerate(keep_idx)}
264
+ new_pv = pv_arr[keep_idx]
265
+ new_pe = [(old_to_new[int(a)], old_to_new[int(b)])
266
+ for a, b in pe
267
+ if int(a) in old_to_new and int(b) in old_to_new]
268
+ if len(new_pv) < 2 or len(new_pe) < 1:
269
+ return pv, pe
270
+ return new_pv, new_pe
271
+ except Exception:
272
+ return pv, pe
273
+
274
+
275
+ def load_classifier_v4(path: str, device: str = "cpu"):
276
+ blob = torch.load(path, map_location=device, weights_only=False)
277
+ m = VertexClassifierV4(hidden=blob.get("hidden", 128))
278
+ m.load_state_dict(blob["model"])
279
+ m.to(device).eval()
280
+ return (m, blob.get("feature_mean"), blob.get("feature_std"),
281
+ blob.get("edge_feat_mean"), blob.get("edge_feat_std"))
vertex_regressor_v4.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Vertex position REGRESSOR using DINOv2 features.
2
+
3
+ Predicts a 3D offset (dx, dy, dz) for each vertex to push it toward the
4
+ nearest GT vertex. This is the regression analog of the classifier — instead
5
+ of "drop bad vertices", we "move vertices to where they should be".
6
+
7
+ Loss: MSE on offset to nearest GT vertex (within match_radius), 0 if no GT
8
+ match (don't move).
9
+
10
+ Inference: clamp predicted offset magnitude to `max_move_meters` for safety.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import numpy as np
16
+ import torch
17
+ import torch.nn as nn
18
+
19
+ from edge_classifier_v4 import DINO_FEAT_DIM, get_dino_model
20
+ from vertex_classifier_v4 import (
21
+ V_GEOM_DIM, V_EDGE_FEAT_DIM,
22
+ _vertex_geom_features, extract_vertex_features_v4,
23
+ )
24
+
25
+
26
+ class VertexRegressorV4(nn.Module):
27
+ def __init__(self, geom_dim: int = V_GEOM_DIM, edge_feat_dim: int = V_EDGE_FEAT_DIM,
28
+ hidden: int = 128):
29
+ super().__init__()
30
+ in_dim = geom_dim + edge_feat_dim
31
+ self.net = nn.Sequential(
32
+ nn.Linear(in_dim, hidden),
33
+ nn.GELU(),
34
+ nn.Dropout(0.2),
35
+ nn.Linear(hidden, hidden),
36
+ nn.GELU(),
37
+ nn.Dropout(0.2),
38
+ nn.Linear(hidden, hidden // 2),
39
+ nn.GELU(),
40
+ nn.Linear(hidden // 2, 3), # (dx, dy, dz)
41
+ )
42
+
43
+ def forward(self, geom_feats, dino_feats):
44
+ x = torch.cat([geom_feats, dino_feats], dim=1)
45
+ return self.net(x)
46
+
47
+
48
+ def label_vertex_offsets(pv, gt_v, match_radius: float = 0.5):
49
+ """Return (V, 3) target offsets: vertex moves toward nearest GT within radius.
50
+
51
+ For vertices with no GT match within radius, target = (0, 0, 0).
52
+ Mask (V,) is 1 where we have valid GT to learn from.
53
+ """
54
+ pv_arr = np.asarray(pv, dtype=np.float32)
55
+ gt_v_arr = np.asarray(gt_v, dtype=np.float32)
56
+ V = pv_arr.shape[0]
57
+ offsets = np.zeros((V, 3), dtype=np.float32)
58
+ mask = np.zeros(V, dtype=np.float32)
59
+ if V == 0 or gt_v_arr.shape[0] == 0:
60
+ return offsets, mask
61
+ from scipy.spatial import cKDTree
62
+ tree = cKDTree(gt_v_arr)
63
+ dists, idxs = tree.query(pv_arr)
64
+ keep = dists <= match_radius
65
+ offsets[keep] = gt_v_arr[idxs[keep]] - pv_arr[keep]
66
+ mask[keep] = 1.0
67
+ return offsets, mask
68
+
69
+
70
+ def refine_vertices_with_regressor(pv, pe, sample, regressor, dino, device="cpu",
71
+ feature_mean=None, feature_std=None,
72
+ edge_feat_mean=None, edge_feat_std=None,
73
+ max_move_meters: float = 0.4,
74
+ min_views_for_move: int = 2):
75
+ """Apply regressor to predict per-vertex offsets, clamp by max_move."""
76
+ try:
77
+ pv_arr = np.asarray(pv, dtype=np.float32)
78
+ if pv_arr.shape[0] == 0:
79
+ return pv, pe
80
+ geom, dino_feats = extract_vertex_features_v4(pv, pe, sample, dino, device=device)
81
+ # Only move vertices visible in enough views (sanity)
82
+ in_view_count = geom[:, 4].astype(np.int32) # feature[4] is num views vertex is in-frame
83
+
84
+ if feature_mean is not None and feature_std is not None:
85
+ geom_n = (geom - feature_mean) / (feature_std + 1e-6)
86
+ else:
87
+ geom_n = geom
88
+ if edge_feat_mean is not None and edge_feat_std is not None:
89
+ dino_n = (dino_feats - edge_feat_mean) / (edge_feat_std + 1e-6)
90
+ else:
91
+ dino_n = dino_feats
92
+
93
+ with torch.no_grad():
94
+ g = torch.tensor(geom_n, dtype=torch.float32)
95
+ d = torch.tensor(dino_n, dtype=torch.float32)
96
+ offsets = regressor(g, d).numpy() # (V, 3)
97
+
98
+ # Clamp magnitude
99
+ norms = np.linalg.norm(offsets, axis=1, keepdims=True)
100
+ scale = np.clip(max_move_meters / np.maximum(norms, 1e-9), 0, 1)
101
+ clamped = offsets * scale
102
+
103
+ # Don't move vertices with too few view supports
104
+ no_move = in_view_count < min_views_for_move
105
+ clamped[no_move] = 0
106
+
107
+ new_pv = pv_arr + clamped
108
+ return new_pv.astype(np.float64), pe
109
+ except Exception:
110
+ return pv, pe
111
+
112
+
113
+ def load_regressor_v4(path: str, device: str = "cpu"):
114
+ blob = torch.load(path, map_location=device, weights_only=False)
115
+ m = VertexRegressorV4(hidden=blob.get("hidden", 128))
116
+ m.load_state_dict(blob["model"])
117
+ m.to(device).eval()
118
+ return (m, blob.get("feature_mean"), blob.get("feature_std"),
119
+ blob.get("edge_feat_mean"), blob.get("edge_feat_std"))