Upload folder using huggingface_hub

measure_finger.py

@@ -386,6 +386,81 @@ def _overlay_sam_masks(
     return out
 
 
+def _overlay_card_seeds(
+    image: np.ndarray,
+    seed_debug: Optional[Dict[str, Any]],
+    rotation_matrix: Optional[np.ndarray] = None,
+) -> np.ndarray:
+    """Draw SAM card-detection prompt points on a debug image.
+
+    Positive seeds in green, palm-center negative in red, middle-PIP anchor
+    as a cyan cross. All points are assumed to be in the canonical (pre-
+    precise-rotation) frame; pass ``rotation_matrix`` to align with an image
+    that had the finger rotation applied.
+    """
+    if not seed_debug:
+        return image
+    from src.geometry import transform_points_rotation
+
+    def _xform(points: List[Tuple[int, int]]) -> np.ndarray:
+        arr = np.asarray(points, dtype=np.float32)
+        if rotation_matrix is not None and len(arr) > 0:
+            arr = transform_points_rotation(arr, rotation_matrix)
+        return arr
+
+    out = image
+    h, w = out.shape[:2]
+    # Thin, map-style reference crosses: small, single-pixel-wide, softened.
+    marker_size = max(14, int(round(0.012 * max(h, w))))
+    thickness = 2
+
+    # Pastel variants so the markers read as reference lines rather than
+    # high-contrast callouts.
+    DROP_COLOR = (120, 220, 220)   # soft yellow
+    KEEP_COLOR = (120, 220, 120)   # soft green
+    NEG_COLOR = (120, 120, 220)    # soft red
+    ANCHOR_COLOR = (220, 220, 120)  # soft cyan
+
+    def _plus(pt, color):
+        cv2.drawMarker(
+            out, (int(pt[0]), int(pt[1])), color,
+            markerType=cv2.MARKER_CROSS,
+            markerSize=marker_size,
+            thickness=thickness,
+            line_type=cv2.LINE_AA,
+        )
+
+    # Seeds dropped by the hand-mask filter. Drawn first so any coincident
+    # kept seed paints on top.
+    dropped = seed_debug.get("dropped") or []
+    if dropped:
+        for pt in _xform(dropped).astype(int):
+            _plus(pt, DROP_COLOR)
+
+    seeds = seed_debug.get("seeds") or []
+    if seeds:
+        for pt in _xform(seeds).astype(int):
+            _plus(pt, KEEP_COLOR)
+
+    negatives = seed_debug.get("negatives") or []
+    if negatives:
+        for pt in _xform(negatives).astype(int):
+            _plus(pt, NEG_COLOR)
+
+    anchor = seed_debug.get("anchor")
+    if anchor is not None:
+        ax, ay = _xform([anchor])[0].astype(int)
+        # Tilted cross (X) to distinguish the anchor from the plus-shaped seeds.
+        cv2.drawMarker(
+            out, (int(ax), int(ay)), ANCHOR_COLOR,
+            markerType=cv2.MARKER_TILTED_CROSS,
+            markerSize=marker_size,
+            thickness=thickness,
+            line_type=cv2.LINE_AA,
+        )
+    return out
+
+
 def _save_debug_visualization(path: str, image: np.ndarray) -> None:
     """Downscale + fast-encode a debug overlay image.
 
@@ -434,7 +509,9 @@ def _sam_card_detect(
     middle_pip = landmarks[10, :2]
     anchor_xy = (int(round(middle_pip[0])), int(round(middle_pip[1])))
 
-    seeds = suggest_card_seeds(hand_mask, image_canonical.shape[:2], anchor_xy)
+    seed_info = suggest_card_seeds(hand_mask, image_canonical.shape[:2], anchor_xy)
+    seeds = seed_info["kept"]
+    dropped_seeds = seed_info["dropped"]
     if not seeds:
         return None
 
@@ -445,12 +522,28 @@ def _sam_card_detect(
     ]
 
     prompt_debug = str(debug_root / "sam_card_prompt_debug") if debug_root else None
-    return detect_credit_card_sam_prompt(
+    card_result = detect_credit_card_sam_prompt(
         image_canonical,
         seed_points=seeds,
         negative_points=negatives,
         debug_dir=prompt_debug,
+        hand_mask=hand_mask,
     )
+    # Stash seed geometry so the final result PNG can visualize what was
+    # prompted into SAM, even when card detection fails.
+    seed_debug = {
+        "anchor": anchor_xy,
+        "seeds": list(seeds),
+        "dropped": list(dropped_seeds),
+        "negatives": list(negatives),
+    }
+    if card_result is not None:
+        card_result["seed_debug"] = seed_debug
+    else:
+        # Return a sentinel-less None as before, but tuck seeds where the
+        # caller can still find them via hand_data for the failure overlay.
+        hand_data["_sam_card_seed_debug"] = seed_debug
+    return card_result
 
 
 def measure_finger(
@@ -1032,6 +1125,18 @@ def measure_finger(
                 scale_px_per_cm=px_per_cm,
             )
 
+        # SAM card-detection seed points (prompt geometry). Falls back to
+        # the hand_data slot when card detection returned None so we can
+        # still see what was prompted into SAM.
+        seed_debug = None
+        if card_result is not None:
+            seed_debug = card_result.get("seed_debug")
+        if seed_debug is None and hand_data is not None:
+            seed_debug = hand_data.get("_sam_card_seed_debug")
+        debug_image = _overlay_card_seeds(
+            debug_image, seed_debug, rotation_matrix=rotation_matrix
+        )
+
         # Save result image (downscaled + JPEG-encoded for speed)
         _save_debug_visualization(result_png_path, debug_image)
         print(f"Result visualization saved to: {result_png_path}")
@@ -1355,6 +1460,18 @@ def measure_multi_finger(
         else:
             card_result = detect_credit_card(image_canonical, debug_dir=card_debug_dir)
         if card_result is None:
+            # Emit a diagnostic visualization so the failure is debuggable:
+            # hand mask + card-prompt seeds on the canonical image. Without
+            # this, a card_not_detected failure on HF leaves no PNG to pull.
+            if result_png_path is not None:
+                vis = image_canonical.copy()
+                vis = _overlay_sam_masks(vis, hand_mask=hand_data.get("mask"))
+                vis = _overlay_hand_skeleton(vis, landmarks=hand_data.get("landmarks"))
+                vis = _overlay_card_seeds(
+                    vis, hand_data.get("_sam_card_seed_debug")
+                )
+                _save_debug_visualization(result_png_path, vis)
+                print(f"[multi] Card-not-detected viz saved to: {result_png_path}")
             return {"fail_reason": "card_not_detected", "per_finger": {}, "fingers_measured": 0, "fingers_succeeded": 0}
         px_per_cm, scale_confidence = compute_scale_factor(card_result["corners"])
         view_angle_ok = scale_confidence > 0.9
@@ -1493,6 +1610,14 @@ def _draw_multi_finger_debug(
     if card_result is not None:
         vis = draw_card_overlay(vis, card_result, px_per_cm)
 
+    # SAM card-detection seed points (prompt geometry). Multi-finger viz
+    # runs in the canonical frame with no precise-rotation applied, so no
+    # rotation matrix is needed here.
+    seed_debug = None
+    if card_result is not None:
+        seed_debug = card_result.get("seed_debug")
+    vis = _overlay_card_seeds(vis, seed_debug)
+
     # Draw per-finger Sobel edge overlays
     for fn, result in per_finger_raw.items():
         internal = result.get("_internal")

script/validate_sam_card.py

@@ -102,7 +102,8 @@ def run_one(img_path: Path) -> dict:
             return rec
         mp = landmarks[10, :2]
         anchor_xy = (int(round(mp[0])), int(round(mp[1])))
-        seeds = suggest_card_seeds(hand_data["mask"], canonical.shape[:2], anchor_xy)
+        seed_info = suggest_card_seeds(hand_data["mask"], canonical.shape[:2], anchor_xy)
+        seeds = seed_info["kept"]
         rec["prompt_n_seeds"] = len(seeds)
         negs = _negatives_from_landmarks(hand_data["landmarks"])
         t0 = time.time()
@@ -112,6 +113,7 @@ def run_one(img_path: Path) -> dict:
                 seed_points=seeds,
                 negative_points=negs,
                 debug_dir=str(prompt_debug),
+                hand_mask=hand_data["mask"],
             )
             if pr is not None:
                 px_cm, _ = compute_scale_factor(pr["corners"])

src/sam_card_detection.py

@@ -43,6 +43,7 @@ POINTS_PER_BATCH = 64
 # Candidate filtering
 MIN_RECTANGULARITY = 0.90  # mask_area / minAreaRect_area; card mask is near-perfect rectangle
 ASPECT_RATIO_TOLERANCE = 0.15  # fractional deviation from 1.586
+MAX_HAND_OVERLAP_RATIO = 0.20  # reject candidates that swallow the hand (background paper, tabletop)
 
 _pipeline = None  # lazy singleton
 
@@ -113,6 +114,7 @@ def _mask_to_bool_array(mask: Any, target_shape: Optional[Tuple[int, int]] = Non
 def _score_card_mask(
     mask: np.ndarray,
     image_area: float,
+    hand_mask: Optional[np.ndarray] = None,
 ) -> Optional[Dict[str, Any]]:
     """Score a candidate mask for being a credit card.
 
@@ -142,6 +144,22 @@ def _score_card_mask(
     if contour_area <= 0:
         return None
 
+    # Reject candidates whose convex hull engulfs the hand. When SAM is
+    # prompted to segment the background paper, it returns the paper mask
+    # with the hand carved *out* of it — so raw AND(mask, hand) is ~0
+    # even though the hand sits visually on top of the paper. The convex
+    # hull closes that hand-shaped hole, exposing the engulfment.
+    if hand_mask is not None and mask.shape == hand_mask.shape:
+        hand_bool = hand_mask.astype(bool) if hand_mask.dtype != bool else hand_mask
+        hand_area = float(hand_bool.sum())
+        if hand_area > 0:
+            hull_mask = np.zeros(mask.shape, dtype=np.uint8)
+            cv2.fillPoly(hull_mask, [contour.astype(np.int32)], 255)
+            hull_bool = hull_mask.astype(bool)
+            overlap = float(np.logical_and(hull_bool, hand_bool).sum())
+            if overlap / hand_area > MAX_HAND_OVERLAP_RATIO:
+                return None
+
     rect = cv2.minAreaRect(contour)
     box = cv2.boxPoints(rect)
     rect_area = cv2.contourArea(box.astype(np.float32))
@@ -323,14 +341,18 @@ def suggest_card_seeds(
     hand_mask: np.ndarray,
     image_shape: Tuple[int, int],
     anchor_xy: Tuple[int, int],
-) -> List[Tuple[int, int]]:
+) -> Dict[str, List[Tuple[int, int]]]:
     """Cross-shaped seed points through the anchor (middle-finger PIP).
 
     Users place the credit card either above/below or left/right of the
     middle-finger PIP, so a rake along the horizontal and vertical lines
     through PIP catches it with far fewer prompts than a dense grid.
-    5 points per arm (step 0.15·dim), shared center deduped, hand-mask
-    points dropped.
+    5 points per arm (step 0.15·dim), shared center deduped.
+
+    Returns a dict with two lists:
+        - "kept": seeds that passed the hand-mask filter (sent to SAM).
+        - "dropped": seeds whose (x, y) landed inside the hand mask and
+          were filtered out. Retained purely for debug visualization.
     """
     h, w = image_shape
     mask_bool = hand_mask.astype(bool) if hand_mask.dtype != bool else hand_mask
@@ -346,16 +368,20 @@ def suggest_card_seeds(
     for k in (-2, -1, 1, 2):
         candidates.append((ax, ay + k * dy))
 
-    seeds: List[Tuple[int, int]] = []
+    kept: List[Tuple[int, int]] = []
+    dropped: List[Tuple[int, int]] = []
     seen: set = set()
     for px, py in candidates:
         px = max(0, min(w - 1, px))
         py = max(0, min(h - 1, py))
-        if (px, py) in seen or mask_bool[py, px]:
+        if (px, py) in seen:
             continue
         seen.add((px, py))
-        seeds.append((px, py))
-    return seeds
+        if mask_bool[py, px]:
+            dropped.append((px, py))
+        else:
+            kept.append((px, py))
+    return {"kept": kept, "dropped": dropped}
 
 
 def _downscale_prompt(image_bgr: np.ndarray) -> Tuple[np.ndarray, float]:
@@ -446,6 +472,7 @@ def detect_credit_card_sam_prompt(
     seed_points: List[Tuple[int, int]],
     negative_points: Optional[List[Tuple[int, int]]] = None,
     debug_dir: Optional[str] = None,
+    hand_mask: Optional[np.ndarray] = None,
 ) -> Optional[Dict[str, Any]]:
     """Prompt-based SAM 2.1 credit card detection.
 
@@ -528,13 +555,26 @@ def detect_credit_card_sam_prompt(
     iou_scores = outputs.iou_scores.cpu().numpy()[0]
     infer_time = time.time() - t0
 
+    # Resize the hand mask into the same scaled 1024-space the candidate
+    # masks live in, so overlap rejection works without upscaling every
+    # candidate to full resolution.
+    hand_mask_scaled: Optional[np.ndarray] = None
+    if hand_mask is not None:
+        hand_u8 = (hand_mask.astype(bool).astype(np.uint8) * 255)
+        if hand_u8.shape != (scaled_h, scaled_w):
+            hand_u8 = cv2.resize(
+                hand_u8, (scaled_w, scaled_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+        hand_mask_scaled = hand_u8.astype(bool)
+
     scored: List[Dict[str, Any]] = []
     scaled_candidate_masks: List[np.ndarray] = []
     for prompt_idx in range(masks_tensor.shape[0]):
         for cand_idx in range(masks_tensor.shape[1]):
             mask_scaled = masks_tensor[prompt_idx, cand_idx].numpy().astype(bool)
             scaled_candidate_masks.append(mask_scaled)
-            result = _score_card_mask(mask_scaled, scaled_area)
+            result = _score_card_mask(mask_scaled, scaled_area, hand_mask=hand_mask_scaled)
             if result is not None:
                 result["seed_idx"] = prompt_idx
                 result["cand_idx"] = cand_idx