ring-sizer / src /sam_card_detection.py
feng-x's picture
Upload folder using huggingface_hub
6cd4ed9 verified
"""
SAM 2.1-based credit card detection.
Uses Meta's Segment Anything 2.1 (Hiera Tiny) via HuggingFace transformers
to produce a pixel-accurate card mask, then filters candidate masks by area,
rectangularity, and aspect ratio (~1.586) to pick the credit card.
Drop-in replacement for `card_detection.detect_credit_card`: returns a dict
with the same keys so the downstream pipeline is unchanged.
"""
from __future__ import annotations
import logging
import os
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import cv2
import numpy as np
from .card_detection import (
CARD_ASPECT_RATIO,
MIN_CARD_AREA_RATIO,
get_quad_dimensions,
order_corners,
)
from .sam_backend import INFERENCE_MAX_SIDE as PROMPT_INFERENCE_MAX_SIDE, get_sam2
logger = logging.getLogger(__name__)
# Candidate filtering
MIN_RECTANGULARITY = 0.90 # mask_area / minAreaRect_area; card mask is near-perfect rectangle
ASPECT_RATIO_TOLERANCE = 0.15 # fractional deviation from 1.586
MAX_HAND_OVERLAP_RATIO = 0.20 # reject candidates that swallow the hand (background paper, tabletop)
# Reject candidates whose convex hull is "fattened" by hand-shaped indentations.
# A real card mask is convex, so hull == mask and (hull \ mask) ∩ hand is ~0.
# When SAM segments a chunk of background (e.g. paper towel) bordered by the
# hand, the mask has a hand-shaped notch on one side; the hull closes that
# notch and adds hand pixels. Empirically: real-card winners measure 0.000,
# paper-towel false positives measure ~0.10+.
MAX_HULL_HAND_FILL_RATIO = 0.05
# SAM-specific upper bound on card area. Tighter than the shared
# MAX_CARD_AREA_RATIO (0.5) because SAM happily returns whole-background
# segments (ceilings, walls) as a single rectangular-ish mask when no card
# is actually present — a ~50% half-image mask can pass rectangularity and
# aspect ratio purely by accident. A real credit card held alongside a hand
# is ~5-15% of the frame; 25% is already 2× the realistic maximum.
SAM_MAX_CARD_AREA_RATIO = 0.25
# Reject candidates whose longer side spans more of the image short side
# than any real card photo plausibly would. This catches the distinctive
# SAM failure where a single-prompt mask grabs the entire background paper
# / tabletop: the candidate is long and thin (so its mask area sneaks
# under SAM_MAX_CARD_AREA_RATIO) but its bounding rectangle stretches
# across nearly the full image short side (framing ratio ~0.99). Threshold
# picked from doc/report/framing_ratio_survey.md: max observed in 47 KOL
# successes is 0.532, max in calibration is 0.486; 0.70 leaves ≥30% margin
# above legitimate framing while sitting well below the ~1.0 failure mode.
MAX_CARD_FRAMING_RATIO = 0.70
def _score_card_mask(
mask: np.ndarray,
image_area: float,
hand_mask: Optional[np.ndarray] = None,
image_short_side: float = 0.0,
iou_score: float = 0.0,
) -> Optional[Dict[str, Any]]:
"""Score a candidate mask for being a credit card.
Returns a dict with {corners, width, height, area, aspect_ratio, rectangularity, score}
or None if the mask is rejected.
"""
mask_u8 = mask.astype(np.uint8) * 255
mask_area = float(mask.sum())
area_ratio = mask_area / image_area
if area_ratio < MIN_CARD_AREA_RATIO or area_ratio > SAM_MAX_CARD_AREA_RATIO:
return None
contours, _ = cv2.findContours(mask_u8, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
if not contours:
return None
# Largest external contour is the card body (SAM masks can be slightly disconnected).
# Take its convex hull before scoring: credit cards are convex, so the hull
# erases SAM boundary noise that varies with CPU numerics (the same mask on
# x86 vs Apple Silicon can bump `contour_area / rect_area` below 0.90 purely
# from Torch CPU activation drift). Non-card shapes stay non-rectangular
# under their hull, so this does not create false positives.
largest_contour = max(contours, key=cv2.contourArea)
contour = cv2.convexHull(largest_contour)
contour_area = cv2.contourArea(contour)
if contour_area <= 0:
return None
# Replace the raw multi-blob SAM mask with just the largest connected
# component. The card prompt with multimask_output=True occasionally lassos
# background paper between fingers as part of the same candidate; those
# blobs pass scoring (we only check the largest contour) but pollute every
# downstream consumer of `result["mask"]` (debug overlays, the result PNG).
clean_mask_u8 = np.zeros_like(mask_u8)
cv2.drawContours(clean_mask_u8, [largest_contour], -1, 255, thickness=cv2.FILLED)
mask = clean_mask_u8.astype(bool)
# Reject candidates whose convex hull engulfs the hand. When SAM is
# prompted to segment the background paper, it returns the paper mask
# with the hand carved *out* of it — so raw AND(mask, hand) is ~0
# even though the hand sits visually on top of the paper. The convex
# hull closes that hand-shaped hole, exposing the engulfment.
if hand_mask is not None and mask.shape == hand_mask.shape:
hand_bool = hand_mask.astype(bool) if hand_mask.dtype != bool else hand_mask
hand_area = float(hand_bool.sum())
if hand_area > 0:
hull_mask = np.zeros(mask.shape, dtype=np.uint8)
cv2.fillPoly(hull_mask, [contour.astype(np.int32)], 255)
hull_bool = hull_mask.astype(bool)
overlap = float(np.logical_and(hull_bool, hand_bool).sum())
if overlap / hand_area > MAX_HAND_OVERLAP_RATIO:
return None
if mask_area > 0 and overlap / mask_area > MAX_HULL_HAND_FILL_RATIO:
return None
rect = cv2.minAreaRect(contour)
box = cv2.boxPoints(rect)
rect_area = cv2.contourArea(box.astype(np.float32))
if rect_area <= 0:
return None
rectangularity = contour_area / rect_area
if rectangularity < MIN_RECTANGULARITY:
return None
corners = order_corners(box)
width, height = get_quad_dimensions(corners)
if width <= 0 or height <= 0:
return None
# Reject long-thin SAM false positives that span ~the entire image short
# side. These slip past SAM_MAX_CARD_AREA_RATIO because their pixel
# count is modest (the mask is hollow / not solidly filled), but their
# bounding rectangle gives them away.
if image_short_side > 0:
framing_ratio = max(width, height) / image_short_side
if framing_ratio > MAX_CARD_FRAMING_RATIO:
return None
aspect_ratio = max(width, height) / min(width, height)
ratio_diff = abs(aspect_ratio - CARD_ASPECT_RATIO) / CARD_ASPECT_RATIO
if ratio_diff > ASPECT_RATIO_TOLERANCE:
return None
# Score components — picking weights here is delicate because real
# photos have *perspective foreshortening* that pulls the apparent card
# aspect away from the flat-card ideal of 1.586. A mask that bleeds
# extra background paper onto the short edge can pull aspect *closer*
# to the ideal than a tight mask, so over-weighting ratio_score selects
# fattened masks (the Brooklyn Shields case). The current split:
# * 0.3 ratio — kept as a soft preference but no longer dominant
# * 0.4 rect — primary signal; tight cards are near-perfect rectangles,
# fattened SAM masks always lose a little here
# * 0.1 area — small reward for "actually card-sized"
# * 0.2 iou — SAM's own segmentation confidence; stable across
# platforms because it's decoder-internal, not derived
# from per-pixel boundary noise. Acts as a second opinion
# that breaks the tie when geometry is too close to call.
ratio_score = 1.0 - ratio_diff / ASPECT_RATIO_TOLERANCE
rect_score = (rectangularity - MIN_RECTANGULARITY) / (1.0 - MIN_RECTANGULARITY)
area_score = min(area_ratio / 0.1, 1.0) # caps at 10% of image area
score = (
0.3 * ratio_score
+ 0.4 * rect_score
+ 0.1 * area_score
+ 0.2 * iou_score
)
return {
"corners": corners,
"contour": contour,
"width": width,
"height": height,
"area": mask_area,
"aspect_ratio": aspect_ratio,
"rectangularity": rectangularity,
"score": score,
"mask": mask,
}
# ---------------------------------------------------------------------------
# Prompt-based card detection
# ---------------------------------------------------------------------------
def suggest_card_seeds(
hand_mask: np.ndarray,
image_shape: Tuple[int, int],
y_limit: int,
) -> Dict[str, List[Tuple[int, int]]]:
"""Uniform 4x4 grid seeds in the top band of the canonical image.
In canonical orientation the fingertips point up, so the middle-finger
MCP is the lowest landmark of the middle finger. Users overwhelmingly
place the card in the band between the top of the frame and the MCP
row (beside or above the fingers), so sampling that band catches the
card with a handful of prompts. The grid excludes a 10% border padding
on all four sides of the band and drops any seed that lands on the
hand mask.
Args:
hand_mask: SAM hand mask (HxW, bool or uint8).
image_shape: (H, W) of the canonical image.
y_limit: Y coordinate of the middle-finger MCP; the grid spans
[0.1·H, y_limit] vertically.
Returns:
Dict with two lists:
- "kept": seeds that passed the hand-mask filter (sent to SAM).
- "dropped": seeds whose (x, y) landed inside the hand mask and
were filtered out. Retained purely for debug visualization.
"""
h, w = image_shape
mask_bool = hand_mask.astype(bool) if hand_mask.dtype != bool else hand_mask
x_min = 0.1 * w
x_max = 0.9 * w
y_min = 0.1 * h
y_max = float(y_limit)
# Guard against degenerate bands (e.g., MCP above the 10% top padding).
if y_max <= y_min:
y_max = y_min + 1.0
n = 4
candidates: List[Tuple[int, int]] = []
for iy in range(n):
fy = (iy + 0.5) / n
py = int(round(y_min + fy * (y_max - y_min)))
for ix in range(n):
fx = (ix + 0.5) / n
px = int(round(x_min + fx * (x_max - x_min)))
candidates.append((px, py))
kept: List[Tuple[int, int]] = []
dropped: List[Tuple[int, int]] = []
seen: set = set()
for px, py in candidates:
px = max(0, min(w - 1, px))
py = max(0, min(h - 1, py))
if (px, py) in seen:
continue
seen.add((px, py))
if mask_bool[py, px]:
dropped.append((px, py))
else:
kept.append((px, py))
return {"kept": kept, "dropped": dropped}
def _downscale_prompt(image_bgr: np.ndarray) -> Tuple[np.ndarray, float]:
"""Downscale for prompt inference. Returns (scaled, scale_back)."""
h, w = image_bgr.shape[:2]
long_side = max(h, w)
if long_side <= PROMPT_INFERENCE_MAX_SIDE:
return image_bgr, 1.0
scale = PROMPT_INFERENCE_MAX_SIDE / long_side
new_w = int(round(w * scale))
new_h = int(round(h * scale))
scaled = cv2.resize(image_bgr, (new_w, new_h), interpolation=cv2.INTER_AREA)
return scaled, 1.0 / scale
def _save_prompt_debug(
debug_dir: str,
image_bgr: np.ndarray,
seeds: List[Tuple[int, int]],
negatives: List[Tuple[int, int]],
candidate_masks: List[np.ndarray],
scored: List[Dict[str, Any]],
best: Optional[Dict[str, Any]],
) -> None:
"""Save debug visualizations for prompt-based card detection."""
from .debug_observer import DebugObserver
observer = DebugObserver(debug_dir)
# 01: prompt points on the image
pts_img = image_bgr.copy()
for (px, py) in seeds:
cv2.circle(pts_img, (px, py), 20, (0, 255, 0), -1)
cv2.circle(pts_img, (px, py), 20, (0, 0, 0), 3)
for (nx, ny) in negatives:
cv2.circle(pts_img, (nx, ny), 20, (0, 0, 255), -1)
cv2.circle(pts_img, (nx, ny), 20, (0, 0, 0), 3)
observer.save_stage("01_prompt_points", pts_img)
# 02: all candidate masks overlaid (one color per prompt)
overlay = image_bgr.copy()
rng = np.random.default_rng(7)
for m in candidate_masks:
if m is None or m.sum() == 0:
continue
color = rng.integers(64, 255, size=3).tolist()
overlay[m] = (0.5 * overlay[m] + 0.5 * np.array(color)).astype(np.uint8)
observer.save_stage("02_candidate_masks", overlay)
# 03: scored candidates
cand_img = image_bgr.copy()
for s in scored:
corners = s["corners"].astype(np.int32)
cv2.polylines(cand_img, [corners], True, (0, 255, 0), 3)
cv2.putText(
cand_img,
f"{s['score']:.2f} ar={s['aspect_ratio']:.3f}",
tuple(corners[0]),
cv2.FONT_HERSHEY_SIMPLEX,
1.2,
(0, 255, 0),
3,
cv2.LINE_AA,
)
observer.save_stage("03_scored", cand_img)
if best is not None:
final = image_bgr.copy()
mask_u8 = best["mask"].astype(np.uint8) * 255
tint = np.zeros_like(final)
tint[:, :, 1] = mask_u8
final = cv2.addWeighted(final, 1.0, tint, 0.35, 0)
corners = best["corners"].astype(np.int32)
cv2.polylines(final, [corners], True, (0, 255, 0), 4)
for pt in corners:
cv2.circle(final, tuple(pt), 10, (0, 0, 255), -1)
label = (
f"SAM-prompt card score={best['score']:.3f} "
f"ar={best['aspect_ratio']:.3f} rect={best['rectangularity']:.3f}"
)
cv2.putText(final, label, (30, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.1,
(255, 255, 255), 5, cv2.LINE_AA)
cv2.putText(final, label, (30, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.1,
(0, 255, 0), 2, cv2.LINE_AA)
observer.save_stage("04_final_selection", final)
def detect_credit_card_sam_prompt(
image: np.ndarray,
seed_points: List[Tuple[int, int]],
negative_points: Optional[List[Tuple[int, int]]] = None,
debug_dir: Optional[str] = None,
hand_mask: Optional[np.ndarray] = None,
) -> Optional[Dict[str, Any]]:
"""Prompt-based SAM 2.1 credit card detection.
For each seed point, runs a single-point SAM decoder pass with
`multimask_output=True` and collects all returned masks. Every mask is
then filtered through `_score_card_mask`; the highest-scoring survivor
is returned. This is ~20× faster than the AMG path because it runs the
decoder ~N times (one per seed) instead of 256 times on a dense grid.
Args:
image: Full-resolution BGR image (canonical orientation).
seed_points: List of (x, y) positive-point candidates. Each one is
tried independently. A few well-placed candidates are enough.
negative_points: Optional list of (x, y) negative points applied to
every seed's prompt (e.g., palm center to steer SAM off the hand).
debug_dir: Optional directory to dump debug visualizations.
Returns:
Card dict matching `detect_credit_card`, or None if no seed produced
a valid card mask.
"""
import torch
from PIL import Image as PILImage
if not seed_points:
logger.info("SAM-prompt: no seed points provided")
return None
h, w = image.shape[:2]
image_area = float(h * w)
scaled_bgr, scale_back = _downscale_prompt(image)
scaled_rgb = cv2.cvtColor(scaled_bgr, cv2.COLOR_BGR2RGB)
pil = PILImage.fromarray(scaled_rgb)
scale_down = 1.0 / scale_back # original → scaled
def _to_scaled(pts: List[Tuple[int, int]]) -> List[List[int]]:
return [[int(round(px * scale_down)), int(round(py * scale_down))] for px, py in pts]
seeds_scaled = _to_scaled(seed_points)
negatives_scaled = _to_scaled(negative_points) if negative_points else []
# Build one prompt per seed; each prompt carries (1 positive + all negatives)
# input_points shape: [batch=1, num_prompts, points_per_prompt, 2]
# input_labels shape: [batch=1, num_prompts, points_per_prompt]
points_per_prompt = 1 + len(negatives_scaled)
input_points = [[[seed] + negatives_scaled for seed in seeds_scaled]]
input_labels = [[[1] + [0] * len(negatives_scaled) for _ in seeds_scaled]]
model, processor = get_sam2()
t0 = time.time()
inputs = processor(
images=pil,
input_points=input_points,
input_labels=input_labels,
return_tensors="pt",
)
with torch.inference_mode():
# multimask_output=True gives 3 masks per seed (small / medium / large
# disambiguation of the prompt). Empirically this matters for card
# detection: SAM's single-best IoU mask sometimes latches onto a
# sub-region or a nearby distractor, but one of the other two
# candidates is the full card. Scoring cost is fine because we score
# in the scaled 1024-space, not full resolution.
outputs = model(**inputs, multimask_output=True)
# Score masks in the scaled 1024-space. Only the single winner is
# upscaled to full resolution afterward, which avoids O(N) 12 MP resizes.
scaled_h = inputs["original_sizes"][0][0].item()
scaled_w = inputs["original_sizes"][0][1].item()
scaled_area = float(scaled_h * scaled_w)
masks_list = processor.post_process_masks(
outputs.pred_masks.cpu(),
inputs["original_sizes"],
mask_threshold=0.0,
)
masks_tensor = masks_list[0] # (num_prompts, num_candidates, H_s, W_s)
iou_scores = outputs.iou_scores.cpu().numpy()[0]
infer_time = time.time() - t0
# Resize the hand mask into the same scaled 1024-space the candidate
# masks live in, so overlap rejection works without upscaling every
# candidate to full resolution.
hand_mask_scaled: Optional[np.ndarray] = None
if hand_mask is not None:
hand_u8 = (hand_mask.astype(bool).astype(np.uint8) * 255)
if hand_u8.shape != (scaled_h, scaled_w):
hand_u8 = cv2.resize(
hand_u8, (scaled_w, scaled_h),
interpolation=cv2.INTER_NEAREST,
)
hand_mask_scaled = hand_u8.astype(bool)
scored: List[Dict[str, Any]] = []
scaled_candidate_masks: List[np.ndarray] = []
for prompt_idx in range(masks_tensor.shape[0]):
for cand_idx in range(masks_tensor.shape[1]):
mask_scaled = masks_tensor[prompt_idx, cand_idx].numpy().astype(bool)
scaled_candidate_masks.append(mask_scaled)
iou = float(iou_scores[prompt_idx, cand_idx])
result = _score_card_mask(
mask_scaled, scaled_area, hand_mask=hand_mask_scaled,
image_short_side=float(min(scaled_h, scaled_w)),
iou_score=iou,
)
if result is not None:
result["seed_idx"] = prompt_idx
result["cand_idx"] = cand_idx
result["iou_score"] = iou
# `result["mask"]` is the cleaned (largest-component) mask;
# keep that as the scaled-space mask so upscaling and debug
# rendering both see the cleaned version.
result["mask_scaled"] = result["mask"]
scored.append(result)
scored.sort(key=lambda d: d["score"], reverse=True)
best = scored[0] if scored else None
# Upscale only the winning mask + corners to full resolution
if best is not None:
mask_scaled_best = best["mask_scaled"]
if mask_scaled_best.shape != (h, w):
mask_full = cv2.resize(
mask_scaled_best.astype(np.uint8), (w, h),
interpolation=cv2.INTER_NEAREST,
).astype(bool)
else:
mask_full = mask_scaled_best
best["mask"] = mask_full
best["corners"] = best["corners"] * scale_back
best["width"] = best["width"] * scale_back
best["height"] = best["height"] * scale_back
logger.info(
"SAM-prompt: %d seeds, %d candidates, %d passed filter, inference=%.2fs",
len(seed_points),
masks_tensor.shape[0] * masks_tensor.shape[1],
len(scored), infer_time,
)
if debug_dir:
# Render debug overlays in the downscaled 1024-space. Upscaling
# ~60 masks to full 12 MP resolution just for PNGs was dominating
# end-to-end time (8–10s out of ~9s total). The debug images are
# for human inspection; 1024 is plenty.
dh, dw = scaled_bgr.shape[:2]
debug_seeds = [
(int(round(px / scale_back)), int(round(py / scale_back)))
for px, py in seed_points
]
debug_negs = [
(int(round(px / scale_back)), int(round(py / scale_back)))
for px, py in (negative_points or [])
]
debug_scored_for_viz = []
for s in scored:
s_copy = dict(s)
s_copy["corners"] = s["corners"] # already scaled-space
s_copy["mask"] = s["mask_scaled"]
debug_scored_for_viz.append(s_copy)
best_for_viz = None
if best is not None:
best_for_viz = dict(best)
best_for_viz["corners"] = best["corners"] / scale_back # back to scaled
best_for_viz["mask"] = best["mask_scaled"]
_save_prompt_debug(
debug_dir, scaled_bgr, debug_seeds, debug_negs,
scaled_candidate_masks, debug_scored_for_viz, best_for_viz,
)
if best is None:
return None
logger.info(
"SAM-prompt card: score=%.3f, aspect=%.3f, rect=%.3f, %.0fx%.0fpx (seed %d)",
best["score"], best["aspect_ratio"], best["rectangularity"],
best["width"], best["height"], best["seed_idx"],
)
return {
"corners": best["corners"],
"contour": best["corners"],
"confidence": float(best["score"]),
"width_px": float(best["width"]),
"height_px": float(best["height"]),
"aspect_ratio": float(best["aspect_ratio"]),
"mask": best["mask"], # bool HxW, canonical-image coords
"mask_source": "sam_prompt",
}