""" SAM2 two-fighter tracker — Space GPU path (occlusion-robust identity + crop-pose). Validated in Colab: two-fighter coverage 23% (pose-only) -> 58% (this) on hard footage. SAM2 segments + tracks each fighter through the clip; pose runs on each fighter's mask CROP (isolated fighter = easier detection) and the result feeds the UNCHANGED feature extractor + overlay renderer. Space-specific differences from the Colab original: - Frames are extracted to JPEGs with cv2 and fed to SAM2's init_state as a directory (SAM2's mp4 loader needs decord, which has no py3.13 wheels). - Identity seeds at the BEST frame (both fighters big, near — same idea as the pose path's _seed_frame), then propagates FORWARD AND BACKWARD; the Colab version seeded at frame 0 and inherited the bystander bug. - fp16 autocast on pre-Ampere GPUs (T4 has no bfloat16), bf16 on Ampere+. - Long clips are strided to ~TARGET_FRAMES like the pose path. """ import os import shutil import tempfile import numpy as np import cv2 import torch import pose_features as pf SAM2_ID = "facebook/sam2-hiera-base-plus" SAM2_POSE_MODEL = "yolo11m-pose.pt" # bigger model is fine: it only sees small crops # (distinct name: POSE_MODEL is the env knob for # pose_features' full-frame model) DET_MODEL = "yolo11n.pt" KP_CONF = 0.30 # Frame BUDGET, not a frame RATE -- so effective fps is TARGET_FRAMES divided # by clip length, and a long clip is measured more coarsely than a short one. # At 300 a 30s clip was sampled at 10 fps and a 120s clip at 2.5 fps, which is # below the floor where a punch can be detected at all: a hand travels further # between two samples than the whole extension the detector looks for. 750 # holds every clip up to the 60s limit at >=12.5 fps. # # This is a GPU-cost decision as much as an accuracy one -- it is ~2.5x the # frames of the old budget. Lower it and clips get coarser, do not get faster # per frame. See docs/research/accuracy-for-combat-sports-athletes.md section 1. TARGET_FRAMES = int(os.environ.get("SAM2_TARGET_FRAMES", "750")) SEED_DET_STRIDE = 10 # run the person detector on every Nth sampled frame # Per-process caches — constructing YOLO/SAM2 per request costs seconds each, # and _tracks_collapsed retries were re-paying it up to 3x per clip. _models = None _predictor = None def _get_models(): # Imports stay lazy (heavy CUDA init) but must live HERE: at call time this # runs at module scope, where a function-local import in the caller is not # visible. global _models if _models is None: from ultralytics import YOLO _models = (YOLO(DET_MODEL), YOLO(SAM2_POSE_MODEL)) return _models def _get_predictor(): global _predictor if _predictor is None: from sam2.sam2_video_predictor import SAM2VideoPredictor _predictor = SAM2VideoPredictor.from_pretrained(SAM2_ID) return _predictor def _extract_jpeg_frames(video_path, tmpdir): """Sample the clip to ~TARGET_FRAMES and write JPEGs SAM2 can read. Returns (frame_paths, stride, fps, (w, h)).""" cap = cv2.VideoCapture(video_path) total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) fps = cap.get(cv2.CAP_PROP_FPS) or 30.0 stride = max(1, total // TARGET_FRAMES) if total > TARGET_FRAMES else 1 paths, idx, si = [], 0, 0 size = None while True: ok, frame = cap.read() if not ok: break if idx % stride == 0: p = os.path.join(tmpdir, f"{si:05d}.jpg") cv2.imwrite(p, frame, [cv2.IMWRITE_JPEG_QUALITY, 92]) paths.append(p) size = (frame.shape[1], frame.shape[0]) si += 1 idx += 1 cap.release() return paths, stride, fps, size def _iou(a, b): ix0, iy0 = max(a[0], b[0]), max(a[1], b[1]) ix1, iy1 = min(a[2], b[2]), min(a[3], b[3]) iw, ih = max(0.0, ix1 - ix0), max(0.0, iy1 - iy0) inter = iw * ih ua = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter return inter / ua if ua > 0 else 0.0 MAX_SEED_IOU = 0.25 # the two seed boxes must be DIFFERENT people: MIN_CENTER_SEP = 0.6 # low overlap AND centers > this many box-widths apart def _valid_pair(bi, bj): """True if two person boxes are plausibly two DIFFERENT people. YOLO can emit two boxes for one person (part + whole); seeding SAM2 with such a pair tracks the same fighter twice — 'Fighter A and B are the same guy'.""" if _iou(bi, bj) > MAX_SEED_IOU: return False wi, wj = bi[2] - bi[0], bj[2] - bj[0] ci = ((bi[0] + bi[2]) / 2, (bi[1] + bi[3]) / 2) cj = ((bj[0] + bj[2]) / 2, (bj[1] + bj[3]) / 2) sep = float(np.hypot(ci[0] - cj[0], ci[1] - cj[1])) return sep > MIN_CENTER_SEP * (wi + wj) / 2 def _seed_candidates(det, frame_paths, k=3): """Ranked [(seed_idx, two boxes left->right), ...] of frames where two genuinely DISTINCT large people are visible. Candidates are spaced apart in time so a retry after a collapsed pair tries a different moment, not the neighbouring frame with the same problem.""" scored = [] for i in range(0, len(frame_paths), SEED_DET_STRIDE): frame = cv2.imread(frame_paths[i]) r = det.predict(frame, classes=[0], verbose=False, imgsz=640, conf=0.25, **pf.predict_kwargs())[0] if r.boxes is None or len(r.boxes) < 2: continue b = r.boxes.xyxy.cpu().numpy() areas = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1]) order = np.argsort(-areas)[:4] # consider the 4 largest best = None for x in range(len(order)): for y in range(x + 1, len(order)): bi, bj = b[order[x]], b[order[y]] if not _valid_pair(bi, bj): continue score = float(np.sqrt(min(areas[order[x]], areas[order[y]]))) if best is None or score > best[0]: best = (score, bi, bj) if best is not None: two = np.stack([best[1], best[2]]) scored.append((best[0], i, two[np.argsort(two[:, 0])])) scored.sort(key=lambda t: -t[0]) picked = [] for s, i, boxes in scored: if all(abs(i - j) >= SEED_DET_STRIDE * 3 for _, j, _ in picked): picked.append((s, i, boxes)) if len(picked) >= k: break return [(i, boxes) for _, i, boxes in picked] def _tracks_collapsed(poses): """True if the two tracks are really the same person: among frames where both are present, their hips stay within a fraction of a shoulder-width.""" hipA = np.nanmean(poses[:, 0, [pf.L_HIP, pf.R_HIP]], axis=1) hipB = np.nanmean(poses[:, 1, [pf.L_HIP, pf.R_HIP]], axis=1) both = ~np.isnan(hipA).any(1) & ~np.isnan(hipB).any(1) if both.sum() < 10: return True # too little signal to trust gaps = np.linalg.norm(hipA[both] - hipB[both], axis=1) sh = np.concatenate([ np.linalg.norm(poses[:, 0, pf.L_SH] - poses[:, 0, pf.R_SH], axis=1), np.linalg.norm(poses[:, 1, pf.L_SH] - poses[:, 1, pf.R_SH], axis=1)]) scale = np.nanmedian(sh) return bool(np.median(gaps) < 0.6 * scale) def _autocast_dtype(): if torch.cuda.is_available() and torch.cuda.is_bf16_supported(): return torch.bfloat16 return torch.float16 # T4 (sm_75) has no bf16 def _pose_in_mask(frame, mask, pose): """Pose on the crop around a fighter's mask -> full-frame keypoints or None.""" ys, xs = np.where(mask) if len(xs) < 50: return None x0, x1, y0, y1 = int(xs.min()), int(xs.max()), int(ys.min()), int(ys.max()) pad = int(0.12 * max(x1 - x0, y1 - y0)) h, w = frame.shape[:2] x0, y0 = max(0, x0 - pad), max(0, y0 - pad) x1, y1 = min(w, x1 + pad), min(h, y1 + pad) crop = frame[y0:y1, x0:x1] if crop.size == 0: return None r = pose.predict(crop, classes=[0], verbose=False, imgsz=640, conf=0.25, **pf.predict_kwargs())[0] if r.keypoints is None or r.boxes is None or len(r.boxes) == 0: return None # pick the detection ON THIS FIGHTER'S MASK, not the largest in the crop: # at close range both fighters appear in both crops, and largest-wins gave # BOTH tracks the same person's pose exactly during exchanges sub = mask[y0:y1, x0:x1] best_i, best_ov = -1, 0.0 for j, (bx0, by0, bx1, by1) in enumerate(r.boxes.xyxy.cpu().numpy().astype(int)): bx0, by0 = max(0, bx0), max(0, by0) bx1, by1 = min(sub.shape[1], bx1), min(sub.shape[0], by1) if bx1 <= bx0 or by1 <= by0: continue ov = float(sub[by0:by1, bx0:bx1].mean()) # share of box on the mask if ov > best_ov: best_ov, best_i = ov, j if best_i < 0 or best_ov < 0.15: return None # nobody actually on the mask i = best_i kp = r.keypoints.xy[i].cpu().numpy() cf = r.keypoints.conf[i].cpu().numpy() kp[cf < KP_CONF] = np.nan kp[:, 0] += x0 kp[:, 1] += y0 return kp def sam2_tracks(video_path, progress_cb=None): """RawTracks from SAM2 mask propagation + crop pose. Naming, left/right ordering, features and metadata are the Tracker seam's shared tail (trackers.finalize) — this only produces tracks. Raises on failure; the chain logs it and falls back. SAM2 has no idea which fighter the operator marked, so `seed_a_track` is left None: on a tapped clip the tail recovers the subject from the taps, and says so honestly when it can't. progress_cb(frac_0_to_1, desc) — optional; drives the app's loading bar through the three long stages (seed scan / SAM2 propagation / crop pose).""" from trackers import RawTracks, TAG_SAM2 cb = progress_cb or (lambda f, d: None) tmpdir = tempfile.mkdtemp(prefix="sam2_frames_") try: cb(0.01, "sampling frames from the clip") frame_paths, stride, fps, size = _extract_jpeg_frames(video_path, tmpdir) n = len(frame_paths) if n < pf.MIN_PAIR_FRAMES: raise RuntimeError(f"clip too short after sampling ({n} frames)") print(f"[sam2] {n} frames (stride {stride}) at {size}", flush=True) cb(0.04, "finding the two fighters") det, pose = _get_models() candidates = _seed_candidates(det, frame_paths) if not candidates: raise RuntimeError("no frame with two distinct people found to seed SAM2") predictor = _get_predictor() poses = cols = None for attempt, (seed, boxes) in enumerate(candidates, 1): print(f"[sam2] attempt {attempt}: seeding at frame {seed}/{n}", flush=True) retry = f" (retry {attempt})" if attempt > 1 else "" masks = {} done = 0 with torch.inference_mode(), torch.autocast("cuda", dtype=_autocast_dtype()): # Offload frames + inference state to CPU RAM: init_state # otherwise loads every sampled frame onto the GPU at once — # ~480 full-res frames is a real OOM risk on a 16 GB T4. # Per-frame transfer cost is minor next to propagation. state = predictor.init_state( tmpdir, offload_video_to_cpu=True, offload_state_to_cpu=True, ) for oid, box in enumerate(boxes): predictor.add_new_points_or_box(state, frame_idx=seed, obj_id=oid, box=box) for fidx, obj_ids, logits in predictor.propagate_in_video(state): masks[fidx] = {int(o): (logits[i, 0] > 0).cpu().numpy() for i, o in enumerate(obj_ids)} done += 1 if done % 10 == 0: cb(0.10 + 0.40 * min(done / max(n, 1), 1.0), f"tracking both fighters{retry} — frame {done}/{n}") for fidx, obj_ids, logits in predictor.propagate_in_video(state, reverse=True): masks.setdefault(fidx, {}).update( {int(o): (logits[i, 0] > 0).cpu().numpy() for i, o in enumerate(obj_ids)}) done += 1 if done % 10 == 0: cb(0.10 + 0.40 * min(done / max(n, 1), 1.0), f"tracking both fighters{retry} — frame {done}/{n}") print(f"[sam2] propagation done ({len(masks)} frames with masks)", flush=True) poses = np.full((n, 2, 17, 2), np.nan) overlap = np.full(n, np.nan) # per-frame mask overlap -> clinch metric cols = ([], []) for si, p in enumerate(frame_paths): if si % 10 == 0: cb(0.50 + 0.50 * si / max(n, 1), f"reading each fighter's pose{retry} — frame {si}/{n}") fm = masks.get(si) if not fm: continue frame = cv2.imread(p) rm = {} for oid, m in fm.items(): if oid > 1 or m is None: continue if m.shape[:2] != frame.shape[:2]: m = cv2.resize(m.astype(np.uint8), (frame.shape[1], frame.shape[0]), interpolation=cv2.INTER_NEAREST).astype(bool) rm[oid] = m kp = _pose_in_mask(frame, m, pose) if kp is not None: poses[si, oid] = kp cols[oid].append(pf._torso_bgr(frame, kp)) if 0 in rm and 1 in rm: smaller = min(int(rm[0].sum()), int(rm[1].sum())) if smaller > 50: overlap[si] = float(np.logical_and(rm[0], rm[1]).sum()) / smaller if not _tracks_collapsed(poses): break print("[sam2] tracks collapsed onto one person; retrying from another seed", flush=True) poses = None if poses is None: raise RuntimeError("every SAM2 seeding attempt tracked one person twice") return RawTracks(poses=poses, cols=cols, fps=fps, stride=stride, tag=TAG_SAM2, frame_size=size, mask_overlap=overlap) finally: shutil.rmtree(tmpdir, ignore_errors=True)