VAR offside visualizer

Dockerfile

@@ -0,0 +1,39 @@
+# A100 / dedicated-GPU Hugging Face Space (Docker SDK).
+# Models load once at boot and stay warm; pause the Space to stop billing.
+
+FROM pytorch/pytorch:2.4.1-cuda12.1-cudnn9-runtime
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYOPENGL_PLATFORM=egl \
+    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    SAM3D_DIR=/app/sam-3d-body \
+    HF_HOME=/app/.cache/huggingface
+
+# System libs for OpenCV / pyrender / video decoding
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git ffmpeg libgl1 libglib2.0-0 libosmesa6 libegl1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Clone the model repo (added to sys.path by pipeline/gpu.py via SAM3D_DIR)
+RUN git clone https://github.com/facebookresearch/sam-3d-body.git /app/sam-3d-body
+
+# Python deps. detectron2 is pinned to a1ce2f9 and built --no-build-isolation
+# --no-deps so it compiles against the torch already in the base image — the
+# single most fragile step; the pinned base keeps it reproducible.
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt \
+    && pip install --no-cache-dir \
+       "git+https://github.com/facebookresearch/detectron2.git@a1ce2f9" --no-build-isolation --no-deps \
+    && pip install --no-cache-dir "git+https://github.com/microsoft/MoGe.git"
+
+# App (UI + the pipeline package; GPU code is isolated in pipeline/gpu.py)
+COPY app.py /app/app.py
+COPY pipeline /app/pipeline
+
+# Writable cache (HF Spaces runs as a non-root user)
+RUN mkdir -p /app/.cache/huggingface && chmod -R 777 /app/.cache
+
+EXPOSE 7860
+CMD ["python", "app.py"]

README.md

@@ -1,12 +1,71 @@
 ---
-title: VAR
-emoji: 🔥
-colorFrom: blue
-colorTo: purple
+title: VAR Offside Visualizer
+emoji: 🥅
+colorFrom: green
+colorTo: blue
 sdk: docker
+app_port: 7860
 pinned: false
-license: apache-2.0
-short_description: VAR Offside - SAM3
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# VAR-style Offside Visualizer
+
+Upload a match clip, scrub to the moment the ball is played, reconstruct selected
+players in 3D with SAM 3D Body, and place them on a virtual pitch with a draggable
+offside plane.
+
+## Pipeline
+
+1. **Upload** a video clip.
+2. **Scrub** to the offside frame (slider + prev/next, frames seeked on demand).
+3. **Detect** players on that frame — the only GPU step, cached per (video, frame, threshold).
+4. **Select** the players to analyze and mark the defenders (incl. GK).
+5. **Click two goal-parallel lines** (4 points) on the detected frame to fix the offside axis.
+6. **Build** the 3D scene; drag the offside plane and read the OFFSIDE / NO-OFFSIDE verdict.
+
+The GPU runs once per frame (`pipeline/gpu.py`). Scrubbing, line geometry,
+placement, plotting, and the draggable plane are all CPU on the cached result.
+
+## Code layout
+
+```
+app.py            Gradio UI + event wiring (CPU)
+pipeline/
+  video.py        frame seek/probe (CPU)
+  gpu.py          model load + reconstruct_frame  ← the ONLY GPU code
+  geometry.py     vanishing point, ground fit, field frame, scene (CPU)
+  overlay.py      detection boxes + line-click drawing (CPU)
+```
+
+Isolating the GPU in `pipeline/gpu.py` means moving inference to a serverless
+backend (Modal / ZeroGPU) later only touches `reconstruct_frame`.
+
+## Deploy
+
+This is a **Docker SDK** Space for **dedicated GPU hardware** (A100 recommended):
+
+1. Set hardware to an A100 tier.
+2. Add a secret `HF_TOKEN` — a read token for an account with approved access to
+   the gated **`facebook/sam-3d-body-dinov3`**. (Override the repo with the
+   `SAM3D_REPO_ID` env var if you use a different checkpoint.)
+3. First boot builds the image and downloads ~7 GB of weights — give it time.
+   After that, the model stays warm until you pause the Space.
+
+**Cost control:** dedicated GPU bills while the Space is running, with no
+auto-shutoff. Pause the Space from its settings when you are not using it.
+
+### Why not ZeroGPU?
+
+ZeroGPU allocates the GPU per call, caps call duration, enforces a daily quota,
+and cold-loads the ~7 GB model stack on each allocation — a poor fit for an
+interactive video-scrubbing session, and it requires the Gradio SDK (not Docker).
+
+## Notes / limits
+
+- Scale comes from the reconstructed body height, so positions are approximate
+  metres — good for relative offside ordering, **not** sub-10 cm officiating calls.
+  The verdict surfaces a "too close to call" band rather than implying false precision.
+- The offside point currently uses the forward-most body vertex **including arms**;
+  excluding arms (via MHR body-part labels) is planned — see `TODO.md`.
+- "Find the offside moment" is manual scrubbing; automatic pass-instant detection
+  (ball tracking) is future work.

TODO.md

@@ -0,0 +1,21 @@
+# Parked — explicitly out of scope for now (SPEC §7)
+
+These are deliberately NOT built yet. Listed so they aren't lost.
+
+- **Automatic pass-instant detection** — needs ball tracking to replace manual
+  frame scrubbing.
+- **Exclude arms from the forward-most point** — needs MHR body-part vertex
+  labels; today `build_scene`/`offside_plane_x` use the forward-most body vertex
+  including arms.
+- **three.js broadcast frontend** (fog/bloom/camera moves) replacing Plotly.
+- **Team identification by jersey colour** — currently the user labels defenders
+  manually.
+- **Multi-frame tracking.**
+
+# Known nuances carried from the notebook
+
+- `flip up` toggle exists because the ground-normal sign can invert; if median
+  player height < 1.0 m the scene warns to toggle it.
+- The offside line uses the **2nd-last** defender's forward-most point (last is
+  usually the GK). With only one labeled defender it falls back to that defender;
+  with none, to the scene's median X (then drag the plane).

app.py

@@ -0,0 +1,215 @@
+"""
+VAR-style offside visualizer — Gradio app.
+
+Pipeline (matches the working Colab notebook):
+  upload video -> scrub to the offside frame -> detect players (GPU, once/frame)
+  -> select players -> click 2 goal-parallel lines -> place 3D poses on a field
+  -> Plotly scene with a draggable offside plane.
+
+The GPU is touched ONLY inside pipeline.gpu.reconstruct_frame(); every other
+callback in this file runs on cached numpy and stays on the CPU.
+"""
+
+import os
+
+# headless GL + CUDA fragmentation hygiene must be set before any heavy import
+os.environ.setdefault("PYOPENGL_PLATFORM", "egl")
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import numpy as np
+import gradio as gr
+
+from pipeline.video import probe_video, grab_frame
+from pipeline.overlay import annotate_detections, draw_lines
+from pipeline import geometry as G
+
+
+# ============================================================================
+# Stage (a): upload + frame scrubbing  (pure CPU)
+# ============================================================================
+def on_upload(video_path):
+    """New clip: size the slider, show frame 0, report length, remember max idx."""
+    if not video_path:
+        return gr.update(maximum=1, value=0), None, "Upload a clip to begin.", 0
+    n, fps = probe_video(video_path)
+    n_max = max(n - 1, 0)
+    return (
+        gr.update(maximum=max(n_max, 1), value=0),
+        grab_frame(video_path, 0),
+        f"{n} frames @ {fps:.1f} fps — scrub to the moment the ball is played.",
+        n_max,
+    )
+
+
+def on_scrub(video_path, idx):
+    if not video_path:
+        return None
+    return grab_frame(video_path, int(idx))
+
+
+def step_frame(idx, delta, n_max):
+    """Clamp prev/next within [0, slider max]."""
+    return max(0, min(int(n_max), int(idx) + delta))
+
+
+# ============================================================================
+# Stage (b)+(c): detect (GPU, cached) + populate selection  (CPU after the call)
+# ============================================================================
+def on_detect(video_path, idx, bbox_thr):
+    """The one GPU step. Imported lazily so the app boots without the model."""
+    from pipeline.gpu import reconstruct_frame
+    if not video_path:
+        return None, gr.update(choices=[], value=[]), gr.update(choices=[], value=[]), [], "Upload a clip first."
+    people = reconstruct_frame(video_path, idx, bbox_thr)
+    if not people:
+        return (None, gr.update(choices=[], value=[]), gr.update(choices=[], value=[]),
+                [], "No players detected — try a lower threshold.")
+    annotated = annotate_detections(grab_frame(video_path, idx), people)
+    choices = list(range(len(people)))
+    msg = (f"Detected {len(people)} players. Tick the ones to analyze, mark defenders, "
+           "then click 2 goal-parallel lines on the image above.")
+    return (annotated,
+            gr.update(choices=choices, value=[]),
+            gr.update(choices=choices, value=[]),
+            people, msg)
+
+
+# ============================================================================
+# Stage (d): click two goal-parallel lines (4 points)  (CPU)
+# ============================================================================
+def on_line_click(line_pts, frame_rgb, evt: gr.SelectData):
+    """Collect 4 clicks = 2 lines; redraw dots + segments. Resets after a full set."""
+    pts = list(line_pts) if line_pts else []
+    if len(pts) >= 4:
+        pts = []
+    pts.append([float(evt.index[0]), float(evt.index[1])])
+    img = draw_lines(frame_rgb, pts)
+    status = {1: "line 1: 1/2", 2: "line 1 set", 3: "line 2: 1/2",
+              4: "both lines set — build the scene"}.get(len(pts), "")
+    return pts, img, status
+
+
+# ============================================================================
+# Stage (e)+(f): place players + build the Plotly scene with a draggable plane
+# ============================================================================
+def on_build(video_path, idx, bbox_thr, selected_ids, line_pts, flip_up,
+             attack_dir, defender_ids):
+    from pipeline.gpu import reconstruct_frame, get_faces
+    if not selected_ids:
+        return None, "Select at least one player.", gr.update(), None, +1, []
+    if not line_pts or len(line_pts) < 4:
+        return None, "Click 2 goal-parallel lines (4 points) first.", gr.update(), None, +1, []
+
+    people = reconstruct_frame(video_path, idx, bbox_thr)
+    faces = get_faces()
+    h, w = grab_frame(video_path, idx).shape[:2]
+    focal = people[selected_ids[0]]["focal_length"]
+    gdir = G.goal_dir_from_lines(line_pts, focal, w, h)
+
+    placed = G.place_players(people, list(selected_ids), gdir, flip_up=flip_up)
+    med = float(np.median([placed[i][:, 2].max() for i in placed]))
+
+    attack_sign = +1 if attack_dir == "toward +X" else -1
+    dset = [int(d) for d in (defender_ids or [])]
+    plane_x = G.offside_plane_x(placed, attack_sign, dset)
+
+    fig = G.build_scene(placed, faces, plane_x, attack_sign, dset)
+
+    allX = np.vstack(list(placed.values()))[:, 0]
+    x0, x1 = float(allX.min() - 4), float(allX.max() + 4)
+    plane_update = gr.update(minimum=x0, maximum=x1, value=float(plane_x),
+                             visible=True, label="Drag the offside plane (X, m)")
+
+    warn = "  ⚠ heights look wrong — toggle 'flip up'." if med < 1.0 else ""
+    msg = f"Median player height {med:.2f} m (expect ~1.7–1.9).{warn}"
+    return fig, msg, plane_update, placed, attack_sign, dset
+
+
+def on_plane(placed, plane_x, attack_sign, defender_ids):
+    """Re-render the scene at a new plane X — pure CPU on the cached placement."""
+    from pipeline.gpu import get_faces
+    if not placed:
+        return gr.update()
+    fig = G.build_scene(placed, get_faces(), float(plane_x),
+                        int(attack_sign), defender_ids or [])
+    return fig
+
+
+# ============================================================================
+# UI
+# ============================================================================
+with gr.Blocks(title="VAR Offside Visualizer") as demo:
+    gr.Markdown(
+        "## VAR-style Offside Visualizer\n"
+        "Upload a clip, scrub to the moment the ball is played, then "
+        "detect → select → click 2 goal-parallel lines → build the 3D scene.\n\n"
+        "_Scale comes from reconstructed body height, so positions are approximate "
+        "metres — good for relative offside ordering, not sub-10 cm calls._"
+    )
+
+    # session state
+    st_nmax = gr.State(0)        # last valid frame index
+    st_people = gr.State([])     # slim detections for the current frame
+    st_lines = gr.State([])      # clicked line points
+    st_placed = gr.State(None)   # placed meshes (field frame) after build
+    st_attack = gr.State(+1)     # attack_sign
+    st_defenders = gr.State([])  # defender ids used for the verdict
+
+    # --- stage (a) ---
+    video = gr.Video(label="1. Upload match clip")
+    status = gr.Markdown()
+    with gr.Row():
+        frame_slider = gr.Slider(0, 1, value=0, step=1,
+                                 label="2. Scrub to the offside frame")
+    with gr.Row():
+        prev_btn = gr.Button("◀ prev frame")
+        next_btn = gr.Button("next frame ▶")
+    frame_view = gr.Image(label="Current frame", interactive=False)
+
+    # --- stage (b)/(c) ---
+    with gr.Row():
+        thr = gr.Slider(0.3, 0.95, value=0.85, step=0.05, label="Detection confidence")
+        detect_btn = gr.Button("3. Detect players (GPU)", variant="primary")
+    detect_view = gr.Image(
+        label="Detected players — click 2 goal-parallel lines here (4 points)",
+        interactive=True)
+    line_status = gr.Markdown()
+    sel = gr.CheckboxGroup(choices=[], label="4. Select players to analyze (indices)")
+
+    # --- stage (e)/(f) controls ---
+    with gr.Row():
+        flip = gr.Checkbox(False, label="flip up (if players are upside-down)")
+        attack = gr.Radio(["toward +X", "toward -X"], value="toward +X",
+                          label="Attacking direction")
+    defenders = gr.CheckboxGroup(
+        choices=[], label="Defenders (incl. GK) — sets the offside line")
+    build_btn = gr.Button("5. Build 3D scene + offside line", variant="primary")
+
+    scene = gr.Plot(label="3D scene")
+    plane_slider = gr.Slider(-10, 10, value=0, step=0.05, visible=False,
+                             label="Drag the offside plane (X, m)")
+    build_status = gr.Markdown()
+
+    # --- wiring ---
+    video.change(on_upload, [video], [frame_slider, frame_view, status, st_nmax])
+    frame_slider.change(on_scrub, [video, frame_slider], [frame_view])
+    prev_btn.click(lambda i, m: step_frame(i, -1, m),
+                   [frame_slider, st_nmax], [frame_slider])
+    next_btn.click(lambda i, m: step_frame(i, +1, m),
+                   [frame_slider, st_nmax], [frame_slider])
+
+    detect_btn.click(on_detect, [video, frame_slider, thr],
+                     [detect_view, sel, defenders, st_people, status])
+    detect_view.select(on_line_click, [st_lines, detect_view],
+                       [st_lines, detect_view, line_status])
+
+    build_btn.click(
+        on_build,
+        [video, frame_slider, thr, sel, st_lines, flip, attack, defenders],
+        [scene, build_status, plane_slider, st_placed, st_attack, st_defenders])
+    plane_slider.change(on_plane, [st_placed, plane_slider, st_attack, st_defenders],
+                        [scene])
+
+
+if __name__ == "__main__":
+    demo.queue().launch(server_name="0.0.0.0", server_port=7860)

pipeline/__init__.py

@@ -0,0 +1,8 @@
+"""VAR offside visualizer pipeline.
+
+Module layout keeps the GPU strictly isolated:
+  video.py     — frame seek/probe (CPU)
+  gpu.py       — model load + reconstruct_frame (the ONLY GPU code)
+  geometry.py  — vanishing point, ground fit, field frame, scene (CPU)
+  overlay.py   — cv2 detection boxes + line-click drawing (CPU)
+"""

pipeline/geometry.py

@@ -0,0 +1,135 @@
+"""
+Placement geometry — pure CPU, copied verbatim from the verified Colab notebook
+(sections 4 & the placement cell). Do NOT rewrite this math:
+
+  world_verts = pred_vertices + pred_cam_t            (shared camera frame)
+  goal dir    = back-projected vanishing point of two clicked parallel lines
+  ground      = SVD on the lowest ~3% feet vertices (camera Y is down)
+  field frame = ez=up, ey=goal dir in-plane, ex=ey x ez (the offside axis)
+  per player  : drop each player's own feet to Z=0 (not one global shift)
+  offside     : constant-X plane, parallel to the goal line by construction
+"""
+
+import numpy as np
+import plotly.graph_objects as go
+
+
+def goal_dir_from_lines(line_pts, focal, w, h):
+    """Two clicked goal-parallel lines -> goal-line direction in camera coords."""
+    def homog_line(p, q):
+        return np.cross([p[0], p[1], 1.0], [q[0], q[1], 1.0])
+    lp = np.asarray(line_pts, dtype=float)
+    l1 = homog_line(lp[0], lp[1])
+    l2 = homog_line(lp[2], lp[3])
+    vp = np.cross(l1, l2)
+    K = np.array([[focal, 0, w / 2], [0, focal, h / 2], [0, 0, 1.0]])
+    d_img = (np.array([vp[0], vp[1], 0.0]) if abs(vp[2]) < 1e-9
+             else np.array([vp[0] / vp[2], vp[1] / vp[2], 1.0]))
+    g = np.linalg.inv(K) @ d_img
+    return g / np.linalg.norm(g)
+
+
+def world_verts(p):
+    """Vertices in the shared camera frame = local mesh + per-player cam translation."""
+    return np.asarray(p["pred_vertices"]) + np.asarray(p["pred_cam_t"]).reshape(1, 3)
+
+
+def place_players(people, selected_ids, goal_dir_cam, flip_up=False):
+    """Place selected meshes on a common field frame: X=offside axis, Y=goal line, Z=up."""
+    def feet_pts(Vc, frac=0.03):
+        thr = np.quantile(Vc[:, 1], 1 - frac)   # camera Y is down -> feet = largest Y
+        return Vc[Vc[:, 1] >= thr]
+
+    feet_all = np.vstack([feet_pts(world_verts(people[i])) for i in selected_ids])
+    o = feet_all.mean(0)
+    _, _, Vt = np.linalg.svd(feet_all - o)
+    n = Vt[-1]
+    if n @ np.array([0, -1, 0]) < 0:
+        n = -n
+    if flip_up:
+        n = -n
+
+    g = goal_dir_cam - (goal_dir_cam @ n) * n
+    g /= np.linalg.norm(g)
+    ez, ey = n, g
+    ex = np.cross(ey, ez); ex /= np.linalg.norm(ex)
+    ey = np.cross(ez, ex)
+    Rwf = np.stack([ex, ey, ez], axis=1)
+
+    placed = {i: (world_verts(people[i]) - o) @ Rwf for i in selected_ids}
+    for i in placed:                       # drop each player's feet to Z=0 individually
+        placed[i][:, 2] -= placed[i][:, 2].min()
+    return placed
+
+
+def offside_plane_x(placed, attack_sign, defender_ids):
+    """X of the offside line = 2nd-last defender's forward-most point.
+
+    Forward-most is measured in the ATTACK direction (fwd = attack_sign * X), the
+    same quantity build_scene uses for the per-player verdict, so the drawn plane
+    and the colors always agree — matches the notebook. Falls back to the only
+    defender, then to the scene's median X if no defenders are labeled.
+    """
+    dset = set(int(d) for d in (defender_ids or []))
+    fmost = {i: float((attack_sign * placed[i][:, 0]).max()) for i in placed}
+    dfwd = sorted([fmost[i] for i in placed if i in dset], reverse=True)
+    if len(dfwd) >= 2:
+        return attack_sign * dfwd[1]
+    if dfwd:
+        return attack_sign * dfwd[0]
+    return float(np.median(np.vstack(list(placed.values()))[:, 0]))
+
+
+def build_scene(placed, faces, plane_x, attack_sign, defender_ids, too_close=0.30):
+    allP = np.vstack(list(placed.values()))
+    x0, x1 = allP[:, 0].min() - 4, allP[:, 0].max() + 4
+    y0, y1 = allP[:, 1].min() - 4, allP[:, 1].max() + 4
+
+    def fwd(x):
+        return attack_sign * x
+
+    fmost = {i: float(fwd(placed[i][:, 0]).max()) for i in placed}
+
+    fig = go.Figure()
+    # pitch
+    fig.add_trace(go.Mesh3d(x=[x0, x1, x1, x0], y=[y0, y0, y1, y1], z=[0, 0, 0, 0],
+                            i=[0, 0], j=[1, 2], k=[2, 3], color="seagreen",
+                            opacity=0.5, showlegend=False))
+    # offside plane
+    if plane_x is not None:
+        fig.add_trace(go.Mesh3d(x=[plane_x] * 4, y=[y0, y1, y1, y0], z=[0, 0, 3, 3],
+                                i=[0, 0], j=[1, 2], k=[2, 3], color="red",
+                                opacity=0.3, showlegend=False))
+    # players
+    palette = ["crimson", "royalblue", "gold", "darkorange", "mediumpurple",
+               "deepskyblue", "hotpink", "mediumspringgreen", "tomato", "slateblue"]
+    dset = set(int(d) for d in (defender_ids or []))
+    any_off = False
+    line_fwd = fwd(plane_x) if plane_x is not None else None
+    for n_, i in enumerate(placed):
+        if i in dset:
+            col, lab = "royalblue", "defender"
+        elif line_fwd is None:
+            col, lab = palette[n_ % len(palette)], ""
+        else:
+            m = fmost[i] - line_fwd
+            if m > too_close:
+                col, lab, any_off = "red", f"OFFSIDE +{m:.2f}m", True
+            elif m < -too_close:
+                col, lab = "seagreen", f"onside {m:.2f}m"
+            else:
+                col, lab = "orange", f"close {m:+.2f}m"
+        V = placed[i]
+        fig.add_trace(go.Mesh3d(x=V[:, 0], y=V[:, 1], z=V[:, 2],
+                                i=faces[:, 0], j=faces[:, 1], k=faces[:, 2],
+                                color=col, opacity=1.0, name=f"#{i} {lab}"))
+
+    title = "VAR — OFFSIDE" if any_off else "VAR — NO OFFSIDE"
+    fig.update_layout(
+        title=dict(text=title, x=0.5, font=dict(size=22, color="white")),
+        paper_bgcolor="#0b1f3a", height=640, margin=dict(l=0, r=0, t=40, b=0),
+        scene=dict(aspectmode="data", bgcolor="#0b1f3a",
+                   xaxis_title="X offside axis (m)", yaxis_title="Y goal line (m)",
+                   zaxis_title="Z (m)",
+                   camera=dict(eye=dict(x=0, y=-2.2, z=1.2), up=dict(x=0, y=0, z=1))))
+    return fig

pipeline/gpu.py

@@ -0,0 +1,89 @@
+"""
+The ONLY module that touches the GPU.
+
+`get_estimator()` loads SAM 3D Body once at first use and keeps it warm on the
+GPU. `reconstruct_frame()` runs `process_one_image` for a single frame and caches
+the slimmed result per (video, frame, threshold). Everything downstream consumes
+that cached numpy on the CPU and must never call back into this module's GPU path.
+
+Isolating the GPU here is deliberate: to move inference to a serverless backend
+(Modal / ZeroGPU) later, only `reconstruct_frame` has to change.
+"""
+
+import os
+import sys
+import functools
+
+import numpy as np
+
+# The sam-3d-body repo is cloned here by the Dockerfile and added to sys.path.
+SAM3D_DIR = os.environ.get("SAM3D_DIR", "/app/sam-3d-body")
+if SAM3D_DIR not in sys.path:
+    sys.path.insert(0, SAM3D_DIR)
+
+HF_REPO_ID = os.environ.get("SAM3D_REPO_ID", "facebook/sam-3d-body-dinov3")
+
+_ESTIMATOR = None
+_FACES = None
+
+
+def get_estimator():
+    """Lazy-load the SAM 3D Body estimator a single time; returns (estimator, faces).
+
+    Verified against facebookresearch/sam-3d-body:
+      - setup_sam_3d_body -> load_sam_3d_body_hf -> load_sam_3d_body(ckpt, mhr_path),
+        which sidesteps the model_config.yaml path bug in the demo.py route.
+      - estimator.faces == model.head_pose.faces (numpy).
+      - The "missing keys" warning at load is benign (MHR rig buffers).
+    """
+    global _ESTIMATOR, _FACES
+    if _ESTIMATOR is None:
+        from huggingface_hub import login
+        token = os.environ.get("HF_TOKEN")
+        if token:
+            login(token=token)
+        # Imported here so module import never fails before the repo is on sys.path.
+        from notebook.utils import setup_sam_3d_body
+        _ESTIMATOR = setup_sam_3d_body(hf_repo_id=HF_REPO_ID)
+        _FACES = np.asarray(_ESTIMATOR.faces)
+    return _ESTIMATOR, _FACES
+
+
+def get_faces():
+    """Triangle faces for the body mesh (CPU-only consumers use this)."""
+    return get_estimator()[1]
+
+
+@functools.lru_cache(maxsize=8)
+def _reconstruct_cached(video_path, idx, bbox_thr):
+    """GPU call, memoized per (video, frame, threshold).
+
+    Returns a list of slim, picklable dicts (one per detected person) holding only
+    the numpy fields used downstream. Returns None if the frame can't be read.
+    """
+    from .video import grab_frame  # local import keeps this module import-light
+
+    est, _ = get_estimator()
+    frame_rgb = grab_frame(video_path, idx)
+    if frame_rgb is None:
+        return None
+
+    # process_one_image accepts an RGB array and a bbox_thr kwarg (verified).
+    # It returns a LIST, one dict per person, keys include:
+    #   bbox, pred_vertices, pred_cam_t, focal_length, pred_keypoints_2d, mask
+    people = est.process_one_image(frame_rgb, bbox_thr=bbox_thr)
+
+    slim = []
+    for p in people:
+        slim.append({
+            "bbox": np.asarray(p["bbox"]).reshape(-1)[:4].astype(float),
+            "pred_vertices": np.asarray(p["pred_vertices"], dtype=np.float32),
+            "pred_cam_t": np.asarray(p["pred_cam_t"], dtype=np.float32).reshape(3),
+            "focal_length": float(np.asarray(p["focal_length"]).reshape(-1)[0]),
+        })
+    return slim
+
+
+def reconstruct_frame(video_path, idx, bbox_thr=0.85):
+    """CPU-cheap wrapper: normalizes the cache key and returns cached people."""
+    return _reconstruct_cached(str(video_path), int(idx), round(float(bbox_thr), 3))

pipeline/overlay.py

@@ -0,0 +1,36 @@
+"""
+cv2 overlays — pure CPU. Draws detection boxes and the two goal-parallel lines
+the user clicks. Works on RGB arrays in, RGB arrays out (Gradio Image is numpy/RGB).
+"""
+
+import cv2
+import numpy as np
+
+
+def annotate_detections(frame_rgb, people):
+    """Draw numbered green boxes for every detected person."""
+    img = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR)
+    for i, p in enumerate(people):
+        x1, y1, x2, y2 = [int(v) for v in p["bbox"]]
+        cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 2)
+        cv2.putText(img, str(i), (x1, max(y1 - 6, 14)),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 3)
+    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+
+def draw_lines(frame_rgb, pts):
+    """Draw the clicked points and the (up to two) line segments.
+
+    pts is a list of [x, y]; points 0-1 form line 1 (red), 2-3 form line 2 (yellow).
+    """
+    if frame_rgb is None:
+        return None
+    img = cv2.cvtColor(frame_rgb.copy(), cv2.COLOR_RGB2BGR)
+    cols = [(0, 0, 255), (0, 255, 255)]  # BGR: line1 red, line2 yellow
+    for k, (px, py) in enumerate(pts):
+        cv2.circle(img, (int(px), int(py)), 6, cols[k // 2], -1)
+    if len(pts) >= 2:
+        cv2.line(img, tuple(map(int, pts[0])), tuple(map(int, pts[1])), cols[0], 2)
+    if len(pts) >= 4:
+        cv2.line(img, tuple(map(int, pts[2])), tuple(map(int, pts[3])), cols[1], 2)
+    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

pipeline/video.py

@@ -0,0 +1,26 @@
+"""
+Video helpers — pure CPU. Frames are seeked on demand (no bulk extraction),
+so scrubbing a long clip stays cheap and never touches the GPU.
+"""
+
+import cv2
+
+
+def probe_video(video_path):
+    """Return (frame_count, fps) for a clip. fps falls back to 25 if unknown."""
+    cap = cv2.VideoCapture(video_path)
+    n = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
+    cap.release()
+    return n, fps
+
+
+def grab_frame(video_path, idx):
+    """Seek to an exact frame index and return it as an RGB array (or None)."""
+    cap = cv2.VideoCapture(video_path)
+    cap.set(cv2.CAP_PROP_POS_FRAMES, int(idx))
+    ok, frame_bgr = cap.read()
+    cap.release()
+    if not ok:
+        return None
+    return cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)

requirements.txt

@@ -0,0 +1,35 @@
+# Pinned to the versions verified during the build (mirrors the Dockerfile's
+# torch pin). Gradio 6 requires Slider minimum < maximum — see app.py.
+gradio==6.19.0
+plotly>=5.20
+opencv-python-headless
+numpy
+huggingface_hub
+pytorch-lightning
+pyrender
+yacs
+scikit-image
+einops
+timm
+dill
+pandas
+rich
+hydra-core
+hydra-submitit-launcher
+hydra-colorlog
+pyrootutils
+webdataset
+chump
+networkx==3.2.1
+roma
+joblib
+seaborn
+appdirs
+cython
+jsonlines
+xtcocotools
+loguru
+optree
+fvcore
+pycocotools
+trimesh