Upload folder using huggingface_hub

README.md

@@ -1,13 +1,15 @@
 ---
 title: SAM3 Video
-emoji: 📉
-colorFrom: gray
-colorTo: green
+emoji: 🎬
+colorFrom: indigo
+colorTo: blue
 sdk: gradio
-sdk_version: 6.15.2
-python_version: '3.12'
+sdk_version: 6.6.0
 app_file: app.py
 pinned: false
+short_description: SAM 3 concept tracking across video frames (API)
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+`api_track(video, concepts, conf, max_frames)` -> JSON tracks (stable object ids
+across frames) with base64-PNG masks. Backed by the transformers `Sam3VideoModel`
+(facebook/sam3). Requires the `HF_TOKEN` secret with gated access to facebook/sam3.

app.py

@@ -0,0 +1,116 @@
+"""SAM 3 video concept-tracking API (ZeroGPU), transformers Sam3VideoModel route.
+
+Tracks every instance of the given concept(s) across video frames with stable
+object ids. Output schema matches the local video_client parser:
+  {version, model, fps, width, height, n_frames, tracks:[{label, object_id,
+   frames:[{frame, score, box, mask_png_b64}]}]}
+"""
+import base64
+import io
+import os
+
+import gradio as gr
+import numpy as np
+import spaces
+from PIL import Image
+from transformers import Sam3VideoModel, Sam3VideoProcessor
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+MODEL_ID = "facebook/sam3"
+
+# Built at import on CPU; moved to CUDA inside the @spaces.GPU function.
+processor = Sam3VideoProcessor.from_pretrained(MODEL_ID, token=HF_TOKEN)
+model = Sam3VideoModel.from_pretrained(MODEL_ID, token=HF_TOKEN)
+model.eval()
+
+
+def _enc(mask_bool: np.ndarray, maxside: int = 512) -> str:
+    h, w = mask_bool.shape
+    img = Image.fromarray((mask_bool.astype(np.uint8) * 255), "L")
+    scale = min(1.0, maxside / max(h, w))
+    if scale < 1.0:
+        img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))))
+    buf = io.BytesIO(); img.save(buf, "PNG")
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+def _np(x):
+    return x.detach().cpu().numpy() if hasattr(x, "detach") else np.asarray(x)
+
+
+def _read_frames(path, max_frames):
+    import imageio
+    reader = imageio.get_reader(path)
+    frames = []
+    try:
+        for i, fr in enumerate(reader):
+            if i >= max_frames:
+                break
+            frames.append(np.asarray(fr))
+    finally:
+        reader.close()
+    return frames
+
+
+@spaces.GPU(duration=300)
+def api_track(video, concepts, conf, max_frames):
+    device = "cuda"
+    model.to(device)
+    concept_list = [c.strip() for c in (concepts or "").split(",") if c.strip()] or ["person"]
+    frames = _read_frames(video, int(max_frames))
+    if not frames:
+        return {"error": "no frames read from video"}
+    H, W = frames[0].shape[:2]
+    session = processor.init_video_session(
+        video=frames, inference_device=device,
+        processing_device="cpu", video_storage_device="cpu",
+    )
+    processor.add_text_prompt(session, concept_list)
+
+    tracks, obj_label, n_frames = {}, {}, 0
+    for mo in model.propagate_in_video_iterator(inference_session=session,
+                                                max_frame_num_to_track=int(max_frames)):
+        proc = processor.postprocess_outputs(session, mo)
+        fi = int(mo.frame_idx); n_frames = max(n_frames, fi + 1)
+        for prompt, oids in (proc.get("prompt_to_obj_ids") or {}).items():
+            for oid in oids:
+                obj_label.setdefault(int(oid), prompt)
+        oids = _np(proc["object_ids"]).tolist()
+        scores = _np(proc["scores"]).tolist()
+        masks = proc["masks"]
+        boxes = _np(proc["boxes"])
+        for k, oid in enumerate(oids):
+            oid = int(oid)
+            m = _np(masks[k])
+            if m.ndim == 3:
+                m = m[0]
+            m = m > 0.5 if m.dtype != bool else m
+            tr = tracks.get(oid)
+            if tr is None:
+                tr = {"label": obj_label.get(oid, concept_list[0]), "object_id": oid, "frames": []}
+                tracks[oid] = tr
+            tr["frames"].append({"frame": fi, "score": float(scores[k]),
+                                 "box": [float(v) for v in boxes[k]],
+                                 "mask_png_b64": _enc(m)})
+
+    out_tracks = []
+    for oid, tr in tracks.items():
+        tr["label"] = obj_label.get(oid, tr["label"])
+        if tr["frames"] and max(f["score"] for f in tr["frames"]) >= float(conf):
+            out_tracks.append(tr)
+    return {"version": "3", "model": MODEL_ID, "fps": 0.0,
+            "width": W, "height": H, "n_frames": n_frames, "tracks": out_tracks}
+
+
+with gr.Blocks(title="SAM3 Video") as demo:
+    gr.Markdown("# SAM 3 Video Tracking API\nUpload a video, enter comma-separated concepts.")
+    with gr.Row():
+        vid = gr.File(file_count="single", type="filepath", label="Video (mp4)")
+        out = gr.JSON(label="Tracks")
+    txt = gr.Textbox(label="Concepts (comma-separated)", value="person")
+    conf = gr.Slider(0.0, 1.0, value=0.4, step=0.05, label="Confidence")
+    mf = gr.Slider(8, 96, value=48, step=8, label="Max frames")
+    gr.Button("Track").click(api_track, [vid, txt, conf, mf], out, api_name="api_track")
+
+if __name__ == "__main__":
+    demo.queue().launch(show_error=True)

requirements.txt

@@ -0,0 +1,11 @@
+transformers==5.9.0
+torch==2.11.0
+torchvision
+gradio==6.6.0
+spaces
+accelerate
+sentencepiece
+pillow
+numpy
+imageio
+imageio-ffmpeg