Add generate_from_image_video: in-Space SAM3 masking (SCAIL-Pose e2e) + ultralytics/moviepy

app.py

@@ -1255,6 +1255,96 @@ def _sampling_controls(prefix: str = ""):
     return steps, cfg, shift, seed, target_size, segment_len, segment_overlap
 
 
+# ============================================================================
+# NEW: generate_from_image_video — ทำ SAM3 mask บน GPU เอง (ส่งแค่รูป+วิดีโอ)
+# ใช้ SCAIL-Pose e2e (process_one e2e_mode → SAM3 ผ่าน ultralytics) → _run_scail_job
+# ============================================================================
+import tempfile as _tempfile
+
+_SCAIL_POSE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "SCAIL-Pose")
+if _SCAIL_POSE_DIR not in sys.path:
+    sys.path.insert(0, _SCAIL_POSE_DIR)
+
+SAM3_MODEL_PATH = os.getenv(
+    "SAM3_MODEL", os.path.join(_SCAIL_POSE_DIR, "pretrained_weights", "sam3.pt")
+)
+
+_sam3_predictor = None
+
+
+def _get_sam3_predictor():
+    global _sam3_predictor
+    if _sam3_predictor is None:
+        from ultralytics.models.sam import SAM3VideoSemanticPredictor
+        overrides = dict(
+            conf=0.25, task="segment", mode="predict", imgsz=640,
+            model=SAM3_MODEL_PATH, half=True, save=False, verbose=False,
+        )
+        _sam3_predictor = SAM3VideoSemanticPredictor(overrides=overrides, new_det_thresh=1.0)
+    return _sam3_predictor
+
+
+def _make_masks_e2e(subdir, video_name="driving.mp4",
+                    text=("human", "character"), max_persons=1):
+    from NLFPoseExtract.process_animation_aio import process_one
+    predictor = _get_sam3_predictor()
+    process_one(
+        subdir, video_name, e2e_mode=True, crop_kind=None,
+        max_persons=int(max_persons), text=list(text),
+        model_nlf=None, detector=None, predictor=predictor, image_predictor=None,
+    )
+    return (
+        os.path.join(subdir, "ref_mask.jpg"),
+        os.path.join(subdir, "rendered_mask_v2.mp4"),
+        os.path.join(subdir, "rendered_v2.mp4"),
+    )
+
+
+def _ensure_sam3():
+    if os.path.exists(SAM3_MODEL_PATH):
+        return
+    os.makedirs(os.path.dirname(SAM3_MODEL_PATH), exist_ok=True)
+    tok = os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN")
+    f = hf_hub_download("facebook/sam3", "sam3.pt", token=tok)
+    try:
+        os.symlink(f, SAM3_MODEL_PATH)
+    except OSError:
+        shutil.copyfile(f, SAM3_MODEL_PATH)
+
+
+@spaces.GPU(duration=_duration_for_job, size=GPU_SIZE)
+def generate_from_image_video(
+    image, pose_video, prompt, mode,
+    sample_steps, guide_scale, sample_shift, seed,
+    target_size, segment_len, segment_overlap,
+    progress=gr.Progress(track_tqdm=True),
+):
+    try:
+        if image is None or pose_video is None:
+            raise RuntimeError("Missing required input(s): reference image / pose video")
+        target_w, target_h = [int(v) for v in str(target_size).split("x")]
+        subdir = _tempfile.mkdtemp(prefix="scail_iv_")
+        ref_path = os.path.join(subdir, "ref.png")
+        drv_path = os.path.join(subdir, "driving.mp4")
+        shutil.copyfile(image, ref_path)
+        shutil.copyfile(pose_video, drv_path)
+        progress(0.05, desc="Masking with SAM3 on GPU ...")
+        ref_mask, drv_mask, rendered = _make_masks_e2e(
+            subdir, "driving.mp4", text=("human", "character"), max_persons=1,
+        )
+        progress(0.35, desc="Running SCAIL-2 generation ...")
+        output = _run_scail_job(
+            ref_path, ref_mask, rendered, drv_mask,
+            prompt, (mode == "replacement"), target_h, target_w,
+            sample_steps, guide_scale, sample_shift, seed,
+            segment_len, segment_overlap, progress=progress,
+        )
+        return output, "Done."
+    except Exception:
+        logging.exception("generate_from_image_video failed")
+        return None, traceback.format_exc()
+
+
 def build_ui():
     examples = _existing_examples()
     example_names = list(examples.keys())
@@ -1371,6 +1461,41 @@ def build_ui():
                 outputs=[upload_output, upload_status],
             )
 
+        with gr.Tab("Image + Video (auto-mask)"):
+            gr.Markdown(
+                "Send only a **reference image** + **driving video**. "
+                "The Space runs SAM3 masking on the GPU (SCAIL-Pose e2e) and generates."
+            )
+            iv_image = gr.Image(type="filepath", label="Reference image")
+            iv_video = gr.Video(label="Driving / pose video")
+            iv_prompt = gr.Textbox(label="Prompt", value="A young woman with natural body movement.")
+            iv_mode = gr.Radio(["animation", "replacement"], value="animation", label="Mode")
+            with gr.Row():
+                iv_steps = gr.Slider(1, 50, value=8, step=1, label="Steps")
+                iv_cfg = gr.Slider(1, 10, value=1.0, step=0.5, label="CFG (guide_scale)")
+                iv_shift = gr.Slider(1, 10, value=3.0, step=0.5, label="Shift")
+            with gr.Row():
+                iv_seed = gr.Number(value=42, precision=0, label="Seed")
+                iv_target = gr.Dropdown(
+                    ["512x896", "896x512", "704x1280", "1280x704"],
+                    value="512x896", label="Target size",
+                )
+                iv_segment_len = gr.Number(value=81, precision=0, label="Segment length")
+                iv_segment_overlap = gr.Number(value=5, precision=0, label="Segment overlap")
+            run_iv = gr.Button("Generate (auto-mask)", variant="primary")
+            iv_output = gr.Video(label="Output")
+            iv_status = gr.Textbox(label="Run status")
+            run_iv.click(
+                generate_from_image_video,
+                inputs=[
+                    iv_image, iv_video, iv_prompt, iv_mode,
+                    iv_steps, iv_cfg, iv_shift, iv_seed,
+                    iv_target, iv_segment_len, iv_segment_overlap,
+                ],
+                outputs=[iv_output, iv_status],
+                api_name="generate_from_image_video",
+            )
+
         with gr.Tab("Advanced Pack"):
             gr.Markdown(
                 "Upload a `.zip` pack for multi-reference or multi-character inputs. "
@@ -1491,6 +1616,10 @@ if __name__ == "__main__":
         _prepare_runtime_for_startup()
     if PRELOAD_PIPELINE:
         _prepare_pipeline_for_startup()
+    try:
+        _ensure_sam3()   # โหลด sam3.pt (ครั้งแรก) — ต้องมี secret HF_TOKEN ที่ได้สิทธิ์ facebook/sam3
+    except Exception as _e:
+        logging.warning("SAM3 weights not ready: %s", _e)
     build_ui().queue(max_size=8).launch(
         allowed_paths=[str(OUTPUT_DIR.resolve())],
         show_error=True,

requirements.txt

@@ -18,3 +18,6 @@ ftfy
 tqdm
 einops
 numpy>=1.23.5,<2
+
+ultralytics
+moviepy>=1.0.3