Upload folder using huggingface_hub

README.md

@@ -1,13 +1,35 @@
 ---
 title: StreamDiffusionV2 Realtime
-emoji: 🌖
-colorFrom: red
-colorTo: purple
+emoji: 🌀
+colorFrom: indigo
+colorTo: pink
 sdk: gradio
-sdk_version: 6.19.0
-python_version: '3.13'
+sdk_version: 6.10.0
 app_file: app.py
+python_version: "3.10"
+short_description: Realtime webcam video diffusion (StreamDiffusionV2 / Wan2.1)
+startup_duration_timeout: 1h
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# StreamDiffusionV2 · Realtime Webcam Diffusion
+
+Live ZeroGPU demo of [**StreamDiffusionV2**](https://streamdiffusionv2.github.io/)
+(MLSys 2026 Best Paper) on **Wan2.1-T2V-1.3B**, with a custom `gradio.Server`
+frontend.
+
+Unlike a fixed-length generator, StreamDiffusionV2 is **designed for continuous
+streaming**: a causal Diffusion-Transformer with a **sink-token-guided rolling KV
+cache**, a motion-aware noise controller, and StreamVAE. Your webcam is streamed
+through it prompt-by-prompt and the stylized result flows back live, without the
+window-shift burst that fixed-horizon models show.
+
+The browser captures the webcam and posts frames to a lightweight FastAPI route;
+a held `@spaces.GPU` session runs StreamDiffusionV2's single-GPU streaming loop
+(`start_stream_session` → `run_stream_batch`) and streams frames back over the
+Gradio JS client, paced by a client jitter buffer.
+
+One of three rolling/streaming demos:
+- StreamDiffusionV2 (this) — video-to-video webcam.
+- LongLive — interactive long text-to-video.
+- Rolling Forcing — real-time multi-minute text-to-video.

app.py

@@ -0,0 +1,197 @@
+import os
+import time
+import base64
+import tempfile
+import queue as pyqueue
+import multiprocessing as mp
+from io import BytesIO
+
+import spaces  # before torch / CUDA imports
+
+import torch
+import numpy as np
+from PIL import Image
+from huggingface_hub import snapshot_download, hf_hub_download
+
+from streamdiffusionv2 import StreamDiffusionV2Pipeline
+
+# ----------------------------------------------------------------------------
+# Config
+# ----------------------------------------------------------------------------
+WAN_REPO = "Wan-AI/Wan2.1-T2V-1.3B"
+WAN_DIR = "wan_models/Wan2.1-T2V-1.3B"
+SDV2_REPO = "jerryfeng/StreamDiffusionV2"
+CKPT_DIR = "ckpts"
+CKPT_FOLDER = os.path.join(CKPT_DIR, "wan_causal_dmd_v2v")  # 1.3B v2v checkpoint
+
+HEIGHT, WIDTH = 480, 832
+SESSION_DURATION = 58
+POLL_INTERVAL = 0.005
+DEFAULT_PROMPT = "a psychedelic neon dream, vivid saturated colors, glowing"
+NOISE_SCALE = 0.8
+
+SESSION_DIR = tempfile.gettempdir()
+INSTRUCTION_FILE = os.path.join(SESSION_DIR, "sdv2_prompt.txt")
+READY_SENTINEL = "__READY__"
+
+# Fork-safe live frame queue (created before any ZeroGPU fork).
+FRAME_Q = mp.get_context("fork").Queue(maxsize=512)
+
+# ----------------------------------------------------------------------------
+# Weights + pipeline at module scope (ZeroGPU snapshot preload)
+# ----------------------------------------------------------------------------
+snapshot_download(
+    repo_id=WAN_REPO, local_dir=WAN_DIR,
+    allow_patterns=[
+        "config.json", "diffusion_pytorch_model.safetensors", "Wan2.1_VAE.pth",
+        "models_t5_umt5-xxl-enc-bf16.pth", "google/umt5-xxl/*",
+    ],
+)
+snapshot_download(repo_id=SDV2_REPO, local_dir=CKPT_DIR,
+                  allow_patterns=["wan_causal_dmd_v2v/*"])
+
+device = torch.device("cuda")
+
+# StreamDiffusionV2 single-GPU streaming pipeline (rolling KV + sink tokens are
+# built into the model -> continuous streaming without the window-shift burst).
+stream = StreamDiffusionV2Pipeline(
+    checkpoint_folder=CKPT_FOLDER,
+    mode="single",
+    device=device,
+    height=HEIGHT,
+    width=WIDTH,
+    step=2,
+    noise_scale=NOISE_SCALE,
+    model_type="T2V-1.3B",
+    use_taehv=False,
+)
+PM = stream.pipeline_manager
+CHUNK = PM.base_chunk_size * PM.pipeline.num_frame_per_block   # 4 px frames / chunk
+FIRST_BATCH = 1 + CHUNK                                        # 5 px frames first
+
+
+def _read_prompt():
+    try:
+        with open(INSTRUCTION_FILE, encoding="utf-8") as f:
+            return f.read().strip()
+    except FileNotFoundError:
+        return ""
+
+
+def _decode_jpeg_to_tensor(jpeg_bytes):
+    """JPEG bytes -> [C, H, W] in [-1, 1]."""
+    im = Image.open(BytesIO(jpeg_bytes)).convert("RGB").resize((WIDTH, HEIGHT), Image.BICUBIC)
+    arr = torch.from_numpy(np.asarray(im)).float().permute(2, 0, 1) / 255.0
+    return arr * 2.0 - 1.0
+
+
+def _frames_to_video_tensor(frame_list):
+    """list of [C,H,W] -> [B, C, T, H, W] bf16 on device."""
+    vid = torch.stack(frame_list, dim=1).unsqueeze(0)  # [1, C, T, H, W]
+    return vid.to(device=device, dtype=torch.bfloat16)
+
+
+def _to_data_uri(frame01):
+    im = Image.fromarray((np.clip(frame01, 0, 1) * 255.0).astype(np.uint8))
+    buf = BytesIO()
+    im.save(buf, format="JPEG", quality=80)
+    return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
+
+
+# ----------------------------------------------------------------------------
+# Gradio Server
+# ----------------------------------------------------------------------------
+from gradio import Server
+from fastapi import Request
+from fastapi.responses import HTMLResponse
+
+app = Server()
+
+
+@app.api(name="run_session")
+@spaces.GPU(duration=60, size="xlarge")
+@torch.inference_mode()
+def run_session() -> str:
+    # Drain stale frames, then signal the client to start streaming.
+    try:
+        while True:
+            FRAME_Q.get_nowait()
+    except pyqueue.Empty:
+        pass
+    yield READY_SENTINEL
+
+    prompt = _read_prompt() or DEFAULT_PROMPT
+    buffer = []
+    session = None
+    deadline = time.time() + SESSION_DURATION
+    last = None
+
+    while time.time() < deadline:
+        drained = 0
+        while drained < 256:
+            try:
+                buffer.append(FRAME_Q.get_nowait())
+            except pyqueue.Empty:
+                break
+            drained += 1
+
+        need = FIRST_BATCH if session is None else CHUNK
+        if len(buffer) < need:
+            time.sleep(POLL_INTERVAL)
+            continue
+
+        frames = [_decode_jpeg_to_tensor(b) for b in buffer[:need]]
+        buffer = buffer[need:]
+        vid = _frames_to_video_tensor(frames)
+
+        t0 = time.time()
+        if session is None:
+            session, init_video = PM.start_stream_session(prompt, vid, NOISE_SCALE)
+            outs = [init_video]
+        else:
+            outs = PM.run_stream_batch(session, vid)
+        dt = time.time() - t0
+
+        n = 0
+        for arr in outs:                       # each arr: [T, H, W, C] in [0,1]
+            for fr in arr:
+                last = _to_data_uri(fr)
+                yield last
+                n += 1
+        if n:
+            print(f"[sdv2] {n} frames in {dt:.2f}s ({n/max(1e-3,dt):.1f} fps)", flush=True)
+
+    if last is not None:
+        yield last
+
+
+@app.post("/frame")
+async def post_frame(request: Request):
+    body = await request.body()
+    if body:
+        try:
+            FRAME_Q.put_nowait(body)
+        except pyqueue.Full:
+            pass
+    return {"ok": True}
+
+
+@app.post("/instruction")
+async def post_instruction(request: Request):
+    data = await request.json()
+    text = (data.get("instruction", "") or "").strip()
+    tmp = INSTRUCTION_FILE + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        f.write(text)
+    os.replace(tmp, INSTRUCTION_FILE)
+    return {"ok": True}
+
+
+@app.get("/", response_class=HTMLResponse)
+async def homepage():
+    here = os.path.dirname(os.path.abspath(__file__))
+    with open(os.path.join(here, "index.html"), encoding="utf-8") as f:
+        return f.read()
+
+
+app.launch(show_error=True)

index.html

@@ -0,0 +1,203 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>LiveEdit · Realtime</title>
+<style>
+  :root { --bg:#0e0f13; --panel:#171922; --line:#2a2e3a; --fg:#e8e8ee; --accent:#c084fc; --good:#86efac; }
+  * { box-sizing: border-box; }
+  body { margin:0; background:var(--bg); color:var(--fg); font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif; }
+  .wrap { max-width:1100px; margin:0 auto; padding:24px 18px 60px; }
+  h1 { font-size:1.5rem; margin:0 0 4px; }
+  .sub { color:#9aa0ad; font-size:.95rem; margin:0 0 18px; line-height:1.5; }
+  .sub a { color:var(--accent); }
+  .controls { display:flex; gap:10px; flex-wrap:wrap; align-items:center; margin-bottom:16px; }
+  input[type=text] { flex:1; min-width:260px; background:var(--panel); border:1px solid var(--line); color:var(--fg); padding:12px 14px; border-radius:10px; font-size:1rem; }
+  button { background:var(--accent); color:#1a1024; border:0; padding:12px 18px; border-radius:10px; font-size:1rem; font-weight:600; cursor:pointer; }
+  button:disabled { opacity:.5; cursor:not-allowed; }
+  .timer { font-family:ui-monospace,Menlo,monospace; color:var(--good); background:#11210f; border:1px solid #234; padding:8px 12px; border-radius:8px; display:none; }
+  .grid { display:grid; grid-template-columns:1fr 1fr; gap:14px; }
+  .card { background:var(--panel); border:1px solid var(--line); border-radius:14px; overflow:hidden; }
+  .card h2 { font-size:.85rem; text-transform:uppercase; letter-spacing:.05em; color:#9aa0ad; margin:0; padding:10px 14px; border-bottom:1px solid var(--line); }
+  .media { aspect-ratio:832/480; background:#000; display:flex; align-items:center; justify-content:center; }
+  .media video, .media img { width:100%; height:100%; object-fit:cover; display:block; }
+  .placeholder { color:#5a6070; font-size:.9rem; }
+  .status { margin-top:14px; color:#9aa0ad; font-size:.9rem; min-height:1.2em; }
+  @media (max-width:780px){ .grid{ grid-template-columns:1fr; } }
+</style>
+</head>
+<body>
+<div class="wrap">
+  <h1>🌀 StreamDiffusionV2 · Realtime Webcam Diffusion</h1>
+  <p class="sub">
+    Live demo of <a href="https://streamdiffusionv2.github.io/" target="_blank">StreamDiffusionV2</a>
+    (MLSys 2026 Best Paper) on Wan2.1-T2V-1.3B. It streams your webcam through a causal video-diffusion
+    model with a <b>sink-token rolling KV cache</b> &mdash; built for <i>continuous</i> streaming, so it
+    keeps flowing without the window-shift burst. Type a style prompt, click <b>Start</b> to grab ZeroGPU
+    for ~60s. (Prompt is fixed for the session &mdash; restart to change it.)
+  </p>
+
+  <div class="controls">
+    <input id="instruction" type="text" placeholder="style prompt · e.g. psychedelic neon dream · van gogh · cyberpunk city" />
+    <button id="startBtn">▶ Start session</button>
+    <span id="timer" class="timer">⏱ <span id="count">58</span>s</span>
+  </div>
+
+  <div class="grid">
+    <div class="card">
+      <h2>Your webcam</h2>
+      <div class="media"><video id="cam" autoplay muted playsinline></video></div>
+    </div>
+    <div class="card">
+      <h2>Edited (live)</h2>
+      <div class="media"><img id="out" alt="" /><span id="outPh" class="placeholder">edited stream appears here</span></div>
+    </div>
+  </div>
+
+  <div id="status" class="status"></div>
+</div>
+
+<canvas id="grab" width="832" height="480" style="display:none"></canvas>
+
+<script type="module">
+import { Client } from "https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js";
+
+const camEl = document.getElementById("cam");
+const outEl = document.getElementById("out");
+const outPh = document.getElementById("outPh");
+const startBtn = document.getElementById("startBtn");
+const instr = document.getElementById("instruction");
+const timerEl = document.getElementById("timer");
+const countEl = document.getElementById("count");
+const statusEl = document.getElementById("status");
+const grab = document.getElementById("grab");
+const gctx = grab.getContext("2d");
+
+let client = null;
+let stream = null;
+let captureTimer = null;
+let countdownTimer = null;
+let running = false;
+
+const FPS = 10;                // webcam frames sent per second
+const SESSION_SECONDS = 58;
+
+// --- jitter buffer: edited frames arrive in bursts (one chunk at a time) but
+// we play them out at a smooth, adaptive rate so the preview doesn't stutter.
+let playQueue = [];
+const MAX_QUEUE = 36;          // bound latency (~3 chunks)
+let lastShown = 0;
+function playLoop(ts){
+  if (playQueue.length > MAX_QUEUE) playQueue = playQueue.slice(-MAX_QUEUE);
+  // drain faster as the backlog grows, slower when nearly empty
+  const n = playQueue.length;
+  const fps = n > 24 ? 18 : n > 14 ? 13 : n > 6 ? 9 : 6;
+  if (n && ts - lastShown >= 1000 / fps){
+    outEl.src = playQueue.shift();
+    outPh.style.display = "none";
+    lastShown = ts;
+  }
+  requestAnimationFrame(playLoop);
+}
+requestAnimationFrame(playLoop);
+
+function setStatus(t){ statusEl.textContent = t; }
+
+async function ensureClient(){
+  if (!client) client = await Client.connect(window.location.origin);
+  return client;
+}
+
+async function ensureCam(){
+  if (stream) return;
+  stream = await navigator.mediaDevices.getUserMedia({ video: { width: 832, height: 480 }, audio: false });
+  camEl.srcObject = stream;
+  await camEl.play().catch(()=>{});
+}
+
+async function sendInstruction(){
+  try {
+    await fetch("/instruction", {
+      method:"POST", headers:{ "Content-Type":"application/json" },
+      body: JSON.stringify({ instruction: instr.value || "" })
+    });
+  } catch(e){}
+}
+
+function startCapture(){
+  captureTimer = setInterval(() => {
+    if (!camEl.videoWidth) return;
+    gctx.drawImage(camEl, 0, 0, grab.width, grab.height);
+    grab.toBlob(async (blob) => {
+      if (!blob) return;
+      try { await fetch("/frame", { method:"POST", body: blob }); } catch(e){}
+    }, "image/jpeg", 0.8);
+  }, 1000 / FPS);
+}
+
+function stopAll(){
+  running = false;
+  if (captureTimer) { clearInterval(captureTimer); captureTimer = null; }
+  if (countdownTimer) { clearInterval(countdownTimer); countdownTimer = null; }
+  timerEl.style.display = "none";
+  startBtn.disabled = false;
+  startBtn.textContent = "▶ Start session";
+}
+
+function startCountdown(){
+  let r = SESSION_SECONDS;
+  countEl.textContent = r;
+  timerEl.style.display = "inline-block";
+  countdownTimer = setInterval(() => {
+    r -= 1; countEl.textContent = Math.max(0, r);
+    if (r <= 0) clearInterval(countdownTimer);
+  }, 1000);
+}
+
+instr.addEventListener("change", () => { if (running) sendInstruction(); });
+instr.addEventListener("input", () => { if (running) sendInstruction(); });
+
+startBtn.addEventListener("click", async () => {
+  if (running) return;
+  startBtn.disabled = true;
+  try {
+    setStatus("Requesting webcam…");
+    await ensureCam();
+    setStatus("Connecting…");
+    await ensureClient();
+    running = true;
+    playQueue = [];
+    startBtn.textContent = "◌ Acquiring ZeroGPU…";
+    await sendInstruction();
+    setStatus("Queued for ZeroGPU — webcam streaming starts once the GPU is acquired…");
+
+    const job = client.submit("/run_session", {});
+    let frames = 0;
+    for await (const msg of job) {
+      if (msg.type !== "data" || !msg.data || msg.data[0] == null) continue;
+      const payload = msg.data[0];
+      if (payload === "__READY__") {
+        // GPU is now allocated — only now start capturing & sending frames.
+        startBtn.textContent = "● Live";
+        await sendInstruction();
+        startCapture();
+        startCountdown();
+        setStatus("ZeroGPU acquired — streaming your webcam through LiveEdit…");
+        continue;
+      }
+      playQueue.push(payload);          // jitter buffer paces actual display
+      frames += 1;
+      if (frames % 12 === 0) setStatus(`Streaming… ${frames} edited frames`);
+    }
+    setStatus(frames ? "Session ended. Click Start to run another ~60s session."
+                     : "Session ended before any frames were produced — try again.");
+  } catch (e) {
+    setStatus("Error: " + (e && e.message ? e.message : e));
+  } finally {
+    stopAll();
+  }
+});
+</script>
+</body>
+</html>

requirements.txt

@@ -0,0 +1 @@
+streamdiffusionv2