Spaces:
Running on Zero
Running on Zero
File size: 13,399 Bytes
d29eed7 ed519a7 d29eed7 ed519a7 d29eed7 ed519a7 d29eed7 ed519a7 d29eed7 ed519a7 d29eed7 ed519a7 d29eed7 ed519a7 d29eed7 b2f3812 d29eed7 c6789f1 d29eed7 c6789f1 d29eed7 c6789f1 d29eed7 c6789f1 d29eed7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 | """VAE Rhythm / CaDet β AI-generated video detection demo (ZeroGPU).
Two detectors from the paper "VAE Rhythm: Temporal Decoding Traces for
AI-Generated Video Detection" (arXiv 2611.21114, Show Lab, NUS):
1. Phase scan (training-free, paper Sec. 3.1): re-encode the clip with the
protocol (x264 CRF 23, GOP 32, short side capped at 720 px), reconstruct
K=24 successive W=13-frame windows with a frozen causal Wan2.1 VAE and
read the periodicity of the reconstruction-error residual. Generated
videos are decoded in frame groups, so their error profile repeats with
the group period.
2. CaDet (trained, paper Sec. 4.2): two parameter-efficient paths on a
frozen DINOv3 ViT-L/16 read 8 frames per clip and output a sum of
z-scores (about 0 for typical real videos, larger = more likely
generated).
Interpretation rules (from the paper's repository): a high z(Ppk) β above
about 2.35, the 95th percentile of real videos β is evidence of VAE decoding;
a LOW value is NOT evidence that a video is real (low motion, frame-rate
conversion, upscaling and re-encoding weaken the rhythm). Neither score is a
definitive real/fake verdict on its own.
"""
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede any torch/CUDA import)
import shutil # noqa: E402
import sys # noqa: E402
import tempfile # noqa: E402
import time # noqa: E402
# --- ffmpeg bootstrap: the scan protocol shells out to ffmpeg/ffprobe -------------
if shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None:
try:
from static_ffmpeg import add_paths
add_paths()
except Exception: # pragma: no cover - the Space image ships ffmpeg
pass
for _tool in ("ffmpeg", "ffprobe"):
_p = shutil.which(_tool)
if _p:
os.environ.setdefault(f"VAE_RHYTHM_{_tool.upper()}", _p)
import gradio as gr # noqa: E402
import numpy as np # noqa: E402
from huggingface_hub import snapshot_download # noqa: E402
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from vae_rhythm.scan import plot_result, scan_video # noqa: E402
from vae_rhythm.vaes import VAE # noqa: E402
from vae_rhythm.cadet.detector import CaDet # noqa: E402
Z_REAL_P95 = 2.35 # 95th percentile of z(Ppk) over real videos
STAT_HEADERS = ["Ppk", "peak period (frames)", "z(Ppk)", "real percentile",
"C4 (period-4 comb)", "C8 (period-8 comb)", "dE4 (lattice gain)",
"scan size", "motion", "time (s)"]
def _fmt(x, nd=3):
try:
x = float(x)
return f"{x:.{nd}f}" if np.isfinite(x) else "β"
except (TypeError, ValueError):
return "β"
# --- module-scope model loads -----------------------------------------------------
# ZeroGPU: at import time `.to("cuda")` is intercepted and tensors stay on CPU;
# they are packed to disk at startup and streamed into VRAM on each GPU call.
print("Loading Wan2.1 VAE (the scan's ruler)...", flush=True)
SCAN_VAE = VAE("wan21", device="cuda", dtype="bf16")
print("Loading CaDet (DINOv3 ViT-L/16 + two paths; builds 96 SVDs)...", flush=True)
CADET_DIR = snapshot_download("Jack97/CaDet", allow_patterns=["seed0/*"])
DET = CaDet(os.path.join(CADET_DIR, "seed0"), device="cuda")
print("Models loaded.", flush=True)
@spaces.GPU(duration=150)
def run_scan(video, cap=720, progress=gr.Progress()):
"""Run the training-free VAE Rhythm phase scan on a video.
Re-encodes the clip with the paper's protocol (x264 CRF 23, GOP 32, short
side capped at 720 px), reconstructs K=24 successive W=13-frame windows
with a frozen Wan2.1 VAE and returns the residual-curve plot, the scan
statistics and a plain-language reading of z(Ppk).
"""
t0 = time.time()
if video is None or not str(video):
raise gr.Error("Upload a video (or pick an example) first.")
res = scan_video(str(video), vae=SCAN_VAE, short_side_cap=int(cap))
plot_path = os.path.join(tempfile.mkdtemp(prefix="scan_"), "scan.png")
plot_result(res, plot_path)
st = res.stats
z, pct = res.z_ppk, res.real_percentile
motion = float(res.segments[0].motion)
cal_note = res.extra.get("calibration_note", "")
if z is not None and z > Z_REAL_P95:
reading = ("**Strong periodicity** β z(Ppk) is above the 95th percentile of real "
"videos. This is evidence that the clip was decoded by a causal video "
"VAE, a hallmark of AI generation.")
elif z is not None and z > 0:
reading = ("**Some periodicity** β above the real-video average but still inside "
"the real range. Inconclusive on its own; treat as a weak signal.")
elif z is not None:
reading = ("**No strong periodicity** β within the range typical of real videos. "
"This is *not* evidence that the video is real: low motion, frame-rate "
"conversion, upscaling and re-encoding all weaken the rhythm.")
else:
reading = f"**z(Ppk) not reported** β {cal_note}."
if motion < 2.0:
reading += "\n\nβ οΈ Low motion energy (< 2): the rhythm trace is unreliable for this clip."
reading += ("\n\n*A high score is evidence of VAE decoding; a low score is not proof a "
"video is real. Never treat the scan alone as a definitive verdict.*")
row = [_fmt(res.ppk), _fmt(st.get("ppk"), 2), _fmt(z, 2),
f"{100 * pct:.0f}%" if pct is not None else "β",
_fmt(st.get("C4")), _fmt(st.get("C8")), _fmt(st.get("dE4")),
f"{res.scan_size[0]}Γ{res.scan_size[1]}", _fmt(motion, 2),
f"{time.time() - t0:.1f}"]
return plot_path, [row], reading
@spaces.GPU(duration=60)
def run_detect(video, progress=gr.Progress()):
"""Score a video with the trained CaDet detector (sum of two z-scores).
Returns the total score, the semantic / artifact z-score breakdown, the
raw path scores and a plain-language reading. The score is standardised
on real validation videos (about 0 for typical real clips, larger = more
likely generated); it is not a probability.
"""
t0 = time.time()
if video is None or not str(video):
raise gr.Error("Upload a video (or pick an example) first.")
out = DET.score_video(str(video))
score = float(out["score"])
if score > 3:
verdict = ("**Likely AI-generated** β the score is well above the range typical "
"of real videos.")
elif score > 0:
verdict = ("**Possibly AI-generated** β above the real-video average; scores this "
"high are uncommon for real clips but not decisive.")
else:
verdict = ("**In the real-video range** β typical of real clips, and not proof of "
"authenticity: the score is a z-sum, not a probability, and operating "
"points shift across real-video sources.")
verdict += ("\n\nHeavily re-encoded, very short or very low-resolution clips are harder; "
"do not use this score alone to make consequential decisions.")
raw = f"semantic {_fmt(out['semantic'], 2)} / artifact {_fmt(out['artifact'], 2)}"
return (_fmt(score, 2), _fmt(out["z_semantic"], 2), _fmt(out["z_artifact"], 2),
raw, f"{time.time() - t0:.1f}", verdict)
# --- examples: VAE pairs built from CC0 clips ------------------------------------
# Each pair is a stock clip from the CC0 dataset linoyts/repo-to-space-example-videos
# (protocol-encoded) plus a copy of the SAME frames through one Wan2.1 VAE round
# trip, re-encoded with the same chain β the paper's negative-pair construction.
# Names are neutral on purpose: the pairs showcase the *contrast* a VAE round trip
# leaves in the traces; no ground-truth authenticity claim is made about the
# stock footage itself.
EX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples")
EXAMPLE_FILES = sorted(
os.path.join(EX_DIR, f) for f in os.listdir(EX_DIR) if f.endswith(".mp4")
) if os.path.isdir(EX_DIR) else []
CSS = ".dark .gradio-container { color: var(--body-text-color); }"
with gr.Blocks(title="VAE Rhythm Β· AI-generated video detection") as demo:
gr.Markdown(
"""
# VAE Rhythm: Temporal Decoding Traces for AI-Generated Video Detection
<p style="font-size: 95%; opacity: 0.85;">Show Lab Β· NUS |
<a href="https://arxiv.org/abs/2611.21114">arXiv 2611.21114</a> |
<a href="https://github.com/showlab/vae-rhythm">code</a> |
<a href="https://showlab.github.io/vae-rhythm/">project page</a> |
detector weights: <a href="https://huggingface.co/Jack97/CaDet">Jack97/CaDet</a></p>
Causal video VAEs decode frames in **groups of 4**. The **phase scan** reconstructs
successive windows of a clip with a frozen Wan2.1 VAE and measures whether the
reconstruction-error profile repeats with that period β a training-free trace of
VAE decoding. **CaDet** is a trained detector (frozen DINOv3 ViT-L/16 plus two
parameter-efficient paths) that reads 8 frames per clip.
"""
)
with gr.Tabs():
with gr.TabItem("Phase scan (training-free)"):
gr.Markdown(
"Upload a clip β at least 36 frames for the full K=24 / W=13 scan; shorter "
"clips shrink the window (down to W=9). The clip is re-encoded with the "
"paper's protocol (x264 CRF 23, GOP 32, β€ 720 px short side) before scanning, "
"so compression history is equalised."
)
with gr.Row():
with gr.Column():
scan_in = gr.Video(label="Video", sources=["upload"])
cap_in = gr.Slider(256, 1088, value=720, step=32,
label="Short-side cap (px)",
info="The bundled real-video calibration matches 720 only.")
scan_btn = gr.Button("Run phase scan", variant="primary")
with gr.Column():
scan_plot = gr.Image(label="Residual curve & spectrum", type="filepath",
interactive=False)
scan_md = gr.Markdown()
scan_df = gr.Dataframe(headers=STAT_HEADERS, datatype=["str"] * len(STAT_HEADERS),
interactive=False, label="Scan statistics")
if EXAMPLE_FILES:
gr.Examples(
examples=[[f] for f in EXAMPLE_FILES],
inputs=[scan_in],
fn=run_scan,
outputs=[scan_plot, scan_df, scan_md],
cache_examples=True,
cache_mode="lazy",
label="Examples β each stock clip and its Wan2.1 round-trip twin",
)
with gr.TabItem("CaDet (trained detector)"):
gr.Markdown(
"CaDet reads 8 frames (0, 4, β¦, 28) of the clip and returns a **sum of "
"z-scores** standardised on real validation videos: β 0 for typical real "
"clips, larger = more likely generated. **It is not a probability.**"
)
with gr.Row():
with gr.Column():
det_in = gr.Video(label="Video", sources=["upload"])
det_btn = gr.Button("Score with CaDet", variant="primary")
with gr.Column():
det_score = gr.Textbox(label="Score (z-sum)", interactive=False)
det_sem = gr.Textbox(label="z(semantic path)", interactive=False)
det_art = gr.Textbox(label="z(artifact path)", interactive=False)
det_raw = gr.Textbox(label="raw path scores", interactive=False)
det_time = gr.Textbox(label="time (s)", interactive=False, visible=False)
det_md = gr.Markdown()
if EXAMPLE_FILES:
gr.Examples(
examples=[[f] for f in EXAMPLE_FILES],
inputs=[det_in],
fn=run_detect,
outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md],
cache_examples=True,
cache_mode="lazy",
label="Examples β each stock clip and its Wan2.1 round-trip twin",
)
gr.Markdown(
"""
---
<p style="font-size: 88%; opacity: 0.75;">Research demo. A high score is evidence of
VAE decoding, not a definitive verdict; a low score is not proof a video is real.
Example clips are from the CC0 dataset
<a href="https://huggingface.co/datasets/linoyts/repo-to-space-example-videos">linoyts/repo-to-space-example-videos</a>;
round-trip twins were made with the paper's own VAE-pair recipe. Code is MIT-licensed
(<a href="https://github.com/showlab/vae-rhythm">showlab/vae-rhythm</a>).</p>
"""
)
scan_btn.click(run_scan, inputs=[scan_in, cap_in],
outputs=[scan_plot, scan_df, scan_md])
det_btn.click(run_detect, inputs=[det_in],
outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md])
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)
|