"""VAE Rhythm / CaDet — AI-generated video detection demo (ZeroGPU). Two detectors from the paper "VAE Rhythm: Temporal Decoding Traces for AI-Generated Video Detection" (arXiv 2611.21114, Show Lab, NUS): 1. Phase scan (training-free, paper Sec. 3.1): re-encode the clip with the protocol (x264 CRF 23, GOP 32, short side capped at 720 px), reconstruct K=24 successive W=13-frame windows with a frozen causal Wan2.1 VAE and read the periodicity of the reconstruction-error residual. Generated videos are decoded in frame groups, so their error profile repeats with the group period. 2. CaDet (trained, paper Sec. 4.2): two parameter-efficient paths on a frozen DINOv3 ViT-L/16 read 8 frames per clip and output a sum of z-scores (about 0 for typical real videos, larger = more likely generated). Interpretation rules (from the paper's repository): a high z(Ppk) — above about 2.35, the 95th percentile of real videos — is evidence of VAE decoding; a LOW value is NOT evidence that a video is real (low motion, frame-rate conversion, upscaling and re-encoding weaken the rhythm). Neither score is a definitive real/fake verdict on its own. """ import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must precede any torch/CUDA import) import shutil # noqa: E402 import sys # noqa: E402 import tempfile # noqa: E402 import time # noqa: E402 # --- ffmpeg bootstrap: the scan protocol shells out to ffmpeg/ffprobe ------------- if shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None: try: from static_ffmpeg import add_paths add_paths() except Exception: # pragma: no cover - the Space image ships ffmpeg pass for _tool in ("ffmpeg", "ffprobe"): _p = shutil.which(_tool) if _p: os.environ.setdefault(f"VAE_RHYTHM_{_tool.upper()}", _p) import gradio as gr # noqa: E402 import numpy as np # noqa: E402 from huggingface_hub import snapshot_download # noqa: E402 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from vae_rhythm.scan import plot_result, scan_video # noqa: E402 from vae_rhythm.vaes import VAE # noqa: E402 from vae_rhythm.cadet.detector import CaDet # noqa: E402 Z_REAL_P95 = 2.35 # 95th percentile of z(Ppk) over real videos STAT_HEADERS = ["Ppk", "peak period (frames)", "z(Ppk)", "real percentile", "C4 (period-4 comb)", "C8 (period-8 comb)", "dE4 (lattice gain)", "scan size", "motion", "time (s)"] def _fmt(x, nd=3): try: x = float(x) return f"{x:.{nd}f}" if np.isfinite(x) else "—" except (TypeError, ValueError): return "—" # --- module-scope model loads ----------------------------------------------------- # ZeroGPU: at import time `.to("cuda")` is intercepted and tensors stay on CPU; # they are packed to disk at startup and streamed into VRAM on each GPU call. print("Loading Wan2.1 VAE (the scan's ruler)...", flush=True) SCAN_VAE = VAE("wan21", device="cuda", dtype="bf16") print("Loading CaDet (DINOv3 ViT-L/16 + two paths; builds 96 SVDs)...", flush=True) CADET_DIR = snapshot_download("Jack97/CaDet", allow_patterns=["seed0/*"]) DET = CaDet(os.path.join(CADET_DIR, "seed0"), device="cuda") print("Models loaded.", flush=True) @spaces.GPU(duration=150) def run_scan(video, cap=720, progress=gr.Progress()): """Run the training-free VAE Rhythm phase scan on a video. Re-encodes the clip with the paper's protocol (x264 CRF 23, GOP 32, short side capped at 720 px), reconstructs K=24 successive W=13-frame windows with a frozen Wan2.1 VAE and returns the residual-curve plot, the scan statistics and a plain-language reading of z(Ppk). """ t0 = time.time() if video is None or not str(video): raise gr.Error("Upload a video (or pick an example) first.") res = scan_video(str(video), vae=SCAN_VAE, short_side_cap=int(cap)) plot_path = os.path.join(tempfile.mkdtemp(prefix="scan_"), "scan.png") plot_result(res, plot_path) st = res.stats z, pct = res.z_ppk, res.real_percentile motion = float(res.segments[0].motion) cal_note = res.extra.get("calibration_note", "") if z is not None and z > Z_REAL_P95: reading = ("**Strong periodicity** — z(Ppk) is above the 95th percentile of real " "videos. This is evidence that the clip was decoded by a causal video " "VAE, a hallmark of AI generation.") elif z is not None and z > 0: reading = ("**Some periodicity** — above the real-video average but still inside " "the real range. Inconclusive on its own; treat as a weak signal.") elif z is not None: reading = ("**No strong periodicity** — within the range typical of real videos. " "This is *not* evidence that the video is real: low motion, frame-rate " "conversion, upscaling and re-encoding all weaken the rhythm.") else: reading = f"**z(Ppk) not reported** — {cal_note}." if motion < 2.0: reading += "\n\n⚠️ Low motion energy (< 2): the rhythm trace is unreliable for this clip." reading += ("\n\n*A high score is evidence of VAE decoding; a low score is not proof a " "video is real. Never treat the scan alone as a definitive verdict.*") row = [_fmt(res.ppk), _fmt(st.get("ppk"), 2), _fmt(z, 2), f"{100 * pct:.0f}%" if pct is not None else "—", _fmt(st.get("C4")), _fmt(st.get("C8")), _fmt(st.get("dE4")), f"{res.scan_size[0]}×{res.scan_size[1]}", _fmt(motion, 2), f"{time.time() - t0:.1f}"] return plot_path, [row], reading @spaces.GPU(duration=60) def run_detect(video, progress=gr.Progress()): """Score a video with the trained CaDet detector (sum of two z-scores). Returns the total score, the semantic / artifact z-score breakdown, the raw path scores and a plain-language reading. The score is standardised on real validation videos (about 0 for typical real clips, larger = more likely generated); it is not a probability. """ t0 = time.time() if video is None or not str(video): raise gr.Error("Upload a video (or pick an example) first.") out = DET.score_video(str(video)) score = float(out["score"]) if score > 3: verdict = ("**Likely AI-generated** — the score is well above the range typical " "of real videos.") elif score > 0: verdict = ("**Possibly AI-generated** — above the real-video average; scores this " "high are uncommon for real clips but not decisive.") else: verdict = ("**In the real-video range** — typical of real clips, and not proof of " "authenticity: the score is a z-sum, not a probability, and operating " "points shift across real-video sources.") verdict += ("\n\nHeavily re-encoded, very short or very low-resolution clips are harder; " "do not use this score alone to make consequential decisions.") raw = f"semantic {_fmt(out['semantic'], 2)} / artifact {_fmt(out['artifact'], 2)}" return (_fmt(score, 2), _fmt(out["z_semantic"], 2), _fmt(out["z_artifact"], 2), raw, f"{time.time() - t0:.1f}", verdict) # --- examples: VAE pairs built from CC0 clips ------------------------------------ # Each pair is a stock clip from the CC0 dataset linoyts/repo-to-space-example-videos # (protocol-encoded) plus a copy of the SAME frames through one Wan2.1 VAE round # trip, re-encoded with the same chain — the paper's negative-pair construction. # Names are neutral on purpose: the pairs showcase the *contrast* a VAE round trip # leaves in the traces; no ground-truth authenticity claim is made about the # stock footage itself. EX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples") EXAMPLE_FILES = sorted( os.path.join(EX_DIR, f) for f in os.listdir(EX_DIR) if f.endswith(".mp4") ) if os.path.isdir(EX_DIR) else [] CSS = ".dark .gradio-container { color: var(--body-text-color); }" with gr.Blocks(title="VAE Rhythm · AI-generated video detection") as demo: gr.Markdown( """ # VAE Rhythm: Temporal Decoding Traces for AI-Generated Video Detection

Show Lab · NUS  |  arXiv 2611.21114  |  code  |  project page  |  detector weights: Jack97/CaDet

Causal video VAEs decode frames in **groups of 4**. The **phase scan** reconstructs successive windows of a clip with a frozen Wan2.1 VAE and measures whether the reconstruction-error profile repeats with that period — a training-free trace of VAE decoding. **CaDet** is a trained detector (frozen DINOv3 ViT-L/16 plus two parameter-efficient paths) that reads 8 frames per clip. """ ) with gr.Tabs(): with gr.TabItem("Phase scan (training-free)"): gr.Markdown( "Upload a clip — at least 36 frames for the full K=24 / W=13 scan; shorter " "clips shrink the window (down to W=9). The clip is re-encoded with the " "paper's protocol (x264 CRF 23, GOP 32, ≤ 720 px short side) before scanning, " "so compression history is equalised." ) with gr.Row(): with gr.Column(): scan_in = gr.Video(label="Video", sources=["upload"]) cap_in = gr.Slider(256, 1088, value=720, step=32, label="Short-side cap (px)", info="The bundled real-video calibration matches 720 only.") scan_btn = gr.Button("Run phase scan", variant="primary") with gr.Column(): scan_plot = gr.Image(label="Residual curve & spectrum", type="filepath", interactive=False) scan_md = gr.Markdown() scan_df = gr.Dataframe(headers=STAT_HEADERS, datatype=["str"] * len(STAT_HEADERS), interactive=False, label="Scan statistics") if EXAMPLE_FILES: gr.Examples( examples=[[f] for f in EXAMPLE_FILES], inputs=[scan_in], fn=run_scan, outputs=[scan_plot, scan_df, scan_md], cache_examples=True, cache_mode="lazy", label="Examples — each stock clip and its Wan2.1 round-trip twin", ) with gr.TabItem("CaDet (trained detector)"): gr.Markdown( "CaDet reads 8 frames (0, 4, …, 28) of the clip and returns a **sum of " "z-scores** standardised on real validation videos: ≈ 0 for typical real " "clips, larger = more likely generated. **It is not a probability.**" ) with gr.Row(): with gr.Column(): det_in = gr.Video(label="Video", sources=["upload"]) det_btn = gr.Button("Score with CaDet", variant="primary") with gr.Column(): det_score = gr.Textbox(label="Score (z-sum)", interactive=False) det_sem = gr.Textbox(label="z(semantic path)", interactive=False) det_art = gr.Textbox(label="z(artifact path)", interactive=False) det_raw = gr.Textbox(label="raw path scores", interactive=False) det_time = gr.Textbox(label="time (s)", interactive=False, visible=False) det_md = gr.Markdown() if EXAMPLE_FILES: gr.Examples( examples=[[f] for f in EXAMPLE_FILES], inputs=[det_in], fn=run_detect, outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md], cache_examples=True, cache_mode="lazy", label="Examples — each stock clip and its Wan2.1 round-trip twin", ) gr.Markdown( """ ---

Research demo. A high score is evidence of VAE decoding, not a definitive verdict; a low score is not proof a video is real. Example clips are from the CC0 dataset linoyts/repo-to-space-example-videos; round-trip twins were made with the paper's own VAE-pair recipe. Code is MIT-licensed (showlab/vae-rhythm).

""" ) scan_btn.click(run_scan, inputs=[scan_in, cap_in], outputs=[scan_plot, scan_df, scan_md]) det_btn.click(run_detect, inputs=[det_in], outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md]) demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)