vae-rhythm / app.py
multimodalart's picture
multimodalart HF Staff
Examples: neutral pair names, honest provenance framing
c6789f1 verified
Raw History Blame Contribute Delete
13.4 kB
"""VAE Rhythm / CaDet β€” AI-generated video detection demo (ZeroGPU).
Two detectors from the paper "VAE Rhythm: Temporal Decoding Traces for
AI-Generated Video Detection" (arXiv 2611.21114, Show Lab, NUS):
1. Phase scan (training-free, paper Sec. 3.1): re-encode the clip with the
protocol (x264 CRF 23, GOP 32, short side capped at 720 px), reconstruct
K=24 successive W=13-frame windows with a frozen causal Wan2.1 VAE and
read the periodicity of the reconstruction-error residual. Generated
videos are decoded in frame groups, so their error profile repeats with
the group period.
2. CaDet (trained, paper Sec. 4.2): two parameter-efficient paths on a
frozen DINOv3 ViT-L/16 read 8 frames per clip and output a sum of
z-scores (about 0 for typical real videos, larger = more likely
generated).
Interpretation rules (from the paper's repository): a high z(Ppk) β€” above
about 2.35, the 95th percentile of real videos β€” is evidence of VAE decoding;
a LOW value is NOT evidence that a video is real (low motion, frame-rate
conversion, upscaling and re-encoding weaken the rhythm). Neither score is a
definitive real/fake verdict on its own.
"""
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede any torch/CUDA import)
import shutil # noqa: E402
import sys # noqa: E402
import tempfile # noqa: E402
import time # noqa: E402
# --- ffmpeg bootstrap: the scan protocol shells out to ffmpeg/ffprobe -------------
if shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None:
try:
from static_ffmpeg import add_paths
add_paths()
except Exception: # pragma: no cover - the Space image ships ffmpeg
pass
for _tool in ("ffmpeg", "ffprobe"):
_p = shutil.which(_tool)
if _p:
os.environ.setdefault(f"VAE_RHYTHM_{_tool.upper()}", _p)
import gradio as gr # noqa: E402
import numpy as np # noqa: E402
from huggingface_hub import snapshot_download # noqa: E402
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from vae_rhythm.scan import plot_result, scan_video # noqa: E402
from vae_rhythm.vaes import VAE # noqa: E402
from vae_rhythm.cadet.detector import CaDet # noqa: E402
Z_REAL_P95 = 2.35 # 95th percentile of z(Ppk) over real videos
STAT_HEADERS = ["Ppk", "peak period (frames)", "z(Ppk)", "real percentile",
"C4 (period-4 comb)", "C8 (period-8 comb)", "dE4 (lattice gain)",
"scan size", "motion", "time (s)"]
def _fmt(x, nd=3):
try:
x = float(x)
return f"{x:.{nd}f}" if np.isfinite(x) else "β€”"
except (TypeError, ValueError):
return "β€”"
# --- module-scope model loads -----------------------------------------------------
# ZeroGPU: at import time `.to("cuda")` is intercepted and tensors stay on CPU;
# they are packed to disk at startup and streamed into VRAM on each GPU call.
print("Loading Wan2.1 VAE (the scan's ruler)...", flush=True)
SCAN_VAE = VAE("wan21", device="cuda", dtype="bf16")
print("Loading CaDet (DINOv3 ViT-L/16 + two paths; builds 96 SVDs)...", flush=True)
CADET_DIR = snapshot_download("Jack97/CaDet", allow_patterns=["seed0/*"])
DET = CaDet(os.path.join(CADET_DIR, "seed0"), device="cuda")
print("Models loaded.", flush=True)
@spaces.GPU(duration=150)
def run_scan(video, cap=720, progress=gr.Progress()):
"""Run the training-free VAE Rhythm phase scan on a video.
Re-encodes the clip with the paper's protocol (x264 CRF 23, GOP 32, short
side capped at 720 px), reconstructs K=24 successive W=13-frame windows
with a frozen Wan2.1 VAE and returns the residual-curve plot, the scan
statistics and a plain-language reading of z(Ppk).
"""
t0 = time.time()
if video is None or not str(video):
raise gr.Error("Upload a video (or pick an example) first.")
res = scan_video(str(video), vae=SCAN_VAE, short_side_cap=int(cap))
plot_path = os.path.join(tempfile.mkdtemp(prefix="scan_"), "scan.png")
plot_result(res, plot_path)
st = res.stats
z, pct = res.z_ppk, res.real_percentile
motion = float(res.segments[0].motion)
cal_note = res.extra.get("calibration_note", "")
if z is not None and z > Z_REAL_P95:
reading = ("**Strong periodicity** β€” z(Ppk) is above the 95th percentile of real "
"videos. This is evidence that the clip was decoded by a causal video "
"VAE, a hallmark of AI generation.")
elif z is not None and z > 0:
reading = ("**Some periodicity** β€” above the real-video average but still inside "
"the real range. Inconclusive on its own; treat as a weak signal.")
elif z is not None:
reading = ("**No strong periodicity** β€” within the range typical of real videos. "
"This is *not* evidence that the video is real: low motion, frame-rate "
"conversion, upscaling and re-encoding all weaken the rhythm.")
else:
reading = f"**z(Ppk) not reported** β€” {cal_note}."
if motion < 2.0:
reading += "\n\n⚠️ Low motion energy (< 2): the rhythm trace is unreliable for this clip."
reading += ("\n\n*A high score is evidence of VAE decoding; a low score is not proof a "
"video is real. Never treat the scan alone as a definitive verdict.*")
row = [_fmt(res.ppk), _fmt(st.get("ppk"), 2), _fmt(z, 2),
f"{100 * pct:.0f}%" if pct is not None else "β€”",
_fmt(st.get("C4")), _fmt(st.get("C8")), _fmt(st.get("dE4")),
f"{res.scan_size[0]}Γ—{res.scan_size[1]}", _fmt(motion, 2),
f"{time.time() - t0:.1f}"]
return plot_path, [row], reading
@spaces.GPU(duration=60)
def run_detect(video, progress=gr.Progress()):
"""Score a video with the trained CaDet detector (sum of two z-scores).
Returns the total score, the semantic / artifact z-score breakdown, the
raw path scores and a plain-language reading. The score is standardised
on real validation videos (about 0 for typical real clips, larger = more
likely generated); it is not a probability.
"""
t0 = time.time()
if video is None or not str(video):
raise gr.Error("Upload a video (or pick an example) first.")
out = DET.score_video(str(video))
score = float(out["score"])
if score > 3:
verdict = ("**Likely AI-generated** β€” the score is well above the range typical "
"of real videos.")
elif score > 0:
verdict = ("**Possibly AI-generated** β€” above the real-video average; scores this "
"high are uncommon for real clips but not decisive.")
else:
verdict = ("**In the real-video range** β€” typical of real clips, and not proof of "
"authenticity: the score is a z-sum, not a probability, and operating "
"points shift across real-video sources.")
verdict += ("\n\nHeavily re-encoded, very short or very low-resolution clips are harder; "
"do not use this score alone to make consequential decisions.")
raw = f"semantic {_fmt(out['semantic'], 2)} / artifact {_fmt(out['artifact'], 2)}"
return (_fmt(score, 2), _fmt(out["z_semantic"], 2), _fmt(out["z_artifact"], 2),
raw, f"{time.time() - t0:.1f}", verdict)
# --- examples: VAE pairs built from CC0 clips ------------------------------------
# Each pair is a stock clip from the CC0 dataset linoyts/repo-to-space-example-videos
# (protocol-encoded) plus a copy of the SAME frames through one Wan2.1 VAE round
# trip, re-encoded with the same chain β€” the paper's negative-pair construction.
# Names are neutral on purpose: the pairs showcase the *contrast* a VAE round trip
# leaves in the traces; no ground-truth authenticity claim is made about the
# stock footage itself.
EX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples")
EXAMPLE_FILES = sorted(
os.path.join(EX_DIR, f) for f in os.listdir(EX_DIR) if f.endswith(".mp4")
) if os.path.isdir(EX_DIR) else []
CSS = ".dark .gradio-container { color: var(--body-text-color); }"
with gr.Blocks(title="VAE Rhythm Β· AI-generated video detection") as demo:
gr.Markdown(
"""
# VAE Rhythm: Temporal Decoding Traces for AI-Generated Video Detection
<p style="font-size: 95%; opacity: 0.85;">Show Lab Β· NUS &nbsp;|&nbsp;
<a href="https://arxiv.org/abs/2611.21114">arXiv 2611.21114</a> &nbsp;|&nbsp;
<a href="https://github.com/showlab/vae-rhythm">code</a> &nbsp;|&nbsp;
<a href="https://showlab.github.io/vae-rhythm/">project page</a> &nbsp;|&nbsp;
detector weights: <a href="https://huggingface.co/Jack97/CaDet">Jack97/CaDet</a></p>
Causal video VAEs decode frames in **groups of 4**. The **phase scan** reconstructs
successive windows of a clip with a frozen Wan2.1 VAE and measures whether the
reconstruction-error profile repeats with that period β€” a training-free trace of
VAE decoding. **CaDet** is a trained detector (frozen DINOv3 ViT-L/16 plus two
parameter-efficient paths) that reads 8 frames per clip.
"""
)
with gr.Tabs():
with gr.TabItem("Phase scan (training-free)"):
gr.Markdown(
"Upload a clip β€” at least 36 frames for the full K=24 / W=13 scan; shorter "
"clips shrink the window (down to W=9). The clip is re-encoded with the "
"paper's protocol (x264 CRF 23, GOP 32, ≀ 720 px short side) before scanning, "
"so compression history is equalised."
)
with gr.Row():
with gr.Column():
scan_in = gr.Video(label="Video", sources=["upload"])
cap_in = gr.Slider(256, 1088, value=720, step=32,
label="Short-side cap (px)",
info="The bundled real-video calibration matches 720 only.")
scan_btn = gr.Button("Run phase scan", variant="primary")
with gr.Column():
scan_plot = gr.Image(label="Residual curve & spectrum", type="filepath",
interactive=False)
scan_md = gr.Markdown()
scan_df = gr.Dataframe(headers=STAT_HEADERS, datatype=["str"] * len(STAT_HEADERS),
interactive=False, label="Scan statistics")
if EXAMPLE_FILES:
gr.Examples(
examples=[[f] for f in EXAMPLE_FILES],
inputs=[scan_in],
fn=run_scan,
outputs=[scan_plot, scan_df, scan_md],
cache_examples=True,
cache_mode="lazy",
label="Examples β€” each stock clip and its Wan2.1 round-trip twin",
)
with gr.TabItem("CaDet (trained detector)"):
gr.Markdown(
"CaDet reads 8 frames (0, 4, …, 28) of the clip and returns a **sum of "
"z-scores** standardised on real validation videos: β‰ˆ 0 for typical real "
"clips, larger = more likely generated. **It is not a probability.**"
)
with gr.Row():
with gr.Column():
det_in = gr.Video(label="Video", sources=["upload"])
det_btn = gr.Button("Score with CaDet", variant="primary")
with gr.Column():
det_score = gr.Textbox(label="Score (z-sum)", interactive=False)
det_sem = gr.Textbox(label="z(semantic path)", interactive=False)
det_art = gr.Textbox(label="z(artifact path)", interactive=False)
det_raw = gr.Textbox(label="raw path scores", interactive=False)
det_time = gr.Textbox(label="time (s)", interactive=False, visible=False)
det_md = gr.Markdown()
if EXAMPLE_FILES:
gr.Examples(
examples=[[f] for f in EXAMPLE_FILES],
inputs=[det_in],
fn=run_detect,
outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md],
cache_examples=True,
cache_mode="lazy",
label="Examples β€” each stock clip and its Wan2.1 round-trip twin",
)
gr.Markdown(
"""
---
<p style="font-size: 88%; opacity: 0.75;">Research demo. A high score is evidence of
VAE decoding, not a definitive verdict; a low score is not proof a video is real.
Example clips are from the CC0 dataset
<a href="https://huggingface.co/datasets/linoyts/repo-to-space-example-videos">linoyts/repo-to-space-example-videos</a>;
round-trip twins were made with the paper's own VAE-pair recipe. Code is MIT-licensed
(<a href="https://github.com/showlab/vae-rhythm">showlab/vae-rhythm</a>).</p>
"""
)
scan_btn.click(run_scan, inputs=[scan_in, cap_in],
outputs=[scan_plot, scan_df, scan_md])
det_btn.click(run_detect, inputs=[det_in],
outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md])
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)