File size: 13,399 Bytes
d29eed7
ed519a7
d29eed7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ed519a7
 
 
d29eed7
 
ed519a7
 
 
 
d29eed7
 
ed519a7
d29eed7
ed519a7
 
 
 
 
 
 
 
 
 
 
 
 
d29eed7
 
ed519a7
 
 
d29eed7
ed519a7
d29eed7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2f3812
d29eed7
 
 
c6789f1
d29eed7
 
c6789f1
 
 
d29eed7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c6789f1
d29eed7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c6789f1
d29eed7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
"""VAE Rhythm / CaDet β€” AI-generated video detection demo (ZeroGPU).

Two detectors from the paper "VAE Rhythm: Temporal Decoding Traces for
AI-Generated Video Detection" (arXiv 2611.21114, Show Lab, NUS):

1. Phase scan (training-free, paper Sec. 3.1): re-encode the clip with the
   protocol (x264 CRF 23, GOP 32, short side capped at 720 px), reconstruct
   K=24 successive W=13-frame windows with a frozen causal Wan2.1 VAE and
   read the periodicity of the reconstruction-error residual. Generated
   videos are decoded in frame groups, so their error profile repeats with
   the group period.
2. CaDet (trained, paper Sec. 4.2): two parameter-efficient paths on a
   frozen DINOv3 ViT-L/16 read 8 frames per clip and output a sum of
   z-scores (about 0 for typical real videos, larger = more likely
   generated).

Interpretation rules (from the paper's repository): a high z(Ppk) β€” above
about 2.35, the 95th percentile of real videos β€” is evidence of VAE decoding;
a LOW value is NOT evidence that a video is real (low motion, frame-rate
conversion, upscaling and re-encoding weaken the rhythm). Neither score is a
definitive real/fake verdict on its own.
"""
import os

os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")

import spaces  # noqa: E402  (must precede any torch/CUDA import)

import shutil  # noqa: E402
import sys  # noqa: E402
import tempfile  # noqa: E402
import time  # noqa: E402

# --- ffmpeg bootstrap: the scan protocol shells out to ffmpeg/ffprobe -------------
if shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None:
    try:
        from static_ffmpeg import add_paths

        add_paths()
    except Exception:  # pragma: no cover - the Space image ships ffmpeg
        pass
for _tool in ("ffmpeg", "ffprobe"):
    _p = shutil.which(_tool)
    if _p:
        os.environ.setdefault(f"VAE_RHYTHM_{_tool.upper()}", _p)

import gradio as gr  # noqa: E402
import numpy as np  # noqa: E402
from huggingface_hub import snapshot_download  # noqa: E402

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

from vae_rhythm.scan import plot_result, scan_video  # noqa: E402
from vae_rhythm.vaes import VAE  # noqa: E402
from vae_rhythm.cadet.detector import CaDet  # noqa: E402

Z_REAL_P95 = 2.35  # 95th percentile of z(Ppk) over real videos

STAT_HEADERS = ["Ppk", "peak period (frames)", "z(Ppk)", "real percentile",
                "C4 (period-4 comb)", "C8 (period-8 comb)", "dE4 (lattice gain)",
                "scan size", "motion", "time (s)"]


def _fmt(x, nd=3):
    try:
        x = float(x)
        return f"{x:.{nd}f}" if np.isfinite(x) else "β€”"
    except (TypeError, ValueError):
        return "β€”"


# --- module-scope model loads -----------------------------------------------------
# ZeroGPU: at import time `.to("cuda")` is intercepted and tensors stay on CPU;
# they are packed to disk at startup and streamed into VRAM on each GPU call.
print("Loading Wan2.1 VAE (the scan's ruler)...", flush=True)
SCAN_VAE = VAE("wan21", device="cuda", dtype="bf16")

print("Loading CaDet (DINOv3 ViT-L/16 + two paths; builds 96 SVDs)...", flush=True)
CADET_DIR = snapshot_download("Jack97/CaDet", allow_patterns=["seed0/*"])
DET = CaDet(os.path.join(CADET_DIR, "seed0"), device="cuda")
print("Models loaded.", flush=True)


@spaces.GPU(duration=150)
def run_scan(video, cap=720, progress=gr.Progress()):
    """Run the training-free VAE Rhythm phase scan on a video.

    Re-encodes the clip with the paper's protocol (x264 CRF 23, GOP 32, short
    side capped at 720 px), reconstructs K=24 successive W=13-frame windows
    with a frozen Wan2.1 VAE and returns the residual-curve plot, the scan
    statistics and a plain-language reading of z(Ppk).
    """
    t0 = time.time()
    if video is None or not str(video):
        raise gr.Error("Upload a video (or pick an example) first.")
    res = scan_video(str(video), vae=SCAN_VAE, short_side_cap=int(cap))
    plot_path = os.path.join(tempfile.mkdtemp(prefix="scan_"), "scan.png")
    plot_result(res, plot_path)
    st = res.stats
    z, pct = res.z_ppk, res.real_percentile
    motion = float(res.segments[0].motion)
    cal_note = res.extra.get("calibration_note", "")

    if z is not None and z > Z_REAL_P95:
        reading = ("**Strong periodicity** β€” z(Ppk) is above the 95th percentile of real "
                   "videos. This is evidence that the clip was decoded by a causal video "
                   "VAE, a hallmark of AI generation.")
    elif z is not None and z > 0:
        reading = ("**Some periodicity** β€” above the real-video average but still inside "
                   "the real range. Inconclusive on its own; treat as a weak signal.")
    elif z is not None:
        reading = ("**No strong periodicity** β€” within the range typical of real videos. "
                   "This is *not* evidence that the video is real: low motion, frame-rate "
                   "conversion, upscaling and re-encoding all weaken the rhythm.")
    else:
        reading = f"**z(Ppk) not reported** β€” {cal_note}."
    if motion < 2.0:
        reading += "\n\n⚠️ Low motion energy (< 2): the rhythm trace is unreliable for this clip."
    reading += ("\n\n*A high score is evidence of VAE decoding; a low score is not proof a "
                "video is real. Never treat the scan alone as a definitive verdict.*")

    row = [_fmt(res.ppk), _fmt(st.get("ppk"), 2), _fmt(z, 2),
           f"{100 * pct:.0f}%" if pct is not None else "β€”",
           _fmt(st.get("C4")), _fmt(st.get("C8")), _fmt(st.get("dE4")),
           f"{res.scan_size[0]}Γ—{res.scan_size[1]}", _fmt(motion, 2),
           f"{time.time() - t0:.1f}"]
    return plot_path, [row], reading


@spaces.GPU(duration=60)
def run_detect(video, progress=gr.Progress()):
    """Score a video with the trained CaDet detector (sum of two z-scores).

    Returns the total score, the semantic / artifact z-score breakdown, the
    raw path scores and a plain-language reading. The score is standardised
    on real validation videos (about 0 for typical real clips, larger = more
    likely generated); it is not a probability.
    """
    t0 = time.time()
    if video is None or not str(video):
        raise gr.Error("Upload a video (or pick an example) first.")
    out = DET.score_video(str(video))
    score = float(out["score"])
    if score > 3:
        verdict = ("**Likely AI-generated** β€” the score is well above the range typical "
                   "of real videos.")
    elif score > 0:
        verdict = ("**Possibly AI-generated** β€” above the real-video average; scores this "
                   "high are uncommon for real clips but not decisive.")
    else:
        verdict = ("**In the real-video range** β€” typical of real clips, and not proof of "
                   "authenticity: the score is a z-sum, not a probability, and operating "
                   "points shift across real-video sources.")
    verdict += ("\n\nHeavily re-encoded, very short or very low-resolution clips are harder; "
                "do not use this score alone to make consequential decisions.")
    raw = f"semantic {_fmt(out['semantic'], 2)} / artifact {_fmt(out['artifact'], 2)}"
    return (_fmt(score, 2), _fmt(out["z_semantic"], 2), _fmt(out["z_artifact"], 2),
            raw, f"{time.time() - t0:.1f}", verdict)


# --- examples: VAE pairs built from CC0 clips ------------------------------------
# Each pair is a stock clip from the CC0 dataset linoyts/repo-to-space-example-videos
# (protocol-encoded) plus a copy of the SAME frames through one Wan2.1 VAE round
# trip, re-encoded with the same chain β€” the paper's negative-pair construction.
# Names are neutral on purpose: the pairs showcase the *contrast* a VAE round trip
# leaves in the traces; no ground-truth authenticity claim is made about the
# stock footage itself.
EX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples")
EXAMPLE_FILES = sorted(
    os.path.join(EX_DIR, f) for f in os.listdir(EX_DIR) if f.endswith(".mp4")
) if os.path.isdir(EX_DIR) else []

CSS = ".dark .gradio-container { color: var(--body-text-color); }"

with gr.Blocks(title="VAE Rhythm Β· AI-generated video detection") as demo:
    gr.Markdown(
        """
        # VAE Rhythm: Temporal Decoding Traces for AI-Generated Video Detection
        <p style="font-size: 95%; opacity: 0.85;">Show Lab Β· NUS &nbsp;|&nbsp;
        <a href="https://arxiv.org/abs/2611.21114">arXiv 2611.21114</a> &nbsp;|&nbsp;
        <a href="https://github.com/showlab/vae-rhythm">code</a> &nbsp;|&nbsp;
        <a href="https://showlab.github.io/vae-rhythm/">project page</a> &nbsp;|&nbsp;
        detector weights: <a href="https://huggingface.co/Jack97/CaDet">Jack97/CaDet</a></p>

        Causal video VAEs decode frames in **groups of 4**. The **phase scan** reconstructs
        successive windows of a clip with a frozen Wan2.1 VAE and measures whether the
        reconstruction-error profile repeats with that period β€” a training-free trace of
        VAE decoding. **CaDet** is a trained detector (frozen DINOv3 ViT-L/16 plus two
        parameter-efficient paths) that reads 8 frames per clip.
        """
    )
    with gr.Tabs():
        with gr.TabItem("Phase scan (training-free)"):
            gr.Markdown(
                "Upload a clip β€” at least 36 frames for the full K=24 / W=13 scan; shorter "
                "clips shrink the window (down to W=9). The clip is re-encoded with the "
                "paper's protocol (x264 CRF 23, GOP 32, ≀ 720 px short side) before scanning, "
                "so compression history is equalised."
            )
            with gr.Row():
                with gr.Column():
                    scan_in = gr.Video(label="Video", sources=["upload"])
                    cap_in = gr.Slider(256, 1088, value=720, step=32,
                                       label="Short-side cap (px)",
                                       info="The bundled real-video calibration matches 720 only.")
                    scan_btn = gr.Button("Run phase scan", variant="primary")
                with gr.Column():
                    scan_plot = gr.Image(label="Residual curve & spectrum", type="filepath",
                                         interactive=False)
            scan_md = gr.Markdown()
            scan_df = gr.Dataframe(headers=STAT_HEADERS, datatype=["str"] * len(STAT_HEADERS),
                                   interactive=False, label="Scan statistics")
            if EXAMPLE_FILES:
                gr.Examples(
                    examples=[[f] for f in EXAMPLE_FILES],
                    inputs=[scan_in],
                    fn=run_scan,
                    outputs=[scan_plot, scan_df, scan_md],
                    cache_examples=True,
                    cache_mode="lazy",
                    label="Examples β€” each stock clip and its Wan2.1 round-trip twin",
                )
        with gr.TabItem("CaDet (trained detector)"):
            gr.Markdown(
                "CaDet reads 8 frames (0, 4, …, 28) of the clip and returns a **sum of "
                "z-scores** standardised on real validation videos: β‰ˆ 0 for typical real "
                "clips, larger = more likely generated. **It is not a probability.**"
            )
            with gr.Row():
                with gr.Column():
                    det_in = gr.Video(label="Video", sources=["upload"])
                    det_btn = gr.Button("Score with CaDet", variant="primary")
                with gr.Column():
                    det_score = gr.Textbox(label="Score (z-sum)", interactive=False)
                    det_sem = gr.Textbox(label="z(semantic path)", interactive=False)
                    det_art = gr.Textbox(label="z(artifact path)", interactive=False)
                    det_raw = gr.Textbox(label="raw path scores", interactive=False)
                    det_time = gr.Textbox(label="time (s)", interactive=False, visible=False)
            det_md = gr.Markdown()
            if EXAMPLE_FILES:
                gr.Examples(
                    examples=[[f] for f in EXAMPLE_FILES],
                    inputs=[det_in],
                    fn=run_detect,
                    outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md],
                    cache_examples=True,
                    cache_mode="lazy",
                    label="Examples β€” each stock clip and its Wan2.1 round-trip twin",
                )

    gr.Markdown(
        """
        ---
        <p style="font-size: 88%; opacity: 0.75;">Research demo. A high score is evidence of
        VAE decoding, not a definitive verdict; a low score is not proof a video is real.
        Example clips are from the CC0 dataset
        <a href="https://huggingface.co/datasets/linoyts/repo-to-space-example-videos">linoyts/repo-to-space-example-videos</a>;
        round-trip twins were made with the paper's own VAE-pair recipe. Code is MIT-licensed
        (<a href="https://github.com/showlab/vae-rhythm">showlab/vae-rhythm</a>).</p>
        """
    )

    scan_btn.click(run_scan, inputs=[scan_in, cap_in],
                   outputs=[scan_plot, scan_df, scan_md])
    det_btn.click(run_detect, inputs=[det_in],
                  outputs=[det_score, det_sem, det_art, det_raw, det_time, det_md])

demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)