File size: 19,365 Bytes
ed519a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
"""Command line interface: ``vae-rhythm <command> ...`` (or ``python -m vae_rhythm ...``).

Commands
  doctor          check ffmpeg, Python packages and the GPU
  selftest        GPU check without data (synthetic clip vs. its Wan2.1 round trip)
  scan            phase scan of one or more videos (zero-shot rhythm score, curve plot)
  scan-manifest   phase scan of every video in a manifest CSV -> statistics CSV (+ NPZ tables)
  make-pairs      VAE-pair engine: real videos -> (real, VAE round trip) pairs
  cache-frames    16-frame JPEG cache used by CaDet training and evaluation
  train-artifact  train the artifact path of CaDet (benchmark training set + VAE pairs)
  train-semantic  train the semantic path of CaDet (benchmark training set only)
  calibrate       fusion statistics of a trained CaDet on validation real videos
  predict         CaDet scores for videos or for a manifest
  plugin          VAE Rhythm plug-in (Eq. 3) on detector scores + Ppk, with AUROC
  evaluate        per-generator AUROC table of a score CSV against a manifest
"""
from __future__ import annotations

import argparse
import csv
import json
import os
import sys


def _rows(spec: str) -> list[dict]:
    path, _, split = spec.partition(":")
    with open(path) as fh:
        rows = list(csv.DictReader(fh))
    return [r for r in rows if not split or r.get("split", "") == split]


def _write_csv(path: str, rows: list[dict]) -> None:
    keys = list(dict.fromkeys(k for r in rows for k in r))
    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
    with open(path, "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=keys)
        w.writeheader()
        w.writerows(rows)


def cmd_scan(a):
    from .scan import plot_result, scan_video
    from .vaes import VAE

    vae = VAE(a.vae, a.device, a.dtype, source=a.vae_path)
    for path in a.videos:
        res = scan_video(path, vae, a.cap, a.K, a.W, a.segments, a.batch, portable=a.portable)
        d = res.to_dict()
        line = f"{path}: Ppk {res.ppk:.4f} (period {res.stats['ppk']:.2f}), C4 {res.stats['C4']:.4f}, C8 {res.stats['C8']:.4f}"
        if res.z_ppk is not None:
            line += f", z(Ppk) {res.z_ppk:+.2f}, above {100 * res.real_percentile:.0f}% of reference real videos"
        print(line, flush=True)
        if a.out:
            os.makedirs(a.out, exist_ok=True)
            stem = os.path.splitext(os.path.basename(path))[0]
            json.dump(d, open(os.path.join(a.out, stem + ".json"), "w"), indent=1)
            plot_result(res, os.path.join(a.out, stem + ".png"))


def cmd_scan_manifest(a):
    from .scan import scan_manifest
    from .vaes import VAE

    rows = _rows(a.manifest)[a.shard::a.nshards]
    scan_manifest(rows, VAE(a.vae, a.device, a.dtype, source=a.vae_path), a.out, a.npz_dir, a.cap, a.K, a.W, a.batch,
                  portable=a.portable)


def cmd_make_pairs(a):
    from .engine.pairs import make_pairs

    if a.reals_manifest:
        paths = [r["path"] for r in _rows(a.reals_manifest) if int(r.get("label", 0)) == 0]
    else:
        paths = [os.path.join(a.reals, f) for f in os.listdir(a.reals) if f.lower().endswith((".mp4", ".mov", ".mkv", ".webm"))]
    make_pairs(paths, a.out, a.vaes.split(","), [int(x) for x in a.gen_scales.split(",")],
               [int(x) for x in a.out_scales.split(",")], a.allow_upscale, a.interp_prob, a.dup_prob, a.crf_max,
               a.frames, a.n, a.seed, a.tag, a.shard, a.nshards, a.device)


def cmd_cache_frames(a):
    from .cadet.frames import cache_manifest

    cache_manifest(a.manifest, a.out, a.workers)


def cmd_train_artifact(a):
    from .cadet.train import read_manifest, train_artifact

    train = [r for s in a.train for r in read_manifest(s)]
    print(train_artifact(train, read_manifest(a.val), a.frames_root, a.out, a.seed, a.epochs, a.bs, a.accum, a.workers,
                         single_head=a.single_head))


def cmd_train_semantic(a):
    from .cadet.train import read_manifest, train_semantic

    train_semantic(read_manifest(a.train), a.frames_root, a.out, a.seed, a.epochs, a.bs, a.workers)


def cmd_calibrate(a):
    """Write <weights>/calibration.json: head and fusion statistics on validation reals."""
    import numpy as np
    import torch

    from .cadet.detector import _find, _load_partial, _load_tensors
    from .cadet.models import ArtifactPath, SemanticPath
    from .cadet.train import ClipDataset, predict_artifact, predict_semantic, read_manifest

    reals = [r for r in read_manifest(a.val) if int(r["label"]) == 0]
    art = ArtifactPath().to(a.device)
    _load_partial(art, _load_tensors(_find(a.weights, "artifact_path")), "artifact path")
    A, B = predict_artifact(art, torch.utils.data.DataLoader(ClipDataset(reals, a.frames_root, False), batch_size=8, num_workers=8), a.device)
    heads = {"A_mean": float(A.mean()), "A_std": float(A.std() + 1e-6), "B_mean": float(B.mean()), "B_std": float(B.std() + 1e-6)}
    s_art = (A - heads["A_mean"]) / heads["A_std"] + (B - heads["B_mean"]) / heads["B_std"]
    del art
    sem = SemanticPath(device=a.device).to(a.device)
    _load_partial(sem, _load_tensors(_find(a.weights, "semantic_path")), "semantic path")
    s_sem = predict_semantic(sem, reals, a.frames_root, a.device)
    cal = {"heads": heads, "fusion": {"sem": [float(np.mean(s_sem)), float(np.std(s_sem, ddof=1))],
                                      "art": [float(np.mean(s_art)), float(np.std(s_art, ddof=1))]},
           "n_val_reals": len(reals)}
    json.dump(cal, open(os.path.join(a.weights, "calibration.json"), "w"), indent=1)
    print(json.dumps(cal, indent=1))


def cmd_predict(a):
    from .cadet.detector import CaDet

    det = CaDet(a.weights, a.device)
    out = []
    if a.manifest:
        for r in _rows(a.manifest)[a.shard::a.nshards]:
            try:
                s = det.score_cached(a.frames_root, r) if a.frames_root else det.score_video(r["path"])
                out.append({"video_id": r["video_id"], "label": r.get("label", ""), **s})
            except Exception as e:  # noqa: BLE001
                out.append({"video_id": r["video_id"], "label": r.get("label", ""), "error": str(e)[:200]})
    for path in a.videos:
        s = det.score_video(path)
        if "z_semantic" in s:
            print(f"{path}: CaDet score {s['score']:+.2f} (semantic z {s['z_semantic']:+.2f}, artifact z {s['z_artifact']:+.2f})")
        else:
            print(f"{path}: CaDet-GF logit {s['score']:+.2f}")
        out.append({"video_id": path, **s})
    if a.out:
        _write_csv(a.out, out)


def cmd_plugin(a):
    import numpy as np

    from .metrics import per_subset, read_scores, summarize
    from .plugin import HybridStandardizer, MeanStdStandardizer, fuse, z_ppk
    from .scan import load_calibration

    det = read_scores(a.scores)
    ppk = read_scores(a.ppk, "Ppk")
    real_det = list(read_scores(a.det_real).values())
    cal = load_calibration(a.ppk_calibration)
    std = HybridStandardizer(real_det) if a.norm == "hybrid" else MeanStdStandardizer(real_det)
    ids = [v for v in det if v in ppk]
    zd = std(np.array([det[v] for v in ids]))
    zc = z_ppk(np.array([ppk[v] for v in ids]), cal["ppk_mean"], cal["ppk_std"])
    fused = dict(zip(ids, fuse(zd, zc, a.tau)))
    rows = [{"video_id": v, "score": f, "z_detector": d, "z_ppk": c} for v, f, d, c in zip(ids, fused.values(), zd, zc)]
    if a.out:
        _write_csv(a.out, rows)
    if a.manifest:
        man = _rows(a.manifest)
        for name, sc in (("detector", {v: float(z) for v, z in zip(ids, zd)}), ("rhythm only", {v: float(z) for v, z in zip(ids, zc)}),
                         ("detector + plug-in", fused)):
            print(f"{name:20s}", {k: round(v, 2) for k, v in summarize(per_subset(man, sc)).items()})


def cmd_evaluate(a):
    from .metrics import per_subset, read_scores, summarize

    recs = per_subset(_rows(a.manifest), read_scores(a.scores, a.column))
    for r in recs:
        print(f"{r['subset']:32s} n_fake {r['n_fake']:5d}  AUROC {r['AUROC']:6.2f}" + (f"  2AFC {r['paired_2afc']:6.2f}" if "paired_2afc" in r else ""))
    print({k: round(v, 2) for k, v in summarize(recs).items()})
    if a.out:
        _write_csv(a.out, recs)


def cmd_doctor(a):
    """Check the environment; prints one OK / FAIL line per item and exits non-zero on failures."""
    import importlib
    import shutil
    import subprocess

    from .video import FFMPEG, FFPROBE

    ok = True

    def report(name, good, detail=""):
        nonlocal ok
        ok &= bool(good) or name.startswith("(optional)")
        print(f"{'OK  ' if good else 'FAIL'} {name}{': ' + detail if detail else ''}", flush=True)

    for tool in (FFMPEG, FFPROBE):
        report(tool, shutil.which(tool) or os.path.isfile(tool))
    if shutil.which(FFMPEG) or os.path.isfile(FFMPEG):
        enc = subprocess.run([FFMPEG, "-hide_banner", "-encoders"], capture_output=True, text=True).stdout
        report("ffmpeg libx264", "libx264" in enc)
        report("ffmpeg libx265 (VAE-pair engine)", "libx265" in enc)
        filt = subprocess.run([FFMPEG, "-hide_banner", "-filters"], capture_output=True, text=True).stdout
        report("ffmpeg minterpolate (VAE-pair engine)", "minterpolate" in filt)
    for mod, need in (("numpy", None), ("torch", None), ("diffusers", "0.33"), ("av", None), ("PIL", None),
                      ("matplotlib", None), ("safetensors", None)):
        try:
            m = importlib.import_module(mod)
            ver = getattr(m, "__version__", "")
            good = need is None or tuple(int(x) for x in ver.split(".")[:2]) >= tuple(int(x) for x in need.split("."))
            report(f"{mod} {ver}", good, f"needs >= {need}" if not good else "")
        except Exception as e:  # noqa: BLE001
            report(mod, False, str(e)[:120])
    for mod in ("timm", "peft", "cv2", "sklearn"):
        try:
            m = importlib.import_module(mod)
            report(f"(optional) {mod} {getattr(m, '__version__', '')} [CaDet]", True)
        except Exception as e:  # noqa: BLE001
            report(f"(optional) {mod} [CaDet]", False, str(e)[:80])
    try:
        import diffusers

        report("diffusers AutoencoderKLWan", hasattr(diffusers, "AutoencoderKLWan"))
    except Exception:  # noqa: BLE001
        pass
    try:
        import torch

        cuda = torch.cuda.is_available()
        report("CUDA GPU", cuda, torch.cuda.get_device_name(0) if cuda else "scans on CPU are very slow")
        if cuda:
            report("bf16 support", torch.cuda.is_bf16_supported())
    except Exception:  # noqa: BLE001
        pass
    sys.exit(0 if ok else 1)


def cmd_selftest(a):
    """No dataset needed: synthesise a moving clip, make a Wan2.1 round-trip copy of it, scan both.
    The round-trip copy carries a VAE lattice, so its rhythm must be clearly stronger."""
    import subprocess
    import tempfile

    from .engine.pairs import vae_pass, vae_prep
    from .scan import scan_video
    from .vaes import VAE
    from .video import FFMPEG, read_frames, write_frames

    vae = VAE("wan21", a.device, "bf16", source=a.vae_path)
    with tempfile.TemporaryDirectory() as tmp:
        src, dec = os.path.join(tmp, "source.mp4"), os.path.join(tmp, "vae_roundtrip.mp4")
        subprocess.run([FFMPEG, "-nostdin", "-v", "error", "-y", "-f", "lavfi", "-i", f"{a.pattern}=size=640x360:rate=24",
                        "-t", "3", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "12", src], check=True)
        frames = vae_prep(read_frames(src, num=49), vae)
        write_frames(vae_pass(frames, vae), dec, 24.0, crf=12)
        res = {name: scan_video(p, vae, 720, segments=1) for name, p in (("source", src), ("VAE round trip", dec))}
        for name, r in res.items():
            print(f"{name:15s} Ppk {r.ppk:.3f}  period {r.stats['ppk']:.2f}  C4 {r.stats['C4']:.3f}  z(Ppk) {r.z_ppk:+.2f}", flush=True)
        good = res["VAE round trip"].ppk > res["source"].ppk and res["VAE round trip"].stats["C4"] > res["source"].stats["C4"]
        print("SELFTEST", "PASSED" if good else "FAILED", flush=True)
        sys.exit(0 if good else 1)


def main(argv=None):
    os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
    p = argparse.ArgumentParser(prog="vae-rhythm", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = p.add_subparsers(dest="cmd", required=True)

    q = sub.add_parser("doctor", help="check ffmpeg, Python packages and the GPU")
    q.set_defaults(fn=cmd_doctor)

    q = sub.add_parser("selftest", help="GPU check without data: synthetic clip vs. its Wan2.1 round trip")
    q.add_argument("--device", default="cuda")
    q.add_argument("--vae-path", default=None)
    q.add_argument("--pattern", default="testsrc2", help="ffmpeg lavfi source (testsrc2, mandelbrot, ...)")
    q.set_defaults(fn=cmd_selftest)

    def scan_args(q):
        q.add_argument("--vae", default="wan21")
        q.add_argument("--vae-path", default=None, help="local diffusers folder instead of the Hugging Face repository")
        q.add_argument("--cap", type=int, default=720, help="short-side cap of the scan (paper: 720 for the plug-in, 480 for the census)")
        q.add_argument("--K", type=int, default=24)
        q.add_argument("--W", type=int, default=13)
        q.add_argument("--batch", type=int, default=1, help="windows per VAE call (2 needs ~40 GB at 720p)")
        q.add_argument("--device", default="cuda")
        q.add_argument("--dtype", default="bf16", choices=["bf16", "fp32", "fp16"])
        q.add_argument("--portable", action="store_true",
                       help="single-threaded x264 without assembly: identical results on every machine (slower encode)")

    q = sub.add_parser("scan", help="phase scan of videos")
    q.add_argument("videos", nargs="+")
    scan_args(q)
    q.add_argument("--segments", type=int, default=1, help="1 = first 36 frames (paper); 3 = start / middle / end, median")
    q.add_argument("--out", default="", help="folder for <video>.json and <video>.png")
    q.set_defaults(fn=cmd_scan)

    q = sub.add_parser("scan-manifest", help="phase scan of a manifest")
    q.add_argument("manifest", help="CSV with video_id, path[, label, subset]; 'file.csv:split' filters by split")
    q.add_argument("--out", required=True)
    q.add_argument("--npz-dir", default=None)
    q.add_argument("--shard", type=int, default=0)
    q.add_argument("--nshards", type=int, default=1)
    scan_args(q)
    q.set_defaults(fn=cmd_scan_manifest)

    q = sub.add_parser("make-pairs", help="VAE-pair engine")
    q.add_argument("--reals", default="", help="folder of real videos")
    q.add_argument("--reals-manifest", default="", help="or a manifest (rows with label 0), e.g. 'aigvd.csv:train'")
    q.add_argument("--out", required=True)
    q.add_argument("--vaes", default="wan21,hunyuan,cogx,ltx")
    q.add_argument("--gen-scales", default="256,256,256,256,360,360,480,480,720", help="reconstruction short sides G (repeat = weight)")
    q.add_argument("--out-scales", default="480,720,1080", help="delivery short sides S")
    q.add_argument("--allow-upscale", action="store_true", help="S may exceed G (upscaled delivery)")
    q.add_argument("--interp-prob", type=float, default=0.2)
    q.add_argument("--dup-prob", type=float, default=0.25)
    q.add_argument("--crf-max", type=int, default=28)
    q.add_argument("--frames", type=int, default=81)
    q.add_argument("--n", type=int, default=0)
    q.add_argument("--seed", type=int, default=0)
    q.add_argument("--tag", default="")
    q.add_argument("--shard", type=int, default=0)
    q.add_argument("--nshards", type=int, default=1)
    q.add_argument("--device", default="cuda")
    q.set_defaults(fn=cmd_make_pairs)

    q = sub.add_parser("cache-frames", help="16-frame cache for CaDet")
    q.add_argument("manifest")
    q.add_argument("--out", required=True)
    q.add_argument("--workers", type=int, default=16)
    q.set_defaults(fn=cmd_cache_frames)

    q = sub.add_parser("train-artifact", help="train the artifact path")
    q.add_argument("--train", nargs="+", required=True, help="e.g. aigvd.csv:train pairs.csv:train")
    q.add_argument("--val", required=True)
    q.add_argument("--frames-root", required=True)
    q.add_argument("--out", required=True)
    q.add_argument("--seed", type=int, default=0)
    q.add_argument("--epochs", type=int, default=8)
    q.add_argument("--bs", type=int, default=8)
    q.add_argument("--accum", type=int, default=3)
    q.add_argument("--workers", type=int, default=10)
    q.add_argument("--single-head", action="store_true", help="CaDet-GF: one head, original labels, best epoch on --val")
    q.set_defaults(fn=cmd_train_artifact)

    q = sub.add_parser("train-semantic", help="train the semantic path")
    q.add_argument("--train", required=True, help="benchmark training split only, e.g. aigvd.csv:train")
    q.add_argument("--frames-root", required=True)
    q.add_argument("--out", required=True)
    q.add_argument("--seed", type=int, default=0)
    q.add_argument("--epochs", type=int, default=11)
    q.add_argument("--bs", type=int, default=32)
    q.add_argument("--workers", type=int, default=8)
    q.set_defaults(fn=cmd_train_semantic)

    q = sub.add_parser("calibrate", help="fusion statistics on validation reals")
    q.add_argument("--weights", required=True, help="folder with artifact_path.* and semantic_path.*")
    q.add_argument("--val", required=True, help="validation manifest, e.g. aigvd.csv:val")
    q.add_argument("--frames-root", required=True)
    q.add_argument("--device", default="cuda")
    q.set_defaults(fn=cmd_calibrate)

    q = sub.add_parser("predict", help="CaDet scores")
    q.add_argument("videos", nargs="*")
    q.add_argument("--weights", required=True)
    q.add_argument("--manifest", default="")
    q.add_argument("--frames-root", default="", help="use cached frames for manifest rows")
    q.add_argument("--out", default="")
    q.add_argument("--device", default="cuda")
    q.add_argument("--shard", type=int, default=0, help="score rows shard::nshards (run several processes / GPUs)")
    q.add_argument("--nshards", type=int, default=1)
    q.set_defaults(fn=cmd_predict)

    q = sub.add_parser("plugin", help="VAE Rhythm plug-in (Eq. 3)")
    q.add_argument("--scores", required=True, help="detector score CSV (video_id, score)")
    q.add_argument("--det-real", required=True, help="the same detector's scores on calibration real videos")
    q.add_argument("--ppk", required=True, help="CSV with video_id, Ppk (from scan-manifest)")
    q.add_argument("--ppk-calibration", default="wan21_k24w13_cap720.json")
    q.add_argument("--norm", default="hybrid", choices=["hybrid", "meanstd"])
    q.add_argument("--tau", type=float, default=0.0)
    q.add_argument("--manifest", default="", help="manifest for AUROC (video_id, label, subset)")
    q.add_argument("--out", default="")
    q.set_defaults(fn=cmd_plugin)

    q = sub.add_parser("evaluate", help="per-generator AUROC")
    q.add_argument("scores")
    q.add_argument("--manifest", required=True)
    q.add_argument("--column", default="score")
    q.add_argument("--out", default="")
    q.set_defaults(fn=cmd_evaluate)

    a = p.parse_args(argv)
    a.fn(a)


if __name__ == "__main__":
    sys.exit(main())