"""Command line interface: ``vae-rhythm ...`` (or ``python -m vae_rhythm ...``). Commands doctor check ffmpeg, Python packages and the GPU selftest GPU check without data (synthetic clip vs. its Wan2.1 round trip) scan phase scan of one or more videos (zero-shot rhythm score, curve plot) scan-manifest phase scan of every video in a manifest CSV -> statistics CSV (+ NPZ tables) make-pairs VAE-pair engine: real videos -> (real, VAE round trip) pairs cache-frames 16-frame JPEG cache used by CaDet training and evaluation train-artifact train the artifact path of CaDet (benchmark training set + VAE pairs) train-semantic train the semantic path of CaDet (benchmark training set only) calibrate fusion statistics of a trained CaDet on validation real videos predict CaDet scores for videos or for a manifest plugin VAE Rhythm plug-in (Eq. 3) on detector scores + Ppk, with AUROC evaluate per-generator AUROC table of a score CSV against a manifest """ from __future__ import annotations import argparse import csv import json import os import sys def _rows(spec: str) -> list[dict]: path, _, split = spec.partition(":") with open(path) as fh: rows = list(csv.DictReader(fh)) return [r for r in rows if not split or r.get("split", "") == split] def _write_csv(path: str, rows: list[dict]) -> None: keys = list(dict.fromkeys(k for r in rows for k in r)) os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) with open(path, "w", newline="") as fh: w = csv.DictWriter(fh, fieldnames=keys) w.writeheader() w.writerows(rows) def cmd_scan(a): from .scan import plot_result, scan_video from .vaes import VAE vae = VAE(a.vae, a.device, a.dtype, source=a.vae_path) for path in a.videos: res = scan_video(path, vae, a.cap, a.K, a.W, a.segments, a.batch, portable=a.portable) d = res.to_dict() line = f"{path}: Ppk {res.ppk:.4f} (period {res.stats['ppk']:.2f}), C4 {res.stats['C4']:.4f}, C8 {res.stats['C8']:.4f}" if res.z_ppk is not None: line += f", z(Ppk) {res.z_ppk:+.2f}, above {100 * res.real_percentile:.0f}% of reference real videos" print(line, flush=True) if a.out: os.makedirs(a.out, exist_ok=True) stem = os.path.splitext(os.path.basename(path))[0] json.dump(d, open(os.path.join(a.out, stem + ".json"), "w"), indent=1) plot_result(res, os.path.join(a.out, stem + ".png")) def cmd_scan_manifest(a): from .scan import scan_manifest from .vaes import VAE rows = _rows(a.manifest)[a.shard::a.nshards] scan_manifest(rows, VAE(a.vae, a.device, a.dtype, source=a.vae_path), a.out, a.npz_dir, a.cap, a.K, a.W, a.batch, portable=a.portable) def cmd_make_pairs(a): from .engine.pairs import make_pairs if a.reals_manifest: paths = [r["path"] for r in _rows(a.reals_manifest) if int(r.get("label", 0)) == 0] else: paths = [os.path.join(a.reals, f) for f in os.listdir(a.reals) if f.lower().endswith((".mp4", ".mov", ".mkv", ".webm"))] make_pairs(paths, a.out, a.vaes.split(","), [int(x) for x in a.gen_scales.split(",")], [int(x) for x in a.out_scales.split(",")], a.allow_upscale, a.interp_prob, a.dup_prob, a.crf_max, a.frames, a.n, a.seed, a.tag, a.shard, a.nshards, a.device) def cmd_cache_frames(a): from .cadet.frames import cache_manifest cache_manifest(a.manifest, a.out, a.workers) def cmd_train_artifact(a): from .cadet.train import read_manifest, train_artifact train = [r for s in a.train for r in read_manifest(s)] print(train_artifact(train, read_manifest(a.val), a.frames_root, a.out, a.seed, a.epochs, a.bs, a.accum, a.workers, single_head=a.single_head)) def cmd_train_semantic(a): from .cadet.train import read_manifest, train_semantic train_semantic(read_manifest(a.train), a.frames_root, a.out, a.seed, a.epochs, a.bs, a.workers) def cmd_calibrate(a): """Write /calibration.json: head and fusion statistics on validation reals.""" import numpy as np import torch from .cadet.detector import _find, _load_partial, _load_tensors from .cadet.models import ArtifactPath, SemanticPath from .cadet.train import ClipDataset, predict_artifact, predict_semantic, read_manifest reals = [r for r in read_manifest(a.val) if int(r["label"]) == 0] art = ArtifactPath().to(a.device) _load_partial(art, _load_tensors(_find(a.weights, "artifact_path")), "artifact path") A, B = predict_artifact(art, torch.utils.data.DataLoader(ClipDataset(reals, a.frames_root, False), batch_size=8, num_workers=8), a.device) heads = {"A_mean": float(A.mean()), "A_std": float(A.std() + 1e-6), "B_mean": float(B.mean()), "B_std": float(B.std() + 1e-6)} s_art = (A - heads["A_mean"]) / heads["A_std"] + (B - heads["B_mean"]) / heads["B_std"] del art sem = SemanticPath(device=a.device).to(a.device) _load_partial(sem, _load_tensors(_find(a.weights, "semantic_path")), "semantic path") s_sem = predict_semantic(sem, reals, a.frames_root, a.device) cal = {"heads": heads, "fusion": {"sem": [float(np.mean(s_sem)), float(np.std(s_sem, ddof=1))], "art": [float(np.mean(s_art)), float(np.std(s_art, ddof=1))]}, "n_val_reals": len(reals)} json.dump(cal, open(os.path.join(a.weights, "calibration.json"), "w"), indent=1) print(json.dumps(cal, indent=1)) def cmd_predict(a): from .cadet.detector import CaDet det = CaDet(a.weights, a.device) out = [] if a.manifest: for r in _rows(a.manifest)[a.shard::a.nshards]: try: s = det.score_cached(a.frames_root, r) if a.frames_root else det.score_video(r["path"]) out.append({"video_id": r["video_id"], "label": r.get("label", ""), **s}) except Exception as e: # noqa: BLE001 out.append({"video_id": r["video_id"], "label": r.get("label", ""), "error": str(e)[:200]}) for path in a.videos: s = det.score_video(path) if "z_semantic" in s: print(f"{path}: CaDet score {s['score']:+.2f} (semantic z {s['z_semantic']:+.2f}, artifact z {s['z_artifact']:+.2f})") else: print(f"{path}: CaDet-GF logit {s['score']:+.2f}") out.append({"video_id": path, **s}) if a.out: _write_csv(a.out, out) def cmd_plugin(a): import numpy as np from .metrics import per_subset, read_scores, summarize from .plugin import HybridStandardizer, MeanStdStandardizer, fuse, z_ppk from .scan import load_calibration det = read_scores(a.scores) ppk = read_scores(a.ppk, "Ppk") real_det = list(read_scores(a.det_real).values()) cal = load_calibration(a.ppk_calibration) std = HybridStandardizer(real_det) if a.norm == "hybrid" else MeanStdStandardizer(real_det) ids = [v for v in det if v in ppk] zd = std(np.array([det[v] for v in ids])) zc = z_ppk(np.array([ppk[v] for v in ids]), cal["ppk_mean"], cal["ppk_std"]) fused = dict(zip(ids, fuse(zd, zc, a.tau))) rows = [{"video_id": v, "score": f, "z_detector": d, "z_ppk": c} for v, f, d, c in zip(ids, fused.values(), zd, zc)] if a.out: _write_csv(a.out, rows) if a.manifest: man = _rows(a.manifest) for name, sc in (("detector", {v: float(z) for v, z in zip(ids, zd)}), ("rhythm only", {v: float(z) for v, z in zip(ids, zc)}), ("detector + plug-in", fused)): print(f"{name:20s}", {k: round(v, 2) for k, v in summarize(per_subset(man, sc)).items()}) def cmd_evaluate(a): from .metrics import per_subset, read_scores, summarize recs = per_subset(_rows(a.manifest), read_scores(a.scores, a.column)) for r in recs: print(f"{r['subset']:32s} n_fake {r['n_fake']:5d} AUROC {r['AUROC']:6.2f}" + (f" 2AFC {r['paired_2afc']:6.2f}" if "paired_2afc" in r else "")) print({k: round(v, 2) for k, v in summarize(recs).items()}) if a.out: _write_csv(a.out, recs) def cmd_doctor(a): """Check the environment; prints one OK / FAIL line per item and exits non-zero on failures.""" import importlib import shutil import subprocess from .video import FFMPEG, FFPROBE ok = True def report(name, good, detail=""): nonlocal ok ok &= bool(good) or name.startswith("(optional)") print(f"{'OK ' if good else 'FAIL'} {name}{': ' + detail if detail else ''}", flush=True) for tool in (FFMPEG, FFPROBE): report(tool, shutil.which(tool) or os.path.isfile(tool)) if shutil.which(FFMPEG) or os.path.isfile(FFMPEG): enc = subprocess.run([FFMPEG, "-hide_banner", "-encoders"], capture_output=True, text=True).stdout report("ffmpeg libx264", "libx264" in enc) report("ffmpeg libx265 (VAE-pair engine)", "libx265" in enc) filt = subprocess.run([FFMPEG, "-hide_banner", "-filters"], capture_output=True, text=True).stdout report("ffmpeg minterpolate (VAE-pair engine)", "minterpolate" in filt) for mod, need in (("numpy", None), ("torch", None), ("diffusers", "0.33"), ("av", None), ("PIL", None), ("matplotlib", None), ("safetensors", None)): try: m = importlib.import_module(mod) ver = getattr(m, "__version__", "") good = need is None or tuple(int(x) for x in ver.split(".")[:2]) >= tuple(int(x) for x in need.split(".")) report(f"{mod} {ver}", good, f"needs >= {need}" if not good else "") except Exception as e: # noqa: BLE001 report(mod, False, str(e)[:120]) for mod in ("timm", "peft", "cv2", "sklearn"): try: m = importlib.import_module(mod) report(f"(optional) {mod} {getattr(m, '__version__', '')} [CaDet]", True) except Exception as e: # noqa: BLE001 report(f"(optional) {mod} [CaDet]", False, str(e)[:80]) try: import diffusers report("diffusers AutoencoderKLWan", hasattr(diffusers, "AutoencoderKLWan")) except Exception: # noqa: BLE001 pass try: import torch cuda = torch.cuda.is_available() report("CUDA GPU", cuda, torch.cuda.get_device_name(0) if cuda else "scans on CPU are very slow") if cuda: report("bf16 support", torch.cuda.is_bf16_supported()) except Exception: # noqa: BLE001 pass sys.exit(0 if ok else 1) def cmd_selftest(a): """No dataset needed: synthesise a moving clip, make a Wan2.1 round-trip copy of it, scan both. The round-trip copy carries a VAE lattice, so its rhythm must be clearly stronger.""" import subprocess import tempfile from .engine.pairs import vae_pass, vae_prep from .scan import scan_video from .vaes import VAE from .video import FFMPEG, read_frames, write_frames vae = VAE("wan21", a.device, "bf16", source=a.vae_path) with tempfile.TemporaryDirectory() as tmp: src, dec = os.path.join(tmp, "source.mp4"), os.path.join(tmp, "vae_roundtrip.mp4") subprocess.run([FFMPEG, "-nostdin", "-v", "error", "-y", "-f", "lavfi", "-i", f"{a.pattern}=size=640x360:rate=24", "-t", "3", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "12", src], check=True) frames = vae_prep(read_frames(src, num=49), vae) write_frames(vae_pass(frames, vae), dec, 24.0, crf=12) res = {name: scan_video(p, vae, 720, segments=1) for name, p in (("source", src), ("VAE round trip", dec))} for name, r in res.items(): print(f"{name:15s} Ppk {r.ppk:.3f} period {r.stats['ppk']:.2f} C4 {r.stats['C4']:.3f} z(Ppk) {r.z_ppk:+.2f}", flush=True) good = res["VAE round trip"].ppk > res["source"].ppk and res["VAE round trip"].stats["C4"] > res["source"].stats["C4"] print("SELFTEST", "PASSED" if good else "FAILED", flush=True) sys.exit(0 if good else 1) def main(argv=None): os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") p = argparse.ArgumentParser(prog="vae-rhythm", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="cmd", required=True) q = sub.add_parser("doctor", help="check ffmpeg, Python packages and the GPU") q.set_defaults(fn=cmd_doctor) q = sub.add_parser("selftest", help="GPU check without data: synthetic clip vs. its Wan2.1 round trip") q.add_argument("--device", default="cuda") q.add_argument("--vae-path", default=None) q.add_argument("--pattern", default="testsrc2", help="ffmpeg lavfi source (testsrc2, mandelbrot, ...)") q.set_defaults(fn=cmd_selftest) def scan_args(q): q.add_argument("--vae", default="wan21") q.add_argument("--vae-path", default=None, help="local diffusers folder instead of the Hugging Face repository") q.add_argument("--cap", type=int, default=720, help="short-side cap of the scan (paper: 720 for the plug-in, 480 for the census)") q.add_argument("--K", type=int, default=24) q.add_argument("--W", type=int, default=13) q.add_argument("--batch", type=int, default=1, help="windows per VAE call (2 needs ~40 GB at 720p)") q.add_argument("--device", default="cuda") q.add_argument("--dtype", default="bf16", choices=["bf16", "fp32", "fp16"]) q.add_argument("--portable", action="store_true", help="single-threaded x264 without assembly: identical results on every machine (slower encode)") q = sub.add_parser("scan", help="phase scan of videos") q.add_argument("videos", nargs="+") scan_args(q) q.add_argument("--segments", type=int, default=1, help="1 = first 36 frames (paper); 3 = start / middle / end, median") q.add_argument("--out", default="", help="folder for