viper/make_comparison.py

"""Build the qualitative comparison set for a set of validation pairs.

For each pair we save every artifact needed to judge the method by eye:

  reference.mp4   the reference video -- the physical process to transfer
  target_gt.mp4   the ground-truth target video (what "right" looks like)
  target_img.png  the target image the generator is actually conditioned on
  baseline.mp4    Wan2.2-I2V with the SAME image + prompt and NO reference
                  (physics tokens zeroed = the untouched base-model context)
  viper.mp4       Wan2.2-I2V + VIPER physics tokens from the reference
  grid.mp4        all four videos tiled 2x2 with labels, for side-by-side viewing

baseline and viper use identical noise seeds, so any difference between them is
attributable to the reference stream alone.
"""

from __future__ import annotations

import argparse
import json
import shutil
import sys
from pathlib import Path

import numpy as np
import torch

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "third_party" / "Wan2.2"))

from viper.infer import generate, save_video  # noqa: E402
from viper.physics_encoder import VisualPhysicsEncoder  # noqa: E402
from viper.wan_viper import add_lora, patch_wan_model  # noqa: E402

LABELS = {
    "reference": "REFERENCE (physics source)",
    "target_gt": "TARGET ground truth",
    "baseline": "BASELINE  Wan2.2 (no reference)",
    "viper": "VIPER (reference-conditioned)",
}


def label_frames(frames: np.ndarray, text: str) -> np.ndarray:
    """Burn a caption bar onto the top of every frame."""
    import cv2

    out = frames.copy()
    for i in range(len(out)):
        f = np.ascontiguousarray(out[i])
        cv2.rectangle(f, (0, 0), (f.shape[1], 28), (0, 0, 0), -1)
        cv2.putText(f, text, (8, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.55,
                    (255, 255, 255), 1, cv2.LINE_AA)
        out[i] = f
    return out


def make_grid(paths: dict, out_path: str, fps: int = 16):
    """Tile four videos 2x2 into a single labelled mp4."""
    import imageio
    import imageio.v3 as iio

    vids = {}
    for k in ("reference", "target_gt", "baseline", "viper"):
        v = iio.imread(paths[k], plugin="pyav")
        vids[k] = label_frames(v, LABELS[k])

    n = min(len(v) for v in vids.values())
    H, W = vids["viper"].shape[1:3]

    def fit(v):
        import cv2
        return np.stack([cv2.resize(f, (W, H)) for f in v[:n]])

    a, b, c, d = (fit(vids[k]) for k in
                  ("reference", "target_gt", "baseline", "viper"))
    top = np.concatenate([a, b], axis=2)
    bot = np.concatenate([c, d], axis=2)
    grid = np.concatenate([top, bot], axis=1)
    imageio.mimsave(out_path, list(grid), fps=fps, quality=7)
    print(f"  grid -> {out_path}")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--ckpt", required=True)
    ap.add_argument("--pairs", default="data/pairs_val.jsonl")
    ap.add_argument("--out_dir", default="results/comparison")
    ap.add_argument("--wan_ckpt", default="models/Wan2.2-I2V-A14B")
    ap.add_argument("--mllm", default="models/Qwen3-VL-4B-Instruct")
    ap.add_argument("--steps", type=int, default=50)
    ap.add_argument("--guide_scale", type=float, default=6.0)
    ap.add_argument("--shift", type=float, default=5.0)
    ap.add_argument("--frames", type=int, default=81)
    ap.add_argument("--num_ref_frames", type=int, default=8)
    ap.add_argument("--seed", type=int, default=0)
    ap.add_argument("--limit", type=int, default=0)
    ap.add_argument("--shard", type=int, default=0)
    ap.add_argument("--num_shards", type=int, default=1)
    args = ap.parse_args()

    import imageio.v3 as iio
    from PIL import Image
    from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
    from wan.modules.model import WanModel
    from wan.modules.t5 import T5EncoderModel
    from wan.modules.vae2_1 import Wan2_1_VAE

    device = torch.device("cuda")
    ckpt_dir = Path(args.wan_ckpt)

    rows = [json.loads(l) for l in open(args.pairs)]
    if args.limit:
        rows = rows[: args.limit]
    rows = rows[args.shard::args.num_shards]
    print(f"[{args.shard}] {len(rows)} pairs")

    vae = Wan2_1_VAE(vae_pth=str(ckpt_dir / "Wan2.1_VAE.pth"), device=device)
    t5 = T5EncoderModel(
        text_len=512, dtype=torch.bfloat16, device=device,
        checkpoint_path=str(ckpt_dir / "models_t5_umt5-xxl-enc-bf16.pth"),
        tokenizer_path=str(ckpt_dir / "google/umt5-xxl"))

    sd = torch.load(args.ckpt, map_location="cpu")
    nq = sd["args"]["num_queries"]
    proc = AutoProcessor.from_pretrained(args.mllm)
    mllm = Qwen3VLForConditionalGeneration.from_pretrained(
        args.mllm, dtype=torch.bfloat16, attn_implementation="sdpa").to(device)
    enc = VisualPhysicsEncoder(mllm, nq, out_dim=5120).to(device)
    enc.query_tokens.data.copy_(sd["query_tokens"].to(device))
    enc.connector.load_state_dict(sd["connector"])
    enc.connector.to(device, torch.float32)
    enc.eval()

    n_gpu = torch.cuda.device_count()
    dev_low = torch.device("cuda:0")
    dev_high = torch.device(f"cuda:{1 if n_gpu > 1 else 0}")

    def load_expert(sub, dev):
        m = patch_wan_model(WanModel.from_pretrained(str(ckpt_dir / sub)))
        return m.to(device=dev, dtype=torch.bfloat16).eval().requires_grad_(False)

    dit_low = load_expert("low_noise_model", dev_low)
    dit_high = load_expert("high_noise_model", dev_high)
    if "lora" in sd:
        for m in (dit_low, dit_high):
            add_lora(m, sd["args"]["lora_rank"], sd["args"]["lora_alpha"])
            msd = dict(m.named_parameters())
            for k, v in sd["lora"].items():
                if k in msd:
                    msd[k].data.copy_(v.to(next(m.parameters()).device))

    out_root = Path(args.out_dir)
    out_root.mkdir(parents=True, exist_ok=True)

    for i, r in enumerate(rows):
        tag = f"{r['ref_id']}__{r['tgt_id']}"
        d = out_root / tag
        d.mkdir(exist_ok=True)
        print(f"[{args.shard}] {i+1}/{len(rows)} {tag}", flush=True)

        # --- inputs -------------------------------------------------
        shutil.copy(r["ref_video"], d / "reference.mp4")
        shutil.copy(r["tgt_video"], d / "target_gt.mp4")

        tgt = iio.imread(r["tgt_video"], plugin="pyav")
        Image.fromarray(tgt[0]).save(d / "target_img.png")
        image = (torch.from_numpy(tgt[0].copy()).permute(2, 0, 1)
                 .float().div_(127.5).sub_(1))

        ref = iio.imread(r["ref_video"], plugin="pyav")
        idx = np.linspace(0, len(ref) - 1, args.num_ref_frames).astype(int)
        ref_frames = ref[idx]

        prompt = r.get("tgt_summary") or r.get("tgt_caption", "")[:300]

        # --- generate both variants with identical seeds -------------
        for name, zero in (("baseline", True), ("viper", False)):
            saved = enc.query_tokens.data.clone()
            if zero:
                enc.query_tokens.data.zero_()
            out = generate(dit_low, dit_high, vae, t5, enc, proc, image, prompt,
                           ref_frames, device, num_queries=nq, steps=args.steps,
                           guide_scale=args.guide_scale, shift=args.shift,
                           frames=args.frames, seed=args.seed)
            enc.query_tokens.data.copy_(saved)
            save_video(out, str(d / f"{name}.mp4"))

        paths = {k: str(d / f"{k}.mp4") for k in
                 ("reference", "target_gt", "baseline", "viper")}
        make_grid(paths, str(d / "grid.mp4"))

        json.dump({
            "id": tag, "prompt": prompt,
            "physical_impact": r.get("physical_impact"),
            "trajectory": r.get("trajectory"),
            "transferability": r.get("transferability"),
            "judge_reason": r.get("judge_reason"),
            "ckpt": args.ckpt, "steps": args.steps,
            "guide_scale": args.guide_scale, "seed": args.seed,
        }, open(d / "info.json", "w"), indent=2, ensure_ascii=False)

    print(f"[{args.shard}] DONE -> {out_root}")


if __name__ == "__main__":
    main()