Parallax-1.3B-preview / tools /gpu_tools.py
hyper88's picture
Research preview
ca6a1ef verified
Raw History Blame Contribute Delete
13.7 kB
#!/usr/bin/env python3
# Copyright 2026 The Parallax Authors. Licensed under the Apache License, Version 2.0.
"""Offline helpers that need the Wan2.1 text encoder or VAE (reference code from Causal Forcing, Apache-2.0).
prompts write a list of text prompts (generated here; no third-party prompt set)
encode prompts -> prompt bank (.safetensors) with the umT5-XXL encoder
preview roll the generator out, decode with the Wan VAE and with the pixel decoder,
report their agreement and write contact sheets
None of this is on the real-time path; infer_realtime.py does not import it.
"""
from __future__ import annotations
import argparse
import base64
import importlib
import io
import json
import os
import random
import sys
import types
import torch
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
import infer_realtime as ir # noqa: E402
VAE_MEAN = [-0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921]
VAE_STD = [2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160]
def ref_module(name: str, cf_dir: str):
"""Import wan.modules.<name> from the reference checkout without running its package __init__ files."""
for pkg, path in (("wan", os.path.join(cf_dir, "wan")), ("wan.modules", os.path.join(cf_dir, "wan", "modules"))):
if pkg not in sys.modules:
mod = types.ModuleType(pkg)
mod.__path__ = [path]
sys.modules[pkg] = mod
return importlib.import_module(f"wan.modules.{name}")
# ---- prompts ----------------------------------------------------------------
SUBJECTS = [
"a wooden desk with a laptop, a notebook and a ceramic mug", "a kitchen counter with fresh vegetables and a cutting board",
"a pair of hands assembling a small mechanical clock", "a pair of hands typing on a mechanical keyboard",
"a hand pouring coffee into a glass cup", "a living room with a sofa, a bookshelf and large windows",
"a narrow corridor lined with framed photographs", "a workshop bench covered with tools and wood shavings",
"a bedroom with an unmade bed and morning light", "a library reading room with long tables and green lamps",
"a red fox walking through tall grass", "a golden retriever running along a beach", "a grey cat sitting on a windowsill",
"a flock of birds taking off from a lake", "a horse galloping across an open field", "a sea turtle swimming over a coral reef",
"a cyclist riding down a mountain road", "a street musician playing a violin", "a chef plating a dessert",
"a child flying a kite on a hill", "an elderly man repairing a bicycle", "a woman painting a large canvas",
"a potter shaping clay on a wheel", "a barista steaming milk behind a counter", "two people playing chess in a park",
"a sailboat crossing a calm bay", "a steam train crossing a stone viaduct", "a tram moving through a rainy city street",
"a small plane landing on a grass airstrip", "a lighthouse on a rocky headland", "a waterfall in a mossy gorge",
"a snow-covered pine forest", "a desert canyon with layered red rock", "a field of sunflowers moving in the wind",
"a busy night market with lanterns and food stalls", "an empty subway platform with a train arriving",
"a glass greenhouse full of tropical plants", "a stone bridge over a slow river", "a rooftop garden above a city skyline",
"a harbour with fishing boats at dawn", "a mountain lake reflecting snowy peaks", "a vineyard on rolling hills",
"an old bookshop with ladders and stacked books", "a museum hall with marble statues", "a robot arm sorting coloured blocks",
"a drone view of a winding coastal road", "an aquarium tank with jellyfish", "a campfire beside a tent at night",
"a bowl of ramen with rising steam", "a glass of water with ice and a slice of lemon",
]
ACTIONS = [
"", "in slow motion", "as the wind picks up", "while light rain falls", "as the sun breaks through clouds", "while dust drifts in the air",
"as leaves fall around", "while people pass in the background", "as steam rises", "while shadows move across the scene",
]
CAMERA = [
"the camera slowly pans to the right", "the camera slowly pans to the left", "the camera moves forward steadily",
"the camera pulls back to reveal the surroundings", "the camera orbits around the subject", "the camera tilts up",
"the camera turns around and then returns to the starting view", "handheld camera with gentle motion", "static camera",
"the camera tracks sideways at walking pace", "the camera rises above the scene", "first-person view walking forward",
]
LIGHT = [
"soft morning light", "golden hour light", "overcast daylight", "bright midday sun", "blue hour", "warm indoor lamps",
"neon signs at night", "fog and diffuse light", "after rain with wet reflections", "dramatic side lighting",
]
STYLE = [
"photorealistic, sharp focus", "cinematic, shallow depth of field", "documentary footage", "35mm film look",
"high detail, natural colours", "wide-angle lens", "macro lens, fine textures", "clean studio look",
]
def make_prompts(n: int, seed: int = 0):
rng = random.Random(seed)
seen, out = set(), []
while len(out) < n:
parts = [rng.choice(SUBJECTS), rng.choice(ACTIONS), rng.choice(CAMERA), rng.choice(LIGHT), rng.choice(STYLE)]
text = ", ".join(p for p in parts if p)
text = text[0].upper() + text[1:] + "."
if text not in seen:
seen.add(text)
out.append(text)
return out
# ---- text encoder -----------------------------------------------------------
class TextEncoder:
def __init__(self, wan_dir: str, cf_dir: str, device, dtype=torch.bfloat16):
t5 = ref_module("t5", cf_dir)
tok = ref_module("tokenizers", cf_dir)
self.model = t5.umt5_xxl(encoder_only=True, return_tokenizer=False, dtype=torch.float32, device=torch.device("cpu")).eval().requires_grad_(False)
self.model.load_state_dict(torch.load(os.path.join(wan_dir, "models_t5_umt5-xxl-enc-bf16.pth"), map_location="cpu", weights_only=False))
self.model = self.model.to(device=device, dtype=dtype)
self.tok = tok.HuggingfaceTokenizer(name=os.path.join(wan_dir, "google", "umt5-xxl"), seq_len=512, clean="whitespace")
self.device = device
@torch.no_grad()
def __call__(self, texts):
ids, mask = self.tok(texts, return_mask=True, add_special_tokens=True)
ids, mask = ids.to(self.device), mask.to(self.device)
lens = mask.gt(0).sum(dim=1).tolist()
ctx = self.model(ids, mask)
return [c[:n] for c, n in zip(ctx, lens)]
def cmd_encode(a):
from safetensors.torch import save_file
texts = [l.strip() for l in open(a.prompts) if l.strip()]
enc = TextEncoder(a.wan_dir, a.cf_dir, torch.device(a.device))
chunks, offsets = [], [0]
for i in range(0, len(texts), a.batch):
for e in enc(texts[i:i + a.batch]):
chunks.append(e.to(torch.bfloat16).cpu())
offsets.append(offsets[-1] + e.shape[0])
print(f"encoded {min(i + a.batch, len(texts))}/{len(texts)}", flush=True)
save_file({"embeds": torch.cat(chunks), "offsets": torch.tensor(offsets, dtype=torch.int64)}, a.out)
print(f"saved {a.out}: {len(texts)} prompts, {offsets[-1]} tokens, mean length {offsets[-1] / len(texts):.1f}")
# ---- preview ----------------------------------------------------------------
def load_vae(wan_dir: str, cf_dir: str, device):
vae = ref_module("vae", cf_dir)
model = vae._video_vae(pretrained_path=os.path.join(wan_dir, "Wan2.1_VAE.pth"), z_dim=16).eval().requires_grad_(False).to(device)
scale = [torch.tensor(VAE_MEAN, device=device), 1.0 / torch.tensor(VAE_STD, device=device)]
return model, scale
def psnr(a: torch.Tensor, b: torch.Tensor) -> float:
return float(10 * torch.log10(1.0 / (a.float() - b.float()).pow(2).mean().clamp_min(1e-12)))
def sheet(rows, width=208):
"""rows: list of [T, 3, H, W] tensors in [0, 1]; returns a JPEG (bytes) with one row per tensor."""
from PIL import Image
imgs = []
for r in rows:
r = torch.nn.functional.interpolate(r.float(), size=(int(width * r.shape[-2] / r.shape[-1]), width), mode="bilinear", antialias=True)
imgs.append(torch.cat(list(r), dim=-1))
grid = (torch.cat(imgs, dim=-2).clamp(0, 1) * 255).round().byte().permute(1, 2, 0).cpu().numpy()
buf = io.BytesIO()
Image.fromarray(grid).save(buf, format="JPEG", quality=80)
return buf.getvalue()
@torch.no_grad()
def cmd_preview(a):
from safetensors.torch import load_file
dev, dt = torch.device(a.device), torch.bfloat16
cfg = ir.DiTConfig()
dit = ir.load_generator(a.checkpoint, cfg, dev, dt)
bank = load_file(a.bank)
hw = (a.height // 8, a.width // 8)
refiner = ir.load_refiner(a.refiner, cfg, dev, dt) if a.refiner else None
decoder = ir.PixelDecoder()
decoder.load_taehv(torch.load(a.decoder, map_location="cpu", weights_only=True))
decoder = decoder.to(device=dev, dtype=dt).eval()
vae, scale = load_vae(a.wan_dir, a.cf_dir, dev)
os.makedirs(a.out, exist_ok=True)
report = []
for pi in a.indices:
pe = bank["embeds"][bank["offsets"][pi]:bank["offsets"][pi + 1]][None].to(device=dev, dtype=dt)
rows, lat = [], {}
for mode in a.modes.split(","):
eng = ir.StreamEngine(dit, ir.EngineConfig(slots=a.frames, cache_update=mode), hw, refiner=refiner)
eng.set_prompt(pe)
g = torch.Generator().manual_seed(a.seed + pi)
zs = []
for _ in range(a.frames):
n_re = len(eng.cfg.first_steps if eng.empty() else eng.cfg.steps) - 1
noise = torch.randn(1, 16, *hw, generator=g).to(device=dev, dtype=dt)
ren = [torch.randn(1, 16, *hw, generator=g).to(device=dev, dtype=dt) for _ in range(n_re)]
zs.append(eng.step(noise, renoise=ren))
lat[mode] = torch.cat(zs) # [F, 16, h, w]
del eng
z = lat[a.modes.split(",")[0]]
ref = vae.decode(z.float().permute(1, 0, 2, 3)[None], scale).float().clamp(-1, 1)[0].permute(1, 0, 2, 3) * 0.5 + 0.5 # [T, 3, H, W]
decoder.reset()
fast = torch.cat([decoder(z[i:i + 1]) for i in range(z.shape[0])]).float()
rec = dict(prompt=pi, frames_vae=ref.shape[0], frames_decoder=fast.shape[0], psnr_decoder_vs_vae=round(psnr(fast, ref), 2),
latent_std=round(float(z.float().std()), 4))
pick = torch.linspace(0, ref.shape[0] - 1, a.cols).round().long()
rows = [ref[pick], fast[pick]]
for mode in a.modes.split(",")[1:]:
other = vae.decode(lat[mode].float().permute(1, 0, 2, 3)[None], scale).float().clamp(-1, 1)[0].permute(1, 0, 2, 3) * 0.5 + 0.5
rec[f"latent_rel_l2_{mode}_vs_{a.modes.split(',')[0]}"] = round(float((lat[mode].float() - z.float()).norm() / z.float().norm()), 4)
rec[f"psnr_{mode}_vs_{a.modes.split(',')[0]}"] = round(psnr(other, ref), 2)
rows.append(other[pick])
jpg = sheet(rows, a.thumb)
path = os.path.join(a.out, f"sheet_{pi:04d}.jpg")
open(path, "wb").write(jpg)
rec["sheet"] = path
report.append(rec)
print(json.dumps(rec), flush=True)
if a.print_b64:
print(f"B64BEGIN {os.path.basename(path)}")
print(base64.b64encode(jpg).decode())
print("B64END", flush=True)
json.dump(report, open(os.path.join(a.out, "preview.json"), "w"), indent=2)
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="cmd", required=True)
p = sub.add_parser("prompts")
p.add_argument("--n", type=int, default=2000)
p.add_argument("--seed", type=int, default=0)
p.add_argument("--out", required=True)
p = sub.add_parser("encode")
p.add_argument("--prompts", required=True)
p.add_argument("--wan-dir", required=True)
p.add_argument("--cf-dir", default=os.path.join(ROOT, "third_party", "cf"))
p.add_argument("--out", required=True)
p.add_argument("--batch", type=int, default=32)
p.add_argument("--device", default="cuda")
p = sub.add_parser("preview")
p.add_argument("--checkpoint", required=True)
p.add_argument("--bank", required=True)
p.add_argument("--decoder", required=True)
p.add_argument("--refiner")
p.add_argument("--wan-dir", required=True)
p.add_argument("--cf-dir", default=os.path.join(ROOT, "third_party", "cf"))
p.add_argument("--out", required=True)
p.add_argument("--indices", type=int, nargs="+", default=[0, 1, 2])
p.add_argument("--modes", default="clean", help="first mode is the reference; others are compared with it")
p.add_argument("--frames", type=int, default=21)
p.add_argument("--height", type=int, default=480)
p.add_argument("--width", type=int, default=832)
p.add_argument("--cols", type=int, default=6)
p.add_argument("--thumb", type=int, default=208)
p.add_argument("--seed", type=int, default=0)
p.add_argument("--device", default="cuda")
p.add_argument("--print-b64", action="store_true")
a = ap.parse_args()
if a.cmd == "prompts":
texts = make_prompts(a.n, a.seed)
open(a.out, "w").write("\n".join(texts) + "\n")
print(f"wrote {len(texts)} prompts to {a.out}")
elif a.cmd == "encode":
cmd_encode(a)
else:
cmd_preview(a)
if __name__ == "__main__":
main()