"""H3 clips -> per-frame image latents at 8 fps + Anima text conditioning. Run with the comfy env from the ComfyUI dir.""" import json import subprocess import sys from pathlib import Path import numpy as np import torch from PIL import Image COMFY = Path.home() / "AI/Comfy/ComfyUI" sys.path.insert(0, str(COMFY)) import comfy.latent_formats # noqa: E402 import comfy.sd # noqa: E402 import comfy.utils # noqa: E402 ROOT = Path(__file__).parent MODELS = Path("/mnt/models/comfyui") WIDTH, HEIGHT, FRAMES, STRIDE = 384, 576, 16, 3 # 24 fps -> 8 fps def frames(video): raw = subprocess.run(["ffmpeg", "-loglevel", "error", "-i", str(video), "-vf", f"select='not(mod(n\\,{STRIDE}))',scale={WIDTH}:{HEIGHT}:flags=lanczos", "-vsync", "0", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], capture_output=True, check=True).stdout a = np.frombuffer(raw, np.uint8).reshape(-1, HEIGHT, WIDTH, 3)[:FRAMES] assert len(a) == FRAMES, (video, len(a)) return torch.from_numpy(a.copy()).float() / 255.0 def ref_latent(vae, latent_format, path): # as Anima Ref Adapter: the character sheet resized to ~1 MP, one image encode image = torch.from_numpy(np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255.0).unsqueeze(0).movedim(-1, 1) scale = (1024 * 1024 / (image.shape[-1] * image.shape[-2])) ** 0.5 width, height = (max(16, round(d * scale / 16) * 16) for d in (image.shape[-1], image.shape[-2])) image = comfy.utils.common_upscale(image, width, height, "lanczos", "disabled").movedim(1, -1) return latent_format.process_in(vae.encode(image))[0].to(torch.bfloat16) def main(): vae = comfy.sd.VAE(sd=comfy.utils.load_torch_file(str(MODELS / "vae/qwen_image_vae.safetensors"))) clip = comfy.sd.load_clip([str(MODELS / "text_encoders/qwen_3_06b_base.safetensors")], clip_type=comfy.sd.CLIPType.STABLE_DIFFUSION) latent_format = comfy.latent_formats.Wan21() out_dir = ROOT / "latents" out_dir.mkdir(exist_ok=True) for info_path in sorted((ROOT / "clips").glob("*.json")): out = out_dir / f"{info_path.stem}.pt" info = json.loads(info_path.read_text()) if out.exists(): d = torch.load(out) if "ref" not in d: d["ref"] = ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png") torch.save(d, out) continue meta = json.loads((Path(info["keyframe"]).parent / "meta.json").read_text()) motion = info["prompt"].split("The character ")[1].split(". Anime style")[0] prompt = f"masterpiece, best quality, score_7, safe, {meta['character_tags']}, {motion}, anime screencap" images = frames(COMFY / "output" / info["video"]) # one encode per frame: the Wan VAE reads a batch as video latents = torch.cat([vae.encode(images[i:i + 1]) for i in range(FRAMES)], dim=2) latents = latent_format.process_in(latents)[0].to(torch.bfloat16) cond, extra = clip.encode_from_tokens_scheduled(clip.tokenize(prompt))[0] torch.save({"latents": latents, "ref": ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png"), "cross_attn": cond[0].to(torch.bfloat16), "t5xxl_ids": extra["t5xxl_ids"], "t5xxl_weights": extra.get("t5xxl_weights"), "prompt": prompt}, out) print(out.name, tuple(latents.shape), prompt, flush=True) if __name__ == "__main__": main()