File size: 3,510 Bytes
5cb23ae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """H3 clips -> per-frame image latents at 8 fps + Anima text conditioning. Run with the comfy env from the ComfyUI dir."""
import json
import subprocess
import sys
from pathlib import Path
import numpy as np
import torch
from PIL import Image
COMFY = Path.home() / "AI/Comfy/ComfyUI"
sys.path.insert(0, str(COMFY))
import comfy.latent_formats # noqa: E402
import comfy.sd # noqa: E402
import comfy.utils # noqa: E402
ROOT = Path(__file__).parent
MODELS = Path("/mnt/models/comfyui")
WIDTH, HEIGHT, FRAMES, STRIDE = 384, 576, 16, 3 # 24 fps -> 8 fps
def frames(video):
raw = subprocess.run(["ffmpeg", "-loglevel", "error", "-i", str(video), "-vf",
f"select='not(mod(n\\,{STRIDE}))',scale={WIDTH}:{HEIGHT}:flags=lanczos",
"-vsync", "0", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], capture_output=True, check=True).stdout
a = np.frombuffer(raw, np.uint8).reshape(-1, HEIGHT, WIDTH, 3)[:FRAMES]
assert len(a) == FRAMES, (video, len(a))
return torch.from_numpy(a.copy()).float() / 255.0
def ref_latent(vae, latent_format, path):
# as Anima Ref Adapter: the character sheet resized to ~1 MP, one image encode
image = torch.from_numpy(np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255.0).unsqueeze(0).movedim(-1, 1)
scale = (1024 * 1024 / (image.shape[-1] * image.shape[-2])) ** 0.5
width, height = (max(16, round(d * scale / 16) * 16) for d in (image.shape[-1], image.shape[-2]))
image = comfy.utils.common_upscale(image, width, height, "lanczos", "disabled").movedim(1, -1)
return latent_format.process_in(vae.encode(image))[0].to(torch.bfloat16)
def main():
vae = comfy.sd.VAE(sd=comfy.utils.load_torch_file(str(MODELS / "vae/qwen_image_vae.safetensors")))
clip = comfy.sd.load_clip([str(MODELS / "text_encoders/qwen_3_06b_base.safetensors")],
clip_type=comfy.sd.CLIPType.STABLE_DIFFUSION)
latent_format = comfy.latent_formats.Wan21()
out_dir = ROOT / "latents"
out_dir.mkdir(exist_ok=True)
for info_path in sorted((ROOT / "clips").glob("*.json")):
out = out_dir / f"{info_path.stem}.pt"
info = json.loads(info_path.read_text())
if out.exists():
d = torch.load(out)
if "ref" not in d:
d["ref"] = ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png")
torch.save(d, out)
continue
meta = json.loads((Path(info["keyframe"]).parent / "meta.json").read_text())
motion = info["prompt"].split("The character ")[1].split(". Anime style")[0]
prompt = f"masterpiece, best quality, score_7, safe, {meta['character_tags']}, {motion}, anime screencap"
images = frames(COMFY / "output" / info["video"])
# one encode per frame: the Wan VAE reads a batch as video
latents = torch.cat([vae.encode(images[i:i + 1]) for i in range(FRAMES)], dim=2)
latents = latent_format.process_in(latents)[0].to(torch.bfloat16)
cond, extra = clip.encode_from_tokens_scheduled(clip.tokenize(prompt))[0]
torch.save({"latents": latents, "ref": ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png"), "cross_attn": cond[0].to(torch.bfloat16), "t5xxl_ids": extra["t5xxl_ids"],
"t5xxl_weights": extra.get("t5xxl_weights"), "prompt": prompt}, out)
print(out.name, tuple(latents.shape), prompt, flush=True)
if __name__ == "__main__":
main()
|