paralaif's picture
Add Anima motion module v0.1 (I2V and R2V), ComfyUI node, workflow, scripts and examples
5cb23ae verified
Raw History Blame Contribute Delete
3.51 kB
"""H3 clips -> per-frame image latents at 8 fps + Anima text conditioning. Run with the comfy env from the ComfyUI dir."""
import json
import subprocess
import sys
from pathlib import Path
import numpy as np
import torch
from PIL import Image
COMFY = Path.home() / "AI/Comfy/ComfyUI"
sys.path.insert(0, str(COMFY))
import comfy.latent_formats # noqa: E402
import comfy.sd # noqa: E402
import comfy.utils # noqa: E402
ROOT = Path(__file__).parent
MODELS = Path("/mnt/models/comfyui")
WIDTH, HEIGHT, FRAMES, STRIDE = 384, 576, 16, 3 # 24 fps -> 8 fps
def frames(video):
raw = subprocess.run(["ffmpeg", "-loglevel", "error", "-i", str(video), "-vf",
f"select='not(mod(n\\,{STRIDE}))',scale={WIDTH}:{HEIGHT}:flags=lanczos",
"-vsync", "0", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], capture_output=True, check=True).stdout
a = np.frombuffer(raw, np.uint8).reshape(-1, HEIGHT, WIDTH, 3)[:FRAMES]
assert len(a) == FRAMES, (video, len(a))
return torch.from_numpy(a.copy()).float() / 255.0
def ref_latent(vae, latent_format, path):
# as Anima Ref Adapter: the character sheet resized to ~1 MP, one image encode
image = torch.from_numpy(np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255.0).unsqueeze(0).movedim(-1, 1)
scale = (1024 * 1024 / (image.shape[-1] * image.shape[-2])) ** 0.5
width, height = (max(16, round(d * scale / 16) * 16) for d in (image.shape[-1], image.shape[-2]))
image = comfy.utils.common_upscale(image, width, height, "lanczos", "disabled").movedim(1, -1)
return latent_format.process_in(vae.encode(image))[0].to(torch.bfloat16)
def main():
vae = comfy.sd.VAE(sd=comfy.utils.load_torch_file(str(MODELS / "vae/qwen_image_vae.safetensors")))
clip = comfy.sd.load_clip([str(MODELS / "text_encoders/qwen_3_06b_base.safetensors")],
clip_type=comfy.sd.CLIPType.STABLE_DIFFUSION)
latent_format = comfy.latent_formats.Wan21()
out_dir = ROOT / "latents"
out_dir.mkdir(exist_ok=True)
for info_path in sorted((ROOT / "clips").glob("*.json")):
out = out_dir / f"{info_path.stem}.pt"
info = json.loads(info_path.read_text())
if out.exists():
d = torch.load(out)
if "ref" not in d:
d["ref"] = ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png")
torch.save(d, out)
continue
meta = json.loads((Path(info["keyframe"]).parent / "meta.json").read_text())
motion = info["prompt"].split("The character ")[1].split(". Anime style")[0]
prompt = f"masterpiece, best quality, score_7, safe, {meta['character_tags']}, {motion}, anime screencap"
images = frames(COMFY / "output" / info["video"])
# one encode per frame: the Wan VAE reads a batch as video
latents = torch.cat([vae.encode(images[i:i + 1]) for i in range(FRAMES)], dim=2)
latents = latent_format.process_in(latents)[0].to(torch.bfloat16)
cond, extra = clip.encode_from_tokens_scheduled(clip.tokenize(prompt))[0]
torch.save({"latents": latents, "ref": ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png"), "cross_attn": cond[0].to(torch.bfloat16), "t5xxl_ids": extra["t5xxl_ids"],
"t5xxl_weights": extra.get("t5xxl_weights"), "prompt": prompt}, out)
print(out.name, tuple(latents.shape), prompt, flush=True)
if __name__ == "__main__":
main()