Download scripts/encode.py from paralaif/anima-motion-module: direct link, hf CLI and curl.
- Browser
- Download file 3.51 kB
-
https://huggingface.co/paralaif/anima-motion-module/resolve/main/scripts/encode.py
- Command line
-
hf download hf://paralaif/anima-motion-module/scripts/encode.py
-
curl -L -o encode.py https://huggingface.co/paralaif/anima-motion-module/resolve/main/scripts/encode.py
3.51 kB
| """H3 clips -> per-frame image latents at 8 fps + Anima text conditioning. Run with the comfy env from the ComfyUI dir.""" | |
| import json | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| COMFY = Path.home() / "AI/Comfy/ComfyUI" | |
| sys.path.insert(0, str(COMFY)) | |
| import comfy.latent_formats # noqa: E402 | |
| import comfy.sd # noqa: E402 | |
| import comfy.utils # noqa: E402 | |
| ROOT = Path(__file__).parent | |
| MODELS = Path("/mnt/models/comfyui") | |
| WIDTH, HEIGHT, FRAMES, STRIDE = 384, 576, 16, 3 # 24 fps -> 8 fps | |
| def frames(video): | |
| raw = subprocess.run(["ffmpeg", "-loglevel", "error", "-i", str(video), "-vf", | |
| f"select='not(mod(n\\,{STRIDE}))',scale={WIDTH}:{HEIGHT}:flags=lanczos", | |
| "-vsync", "0", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], capture_output=True, check=True).stdout | |
| a = np.frombuffer(raw, np.uint8).reshape(-1, HEIGHT, WIDTH, 3)[:FRAMES] | |
| assert len(a) == FRAMES, (video, len(a)) | |
| return torch.from_numpy(a.copy()).float() / 255.0 | |
| def ref_latent(vae, latent_format, path): | |
| # as Anima Ref Adapter: the character sheet resized to ~1 MP, one image encode | |
| image = torch.from_numpy(np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255.0).unsqueeze(0).movedim(-1, 1) | |
| scale = (1024 * 1024 / (image.shape[-1] * image.shape[-2])) ** 0.5 | |
| width, height = (max(16, round(d * scale / 16) * 16) for d in (image.shape[-1], image.shape[-2])) | |
| image = comfy.utils.common_upscale(image, width, height, "lanczos", "disabled").movedim(1, -1) | |
| return latent_format.process_in(vae.encode(image))[0].to(torch.bfloat16) | |
| def main(): | |
| vae = comfy.sd.VAE(sd=comfy.utils.load_torch_file(str(MODELS / "vae/qwen_image_vae.safetensors"))) | |
| clip = comfy.sd.load_clip([str(MODELS / "text_encoders/qwen_3_06b_base.safetensors")], | |
| clip_type=comfy.sd.CLIPType.STABLE_DIFFUSION) | |
| latent_format = comfy.latent_formats.Wan21() | |
| out_dir = ROOT / "latents" | |
| out_dir.mkdir(exist_ok=True) | |
| for info_path in sorted((ROOT / "clips").glob("*.json")): | |
| out = out_dir / f"{info_path.stem}.pt" | |
| info = json.loads(info_path.read_text()) | |
| if out.exists(): | |
| d = torch.load(out) | |
| if "ref" not in d: | |
| d["ref"] = ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png") | |
| torch.save(d, out) | |
| continue | |
| meta = json.loads((Path(info["keyframe"]).parent / "meta.json").read_text()) | |
| motion = info["prompt"].split("The character ")[1].split(". Anime style")[0] | |
| prompt = f"masterpiece, best quality, score_7, safe, {meta['character_tags']}, {motion}, anime screencap" | |
| images = frames(COMFY / "output" / info["video"]) | |
| # one encode per frame: the Wan VAE reads a batch as video | |
| latents = torch.cat([vae.encode(images[i:i + 1]) for i in range(FRAMES)], dim=2) | |
| latents = latent_format.process_in(latents)[0].to(torch.bfloat16) | |
| cond, extra = clip.encode_from_tokens_scheduled(clip.tokenize(prompt))[0] | |
| torch.save({"latents": latents, "ref": ref_latent(vae, latent_format, Path(info["keyframe"]).parent / "ref.png"), "cross_attn": cond[0].to(torch.bfloat16), "t5xxl_ids": extra["t5xxl_ids"], | |
| "t5xxl_weights": extra.get("t5xxl_weights"), "prompt": prompt}, out) | |
| print(out.name, tuple(latents.shape), prompt, flush=True) | |
| if __name__ == "__main__": | |
| main() | |