"""Train the Anima motion module on per-frame image latents (encode.py) with Anima Base frozen. Run with the comfy env. Flow matching as ComfyUI's CONST sampling: x_s = (1 - s) x0 + s eps, target eps - x0.""" import argparse import importlib import json import math import random import sys import time from pathlib import Path import torch from safetensors.torch import save_file COMFY = Path.home() / "AI/Comfy/ComfyUI" sys.path.insert(0, str(COMFY)) sys.path.insert(0, str(COMFY / "custom_nodes/ComfyUI-Anima-Motion")) import comfy.model_management # noqa: E402 import comfy.sd # noqa: E402 import comfy.utils # noqa: E402 from motion import MotionModule, forward_with_motion # noqa: E402 ROOT = Path(__file__).parent BASE = "/mnt/models/comfyui/diffusion_models/anima/anima-base-v1.0.safetensors" SHIFT = 3.0 def load_clips(dm, device): clips = [] for path in sorted((ROOT / "latents").glob("*.pt")): d = torch.load(path) weights = d["t5xxl_weights"] with torch.no_grad(): context = dm.preprocess_text_embeds( d["cross_attn"].unsqueeze(0).to(device), d["t5xxl_ids"].unsqueeze(0).to(device), t5xxl_weights=None if weights is None else weights.unsqueeze(0).unsqueeze(-1).to(device, torch.bfloat16)) clips.append((path.stem, d["latents"].to(device), context, d["ref"].unsqueeze(0).to(device))) return clips def sample_sigma(): t = torch.sigmoid(torch.randn(())).item() return SHIFT * t / (1 + (SHIFT - 1) * t) def main(): p = argparse.ArgumentParser() p.add_argument("--run", default="v1") p.add_argument("--steps", type=int, default=2000) p.add_argument("--lr", type=float, default=1e-4) p.add_argument("--save_every", type=int, default=500) p.add_argument("--cond_prob", type=float, default=0.5, help="chance the first frame is a clean keyframe") p.add_argument("--ref_adapter", help="Anima Ref Adapter v2 LoRA: train R2V, every frame attends to the character sheet") p.add_argument("--init", help="motion checkpoint to start from") p.add_argument("--log_every", type=int, default=25) p.add_argument("--smoke", action="store_true", help="check the zero-init module reproduces Anima, then exit") args = p.parse_args() device = torch.device("cuda") torch.manual_seed(0) random.seed(0) patcher = comfy.sd.load_diffusion_model(BASE, model_options={"dtype": torch.bfloat16}) dm = patcher.model.diffusion_model.to(device) dm.requires_grad_(False) motion = MotionModule(len(dm.blocks), dm.blocks[0].x_dim).to(device) if args.init: motion.load_state_dict(comfy.utils.load_torch_file(args.init)) ref_mod = adapter = None if args.ref_adapter: sys.path.insert(0, str(COMFY / "custom_nodes")) ref_mod = importlib.import_module("ComfyUI-Anima-Ref-Adapter") adapter = ref_mod._Adapter(args.ref_adapter, 1.0) adapter.install(dm) ref_mod._active["key"] = adapter.key # LoRA on for the whole run, also when checkpointing recomputes blocks clips = load_clips(dm, device) print(f"{len(clips)} clips, {sum(p.numel() for p in motion.parameters()) / 1e6:.1f}M trainable params", flush=True) if args.smoke: name, x0, context, _ = clips[0] x = x0.unsqueeze(0) sigma = torch.full((1,), 0.6, device=device) with torch.no_grad(), torch.autocast("cuda", torch.bfloat16): ours = forward_with_motion(dm, motion, x, sigma, context) ref = torch.cat([dm(x[:, :, i:i + 1], sigma, context) for i in range(x.shape[2])], dim=2) print("max |motion(zero) - anima per frame| =", (ours.float() - ref.float()).abs().max().item()) return comfy.model_management.in_training = True # unfused rms/rope: comfy-kitchen's has no backward run_dir = ROOT / "runs" / args.run run_dir.mkdir(parents=True, exist_ok=True) (run_dir / "args.json").write_text(json.dumps(vars(args))) opt = torch.optim.AdamW(motion.parameters(), lr=args.lr, weight_decay=0.0) log = open(run_dir / "loss.txt", "a") t0, avg = time.time(), None for step in range(1, args.steps + 1): name, x0, context, ref_latent = random.choice(clips) x0 = x0.unsqueeze(0) T = x0.shape[2] sigma = torch.full((1, T), sample_sigma(), device=device) keyframe = random.random() < args.cond_prob if keyframe: sigma[:, 0] = 0.0 s = sigma.view(1, 1, T, 1, 1) noise = torch.randn_like(x0) xs = ((1 - s) * x0 + s * noise).to(torch.bfloat16) ref = None if adapter is not None: with torch.no_grad(): ref = (ref_mod._asym_block, ref_mod._asym_ref_kv(adapter, dm, [ref_latent], xs, 1.0)) with torch.autocast("cuda", torch.bfloat16): pred = forward_with_motion(dm, motion, xs, sigma.to(torch.bfloat16), context, checkpoint=True, ref=ref) frames = slice(1, None) if keyframe else slice(None) loss = torch.nn.functional.mse_loss(pred[:, :, frames].float(), (noise - x0)[:, :, frames].float()) opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(motion.parameters(), 1.0) opt.step() avg = loss.item() if avg is None else 0.98 * avg + 0.02 * loss.item() log.write(f"{step} {loss.item():.5f} {sigma[0, -1].item():.3f} {int(keyframe)}\n") if step % args.log_every == 0: log.flush() print(f"step {step} loss {avg:.4f} {(time.time() - t0) / step:.2f}s/step " f"{torch.cuda.max_memory_allocated() / 2**30:.1f}GB", flush=True) if step % args.save_every == 0 or step == args.steps: sd = {k: v.detach().to(torch.bfloat16).contiguous() for k, v in motion.state_dict().items()} save_file(sd, str(run_dir / f"motion_{step:05d}.safetensors"), metadata={"anima_motion": json.dumps({"blocks": len(dm.blocks), "dim": dm.blocks[0].x_dim, "inner_dim": 512, "heads": 8, "frames": T, "fps": 8, "r2v": adapter is not None})}) if __name__ == "__main__": main()