Download scripts/train.py from paralaif/anima-motion-module: direct link, hf CLI and curl.
- Browser
- Download file 6.26 kB
-
https://huggingface.co/paralaif/anima-motion-module/resolve/main/scripts/train.py
- Command line
-
hf download hf://paralaif/anima-motion-module/scripts/train.py
-
curl -L -o train.py https://huggingface.co/paralaif/anima-motion-module/resolve/main/scripts/train.py
6.26 kB
| """Train the Anima motion module on per-frame image latents (encode.py) with Anima Base frozen. | |
| Run with the comfy env. Flow matching as ComfyUI's CONST sampling: x_s = (1 - s) x0 + s eps, target eps - x0.""" | |
| import argparse | |
| import importlib | |
| import json | |
| import math | |
| import random | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import torch | |
| from safetensors.torch import save_file | |
| COMFY = Path.home() / "AI/Comfy/ComfyUI" | |
| sys.path.insert(0, str(COMFY)) | |
| sys.path.insert(0, str(COMFY / "custom_nodes/ComfyUI-Anima-Motion")) | |
| import comfy.model_management # noqa: E402 | |
| import comfy.sd # noqa: E402 | |
| import comfy.utils # noqa: E402 | |
| from motion import MotionModule, forward_with_motion # noqa: E402 | |
| ROOT = Path(__file__).parent | |
| BASE = "/mnt/models/comfyui/diffusion_models/anima/anima-base-v1.0.safetensors" | |
| SHIFT = 3.0 | |
| def load_clips(dm, device): | |
| clips = [] | |
| for path in sorted((ROOT / "latents").glob("*.pt")): | |
| d = torch.load(path) | |
| weights = d["t5xxl_weights"] | |
| with torch.no_grad(): | |
| context = dm.preprocess_text_embeds( | |
| d["cross_attn"].unsqueeze(0).to(device), d["t5xxl_ids"].unsqueeze(0).to(device), | |
| t5xxl_weights=None if weights is None else weights.unsqueeze(0).unsqueeze(-1).to(device, torch.bfloat16)) | |
| clips.append((path.stem, d["latents"].to(device), context, d["ref"].unsqueeze(0).to(device))) | |
| return clips | |
| def sample_sigma(): | |
| t = torch.sigmoid(torch.randn(())).item() | |
| return SHIFT * t / (1 + (SHIFT - 1) * t) | |
| def main(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--run", default="v1") | |
| p.add_argument("--steps", type=int, default=2000) | |
| p.add_argument("--lr", type=float, default=1e-4) | |
| p.add_argument("--save_every", type=int, default=500) | |
| p.add_argument("--cond_prob", type=float, default=0.5, help="chance the first frame is a clean keyframe") | |
| p.add_argument("--ref_adapter", help="Anima Ref Adapter v2 LoRA: train R2V, every frame attends to the character sheet") | |
| p.add_argument("--init", help="motion checkpoint to start from") | |
| p.add_argument("--log_every", type=int, default=25) | |
| p.add_argument("--smoke", action="store_true", help="check the zero-init module reproduces Anima, then exit") | |
| args = p.parse_args() | |
| device = torch.device("cuda") | |
| torch.manual_seed(0) | |
| random.seed(0) | |
| patcher = comfy.sd.load_diffusion_model(BASE, model_options={"dtype": torch.bfloat16}) | |
| dm = patcher.model.diffusion_model.to(device) | |
| dm.requires_grad_(False) | |
| motion = MotionModule(len(dm.blocks), dm.blocks[0].x_dim).to(device) | |
| if args.init: | |
| motion.load_state_dict(comfy.utils.load_torch_file(args.init)) | |
| ref_mod = adapter = None | |
| if args.ref_adapter: | |
| sys.path.insert(0, str(COMFY / "custom_nodes")) | |
| ref_mod = importlib.import_module("ComfyUI-Anima-Ref-Adapter") | |
| adapter = ref_mod._Adapter(args.ref_adapter, 1.0) | |
| adapter.install(dm) | |
| ref_mod._active["key"] = adapter.key # LoRA on for the whole run, also when checkpointing recomputes blocks | |
| clips = load_clips(dm, device) | |
| print(f"{len(clips)} clips, {sum(p.numel() for p in motion.parameters()) / 1e6:.1f}M trainable params", flush=True) | |
| if args.smoke: | |
| name, x0, context, _ = clips[0] | |
| x = x0.unsqueeze(0) | |
| sigma = torch.full((1,), 0.6, device=device) | |
| with torch.no_grad(), torch.autocast("cuda", torch.bfloat16): | |
| ours = forward_with_motion(dm, motion, x, sigma, context) | |
| ref = torch.cat([dm(x[:, :, i:i + 1], sigma, context) for i in range(x.shape[2])], dim=2) | |
| print("max |motion(zero) - anima per frame| =", (ours.float() - ref.float()).abs().max().item()) | |
| return | |
| comfy.model_management.in_training = True # unfused rms/rope: comfy-kitchen's has no backward | |
| run_dir = ROOT / "runs" / args.run | |
| run_dir.mkdir(parents=True, exist_ok=True) | |
| (run_dir / "args.json").write_text(json.dumps(vars(args))) | |
| opt = torch.optim.AdamW(motion.parameters(), lr=args.lr, weight_decay=0.0) | |
| log = open(run_dir / "loss.txt", "a") | |
| t0, avg = time.time(), None | |
| for step in range(1, args.steps + 1): | |
| name, x0, context, ref_latent = random.choice(clips) | |
| x0 = x0.unsqueeze(0) | |
| T = x0.shape[2] | |
| sigma = torch.full((1, T), sample_sigma(), device=device) | |
| keyframe = random.random() < args.cond_prob | |
| if keyframe: | |
| sigma[:, 0] = 0.0 | |
| s = sigma.view(1, 1, T, 1, 1) | |
| noise = torch.randn_like(x0) | |
| xs = ((1 - s) * x0 + s * noise).to(torch.bfloat16) | |
| ref = None | |
| if adapter is not None: | |
| with torch.no_grad(): | |
| ref = (ref_mod._asym_block, ref_mod._asym_ref_kv(adapter, dm, [ref_latent], xs, 1.0)) | |
| with torch.autocast("cuda", torch.bfloat16): | |
| pred = forward_with_motion(dm, motion, xs, sigma.to(torch.bfloat16), context, checkpoint=True, ref=ref) | |
| frames = slice(1, None) if keyframe else slice(None) | |
| loss = torch.nn.functional.mse_loss(pred[:, :, frames].float(), (noise - x0)[:, :, frames].float()) | |
| opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(motion.parameters(), 1.0) | |
| opt.step() | |
| avg = loss.item() if avg is None else 0.98 * avg + 0.02 * loss.item() | |
| log.write(f"{step} {loss.item():.5f} {sigma[0, -1].item():.3f} {int(keyframe)}\n") | |
| if step % args.log_every == 0: | |
| log.flush() | |
| print(f"step {step} loss {avg:.4f} {(time.time() - t0) / step:.2f}s/step " | |
| f"{torch.cuda.max_memory_allocated() / 2**30:.1f}GB", flush=True) | |
| if step % args.save_every == 0 or step == args.steps: | |
| sd = {k: v.detach().to(torch.bfloat16).contiguous() for k, v in motion.state_dict().items()} | |
| save_file(sd, str(run_dir / f"motion_{step:05d}.safetensors"), | |
| metadata={"anima_motion": json.dumps({"blocks": len(dm.blocks), "dim": dm.blocks[0].x_dim, | |
| "inner_dim": 512, "heads": 8, "frames": T, "fps": 8, | |
| "r2v": adapter is not None})}) | |
| if __name__ == "__main__": | |
| main() | |