paralaif's picture
Add Anima motion module v0.1 (I2V and R2V), ComfyUI node, workflow, scripts and examples
5cb23ae verified
Raw History Blame Contribute Delete
6.26 kB
"""Train the Anima motion module on per-frame image latents (encode.py) with Anima Base frozen.
Run with the comfy env. Flow matching as ComfyUI's CONST sampling: x_s = (1 - s) x0 + s eps, target eps - x0."""
import argparse
import importlib
import json
import math
import random
import sys
import time
from pathlib import Path
import torch
from safetensors.torch import save_file
COMFY = Path.home() / "AI/Comfy/ComfyUI"
sys.path.insert(0, str(COMFY))
sys.path.insert(0, str(COMFY / "custom_nodes/ComfyUI-Anima-Motion"))
import comfy.model_management # noqa: E402
import comfy.sd # noqa: E402
import comfy.utils # noqa: E402
from motion import MotionModule, forward_with_motion # noqa: E402
ROOT = Path(__file__).parent
BASE = "/mnt/models/comfyui/diffusion_models/anima/anima-base-v1.0.safetensors"
SHIFT = 3.0
def load_clips(dm, device):
clips = []
for path in sorted((ROOT / "latents").glob("*.pt")):
d = torch.load(path)
weights = d["t5xxl_weights"]
with torch.no_grad():
context = dm.preprocess_text_embeds(
d["cross_attn"].unsqueeze(0).to(device), d["t5xxl_ids"].unsqueeze(0).to(device),
t5xxl_weights=None if weights is None else weights.unsqueeze(0).unsqueeze(-1).to(device, torch.bfloat16))
clips.append((path.stem, d["latents"].to(device), context, d["ref"].unsqueeze(0).to(device)))
return clips
def sample_sigma():
t = torch.sigmoid(torch.randn(())).item()
return SHIFT * t / (1 + (SHIFT - 1) * t)
def main():
p = argparse.ArgumentParser()
p.add_argument("--run", default="v1")
p.add_argument("--steps", type=int, default=2000)
p.add_argument("--lr", type=float, default=1e-4)
p.add_argument("--save_every", type=int, default=500)
p.add_argument("--cond_prob", type=float, default=0.5, help="chance the first frame is a clean keyframe")
p.add_argument("--ref_adapter", help="Anima Ref Adapter v2 LoRA: train R2V, every frame attends to the character sheet")
p.add_argument("--init", help="motion checkpoint to start from")
p.add_argument("--log_every", type=int, default=25)
p.add_argument("--smoke", action="store_true", help="check the zero-init module reproduces Anima, then exit")
args = p.parse_args()
device = torch.device("cuda")
torch.manual_seed(0)
random.seed(0)
patcher = comfy.sd.load_diffusion_model(BASE, model_options={"dtype": torch.bfloat16})
dm = patcher.model.diffusion_model.to(device)
dm.requires_grad_(False)
motion = MotionModule(len(dm.blocks), dm.blocks[0].x_dim).to(device)
if args.init:
motion.load_state_dict(comfy.utils.load_torch_file(args.init))
ref_mod = adapter = None
if args.ref_adapter:
sys.path.insert(0, str(COMFY / "custom_nodes"))
ref_mod = importlib.import_module("ComfyUI-Anima-Ref-Adapter")
adapter = ref_mod._Adapter(args.ref_adapter, 1.0)
adapter.install(dm)
ref_mod._active["key"] = adapter.key # LoRA on for the whole run, also when checkpointing recomputes blocks
clips = load_clips(dm, device)
print(f"{len(clips)} clips, {sum(p.numel() for p in motion.parameters()) / 1e6:.1f}M trainable params", flush=True)
if args.smoke:
name, x0, context, _ = clips[0]
x = x0.unsqueeze(0)
sigma = torch.full((1,), 0.6, device=device)
with torch.no_grad(), torch.autocast("cuda", torch.bfloat16):
ours = forward_with_motion(dm, motion, x, sigma, context)
ref = torch.cat([dm(x[:, :, i:i + 1], sigma, context) for i in range(x.shape[2])], dim=2)
print("max |motion(zero) - anima per frame| =", (ours.float() - ref.float()).abs().max().item())
return
comfy.model_management.in_training = True # unfused rms/rope: comfy-kitchen's has no backward
run_dir = ROOT / "runs" / args.run
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / "args.json").write_text(json.dumps(vars(args)))
opt = torch.optim.AdamW(motion.parameters(), lr=args.lr, weight_decay=0.0)
log = open(run_dir / "loss.txt", "a")
t0, avg = time.time(), None
for step in range(1, args.steps + 1):
name, x0, context, ref_latent = random.choice(clips)
x0 = x0.unsqueeze(0)
T = x0.shape[2]
sigma = torch.full((1, T), sample_sigma(), device=device)
keyframe = random.random() < args.cond_prob
if keyframe:
sigma[:, 0] = 0.0
s = sigma.view(1, 1, T, 1, 1)
noise = torch.randn_like(x0)
xs = ((1 - s) * x0 + s * noise).to(torch.bfloat16)
ref = None
if adapter is not None:
with torch.no_grad():
ref = (ref_mod._asym_block, ref_mod._asym_ref_kv(adapter, dm, [ref_latent], xs, 1.0))
with torch.autocast("cuda", torch.bfloat16):
pred = forward_with_motion(dm, motion, xs, sigma.to(torch.bfloat16), context, checkpoint=True, ref=ref)
frames = slice(1, None) if keyframe else slice(None)
loss = torch.nn.functional.mse_loss(pred[:, :, frames].float(), (noise - x0)[:, :, frames].float())
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(motion.parameters(), 1.0)
opt.step()
avg = loss.item() if avg is None else 0.98 * avg + 0.02 * loss.item()
log.write(f"{step} {loss.item():.5f} {sigma[0, -1].item():.3f} {int(keyframe)}\n")
if step % args.log_every == 0:
log.flush()
print(f"step {step} loss {avg:.4f} {(time.time() - t0) / step:.2f}s/step "
f"{torch.cuda.max_memory_allocated() / 2**30:.1f}GB", flush=True)
if step % args.save_every == 0 or step == args.steps:
sd = {k: v.detach().to(torch.bfloat16).contiguous() for k, v in motion.state_dict().items()}
save_file(sd, str(run_dir / f"motion_{step:05d}.safetensors"),
metadata={"anima_motion": json.dumps({"blocks": len(dm.blocks), "dim": dm.blocks[0].x_dim,
"inner_dim": 512, "heads": 8, "frames": T, "fps": 8,
"r2v": adapter is not None})})
if __name__ == "__main__":
main()