File size: 6,262 Bytes
5cb23ae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | """Train the Anima motion module on per-frame image latents (encode.py) with Anima Base frozen.
Run with the comfy env. Flow matching as ComfyUI's CONST sampling: x_s = (1 - s) x0 + s eps, target eps - x0."""
import argparse
import importlib
import json
import math
import random
import sys
import time
from pathlib import Path
import torch
from safetensors.torch import save_file
COMFY = Path.home() / "AI/Comfy/ComfyUI"
sys.path.insert(0, str(COMFY))
sys.path.insert(0, str(COMFY / "custom_nodes/ComfyUI-Anima-Motion"))
import comfy.model_management # noqa: E402
import comfy.sd # noqa: E402
import comfy.utils # noqa: E402
from motion import MotionModule, forward_with_motion # noqa: E402
ROOT = Path(__file__).parent
BASE = "/mnt/models/comfyui/diffusion_models/anima/anima-base-v1.0.safetensors"
SHIFT = 3.0
def load_clips(dm, device):
clips = []
for path in sorted((ROOT / "latents").glob("*.pt")):
d = torch.load(path)
weights = d["t5xxl_weights"]
with torch.no_grad():
context = dm.preprocess_text_embeds(
d["cross_attn"].unsqueeze(0).to(device), d["t5xxl_ids"].unsqueeze(0).to(device),
t5xxl_weights=None if weights is None else weights.unsqueeze(0).unsqueeze(-1).to(device, torch.bfloat16))
clips.append((path.stem, d["latents"].to(device), context, d["ref"].unsqueeze(0).to(device)))
return clips
def sample_sigma():
t = torch.sigmoid(torch.randn(())).item()
return SHIFT * t / (1 + (SHIFT - 1) * t)
def main():
p = argparse.ArgumentParser()
p.add_argument("--run", default="v1")
p.add_argument("--steps", type=int, default=2000)
p.add_argument("--lr", type=float, default=1e-4)
p.add_argument("--save_every", type=int, default=500)
p.add_argument("--cond_prob", type=float, default=0.5, help="chance the first frame is a clean keyframe")
p.add_argument("--ref_adapter", help="Anima Ref Adapter v2 LoRA: train R2V, every frame attends to the character sheet")
p.add_argument("--init", help="motion checkpoint to start from")
p.add_argument("--log_every", type=int, default=25)
p.add_argument("--smoke", action="store_true", help="check the zero-init module reproduces Anima, then exit")
args = p.parse_args()
device = torch.device("cuda")
torch.manual_seed(0)
random.seed(0)
patcher = comfy.sd.load_diffusion_model(BASE, model_options={"dtype": torch.bfloat16})
dm = patcher.model.diffusion_model.to(device)
dm.requires_grad_(False)
motion = MotionModule(len(dm.blocks), dm.blocks[0].x_dim).to(device)
if args.init:
motion.load_state_dict(comfy.utils.load_torch_file(args.init))
ref_mod = adapter = None
if args.ref_adapter:
sys.path.insert(0, str(COMFY / "custom_nodes"))
ref_mod = importlib.import_module("ComfyUI-Anima-Ref-Adapter")
adapter = ref_mod._Adapter(args.ref_adapter, 1.0)
adapter.install(dm)
ref_mod._active["key"] = adapter.key # LoRA on for the whole run, also when checkpointing recomputes blocks
clips = load_clips(dm, device)
print(f"{len(clips)} clips, {sum(p.numel() for p in motion.parameters()) / 1e6:.1f}M trainable params", flush=True)
if args.smoke:
name, x0, context, _ = clips[0]
x = x0.unsqueeze(0)
sigma = torch.full((1,), 0.6, device=device)
with torch.no_grad(), torch.autocast("cuda", torch.bfloat16):
ours = forward_with_motion(dm, motion, x, sigma, context)
ref = torch.cat([dm(x[:, :, i:i + 1], sigma, context) for i in range(x.shape[2])], dim=2)
print("max |motion(zero) - anima per frame| =", (ours.float() - ref.float()).abs().max().item())
return
comfy.model_management.in_training = True # unfused rms/rope: comfy-kitchen's has no backward
run_dir = ROOT / "runs" / args.run
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / "args.json").write_text(json.dumps(vars(args)))
opt = torch.optim.AdamW(motion.parameters(), lr=args.lr, weight_decay=0.0)
log = open(run_dir / "loss.txt", "a")
t0, avg = time.time(), None
for step in range(1, args.steps + 1):
name, x0, context, ref_latent = random.choice(clips)
x0 = x0.unsqueeze(0)
T = x0.shape[2]
sigma = torch.full((1, T), sample_sigma(), device=device)
keyframe = random.random() < args.cond_prob
if keyframe:
sigma[:, 0] = 0.0
s = sigma.view(1, 1, T, 1, 1)
noise = torch.randn_like(x0)
xs = ((1 - s) * x0 + s * noise).to(torch.bfloat16)
ref = None
if adapter is not None:
with torch.no_grad():
ref = (ref_mod._asym_block, ref_mod._asym_ref_kv(adapter, dm, [ref_latent], xs, 1.0))
with torch.autocast("cuda", torch.bfloat16):
pred = forward_with_motion(dm, motion, xs, sigma.to(torch.bfloat16), context, checkpoint=True, ref=ref)
frames = slice(1, None) if keyframe else slice(None)
loss = torch.nn.functional.mse_loss(pred[:, :, frames].float(), (noise - x0)[:, :, frames].float())
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(motion.parameters(), 1.0)
opt.step()
avg = loss.item() if avg is None else 0.98 * avg + 0.02 * loss.item()
log.write(f"{step} {loss.item():.5f} {sigma[0, -1].item():.3f} {int(keyframe)}\n")
if step % args.log_every == 0:
log.flush()
print(f"step {step} loss {avg:.4f} {(time.time() - t0) / step:.2f}s/step "
f"{torch.cuda.max_memory_allocated() / 2**30:.1f}GB", flush=True)
if step % args.save_every == 0 or step == args.steps:
sd = {k: v.detach().to(torch.bfloat16).contiguous() for k, v in motion.state_dict().items()}
save_file(sd, str(run_dir / f"motion_{step:05d}.safetensors"),
metadata={"anima_motion": json.dumps({"blocks": len(dm.blocks), "dim": dm.blocks[0].x_dim,
"inner_dim": 512, "heads": 8, "frames": T, "fps": 8,
"r2v": adapter is not None})})
if __name__ == "__main__":
main()
|