File size: 6,262 Bytes
5cb23ae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
"""Train the Anima motion module on per-frame image latents (encode.py) with Anima Base frozen.
Run with the comfy env. Flow matching as ComfyUI's CONST sampling: x_s = (1 - s) x0 + s eps, target eps - x0."""
import argparse
import importlib
import json
import math
import random
import sys
import time
from pathlib import Path

import torch
from safetensors.torch import save_file

COMFY = Path.home() / "AI/Comfy/ComfyUI"
sys.path.insert(0, str(COMFY))
sys.path.insert(0, str(COMFY / "custom_nodes/ComfyUI-Anima-Motion"))
import comfy.model_management  # noqa: E402
import comfy.sd  # noqa: E402
import comfy.utils  # noqa: E402
from motion import MotionModule, forward_with_motion  # noqa: E402

ROOT = Path(__file__).parent
BASE = "/mnt/models/comfyui/diffusion_models/anima/anima-base-v1.0.safetensors"
SHIFT = 3.0


def load_clips(dm, device):
    clips = []
    for path in sorted((ROOT / "latents").glob("*.pt")):
        d = torch.load(path)
        weights = d["t5xxl_weights"]
        with torch.no_grad():
            context = dm.preprocess_text_embeds(
                d["cross_attn"].unsqueeze(0).to(device), d["t5xxl_ids"].unsqueeze(0).to(device),
                t5xxl_weights=None if weights is None else weights.unsqueeze(0).unsqueeze(-1).to(device, torch.bfloat16))
        clips.append((path.stem, d["latents"].to(device), context, d["ref"].unsqueeze(0).to(device)))
    return clips


def sample_sigma():
    t = torch.sigmoid(torch.randn(())).item()
    return SHIFT * t / (1 + (SHIFT - 1) * t)


def main():
    p = argparse.ArgumentParser()
    p.add_argument("--run", default="v1")
    p.add_argument("--steps", type=int, default=2000)
    p.add_argument("--lr", type=float, default=1e-4)
    p.add_argument("--save_every", type=int, default=500)
    p.add_argument("--cond_prob", type=float, default=0.5, help="chance the first frame is a clean keyframe")
    p.add_argument("--ref_adapter", help="Anima Ref Adapter v2 LoRA: train R2V, every frame attends to the character sheet")
    p.add_argument("--init", help="motion checkpoint to start from")
    p.add_argument("--log_every", type=int, default=25)
    p.add_argument("--smoke", action="store_true", help="check the zero-init module reproduces Anima, then exit")
    args = p.parse_args()
    device = torch.device("cuda")
    torch.manual_seed(0)
    random.seed(0)

    patcher = comfy.sd.load_diffusion_model(BASE, model_options={"dtype": torch.bfloat16})
    dm = patcher.model.diffusion_model.to(device)
    dm.requires_grad_(False)
    motion = MotionModule(len(dm.blocks), dm.blocks[0].x_dim).to(device)
    if args.init:
        motion.load_state_dict(comfy.utils.load_torch_file(args.init))
    ref_mod = adapter = None
    if args.ref_adapter:
        sys.path.insert(0, str(COMFY / "custom_nodes"))
        ref_mod = importlib.import_module("ComfyUI-Anima-Ref-Adapter")
        adapter = ref_mod._Adapter(args.ref_adapter, 1.0)
        adapter.install(dm)
        ref_mod._active["key"] = adapter.key  # LoRA on for the whole run, also when checkpointing recomputes blocks
    clips = load_clips(dm, device)
    print(f"{len(clips)} clips, {sum(p.numel() for p in motion.parameters()) / 1e6:.1f}M trainable params", flush=True)

    if args.smoke:
        name, x0, context, _ = clips[0]
        x = x0.unsqueeze(0)
        sigma = torch.full((1,), 0.6, device=device)
        with torch.no_grad(), torch.autocast("cuda", torch.bfloat16):
            ours = forward_with_motion(dm, motion, x, sigma, context)
            ref = torch.cat([dm(x[:, :, i:i + 1], sigma, context) for i in range(x.shape[2])], dim=2)
        print("max |motion(zero) - anima per frame| =", (ours.float() - ref.float()).abs().max().item())
        return

    comfy.model_management.in_training = True  # unfused rms/rope: comfy-kitchen's has no backward
    run_dir = ROOT / "runs" / args.run
    run_dir.mkdir(parents=True, exist_ok=True)
    (run_dir / "args.json").write_text(json.dumps(vars(args)))
    opt = torch.optim.AdamW(motion.parameters(), lr=args.lr, weight_decay=0.0)
    log = open(run_dir / "loss.txt", "a")
    t0, avg = time.time(), None
    for step in range(1, args.steps + 1):
        name, x0, context, ref_latent = random.choice(clips)
        x0 = x0.unsqueeze(0)
        T = x0.shape[2]
        sigma = torch.full((1, T), sample_sigma(), device=device)
        keyframe = random.random() < args.cond_prob
        if keyframe:
            sigma[:, 0] = 0.0
        s = sigma.view(1, 1, T, 1, 1)
        noise = torch.randn_like(x0)
        xs = ((1 - s) * x0 + s * noise).to(torch.bfloat16)
        ref = None
        if adapter is not None:
            with torch.no_grad():
                ref = (ref_mod._asym_block, ref_mod._asym_ref_kv(adapter, dm, [ref_latent], xs, 1.0))
        with torch.autocast("cuda", torch.bfloat16):
            pred = forward_with_motion(dm, motion, xs, sigma.to(torch.bfloat16), context, checkpoint=True, ref=ref)
        frames = slice(1, None) if keyframe else slice(None)
        loss = torch.nn.functional.mse_loss(pred[:, :, frames].float(), (noise - x0)[:, :, frames].float())
        opt.zero_grad(set_to_none=True)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(motion.parameters(), 1.0)
        opt.step()
        avg = loss.item() if avg is None else 0.98 * avg + 0.02 * loss.item()
        log.write(f"{step} {loss.item():.5f} {sigma[0, -1].item():.3f} {int(keyframe)}\n")
        if step % args.log_every == 0:
            log.flush()
            print(f"step {step} loss {avg:.4f} {(time.time() - t0) / step:.2f}s/step "
                  f"{torch.cuda.max_memory_allocated() / 2**30:.1f}GB", flush=True)
        if step % args.save_every == 0 or step == args.steps:
            sd = {k: v.detach().to(torch.bfloat16).contiguous() for k, v in motion.state_dict().items()}
            save_file(sd, str(run_dir / f"motion_{step:05d}.safetensors"),
                      metadata={"anima_motion": json.dumps({"blocks": len(dm.blocks), "dim": dm.blocks[0].x_dim,
                                                            "inner_dim": 512, "heads": 8, "frames": T, "fps": 8,
                                                            "r2v": adapter is not None})})


if __name__ == "__main__":
    main()