PartLock-CMU / code /gen_ar.py
Yakimask's picture
Upload 31 files
c7ae99a verified
Raw History Blame Contribute Delete
9.9 kB
"""Plan 2: text -> causal-H tokens, one token at a time.
Per 4 frames: 7 tokens in order [global(4096), torso, head, arm_R, arm_L, leg_R, leg_L (128 each)].
- decoder-only Transformer, 8 layers, d=512, RoPE over flattened position (slides forever)
- text (CLIP ViT-B/32, 512) added to EVERY position (per-slot text possible later)
- corruption: replace tau*100% input tokens with random ones, tau~U[0,1] per sample (T2M-GPT)
- CFG: 10% of samples get the null text (MoMask / MotionStreamer)
- training stops by wall-clock (--seconds), cosine schedule on elapsed/budget
"""
import os, sys, math, time, json, argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
BASE = os.environ.get("PARTLOCK_DIR", ".") # folder that holds data/, ckpt/, runs/
GEN = os.path.join(BASE, "data", "gen")
CKPT = os.path.join(BASE, "ckpt")
RUNS = os.path.join(BASE, "runs")
VOC = [4096] + [128] * 6
OFF = [0]
for v in VOC[:-1]:
OFF.append(OFF[-1] + v)
NV = sum(VOC) # 4864
S = 64 # steps of context (= 8.5 s)
L = S * 7
def rope(x, pos): # x (B,h,T,dh)
dh = x.shape[-1]
inv = 1.0 / (10000 ** (torch.arange(0, dh, 2, device=x.device).float() / dh))
a = pos[:, None].float() * inv[None]
c, s = a.cos()[None, None], a.sin()[None, None]
x1, x2 = x[..., ::2], x[..., 1::2]
return torch.stack([x1 * c - x2 * s, x1 * s + x2 * c], -1).flatten(-2)
class Block(nn.Module):
def __init__(self, d, h, drop):
super().__init__()
self.h = h
self.n1, self.n2 = nn.LayerNorm(d), nn.LayerNorm(d)
self.qkv, self.o = nn.Linear(d, 3 * d), nn.Linear(d, d)
self.ff = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
self.drop = drop
def forward(self, x, pos):
B, T, D = x.shape
q, k, v = self.qkv(self.n1(x)).view(B, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
q, k = rope(q, pos), rope(k, pos)
a = F.scaled_dot_product_attention(q, k, v, is_causal=True, dropout_p=self.drop if self.training else 0)
x = x + F.dropout(self.o(a.transpose(1, 2).reshape(B, T, D)), self.drop, self.training)
return x + F.dropout(self.ff(self.n2(x)), self.drop, self.training)
class Gen(nn.Module):
def __init__(self, d=512, layers=8, heads=8, drop=0.1):
super().__init__()
self.tok = nn.Embedding(NV + 1, d) # last = BOS
self.slot = nn.Embedding(7, d) # slot of the token being PREDICTED
self.txt = nn.Linear(512, d)
self.null = nn.Parameter(torch.zeros(512))
self.blocks = nn.ModuleList([Block(d, heads, drop) for _ in range(layers)])
self.nf = nn.LayerNorm(d)
self.head = nn.Linear(d, NV)
mask = torch.full((7, NV), float("-inf"))
for s in range(7):
mask[s, OFF[s]:OFF[s] + VOC[s]] = 0
self.register_buffer("mask", mask)
def forward(self, inp, slots, text):
"""inp (B,T) flat ids (BOS=NV); slots (T,) slot predicted at each position; text (B,T,512)"""
x = self.tok(inp) + self.slot(slots)[None] + self.txt(text)
pos = torch.arange(inp.shape[1], device=inp.device)
for b in self.blocks:
x = b(x, pos)
return self.head(self.nf(x)) + self.mask[slots][None]
def flat(codes): # (n,7) local ids -> (n*7,) global ids
return (codes.long() + torch.tensor(OFF)).reshape(-1)
def batch(rec, bs, dev, g, corrupt=True, drop=0.1, fixed=False):
inps, tgts, txts = [], [], []
lens = torch.tensor([r["codes"].shape[0] for r in rec], dtype=torch.float)
idx = range(bs) if fixed else torch.multinomial(lens, bs, replacement=True, generator=g).tolist()
for i in idx:
r = rec[i]
n = r["codes"].shape[0]
st = 0 if fixed else int(torch.randint(0, max(1, n - S + 1), (1,), generator=g))
c = flat(r["codes"][st:st + S])
e = r["emb"][0 if fixed else int(torch.randint(0, r["emb"].shape[0], (1,), generator=g))].float()
if not fixed and torch.rand(1, generator=g).item() < drop:
e = None
tgt = torch.full((L,), -100, dtype=torch.long); tgt[:len(c)] = c
inp = torch.full((L,), NV, dtype=torch.long); inp[1:len(c)] = c[:-1]
if corrupt and not fixed:
tau = torch.rand(1, generator=g).item()
sel = torch.rand(L, generator=g) < tau
sel[0] = False
slot_of_inp = (torch.arange(L) - 1) % 7
rnd = torch.tensor(OFF)[slot_of_inp] + (torch.rand(L, generator=g) * torch.tensor(VOC)[slot_of_inp]).long()
inp = torch.where(sel, rnd, inp)
inps.append(inp); tgts.append(tgt); txts.append(e)
return torch.stack(inps).to(dev), torch.stack(tgts).to(dev), txts
def text_tensor(m, txts, dev):
rows = [m.null if t is None else t.to(dev) for t in txts]
return torch.stack(rows)[:, None, :].expand(-1, L, -1)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--seconds", type=float, default=7200)
ap.add_argument("--bs", type=int, default=64)
ap.add_argument("--lr", type=float, default=2e-4)
ap.add_argument("--overfit", type=int, default=0, help="N clips, no corruption/drop: memorization check")
ap.add_argument("--tag", default="gen_v1")
ap.add_argument("--val-every", type=int, default=500)
ap.add_argument("--patience", type=int, default=3)
ap.add_argument("--resume", action="store_true")
a = ap.parse_args()
dev = "cuda"
torch.manual_seed(0)
g = torch.Generator().manual_seed(0)
tr = torch.load(os.path.join(GEN, "train.pt"))
va = torch.load(os.path.join(GEN, "val.pt"))
if a.overfit:
tr = tr[: a.overfit]
m = Gen().to(dev)
print("params %.1fM train clips %d val clips %d" % (sum(p.numel() for p in m.parameters()) / 1e6, len(tr), len(va)), flush=True)
opt = torch.optim.AdamW(m.parameters(), lr=a.lr, weight_decay=0.05, betas=(0.9, 0.95))
slots = (torch.arange(L) % 7).to(dev)
vin, vtg, vtx = batch(va, min(256, len(va)), dev, g, fixed=True)
log = []
@torch.no_grad()
def val_loss():
m.eval()
tot, per = 0.0, torch.zeros(7, device=dev); cnt = torch.zeros(7, device=dev)
for i in range(0, vin.shape[0], 64):
with torch.autocast("cuda", dtype=torch.bfloat16):
lo = m(vin[i:i + 64], slots, text_tensor(m, vtx[i:i + 64], dev)).float()
ce = F.cross_entropy(lo.reshape(-1, NV), vtg[i:i + 64].reshape(-1), reduction="none").view(-1, L)
ok = (vtg[i:i + 64] != -100)
for s in range(7):
per[s] += (ce[:, s::7] * ok[:, s::7]).sum(); cnt[s] += ok[:, s::7].sum()
m.train()
p = (per / cnt)
return float(per.sum() / cnt.sum()), [round(float(v), 3) for v in p]
n_win = sum(r["codes"].shape[0] for r in tr) / S # data size in 64-step windows (1 epoch)
best, bad = [float("inf"), None, None], [0]
t0, step, warm = time.time(), 0, 300
if a.resume:
ck = torch.load(os.path.join(CKPT, a.tag + "_last.pt"), weights_only=False)
m.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); g.set_state(ck["rng"])
step, best, bad, log = ck["step"], ck["best"], ck["bad"], ck["log"]
t0 -= log[-1]["sec"] if log else 0
print("resumed at step %d (best val %.4f)" % (step, best[0]), flush=True)
while True:
el = time.time() - t0
if el > a.seconds:
break
lr = a.lr * min(1, (step + 1) / warm) * 0.5 * (1 + math.cos(math.pi * min(1.0, el / a.seconds)))
for pg in opt.param_groups:
pg["lr"] = lr
inp, tgt, txt = batch(tr, a.bs, dev, g, corrupt=not a.overfit, drop=0 if a.overfit else 0.1)
with torch.autocast("cuda", dtype=torch.bfloat16):
lo = m(inp, slots, text_tensor(m, txt, dev))
loss = F.cross_entropy(lo.float().reshape(-1, NV), tgt.reshape(-1))
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(m.parameters(), 1.0)
opt.step()
step += 1
if step == 20:
torch.cuda.synchronize()
print(" speed %.2f step/s -> ~%d steps in budget" % (20 / (time.time() - t0), a.seconds * 20 / (time.time() - t0)), flush=True)
if step % 100 == 0:
row = dict(step=step, sec=round(time.time() - t0), epochs=round(step * a.bs / n_win, 2),
train_loss=round(loss.item(), 4), lr=lr)
if step % a.val_every == 0 or a.overfit:
vl, per = val_loss(); row.update(val_loss=round(vl, 4), val_per_slot=per)
if vl < best[0]:
best[:] = [vl, step, per]; bad[0] = 0
torch.save(m.state_dict(), os.path.join(CKPT, a.tag + ".pt"))
row["saved"] = True
else:
bad[0] += 1
torch.save(dict(model=m.state_dict(), opt=opt.state_dict(), step=step, best=best, bad=bad,
log=log, rng=g.get_state()), os.path.join(CKPT, a.tag + "_last.pt"))
log.append(row)
print(" " + json.dumps(row), flush=True)
if not a.overfit and bad[0] >= a.patience:
print(" early stop: val not improved for %d checks" % a.patience, flush=True)
break
if a.overfit:
torch.save(m.state_dict(), os.path.join(CKPT, a.tag + ".pt"))
json.dump(dict(args=vars(a), steps=step, best_val_loss=best[0], best_step=best[1], best_val_per_slot=best[2],
log=log), open(os.path.join(RUNS, a.tag + ".json"), "w"), indent=1)
print("done steps %d (%.2f epochs) best val %.4f at step %s per-slot %s"
% (step, step * a.bs / n_win, best[0], best[1], best[2]), flush=True)
if __name__ == "__main__":
main()