Download code/gen_prep.py from Yakimask/PartLock-CMU: direct link, hf CLI and curl.
- Browser
- Download file 3.34 kB
-
https://huggingface.co/Yakimask/PartLock-CMU/resolve/main/code/gen_prep.py
- Command line
-
hf download hf://Yakimask/PartLock-CMU/code/gen_prep.py
-
curl -L -o gen_prep.py https://huggingface.co/Yakimask/PartLock-CMU/resolve/main/code/gen_prep.py
3.34 kB
| """Plan 2 / prep: (a) whole clips -> causal-H tokens (n_steps, 7) ; (b) captions -> CLIP ViT-B/32 text embeddings. | |
| Only captions with whole-clip timing (start=end=0) are used. | |
| Output: $PARTLOCK_DIR/data/gen/{train,val}.pt | |
| """ | |
| import os, sys, time | |
| import numpy as np | |
| import torch | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| import tvq, tvq272 as B, tvq272c as C | |
| dev = "cuda" | |
| BASE = os.environ.get("PARTLOCK_DIR", ".") # folder that holds data/, ckpt/, runs/ | |
| OUT = os.path.join(BASE, "data", "gen") | |
| os.makedirs(OUT, exist_ok=True) | |
| t0 = time.time() | |
| from transformers import CLIPTokenizer, CLIPTextModelWithProjection | |
| name = "openai/clip-vit-base-patch32" | |
| tok = CLIPTokenizer.from_pretrained(name) | |
| clip = CLIPTextModelWithProjection.from_pretrained(name, use_safetensors=True).to(dev).eval() | |
| print("clip loaded %.0fs" % (time.time() - t0), flush=True) | |
| mean = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Mean.npy")).astype(np.float32)).to(dev) | |
| std = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Std.npy")).astype(np.float32)).to(dev) | |
| m = C.CHierTok().to(dev) | |
| m.load_state_dict(torch.load(os.path.join(B.CKPT_DIR, os.environ.get("TOK_CKPT", "cmu_tok_v1_H.pt")), map_location=dev)) | |
| m.eval() | |
| mdir = B.find_motion_dir() | |
| tdir = os.path.join(B.ROOT, "texts", "texts") | |
| def codes(z): | |
| ig = tvq.vq_seq(m.gvq, m.genc(z.permute(0, 2, 1)))[2] | |
| ip = [tvq.vq_seq(m.vq[p], m.enc[p](z[..., c].permute(0, 2, 1)))[2] for p, c in enumerate(B.PD)] | |
| return torch.stack([ig] + ip, -1)[0] # (n, 7) | |
| def embed(caps): | |
| t = tok(caps, padding=True, truncation=True, max_length=77, return_tensors="pt").to(dev) | |
| e = clip(**t).text_embeds | |
| return torch.nn.functional.normalize(e, dim=-1).half().cpu() | |
| for split in ("train", "val"): | |
| ids = open(os.path.join(B.ROOT, "split", split + ".txt")).read().split() | |
| rec = [] | |
| skipped = 0 | |
| for i in ids: | |
| mp, tp = os.path.join(mdir, i + ".npy"), os.path.join(tdir, i + ".txt") | |
| if not (os.path.exists(mp) and os.path.exists(tp)): | |
| skipped += 1; continue | |
| a = np.load(mp).astype(np.float32) | |
| if a.ndim != 2 or a.shape[1] != 272 or a.shape[0] < 16: | |
| skipped += 1; continue | |
| caps = [] | |
| for line in open(tp, encoding="utf-8", errors="ignore"): | |
| f = line.strip().split("#") | |
| if len(f) >= 4 and f[0]: | |
| try: | |
| s, e = float(f[2]), float(f[3]) | |
| except ValueError: | |
| continue | |
| if s == 0.0 and e == 0.0: | |
| caps.append(f[0]) | |
| if not caps: | |
| skipped += 1; continue | |
| a = a[: a.shape[0] // 4 * 4] | |
| z = (torch.from_numpy(a).to(dev) - mean) / std | |
| rec.append(dict(id=i, codes=codes(z[None]).short().cpu(), caps=caps, emb=embed(caps))) | |
| if len(rec) % 2000 == 0: | |
| print(" %s %d clips %.0fs" % (split, len(rec), time.time() - t0), flush=True) | |
| torch.save(rec, os.path.join(OUT, split + ".pt")) | |
| steps = sum(r["codes"].shape[0] for r in rec) | |
| print("%s: %d clips (skipped %d), %d steps = %.1f h of motion, %d captions" | |
| % (split, len(rec), skipped, steps, steps * 4 / 30 / 3600, sum(len(r["caps"]) for r in rec)), flush=True) | |
| print("done %.0fs" % (time.time() - t0)) | |