#!/usr/bin/env python3 """GT | NSLP-G | T2M-GPT on vnhn, rendered in render_vsl.py's format (mp4 + png strip). Why a separate script rather than render_vsl.py: that one generates its own three panels (GT / ceiling / generated) from a single T2M-GPT pair and cannot take an external model's poses. This one reads both models from the .npz dumps so the two systems are drawn on the SAME clips, and reuses render_vsl.py's `draw`/`Topology`/`group_err` so the visual style, colours, viewport auto-fit and mp4+png outputs are identical. ALL THREE PANELS ARE DRAWN ON THE SAME 50 JOINTS (8 body + 21 + 21 hands). NSLP-G structurally cannot emit more -- its SpatialVAE is num_joints=50 -- so drawing GT and T2M-GPT at their native 124 (with 68 face points) would make the comparison about keypoint count rather than about signing quality. The 50-joint set is NSLP-G's KEEP_50 mapped back through the `upper` layout, which is exactly what dump_t2mgpt.py sliced T2M-GPT down to. """ import argparse import os import numpy as np import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.animation import FFMpegWriter, FuncAnimation from dataset import dataset_vsl from dataset.layout import Layout from render_vsl import Topology, draw, group_err # NSLP-G's KEEP_50 inside the 124-kpt `upper` layout: body[0:8] + lhand[82:103] + rhand[103:124] KEEP_50 = list(range(8)) + list(range(82, 124)) def keep50_layout(upper): """A Layout describing the 50-joint subset, so Topology draws the right edges. Group ranges are re-expressed for the 50-joint index space; `face` is an empty slice because these 50 joints contain no face points (draw() then scatters nothing). """ keep = [upper.keep[j] for j in KEEP_50] groups = {"body": (0, 8), "face": (8, 8), "lhand": (8, 29), "rhand": (29, 50)} return Layout(keep, groups, "keep50") def load_dump(p): d = np.load(p, allow_pickle=True) return {str(n): np.asarray(q, np.float32) for n, q in zip(d["names"], d["poses"])} def main(): ap = argparse.ArgumentParser() ap.add_argument("--data-dir", default="./dataset/VNHN") ap.add_argument("--split", default="test") ap.add_argument("--nslpg-npz", required=True) ap.add_argument("--t2mgpt-npz", required=True) ap.add_argument("--out-dir", default="qual_vnhn_3way_mp4") ap.add_argument("--n", type=int, default=6) ap.add_argument("--fps", type=int, default=25) ap.add_argument("--strip-frames", type=int, default=6) ap.add_argument("--display-px", type=float, default=0) ap.add_argument("--seed", type=int, default=1) args = ap.parse_args() os.makedirs(args.out_dir, exist_ok=True) store = dataset_vsl.VSLStore(args.data_dir, args.split) upper = store.layout NK = upper.n_kpts lay50 = keep50_layout(upper) topo = Topology(lay50) groups50 = {"all": (0, 50), "body": (0, 8), "hands": (8, 50)} ns = load_dump(args.nslpg_npz) tm = load_dump(args.t2mgpt_npz) by_name = {c["name"]: k for k, c in enumerate(store.index)} common = [n for n in ns if n in tm and n in by_name] if not common: raise SystemExit("no clips shared between the two dumps and the split") rng = np.random.RandomState(args.seed) picks = rng.choice(sorted(common), size=min(args.n, len(common)), replace=False).tolist() print(f"{len(common)} clips shared; rendering {len(picks)}") for k, name in enumerate(picks): i = by_name[name] c = store.index[i] motion, mask = store.get(i) gt124 = (motion * store.std + store.mean).reshape(-1, NK, 2) gt = gt124[:, KEEP_50] valid = mask[:, ::2][:, KEEP_50] a_ns, a_tm = ns[name], tm[name] e_ns = group_err(a_ns, gt, valid, groups50) e_tm = group_err(a_tm, gt, valid, groups50) lo, hi = gt.reshape(-1, 2).min(0), gt.reshape(-1, 2).max(0) span = float(max(hi - lo)) or 1.0 ctr = (lo + hi) / 2.0 scale = args.display_px or (520.0 / span) half = span * scale * 0.62 txt = (c.get("sentence") or "")[:110] title = (f"{txt} [frame-w, 50 joints]\n" f"GT {len(gt)}f | NSLP-G {e_ns:.3f}, {len(a_ns)}f " f"({len(a_ns)/len(gt):.2f}x) | T2M-GPT {e_tm:.3f}, {len(a_tm)}f " f"({len(a_tm)/len(gt):.2f}x)") print(f"[{k}] {name} nslpg={e_ns:.3f} t2mgpt={e_tm:.3f} " f"len={len(a_ns)}/{len(a_tm)}/{len(gt)}") panels = [("GROUND TRUTH", gt, valid, "#111111", "#c0392b"), ("NSLP-G (from text)", a_ns, None, "#7d3c98", "#a569bd"), ("T2M-GPT (from text)", a_tm, None, "#1a4f8a", "#2980b9")] nT = max(len(p[1]) for p in panels) fig, axes = plt.subplots(1, 3, figsize=(11, 4.6)) fig.suptitle(title, fontsize=9) def frame(t): for ax, (label, seq, vd, col, hcol) in zip(axes, panels): tt = min(t, len(seq) - 1) draw(ax, seq[tt], topo, col, hcol, scale, ctr, valid=(vd[tt] if vd is not None else None)) ax.set_xlim(-half, half); ax.set_ylim(half, -half) ax.set_title(f"{label}\nframe {tt+1}/{len(seq)}", fontsize=8) return [] anim = FuncAnimation(fig, frame, frames=nT, interval=1000 / args.fps, blit=False) anim.save(os.path.join(args.out_dir, f"{k:02d}_{name[:48]}.mp4"), writer=FFMpegWriter(fps=args.fps, bitrate=2400)) plt.close(fig) nf = args.strip_frames fig, ax2 = plt.subplots(3, nf, figsize=(1.7 * nf, 5.6)) fig.suptitle(title, fontsize=9) for r, (label, seq, vd, col, hcol) in enumerate(panels): ts = np.linspace(0, len(seq) - 1, nf).astype(int) for ci, tt in enumerate(ts): a = ax2[r, ci] draw(a, seq[tt], topo, col, hcol, scale, ctr, lw=1.1, valid=(vd[tt] if vd is not None else None)) a.set_xlim(-half, half); a.set_ylim(half, -half) if ci == 0: a.set_ylabel(label, fontsize=7) a.set_title(f"{tt+1}", fontsize=6) plt.tight_layout(rect=[0, 0, 1, 0.93]) fig.savefig(os.path.join(args.out_dir, f"{k:02d}_{name[:48]}.png"), dpi=130) plt.close(fig) print(f"\nwrote {args.out_dir}/ ({len(picks)} examples: mp4 + png each)") if __name__ == "__main__": main()