| |
| """GT | NSLP-G | T2M-GPT on vnhn, rendered in render_vsl.py's format (mp4 + png strip). |
| |
| Why a separate script rather than render_vsl.py: that one generates its own three panels |
| (GT / ceiling / generated) from a single T2M-GPT pair and cannot take an external model's |
| poses. This one reads both models from the .npz dumps so the two systems are drawn on the |
| SAME clips, and reuses render_vsl.py's `draw`/`Topology`/`group_err` so the visual style, |
| colours, viewport auto-fit and mp4+png outputs are identical. |
| |
| ALL THREE PANELS ARE DRAWN ON THE SAME 50 JOINTS (8 body + 21 + 21 hands). NSLP-G |
| structurally cannot emit more -- its SpatialVAE is num_joints=50 -- so drawing GT and |
| T2M-GPT at their native 124 (with 68 face points) would make the comparison about keypoint |
| count rather than about signing quality. The 50-joint set is NSLP-G's KEEP_50 mapped back |
| through the `upper` layout, which is exactly what dump_t2mgpt.py sliced T2M-GPT down to. |
| """ |
| import argparse |
| import os |
|
|
| import numpy as np |
| import matplotlib |
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
| from matplotlib.animation import FFMpegWriter, FuncAnimation |
|
|
| from dataset import dataset_vsl |
| from dataset.layout import Layout |
| from render_vsl import Topology, draw, group_err |
|
|
| |
| KEEP_50 = list(range(8)) + list(range(82, 124)) |
|
|
|
|
| def keep50_layout(upper): |
| """A Layout describing the 50-joint subset, so Topology draws the right edges. |
| |
| Group ranges are re-expressed for the 50-joint index space; `face` is an empty slice |
| because these 50 joints contain no face points (draw() then scatters nothing). |
| """ |
| keep = [upper.keep[j] for j in KEEP_50] |
| groups = {"body": (0, 8), "face": (8, 8), "lhand": (8, 29), "rhand": (29, 50)} |
| return Layout(keep, groups, "keep50") |
|
|
|
|
| def load_dump(p): |
| d = np.load(p, allow_pickle=True) |
| return {str(n): np.asarray(q, np.float32) for n, q in zip(d["names"], d["poses"])} |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--data-dir", default="./dataset/VNHN") |
| ap.add_argument("--split", default="test") |
| ap.add_argument("--nslpg-npz", required=True) |
| ap.add_argument("--t2mgpt-npz", required=True) |
| ap.add_argument("--out-dir", default="qual_vnhn_3way_mp4") |
| ap.add_argument("--n", type=int, default=6) |
| ap.add_argument("--fps", type=int, default=25) |
| ap.add_argument("--strip-frames", type=int, default=6) |
| ap.add_argument("--display-px", type=float, default=0) |
| ap.add_argument("--seed", type=int, default=1) |
| args = ap.parse_args() |
|
|
| os.makedirs(args.out_dir, exist_ok=True) |
| store = dataset_vsl.VSLStore(args.data_dir, args.split) |
| upper = store.layout |
| NK = upper.n_kpts |
| lay50 = keep50_layout(upper) |
| topo = Topology(lay50) |
| groups50 = {"all": (0, 50), "body": (0, 8), "hands": (8, 50)} |
|
|
| ns = load_dump(args.nslpg_npz) |
| tm = load_dump(args.t2mgpt_npz) |
| by_name = {c["name"]: k for k, c in enumerate(store.index)} |
| common = [n for n in ns if n in tm and n in by_name] |
| if not common: |
| raise SystemExit("no clips shared between the two dumps and the split") |
| rng = np.random.RandomState(args.seed) |
| picks = rng.choice(sorted(common), size=min(args.n, len(common)), replace=False).tolist() |
| print(f"{len(common)} clips shared; rendering {len(picks)}") |
|
|
| for k, name in enumerate(picks): |
| i = by_name[name] |
| c = store.index[i] |
| motion, mask = store.get(i) |
| gt124 = (motion * store.std + store.mean).reshape(-1, NK, 2) |
| gt = gt124[:, KEEP_50] |
| valid = mask[:, ::2][:, KEEP_50] |
| a_ns, a_tm = ns[name], tm[name] |
|
|
| e_ns = group_err(a_ns, gt, valid, groups50) |
| e_tm = group_err(a_tm, gt, valid, groups50) |
|
|
| lo, hi = gt.reshape(-1, 2).min(0), gt.reshape(-1, 2).max(0) |
| span = float(max(hi - lo)) or 1.0 |
| ctr = (lo + hi) / 2.0 |
| scale = args.display_px or (520.0 / span) |
| half = span * scale * 0.62 |
|
|
| txt = (c.get("sentence") or "")[:110] |
| title = (f"{txt} [frame-w, 50 joints]\n" |
| f"GT {len(gt)}f | NSLP-G {e_ns:.3f}, {len(a_ns)}f " |
| f"({len(a_ns)/len(gt):.2f}x) | T2M-GPT {e_tm:.3f}, {len(a_tm)}f " |
| f"({len(a_tm)/len(gt):.2f}x)") |
| print(f"[{k}] {name} nslpg={e_ns:.3f} t2mgpt={e_tm:.3f} " |
| f"len={len(a_ns)}/{len(a_tm)}/{len(gt)}") |
|
|
| panels = [("GROUND TRUTH", gt, valid, "#111111", "#c0392b"), |
| ("NSLP-G (from text)", a_ns, None, "#7d3c98", "#a569bd"), |
| ("T2M-GPT (from text)", a_tm, None, "#1a4f8a", "#2980b9")] |
| nT = max(len(p[1]) for p in panels) |
|
|
| fig, axes = plt.subplots(1, 3, figsize=(11, 4.6)) |
| fig.suptitle(title, fontsize=9) |
|
|
| def frame(t): |
| for ax, (label, seq, vd, col, hcol) in zip(axes, panels): |
| tt = min(t, len(seq) - 1) |
| draw(ax, seq[tt], topo, col, hcol, scale, ctr, |
| valid=(vd[tt] if vd is not None else None)) |
| ax.set_xlim(-half, half); ax.set_ylim(half, -half) |
| ax.set_title(f"{label}\nframe {tt+1}/{len(seq)}", fontsize=8) |
| return [] |
|
|
| anim = FuncAnimation(fig, frame, frames=nT, interval=1000 / args.fps, blit=False) |
| anim.save(os.path.join(args.out_dir, f"{k:02d}_{name[:48]}.mp4"), |
| writer=FFMpegWriter(fps=args.fps, bitrate=2400)) |
| plt.close(fig) |
|
|
| nf = args.strip_frames |
| fig, ax2 = plt.subplots(3, nf, figsize=(1.7 * nf, 5.6)) |
| fig.suptitle(title, fontsize=9) |
| for r, (label, seq, vd, col, hcol) in enumerate(panels): |
| ts = np.linspace(0, len(seq) - 1, nf).astype(int) |
| for ci, tt in enumerate(ts): |
| a = ax2[r, ci] |
| draw(a, seq[tt], topo, col, hcol, scale, ctr, lw=1.1, |
| valid=(vd[tt] if vd is not None else None)) |
| a.set_xlim(-half, half); a.set_ylim(half, -half) |
| if ci == 0: |
| a.set_ylabel(label, fontsize=7) |
| a.set_title(f"{tt+1}", fontsize=6) |
| plt.tight_layout(rect=[0, 0, 1, 0.93]) |
| fig.savefig(os.path.join(args.out_dir, f"{k:02d}_{name[:48]}.png"), dpi=130) |
| plt.close(fig) |
|
|
| print(f"\nwrote {args.out_dir}/ ({len(picks)} examples: mp4 + png each)") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
