| |
| """One TriVis table across model families and settings, on a single protocol. |
| |
| Both projects already evaluate on `dataset/VSL_upper`, but with three different |
| conventions, so their published numbers are not directly comparable: |
| |
| joints clips n |
| NSLP-G sentence-level 50 (8 body+42 hand) default_rng(0) 200 |
| T2M-GPT eval_vsl.py 124 (incl. 68 face) RandomState(0) 300 |
| |
| This script re-scores T2M-GPT under NSLP-G's convention -- their exact 200 clip ids |
| (read from their results json), their 50-joint layout, their metric code -- and merges |
| the result with their rows into one table. |
| |
| Units. Both report `frame` (the raw per-axis frame-normalized coordinates the pack |
| stores) and `shoulder` (per-clip: divide by that clip's own median shoulder width, |
| after undoing the per-axis anisotropy). The per-clip method is theirs; a single global |
| factor is an approximation that differs by ~17% and is not used here. |
| """ |
| import argparse |
| import csv |
| import json |
| import os |
| import sys |
|
|
| import numpy as np |
| import torch |
| from tqdm import tqdm |
|
|
| import models.t2m_trans as trans |
| from dataset import dataset_vsl |
| from models.text_encoder_vi import ViTextEncoder |
| from train_t2m_trans_vsl import build_vqvae |
|
|
| REPO = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') |
| KEEP_50 = list(range(8)) + list(range(82, 124)) |
| GROUPS_50 = {'all': (0, 50), 'body': (0, 8), 'hands': (8, 50)} |
| NECK, RSHO, LSHO = 1, 2, 5 |
|
|
|
|
| def anchor_of(xy, valid, W, H): |
| """Per-clip neck position and shoulder width, in pixels (NSLP-G's method).""" |
| px = xy * np.array([W, H], np.float32) |
| ok = valid[:, NECK].astype(bool) & valid[:, RSHO].astype(bool) & valid[:, LSHO].astype(bool) |
| if ok.sum() < 3: |
| ok = np.ones(len(xy), bool) |
| neck = np.median(px[ok, NECK, :], axis=0) |
| sw = float(np.median(np.linalg.norm(px[ok, RSHO, :] - px[ok, LSHO, :], axis=-1))) |
| return neck, max(sw, 1e-3) |
|
|
|
|
| def to_shoulder(xy, neck, sw, W, H): |
| return (xy * np.array([W, H], np.float32) - neck[None, None, :].astype(np.float32)) / sw |
|
|
|
|
| def _dtw(pred, gt, valid): |
| """Same DTW as eval_vsl.dtw_mje but with the 50-joint groups.""" |
| import eval_vsl |
| saved = eval_vsl.GROUPS |
| eval_vsl.GROUPS = GROUPS_50 |
| try: |
| return eval_vsl.dtw_mje(pred, gt, valid) |
| finally: |
| eval_vsl.GROUPS = saved |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument('--data-dir', default='./dataset/VSL_upper') |
| ap.add_argument('--vq', default='output_vsl/vq_vsl_upper/net_best.pth') |
| ap.add_argument('--gpt', default='output_vsl/gpt_vsl_upper/net_best.pth') |
| ap.add_argument('--nslpg-json', default=os.path.join( |
| REPO, '0.NSLP-G/sentence-level/results_sent_shoulder.json')) |
| ap.add_argument('--clipid-json', default=os.path.join( |
| REPO, '0.NSLP-G/sentence-level/results_sent_scratch.json')) |
| ap.add_argument('--pred-gloss', default='output_vsl/text2gloss/pred_test.json') |
| ap.add_argument('--frame-w', type=float, default=1176.0) |
| ap.add_argument('--frame-h', type=float, default=1288.0) |
| ap.add_argument('--device', default='cuda') |
| ap.add_argument('--out-json', default='output_vsl/trivis_unified.json') |
| args = ap.parse_args() |
|
|
| W, H = args.frame_w, args.frame_h |
| device = torch.device(args.device) |
|
|
| with open(args.clipid_json) as f: |
| clip_ids = json.load(f)['clip_ids'] |
| print(f'using NSLP-G\'s exact {len(clip_ids)} test clip ids') |
|
|
| net, targs, _ = build_vqvae(args.vq, device) |
| tck = torch.load(args.gpt, map_location='cpu') |
| g = argparse.Namespace(**tck['args']) |
| text_enc = ViTextEncoder(g.text_model, device=args.device) |
| gpt = trans.Text2Motion_Transformer( |
| num_vq=g.nb_code, embed_dim=g.embed_dim_gpt, clip_dim=text_enc.dim, |
| block_size=g.max_tokens + 1, num_layers=g.num_layers, n_head=g.n_head_gpt, |
| drop_out_rate=g.drop_out_rate, fc_rate=g.ff_rate) |
| gpt.load_state_dict(tck['trans'], strict=True) |
| gpt.eval().to(device) |
|
|
| st = dataset_vsl.VSLStore(args.data_dir, 'test') |
| NK = st.layout.n_kpts |
| pred_gloss = {} |
| if os.path.exists(args.pred_gloss): |
| with open(args.pred_gloss, encoding='utf-8') as f: |
| pred_gloss = {k: v['pred_gloss'] for k, v in json.load(f).items()} |
|
|
| conds = ['ceiling', 'gloss', 'pred_gloss', 'sentence', 'shuffled'] |
| acc = {c: {u: {k: [] for k in GROUPS_50} for u in ('frame', 'shoulder')} for c in conds} |
| lens = {c: [] for c in conds} |
| unit_len = 2 ** targs.down_t |
|
|
| with torch.no_grad(): |
| for i in tqdm(clip_ids, desc='T2M-GPT on NSLP-G protocol'): |
| c = st.index[i] |
| motion, mask = st.get(i) |
| valid_full = mask[:, ::2] |
| gt_full = (motion * st.std + st.mean).reshape(-1, NK, 2) |
| gt = gt_full[:, KEEP_50] |
| valid = valid_full[:, KEEP_50] |
| neck, sw = anchor_of(gt, valid, W, H) |
| gt_sh = to_shoulder(gt, neck, sw, W, H) |
|
|
| outs = {} |
| mt = torch.from_numpy(motion).unsqueeze(0).to(device) |
| T = (len(motion) // unit_len) * unit_len |
| rec = net.decode_batch(net.encode(mt[:, :T]))[0].cpu().numpy() |
| outs['ceiling'] = (rec * st.std + st.mean).reshape(-1, NK, 2)[:, KEEP_50] |
|
|
| other = st.index[(i + len(st.index) // 2) % len(st.index)] |
| texts = {'gloss': c['gloss'], |
| 'pred_gloss': pred_gloss.get(c['name'], c['gloss']), |
| 'sentence': c['sentence'], |
| 'shuffled': other['gloss']} |
| for tag, txt in texts.items(): |
| idx = gpt.sample(text_enc([txt]), if_categorial=True) |
| if idx is None or idx.numel() == 0: |
| continue |
| idx = idx.clamp(max=g.nb_code - 1) |
| p = net.decode_batch(idx)[0].cpu().numpy() |
| outs[tag] = (p * st.std + st.mean).reshape(-1, NK, 2)[:, KEEP_50] |
|
|
| for tag, p in outs.items(): |
| for u, (pp, gg) in (('frame', (p, gt)), |
| ('shoulder', (to_shoulder(p, neck, sw, W, H), gt_sh))): |
| d = _dtw(pp.astype(np.float64), gg.astype(np.float64), |
| valid.astype(np.float64)) |
| for k, v in d.items(): |
| acc[tag][u][k].append(v) |
| lens[tag].append(len(p) / len(gt)) |
|
|
| t2m = {} |
| for cd in conds: |
| if not acc[cd]['frame']['hands']: |
| continue |
| t2m[cd] = {u: {k: float(np.mean(v)) for k, v in acc[cd][u].items()} |
| for u in ('frame', 'shoulder')} |
| t2m[cd]['len_ratio'] = float(np.mean(lens[cd])) |
|
|
| |
| ns = json.load(open(args.nslpg_json))['results'] |
| print(f"\n{'system / setting':<40}{'frame_h':>10}{'shldr_h':>10}{'frame_all':>11}{'len_r':>8}") |
| print('-' * 79) |
| rows = [] |
| order = [('NSLP-G stage-1 ceiling', ns.get('pre/ceiling'), None), |
| ('T2M-GPT stage-1 ceiling', None, t2m.get('ceiling')), |
| (None, None, None), |
| ('NSLP-G pretrained on Multi-VSL', ns.get('pre/nslpg_pred'), None), |
| ('NSLP-G from scratch', ns.get('scratch/nslpg_pred'), None), |
| ('NSLP-G pretrained + handshape loss', ns.get('shape/nslpg_pred'), None), |
| ('NSLP-G oracle length', ns.get('pre/nslpg_oracle'), None), |
| (None, None, None), |
| ('T2M-GPT gloss', None, t2m.get('gloss')), |
| ('T2M-GPT predicted gloss (BARTpho)', None, t2m.get('pred_gloss')), |
| ('T2M-GPT raw sentence (direct)', None, t2m.get('sentence')), |
| (None, None, None), |
| ('NSLP-G shuffled gloss (control)', ns.get('pre/shuffled_gloss'), None), |
| ('T2M-GPT shuffled gloss (control)', None, t2m.get('shuffled')), |
| ('NSLP-G random init (floor)', ns.get('pre/random_init'), None), |
| ('NSLP-G global mean pose (floor)', ns.get('pre/global_mean'), None)] |
| for name, a, b in order: |
| if name is None: |
| print('-' * 79); continue |
| if a is not None: |
| fr, sh, al = a['frame']['hands']['mean'], a['shoulder']['hands']['mean'], a['frame']['all']['mean'] |
| lr = float('nan') |
| elif b is not None: |
| fr, sh, al, lr = b['frame']['hands'], b['shoulder']['hands'], b['frame']['all'], b['len_ratio'] |
| else: |
| print(f'{name:<40} (missing)'); continue |
| lrs = '-' if lr != lr else f'{lr:.3f}' |
| print(f'{name:<40}{fr:>10.4f}{sh:>10.4f}{al:>11.4f}{lrs:>8}') |
| rows.append({'setting': name, 'frame_hands': fr, 'shoulder_hands': sh, |
| 'frame_all': al, 'len_ratio': None if lr != lr else lr}) |
|
|
| os.makedirs(os.path.dirname(args.out_json) or '.', exist_ok=True) |
| with open(args.out_json, 'w') as f: |
| json.dump({'n_clips': len(clip_ids), 'joints': 50, 'protocol': 'NSLP-G sentence-level', |
| 't2mgpt': t2m, 'rows': rows}, f, indent=2) |
| print(f'\nwrote {args.out_json}') |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
