#!/usr/bin/env python3 """One table, one unit: every TriVis text->pose system in shoulder widths. Shoulder width is the standard here because it is isotropic and per-clip: it is the unit Multi-VSL is packed in, the unit NSLP-G's tables use, and the only one in which numbers from the two repos can be compared. TriVis frame-widths divide x by W=1176 and y by H=1288, so they understate vertical error, do not normalise for how big the signer is in frame (173-510 px shoulder width on these clips), and cannot be converted to shoulder widths by any constant -- the measured per-condition ratio runs 3.6-3.9. Sources (all on the same 300 test clips, `RandomState(0).choice(...)` sorted): output_vsl/trivis_floors.json model-free floors output_vsl/gpt_vsl_front_lab_v2/on_trivis_sw.json TriVis sentence T2M-GPT output_vsl/gpt_mvsl/on_trivis_sw.json Multi-VSL word T2M-GPT, composed ../0.NSLP-G/.../results_trivis_predgloss_sw.json Multi-VSL word NSLP-G, composed `hands` is the row to read: it is the same 42 keypoints for every system, it carries the lexical content, and unlike `all` it is not diluted by the 68 near-static face keypoints. """ import argparse import json import os REPO = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') NSLPG = os.path.join(REPO, '0.NSLP-G', 'Word-level', 'NSLP-G') def load(p): return json.load(open(p)) if os.path.exists(p) else None def main(): ap = argparse.ArgumentParser() ap.add_argument('--shoulder-px', type=float, default=334.6, help='mean per-clip shoulder width on these 300 clips, for the px column') args = ap.parse_args() fl = load('output_vsl/trivis_floors.json') se = load('output_vsl/gpt_vsl_front_lab_v2/on_trivis_sw.json') mv = load('output_vsl/gpt_mvsl/on_trivis_sw.json') ng = load(os.path.join(NSLPG, 'results_trivis_predgloss_sw.json')) rows = [] # (label, dtw-summary dict, len_ratio) def add(label, res, key, sub=None, lr=None): if not res: rows.append((label, None, None)) return r = res['results'].get(key) if not r: rows.append((label, None, None)) return d = r[sub] if sub else r['dtw'] rows.append((label, d, r.get('len_ratio', lr))) rows.append(('--- upper bounds (stage-1 round trip) ---', None, None)) add(' NSLP-G Spatial VAE, per-frame', ng, 'ceiling') add(' TriVis VQ tokenizer', se, 'ceiling', sub='gt_anchor') add(' Multi-VSL VQ tokenizer', mv, 'ceiling') rows.append(('--- sentence model, trained on TriVis ---', None, None)) add(' T2M-GPT, oracle gloss', se, 'oracle-gloss', sub='gt_anchor') add(' T2M-GPT, predicted gloss', se, 'pred-gloss', sub='gt_anchor') add(' T2M-GPT, shuffled gloss', se, 'shuffled-gloss', sub='gt_anchor') rows.append(('--- word models, trained on Multi-VSL, composed ---', None, None)) add(' NSLP-G, predicted gloss', ng, 'pred-gloss') add(' NSLP-G, oracle gloss', ng, 'gt-gloss') add(' NSLP-G, shuffled gloss', ng, 'shuffled-gloss') add(' T2M-GPT, predicted gloss', mv, 'pred-gloss') add(' T2M-GPT, oracle gloss', mv, 'gt-gloss') add(' T2M-GPT, shuffled gloss', mv, 'shuffled-gloss') rows.append(('--- floors, no model ---', None, None)) if fl: for k, lbl in (('len-matched-clip', ' a real train clip, length-matched'), ('random-train-clip', ' a real train clip, random'), ('global-mean', ' the mean training pose')): r = fl['results'].get(k) rows.append((lbl, r['shoulder'], r['len_ratio']) if r else (lbl, None, None)) W = 52 print(f"{'system':<{W}}{'hands':>9}{'+-':>8}{'px':>7}{'all':>9}{'body':>9}{'len':>7}") for label, d, lr in rows: if d is None: print(label if label.startswith('---') else f'{label:<{W}} (missing)') continue h = d['hands'] print(f"{label:<{W}}{h['mean']:>9.4f}{h['sem']:>8.4f}" f"{h['mean']*args.shoulder_px:>7.0f}" f"{d['all']['mean']:>9.4f}{d['body']['mean']:>9.4f}" + (f"{lr:>7.2f}" if lr else f"{'-':>7}")) print('\nunit: shoulder widths (isotropic, per-clip median). px column uses the ' f'{args.shoulder_px:.0f} px mean shoulder width of these clips.') print('READ `hands` ONLY across systems: it is the same 42 keypoints everywhere. ' '`all` and `body` are NOT comparable across rows -- the floors and the TriVis ' 'sentence model emit all 128 keypoints (body group = 18), the Multi-VSL word ' 'T2M-GPT emits 124 (body 14 of 18), NSLP-G emits 50 (body 8 of 18, no face), ' 'and keypoints a system does not emit are excluded from its own score.') # The bar is the STRONGEST (lowest) model-free baseline, whichever it turns out to be: # at 300 clips a random train clip beats the length-matched one, at 30 it did not. floor, floor_name = None, None if fl: cands = {k: v['shoulder']['hands']['mean'] for k, v in fl['results'].items()} floor_name = min(cands, key=cands.get) floor = cands[floor_name] if floor: print(f'\nstrong floor = {floor_name} (a model-free baseline that knows nothing ' f'about the text): {floor:.4f} hands') for label, d, _ in rows: if d is None or label.startswith('---') or 'floor' in label: continue print(f' {label.strip():<46} {100*(floor - d["hands"]["mean"])/floor:+6.1f}% vs floor') if __name__ == '__main__': main()