summarize_trivis_shoulder.py

#!/usr/bin/env python3
"""One table, one unit: every TriVis text->pose system in shoulder widths.

Shoulder width is the standard here because it is isotropic and per-clip: it is the unit
Multi-VSL is packed in, the unit NSLP-G's tables use, and the only one in which numbers
from the two repos can be compared. TriVis frame-widths divide x by W=1176 and y by
H=1288, so they understate vertical error, do not normalise for how big the signer is in
frame (173-510 px shoulder width on these clips), and cannot be converted to shoulder
widths by any constant -- the measured per-condition ratio runs 3.6-3.9.

Sources (all on the same 300 test clips, `RandomState(0).choice(...)` sorted):
  output_vsl/trivis_floors.json                          model-free floors
  output_vsl/gpt_vsl_front_lab_v2/on_trivis_sw.json      TriVis sentence T2M-GPT
  output_vsl/gpt_mvsl/on_trivis_sw.json                  Multi-VSL word T2M-GPT, composed
  ../0.NSLP-G/.../results_trivis_predgloss_sw.json       Multi-VSL word NSLP-G, composed

`hands` is the row to read: it is the same 42 keypoints for every system, it carries the
lexical content, and unlike `all` it is not diluted by the 68 near-static face keypoints.
"""
import argparse
import json
import os

REPO = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
NSLPG = os.path.join(REPO, '0.NSLP-G', 'Word-level', 'NSLP-G')


def load(p):
    return json.load(open(p)) if os.path.exists(p) else None


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--shoulder-px', type=float, default=334.6,
                    help='mean per-clip shoulder width on these 300 clips, for the px column')
    args = ap.parse_args()

    fl = load('output_vsl/trivis_floors.json')
    se = load('output_vsl/gpt_vsl_front_lab_v2/on_trivis_sw.json')
    mv = load('output_vsl/gpt_mvsl/on_trivis_sw.json')
    ng = load(os.path.join(NSLPG, 'results_trivis_predgloss_sw.json'))

    rows = []                                    # (label, dtw-summary dict, len_ratio)

    def add(label, res, key, sub=None, lr=None):
        if not res:
            rows.append((label, None, None))
            return
        r = res['results'].get(key)
        if not r:
            rows.append((label, None, None))
            return
        d = r[sub] if sub else r['dtw']
        rows.append((label, d, r.get('len_ratio', lr)))

    rows.append(('--- upper bounds (stage-1 round trip) ---', None, None))
    add('  NSLP-G Spatial VAE, per-frame', ng, 'ceiling')
    add('  TriVis VQ tokenizer', se, 'ceiling', sub='gt_anchor')
    add('  Multi-VSL VQ tokenizer', mv, 'ceiling')

    rows.append(('--- sentence model, trained on TriVis ---', None, None))
    add('  T2M-GPT, oracle gloss', se, 'oracle-gloss', sub='gt_anchor')
    add('  T2M-GPT, predicted gloss', se, 'pred-gloss', sub='gt_anchor')
    add('  T2M-GPT, shuffled gloss', se, 'shuffled-gloss', sub='gt_anchor')

    rows.append(('--- word models, trained on Multi-VSL, composed ---', None, None))
    add('  NSLP-G, predicted gloss', ng, 'pred-gloss')
    add('  NSLP-G, oracle gloss', ng, 'gt-gloss')
    add('  NSLP-G, shuffled gloss', ng, 'shuffled-gloss')
    add('  T2M-GPT, predicted gloss', mv, 'pred-gloss')
    add('  T2M-GPT, oracle gloss', mv, 'gt-gloss')
    add('  T2M-GPT, shuffled gloss', mv, 'shuffled-gloss')

    rows.append(('--- floors, no model ---', None, None))
    if fl:
        for k, lbl in (('len-matched-clip', '  a real train clip, length-matched'),
                       ('random-train-clip', '  a real train clip, random'),
                       ('global-mean', '  the mean training pose')):
            r = fl['results'].get(k)
            rows.append((lbl, r['shoulder'], r['len_ratio']) if r else (lbl, None, None))

    W = 52
    print(f"{'system':<{W}}{'hands':>9}{'+-':>8}{'px':>7}{'all':>9}{'body':>9}{'len':>7}")
    for label, d, lr in rows:
        if d is None:
            print(label if label.startswith('---') else f'{label:<{W}}   (missing)')
            continue
        h = d['hands']
        print(f"{label:<{W}}{h['mean']:>9.4f}{h['sem']:>8.4f}"
              f"{h['mean']*args.shoulder_px:>7.0f}"
              f"{d['all']['mean']:>9.4f}{d['body']['mean']:>9.4f}"
              + (f"{lr:>7.2f}" if lr else f"{'-':>7}"))

    print('\nunit: shoulder widths (isotropic, per-clip median). px column uses the '
          f'{args.shoulder_px:.0f} px mean shoulder width of these clips.')
    print('READ `hands` ONLY across systems: it is the same 42 keypoints everywhere. '
          '`all` and `body` are NOT comparable across rows -- the floors and the TriVis '
          'sentence model emit all 128 keypoints (body group = 18), the Multi-VSL word '
          'T2M-GPT emits 124 (body 14 of 18), NSLP-G emits 50 (body 8 of 18, no face), '
          'and keypoints a system does not emit are excluded from its own score.')

    # The bar is the STRONGEST (lowest) model-free baseline, whichever it turns out to be:
    # at 300 clips a random train clip beats the length-matched one, at 30 it did not.
    floor, floor_name = None, None
    if fl:
        cands = {k: v['shoulder']['hands']['mean'] for k, v in fl['results'].items()}
        floor_name = min(cands, key=cands.get)
        floor = cands[floor_name]
    if floor:
        print(f'\nstrong floor = {floor_name} (a model-free baseline that knows nothing '
              f'about the text): {floor:.4f} hands')
        for label, d, _ in rows:
            if d is None or label.startswith('---') or 'floor' in label:
                continue
            print(f'  {label.strip():<46} {100*(floor - d["hands"]["mean"])/floor:+6.1f}% vs floor')


if __name__ == '__main__':
    main()