File size: 5,701 Bytes
8e5456b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | #!/usr/bin/env python3
"""One table, one unit: every TriVis text->pose system in shoulder widths.
Shoulder width is the standard here because it is isotropic and per-clip: it is the unit
Multi-VSL is packed in, the unit NSLP-G's tables use, and the only one in which numbers
from the two repos can be compared. TriVis frame-widths divide x by W=1176 and y by
H=1288, so they understate vertical error, do not normalise for how big the signer is in
frame (173-510 px shoulder width on these clips), and cannot be converted to shoulder
widths by any constant -- the measured per-condition ratio runs 3.6-3.9.
Sources (all on the same 300 test clips, `RandomState(0).choice(...)` sorted):
output_vsl/trivis_floors.json model-free floors
output_vsl/gpt_vsl_front_lab_v2/on_trivis_sw.json TriVis sentence T2M-GPT
output_vsl/gpt_mvsl/on_trivis_sw.json Multi-VSL word T2M-GPT, composed
../0.NSLP-G/.../results_trivis_predgloss_sw.json Multi-VSL word NSLP-G, composed
`hands` is the row to read: it is the same 42 keypoints for every system, it carries the
lexical content, and unlike `all` it is not diluted by the 68 near-static face keypoints.
"""
import argparse
import json
import os
REPO = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
NSLPG = os.path.join(REPO, '0.NSLP-G', 'Word-level', 'NSLP-G')
def load(p):
return json.load(open(p)) if os.path.exists(p) else None
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--shoulder-px', type=float, default=334.6,
help='mean per-clip shoulder width on these 300 clips, for the px column')
args = ap.parse_args()
fl = load('output_vsl/trivis_floors.json')
se = load('output_vsl/gpt_vsl_front_lab_v2/on_trivis_sw.json')
mv = load('output_vsl/gpt_mvsl/on_trivis_sw.json')
ng = load(os.path.join(NSLPG, 'results_trivis_predgloss_sw.json'))
rows = [] # (label, dtw-summary dict, len_ratio)
def add(label, res, key, sub=None, lr=None):
if not res:
rows.append((label, None, None))
return
r = res['results'].get(key)
if not r:
rows.append((label, None, None))
return
d = r[sub] if sub else r['dtw']
rows.append((label, d, r.get('len_ratio', lr)))
rows.append(('--- upper bounds (stage-1 round trip) ---', None, None))
add(' NSLP-G Spatial VAE, per-frame', ng, 'ceiling')
add(' TriVis VQ tokenizer', se, 'ceiling', sub='gt_anchor')
add(' Multi-VSL VQ tokenizer', mv, 'ceiling')
rows.append(('--- sentence model, trained on TriVis ---', None, None))
add(' T2M-GPT, oracle gloss', se, 'oracle-gloss', sub='gt_anchor')
add(' T2M-GPT, predicted gloss', se, 'pred-gloss', sub='gt_anchor')
add(' T2M-GPT, shuffled gloss', se, 'shuffled-gloss', sub='gt_anchor')
rows.append(('--- word models, trained on Multi-VSL, composed ---', None, None))
add(' NSLP-G, predicted gloss', ng, 'pred-gloss')
add(' NSLP-G, oracle gloss', ng, 'gt-gloss')
add(' NSLP-G, shuffled gloss', ng, 'shuffled-gloss')
add(' T2M-GPT, predicted gloss', mv, 'pred-gloss')
add(' T2M-GPT, oracle gloss', mv, 'gt-gloss')
add(' T2M-GPT, shuffled gloss', mv, 'shuffled-gloss')
rows.append(('--- floors, no model ---', None, None))
if fl:
for k, lbl in (('len-matched-clip', ' a real train clip, length-matched'),
('random-train-clip', ' a real train clip, random'),
('global-mean', ' the mean training pose')):
r = fl['results'].get(k)
rows.append((lbl, r['shoulder'], r['len_ratio']) if r else (lbl, None, None))
W = 52
print(f"{'system':<{W}}{'hands':>9}{'+-':>8}{'px':>7}{'all':>9}{'body':>9}{'len':>7}")
for label, d, lr in rows:
if d is None:
print(label if label.startswith('---') else f'{label:<{W}} (missing)')
continue
h = d['hands']
print(f"{label:<{W}}{h['mean']:>9.4f}{h['sem']:>8.4f}"
f"{h['mean']*args.shoulder_px:>7.0f}"
f"{d['all']['mean']:>9.4f}{d['body']['mean']:>9.4f}"
+ (f"{lr:>7.2f}" if lr else f"{'-':>7}"))
print('\nunit: shoulder widths (isotropic, per-clip median). px column uses the '
f'{args.shoulder_px:.0f} px mean shoulder width of these clips.')
print('READ `hands` ONLY across systems: it is the same 42 keypoints everywhere. '
'`all` and `body` are NOT comparable across rows -- the floors and the TriVis '
'sentence model emit all 128 keypoints (body group = 18), the Multi-VSL word '
'T2M-GPT emits 124 (body 14 of 18), NSLP-G emits 50 (body 8 of 18, no face), '
'and keypoints a system does not emit are excluded from its own score.')
# The bar is the STRONGEST (lowest) model-free baseline, whichever it turns out to be:
# at 300 clips a random train clip beats the length-matched one, at 30 it did not.
floor, floor_name = None, None
if fl:
cands = {k: v['shoulder']['hands']['mean'] for k, v in fl['results'].items()}
floor_name = min(cands, key=cands.get)
floor = cands[floor_name]
if floor:
print(f'\nstrong floor = {floor_name} (a model-free baseline that knows nothing '
f'about the text): {floor:.4f} hands')
for label, d, _ in rows:
if d is None or label.startswith('---') or 'floor' in label:
continue
print(f' {label.strip():<46} {100*(floor - d["hands"]["mean"])/floor:+6.1f}% vs floor')
if __name__ == '__main__':
main()
|