File size: 8,364 Bytes
8e5456b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | #!/usr/bin/env python3
"""Pack Multi-VSL (WACV 2025) front-view DWPose skeletons for T2M-GPT training.
Isolated-sign data: one gloss -> one clip, so there is no gloss/pose alignment
problem. That makes this the clean diagnostic for "can T2M-GPT learn gloss->pose
at all", separate from the sentence-level alignment question.
TWO NORMALIZATIONS ARE REQUIRED HERE, unlike Full_TriVis:
1. Geometry. The dataset ships a *different YOLO crop per clip* (resolutions seen:
560x712, 698x986, 1396x2106, ...). DWPose coordinates are normalized per axis to
[0,1], so they are (a) anisotropically scaled, because W != H, and (b) not
comparable across clips, because the crop scale differs. So:
xy_px = xy_norm * [W, H] -> back to isotropic pixels
xy_out = (xy_px - neck) / shoulder_width -> crop-invariant, body-relative
The reference (neck position, shoulder width) is computed ONCE PER CLIP from the
median over valid frames -- deliberately not per frame. A per-clip affine removes
only the crop artifact; a per-frame one would also erase genuine body sway and
hand displacement relative to the torso, which is exactly the signal.
Full_TriVis is left un-normalized; only this dataset needs it.
2. Frame rate. Clips are 29.97 / 30 / 59.94 fps. Anything above --fps-thresh is
decimated by 2 so token duration means the same thing everywhere.
Splits are the dataset's official signer-disjoint ones (20,161 / 3,713 / 4,538, all
1,000 classes present in each), so no split is invented here.
Outputs mirror prepare_vsl_data.py: {split}_xy.npy / {split}_valid.npy /
{split}_index.json / mean.npy / std.npy / layout.json.
"""
import argparse
import json
import os
import numpy as np
from tqdm import tqdm
from dataset.layout import Layout, PRESETS
REPO = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
NECK, RSHO, LSHO = 1, 2, 5 # OpenPose-18 body indices
MIN_SHOULDER_PX = 8.0 # below this the reference is untrustworthy
def load_npz(path):
with np.load(path) as d:
xy = d['all_xy'].astype(np.float32) # [T,128,2] in [0,1] per axis
sc = d['all_score'].astype(np.float32) # [T,128]
wh = d['frame_size'].astype(np.float32) if 'frame_size' in d else None
fps = float(d['fps']) if 'fps' in d else 30.0
return xy, sc, wh, fps
def normalize_clip(xy, sc, wh, score_thr):
"""-> (xy_norm [T,128,2], valid [T,128] bool, info) or (None, None, reason)."""
T = len(xy)
valid = np.isfinite(xy).all(-1) & (sc > score_thr)
xy = np.nan_to_num(xy, nan=0.0, posinf=0.0, neginf=0.0)
# (a) per-axis [0,1] -> isotropic pixels
xy = xy * wh[None, None, :]
# (b) per-clip reference from the median over frames where it is observed
ok_ref = valid[:, NECK] & valid[:, RSHO] & valid[:, LSHO]
if ok_ref.sum() < max(3, 0.1 * T):
return None, None, 'no reliable neck/shoulder reference'
neck = np.median(xy[ok_ref, NECK, :], axis=0) # [2]
sw = np.median(np.linalg.norm(xy[ok_ref, RSHO, :] - xy[ok_ref, LSHO, :], axis=-1))
if not np.isfinite(sw) or sw < MIN_SHOULDER_PX:
return None, None, f'shoulder width {sw:.1f}px too small'
xy = (xy - neck[None, None, :]) / sw
return xy, valid, {'shoulder_px': float(sw), 'neck_px': neck.tolist()}
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--root', default=os.path.join(REPO, 'Multi-VSL_front'))
ap.add_argument('--out-dir', default='./dataset/MVSL')
ap.add_argument('--score-thr', type=float, default=0.3)
ap.add_argument('--layout', default='upper', choices=list(PRESETS))
ap.add_argument('--min-frames', type=int, default=16)
ap.add_argument('--fps-thresh', type=float, default=45.0,
help='clips above this fps are decimated by 2')
ap.add_argument('--limit', type=int, default=0)
args = ap.parse_args()
os.makedirs(args.out_dir, exist_ok=True)
layout = Layout.preset(args.layout)
layout.save(args.out_dir)
print(layout)
NKP, DIM = layout.n_kpts, layout.dim
with open(os.path.join(args.root, 'clips.json'), encoding='utf-8') as f:
clips = json.load(f)
by_split = {}
for c in clips:
by_split.setdefault(c['split'], []).append(c)
print({k: len(v) for k, v in by_split.items()})
skel = os.path.join(args.root, 'skeleton')
stats_all = {}
for split, rows in sorted(by_split.items()):
if args.limit:
rows = rows[:args.limit]
# ---- pass 1: normalize into RAM (these clips are short; ~2.4M frames total)
kept, drop = [], {'missing': 0, 'short': 0, 'noref': 0}
for c in tqdm(rows, desc=f'{split}: load'):
p = os.path.join(skel, os.path.splitext(c['name'])[0] + '.npz')
if not os.path.exists(p):
drop['missing'] += 1
continue
try:
xy, sc, wh, fps = load_npz(p)
except Exception:
drop['missing'] += 1
continue
if wh is None or len(xy) < args.min_frames:
drop['short'] += 1
continue
if fps > args.fps_thresh: # 59.94 -> ~30
xy, sc = xy[::2], sc[::2]
nxy, valid, info = normalize_clip(xy, sc, wh, args.score_thr)
if nxy is None:
drop['noref'] += 1
continue
if len(nxy) < args.min_frames:
drop['short'] += 1
continue
kept.append((c, nxy[:, layout.keep, :].reshape(len(nxy), DIM),
valid[:, layout.keep].astype(np.uint8), info))
print(f'{split}: kept {len(kept)}, dropped {drop}')
total = sum(len(k[1]) for k in kept)
xy_mm = np.lib.format.open_memmap(os.path.join(args.out_dir, f'{split}_xy.npy'),
mode='w+', dtype=np.float16, shape=(total, DIM))
vd_mm = np.lib.format.open_memmap(os.path.join(args.out_dir, f'{split}_valid.npy'),
mode='w+', dtype=np.uint8, shape=(total, NKP))
index, off = [], 0
s1 = np.zeros(DIM, np.float64); s2 = np.zeros(DIM, np.float64)
cnt = np.zeros(DIM, np.float64)
for c, x, v, info in tqdm(kept, desc=f'{split}: pack'):
T = len(x)
xy_mm[off:off + T] = x.astype(np.float16)
vd_mm[off:off + T] = v
if split == 'train':
m = np.repeat(v, 2, axis=1).astype(np.float64)
x64 = x.astype(np.float64)
s1 += (x64 * m).sum(0); s2 += (x64 * x64 * m).sum(0); cnt += m.sum(0)
index.append({'name': os.path.splitext(c['name'])[0], 'start': off, 'length': T,
'label': c['label'], 'gloss': c['word'], 'sentence': c['word'],
'shoulder_px': info['shoulder_px']})
off += T
xy_mm.flush(); vd_mm.flush(); del xy_mm, vd_mm
with open(os.path.join(args.out_dir, f'{split}_index.json'), 'w', encoding='utf-8') as f:
json.dump(index, f, ensure_ascii=False)
if split == 'train':
cnt = np.maximum(cnt, 1.0)
mean = s1 / cnt
std = np.sqrt(np.maximum(s2 / cnt - mean ** 2, 0.0))
std[cnt < 100] = 1.0
std = np.maximum(std, 1e-2)
np.save(os.path.join(args.out_dir, 'mean.npy'), mean.astype(np.float32))
np.save(os.path.join(args.out_dir, 'std.npy'), std.astype(np.float32))
print(f'mean/std saved; std range {std.min():.4f}..{std.max():.4f}')
L = np.array([len(k[1]) for k in kept])
u = 4
print(f'{split}: frames min {L.min()} median {int(np.median(L))} max {L.max()} '
f'| tokens median {int(np.median(L)//u)} max {L.max()//u}')
stats_all[split] = {'clips': len(kept), 'frames': int(total), 'dropped': drop,
'tokens_median': int(np.median(L) // u),
'tokens_max': int(L.max() // u)}
with open(os.path.join(args.out_dir, 'prep_stats.json'), 'w') as f:
json.dump({'layout': layout.name, 'splits': stats_all}, f, indent=2)
print('\n' + json.dumps(stats_all, indent=2))
if __name__ == '__main__':
main()
|