SignLanguageTranslate / transformer_inference.py
spsarolkar's picture
Deploy SignLanguageTranslate with Transformer evaluation
e35b538 verified
Raw History Blame Contribute Delete
13.7 kB
"""CPU replay of the two frozen Apple Vision INCLUDE Transformer checkpoints."""
from bisect import bisect_right
from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
import struct
import numpy as np
WEIGHTS_SHA256 = {
"rolling": "5df06195356eb8b61aff409d556cebcfb81accfe50fce23a3fb8dbb532a8e3d6",
"causal": "6ed79c31aec5c232141aa9cfccf2fa7eec5265ace50fce3a73a8b75471e658f8",
}
VOCABULARY_SHA256 = "caaf0d329a7de21ffc170a19e539d1fe197e9ef7344f8bf57ea752812de43963"
FEATURES = 180
HIDDEN = 256
HEADS = 4
EMBEDDING = 384
CONTEXT = 48
TICK_SECONDS = 0.25
GROUPS = ("body", "leftHand", "rightHand")
GROUP_SIZES = (18, 21, 21)
FREQUENCIES = np.exp(np.arange(0, HIDDEN, 2, dtype=np.float32) * np.float32(-np.log(10_000) / HIDDEN))
def _weights(path, arm):
path = Path(path)
if not 0 < path.stat().st_size <= 20_000_000:
raise ValueError("Unexpected checkpoint length")
data = path.read_bytes()
if hashlib.sha256(data).hexdigest() != WEIGHTS_SHA256[arm]:
raise ValueError(f"Unexpected {arm} checkpoint digest")
header_size, = struct.unpack_from("<Q", data)
header = json.loads(data[8:8 + header_size])
buffer = memoryview(data)[8 + header_size:]
tensors = {}
rolling_prefixes = {0: "inputProjection", 1: "inputNorm", 2: "outputNorm", 3: "outputProjection", 5: "pool"}
for name, metadata in header.items():
if name == "__metadata__":
continue
if arm == "rolling":
_, layer, suffix = name.split(".", 2)
layer = int(layer)
prefix = rolling_prefixes[layer] if layer < 6 else f"encoderLayers.{layer - 6}"
name = f"{prefix}.{suffix}"
start, end = metadata["data_offsets"]
if metadata["dtype"] != "F32" or not 0 <= start < end <= len(buffer):
raise ValueError("Invalid checkpoint tensor")
tensors[name] = np.frombuffer(buffer[start:end], dtype="<f4").reshape(metadata["shape"])
if len(tensors) != (69 if arm == "rolling" else 64):
raise ValueError("Unexpected checkpoint tensor count")
return tensors
def _vocabulary(rows):
if len(rows) != 256 or len({row["word"] for row in rows}) != 256:
raise ValueError("Expected the frozen 256-sign vocabulary")
digest = hashlib.sha256()
for row in rows:
vector = np.asarray(row["vector"], dtype="<f4")
if vector.shape != (EMBEDDING,) or not np.isfinite(vector).all():
raise ValueError("Invalid sign embedding")
digest.update(row["word"].encode("utf-8") + b"\0")
digest.update(vector.tobytes())
if digest.hexdigest() != VOCABULARY_SHA256:
raise ValueError("Unexpected ordered vocabulary digest")
rows = sorted(rows, key=lambda row: row["word"])
vectors = np.asarray([row["vector"] for row in rows], dtype=np.float32)
return [row["word"] for row in rows], vectors, np.linalg.norm(vectors, axis=1)
def _softmax(x):
x = np.exp(x - x.max(axis=-1, keepdims=True))
return x / x.sum(axis=-1, keepdims=True)
def _gelu(x):
# Abramowitz and Stegun 7.1.26 approximates erf to within 1.5e-7 in
# real arithmetic. Float32 output is checked against the native replay.
scaled = x * np.float32(1 / np.sqrt(2))
t = np.float32(1) / (np.float32(1) + np.float32(0.3275911) * np.abs(scaled))
polynomial = np.float32(1.061405429)
for coefficient in (-1.453152027, 1.421413741, -0.284496736, 0.254829592):
polynomial = polynomial * t + np.float32(coefficient)
erf = np.copysign(np.float32(1) - polynomial * t * np.exp(-scaled * scaled), scaled)
return (x * (np.float32(1) + erf)) / np.float32(2)
@dataclass(frozen=True)
class _Group:
names: tuple
values: np.ndarray
@classmethod
def decode(cls, points):
if points is None:
return None
return cls(tuple(p["id"] for p in points),
np.asarray([(p["x"], p["y"], p["confidence"]) for p in points], dtype=np.float32).reshape(-1, 3))
def _interpolate(a, b, fraction):
if a is None or b is None or len(a.names) != len(b.names):
return a if fraction < 0.5 else b
values = a.values + (b.values - a.values) * fraction
if a.names != b.names:
mismatched = [i for i, (name_a, name_b) in enumerate(zip(a.names, b.names)) if name_a != name_b]
values[mismatched] = a.values[mismatched]
return _Group(a.names, values)
def _serialize(groups):
body = groups[0]
offset = np.zeros(2, dtype=np.float32)
for name in ("neck", "nose"):
if name in body.names:
offset = body.values[body.names.index(name), :2]
break
result = np.zeros((FEATURES // 3, 3), dtype=np.float32)
start = 0
for group, minimum_size in zip(groups, GROUP_SIZES):
if group is not None:
length = min(len(group.names), len(result) - start)
result[start:start + length] = group.values[:length]
result[start:start + length, :2] -= offset
start += max(minimum_size, len(group.names))
else:
start += minimum_size
if start >= len(result):
break
return result.reshape(FEATURES)
def _resample(groups, serialized, start, end):
count = end - start
if count == 60:
return serialized[start:end]
result = np.empty((60, FEATURES), dtype=np.float32)
for row in range(60):
position = row / 59 * (count - 1)
lower = int(position)
upper = min(lower + 1, count - 1)
if lower == upper:
result[row] = serialized[start + lower]
continue
fraction = np.float32(position - lower)
result[row] = _serialize([
_interpolate(a, b, fraction) for a, b in zip(groups[start + lower], groups[start + upper])
])
return result
def _position(time):
estimate = int(time * 12)
if estimate / 12 > time:
return estimate - 1
if (estimate + 1) / 12 <= time:
return estimate + 1
return estimate
class _PoseSequence:
def __init__(self, frames):
self.times = [float(frame["timestamp"]) for frame in frames]
if (not self.times or any(not np.isfinite(t) or t < 0 for t in self.times)
or any(a > b for a, b in zip(self.times, self.times[1:]))):
raise ValueError("Expected saved poses with finite chronological timestamps")
self.groups = [tuple(_Group.decode(frame.get(key)) for key in GROUPS) for frame in frames]
if any(groups[0] is None for groups in self.groups):
raise ValueError("Missing body group in saved pose")
self.rows = np.asarray([_serialize(groups) for groups in self.groups])
self.usable = np.isfinite(self.rows).all(axis=1) & (self.rows[:, 2::3] > 0).any(axis=1)
self.next_frame = 0
self.last_rolling_time = -np.inf
def rolling(self, time):
start = bisect_right(self.times, time - 2)
end = bisect_right(self.times, time, lo=start)
if end - start < 2 or self.times[end - 1] <= self.last_rolling_time:
return None
self.last_rolling_time = self.times[end - 1]
return _resample(self.groups, self.rows, start, end)
def causal(self, time):
horizon = _position(time) / 12
positions, values = [], []
while self.next_frame < len(self.times) and self.times[self.next_frame] <= horizon:
first = self.next_frame
frame_time = self.times[first]
position = _position(frame_time)
if position / 12 < frame_time:
position += 1
grid_time = position / 12
self.next_frame = bisect_right(self.times, grid_time, lo=first)
for i in range(self.next_frame - 1, first - 1, -1):
if grid_time - self.times[i] > 0.25:
break
if self.usable[i]:
positions.append(position)
values.append(self.rows[i])
break
return np.asarray(positions, dtype=np.int32), np.asarray(values, dtype=np.float32)
class Transformer:
"""Load immutable model weights; each replay creates its own temporal state."""
def __init__(self, weights_path, vocabulary, arm):
if arm not in WEIGHTS_SHA256:
raise ValueError("Unknown Transformer mode")
self.arm = arm
self.weights = _weights(weights_path, arm)
self.labels, self.vectors, self.vector_norms = _vocabulary(vocabulary)
def _linear(self, x, name):
x = x @ self.weights[name + ".weight"].T
bias = self.weights.get(name + ".bias")
return x if bias is None else x + bias
def _norm(self, x, name):
mean = x.mean(axis=-1, keepdims=True)
centered = x - mean
variance = (centered * centered).mean(axis=-1, keepdims=True)
return centered / np.sqrt(variance + np.float32(1e-5)) * self.weights[name + ".weight"] + self.weights[name + ".bias"]
def _embed(self, rows, positions, cache=None):
x = self._norm(self._linear(rows, "inputProjection"), "inputNorm")
phase = positions.astype(np.float32)[:, None] * FREQUENCIES
x += np.concatenate((np.sin(phase), np.cos(phase)), axis=-1)
if cache is not None:
old_positions, old_layers = cache
key_positions = np.concatenate((old_positions, positions))
recent = positions[-1] - (CONTEXT - 1)
retain = int(np.searchsorted(key_positions, recent))
mask = (key_positions[None, :] <= positions[:, None]) & (key_positions[None, :] >= positions[:, None] - (CONTEXT - 1))
retained = []
for block in range(4):
name = f"encoderLayers.{block}"
z = self._norm(x, name + ".norm1")
attention = name + ".attention."
queries, keys, values = [
self._linear(z, attention + layer).reshape(len(x), HEADS, -1).transpose(1, 0, 2)
for layer in ("query_proj", "key_proj", "value_proj")
]
if cache is not None and old_layers:
old_keys, old_values = old_layers[block]
keys = np.concatenate((old_keys, keys), axis=1)
values = np.concatenate((old_values, values), axis=1)
scores = queries @ keys.swapaxes(-1, -2) * np.float32(1 / 8)
if cache is not None:
scores = np.where(mask, scores, np.float32(-np.inf))
retained.append((keys[:, retain:].copy(), values[:, retain:].copy()))
joined = (_softmax(scores) @ values).transpose(1, 0, 2).reshape(len(x), HIDDEN)
x += self._linear(joined, attention + "out_proj") * self.weights[name + ".gamma1"]
z = self._linear(self._norm(x, name + ".norm2"), name + ".mlp.layers.0")
z = _gelu(z)
x += self._linear(z, name + ".mlp.layers.3") * self.weights[name + ".gamma2"]
x = self._norm(x, "outputNorm")
if cache is None:
keys = self._linear(x, "pool.keyProj")
values = self._linear(x, "pool.valueProj")
scores = self.weights["pool.query"].reshape(1, HIDDEN) @ keys.T * np.float32(1 / 16)
x = _softmax(scores) @ values
next_cache = None
else:
next_cache = key_positions[retain:].copy(), retained
x = self._linear(x, "outputProjection")
x /= np.sqrt((x * x).sum(axis=-1, keepdims=True) + np.float32(1e-6))
if not np.isfinite(x).all():
raise ValueError("Transformer produced non-finite output")
return x, next_cache
def _predict(self, embedding):
norms = np.linalg.norm(embedding) * self.vector_norms
scores = np.divide(self.vectors @ embedding, norms, out=np.zeros_like(norms), where=norms > 0)
best = int(scores.argmax())
return self.labels[best], float(max(0, scores[best]) * 100)
def replay(self, frames, duration):
"""Yield scheduled predictions and the first latched confirmation."""
duration = float(duration)
if not np.isfinite(duration) or not 0 < duration <= 120:
raise ValueError("Expected a finite clip duration up to 120 seconds")
poses = _PoseSequence(frames)
rolling_positions = np.arange(60, dtype=np.int32)
cache = np.empty(0, dtype=np.int32), []
previous = confirmed = confirmed_at = None
streak = 0
index = 2
while index * TICK_SECONDS < duration:
time = index * TICK_SECONDS
label = score = None
if self.arm == "rolling":
rows = poses.rolling(time)
if rows is not None:
output, _ = self._embed(rows, rolling_positions)
label, score = self._predict(output[-1])
else:
positions, rows = poses.causal(time)
for start in range(0, len(positions), CONTEXT):
stop = start + CONTEXT
output, cache = self._embed(rows[start:stop], positions[start:stop], cache)
if len(positions):
label, score = self._predict(output[-1])
if label is None:
streak = 0
else:
streak = streak + 1 if label == previous else 1
if confirmed is None and streak == 3:
confirmed, confirmed_at = label, time
previous = label
yield {"index": index, "time": time, "label": label, "score": score,
"confirmationLabel": confirmed, "confirmationTime": confirmed_at}
index += 1