Download transformer_inference.py from spsarolkar/SignLanguageTranslate: direct link, hf CLI and curl.
- Browser
- Download file 13.7 kB
-
https://huggingface.co/spaces/spsarolkar/SignLanguageTranslate/resolve/main/transformer_inference.py
- Command line
-
hf download hf://spaces/spsarolkar/SignLanguageTranslate/transformer_inference.py
-
curl -L -o transformer_inference.py https://huggingface.co/spaces/spsarolkar/SignLanguageTranslate/resolve/main/transformer_inference.py
13.7 kB
| """CPU replay of the two frozen Apple Vision INCLUDE Transformer checkpoints.""" | |
| from bisect import bisect_right | |
| from dataclasses import dataclass | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import struct | |
| import numpy as np | |
| WEIGHTS_SHA256 = { | |
| "rolling": "5df06195356eb8b61aff409d556cebcfb81accfe50fce23a3fb8dbb532a8e3d6", | |
| "causal": "6ed79c31aec5c232141aa9cfccf2fa7eec5265ace50fce3a73a8b75471e658f8", | |
| } | |
| VOCABULARY_SHA256 = "caaf0d329a7de21ffc170a19e539d1fe197e9ef7344f8bf57ea752812de43963" | |
| FEATURES = 180 | |
| HIDDEN = 256 | |
| HEADS = 4 | |
| EMBEDDING = 384 | |
| CONTEXT = 48 | |
| TICK_SECONDS = 0.25 | |
| GROUPS = ("body", "leftHand", "rightHand") | |
| GROUP_SIZES = (18, 21, 21) | |
| FREQUENCIES = np.exp(np.arange(0, HIDDEN, 2, dtype=np.float32) * np.float32(-np.log(10_000) / HIDDEN)) | |
| def _weights(path, arm): | |
| path = Path(path) | |
| if not 0 < path.stat().st_size <= 20_000_000: | |
| raise ValueError("Unexpected checkpoint length") | |
| data = path.read_bytes() | |
| if hashlib.sha256(data).hexdigest() != WEIGHTS_SHA256[arm]: | |
| raise ValueError(f"Unexpected {arm} checkpoint digest") | |
| header_size, = struct.unpack_from("<Q", data) | |
| header = json.loads(data[8:8 + header_size]) | |
| buffer = memoryview(data)[8 + header_size:] | |
| tensors = {} | |
| rolling_prefixes = {0: "inputProjection", 1: "inputNorm", 2: "outputNorm", 3: "outputProjection", 5: "pool"} | |
| for name, metadata in header.items(): | |
| if name == "__metadata__": | |
| continue | |
| if arm == "rolling": | |
| _, layer, suffix = name.split(".", 2) | |
| layer = int(layer) | |
| prefix = rolling_prefixes[layer] if layer < 6 else f"encoderLayers.{layer - 6}" | |
| name = f"{prefix}.{suffix}" | |
| start, end = metadata["data_offsets"] | |
| if metadata["dtype"] != "F32" or not 0 <= start < end <= len(buffer): | |
| raise ValueError("Invalid checkpoint tensor") | |
| tensors[name] = np.frombuffer(buffer[start:end], dtype="<f4").reshape(metadata["shape"]) | |
| if len(tensors) != (69 if arm == "rolling" else 64): | |
| raise ValueError("Unexpected checkpoint tensor count") | |
| return tensors | |
| def _vocabulary(rows): | |
| if len(rows) != 256 or len({row["word"] for row in rows}) != 256: | |
| raise ValueError("Expected the frozen 256-sign vocabulary") | |
| digest = hashlib.sha256() | |
| for row in rows: | |
| vector = np.asarray(row["vector"], dtype="<f4") | |
| if vector.shape != (EMBEDDING,) or not np.isfinite(vector).all(): | |
| raise ValueError("Invalid sign embedding") | |
| digest.update(row["word"].encode("utf-8") + b"\0") | |
| digest.update(vector.tobytes()) | |
| if digest.hexdigest() != VOCABULARY_SHA256: | |
| raise ValueError("Unexpected ordered vocabulary digest") | |
| rows = sorted(rows, key=lambda row: row["word"]) | |
| vectors = np.asarray([row["vector"] for row in rows], dtype=np.float32) | |
| return [row["word"] for row in rows], vectors, np.linalg.norm(vectors, axis=1) | |
| def _softmax(x): | |
| x = np.exp(x - x.max(axis=-1, keepdims=True)) | |
| return x / x.sum(axis=-1, keepdims=True) | |
| def _gelu(x): | |
| # Abramowitz and Stegun 7.1.26 approximates erf to within 1.5e-7 in | |
| # real arithmetic. Float32 output is checked against the native replay. | |
| scaled = x * np.float32(1 / np.sqrt(2)) | |
| t = np.float32(1) / (np.float32(1) + np.float32(0.3275911) * np.abs(scaled)) | |
| polynomial = np.float32(1.061405429) | |
| for coefficient in (-1.453152027, 1.421413741, -0.284496736, 0.254829592): | |
| polynomial = polynomial * t + np.float32(coefficient) | |
| erf = np.copysign(np.float32(1) - polynomial * t * np.exp(-scaled * scaled), scaled) | |
| return (x * (np.float32(1) + erf)) / np.float32(2) | |
| class _Group: | |
| names: tuple | |
| values: np.ndarray | |
| def decode(cls, points): | |
| if points is None: | |
| return None | |
| return cls(tuple(p["id"] for p in points), | |
| np.asarray([(p["x"], p["y"], p["confidence"]) for p in points], dtype=np.float32).reshape(-1, 3)) | |
| def _interpolate(a, b, fraction): | |
| if a is None or b is None or len(a.names) != len(b.names): | |
| return a if fraction < 0.5 else b | |
| values = a.values + (b.values - a.values) * fraction | |
| if a.names != b.names: | |
| mismatched = [i for i, (name_a, name_b) in enumerate(zip(a.names, b.names)) if name_a != name_b] | |
| values[mismatched] = a.values[mismatched] | |
| return _Group(a.names, values) | |
| def _serialize(groups): | |
| body = groups[0] | |
| offset = np.zeros(2, dtype=np.float32) | |
| for name in ("neck", "nose"): | |
| if name in body.names: | |
| offset = body.values[body.names.index(name), :2] | |
| break | |
| result = np.zeros((FEATURES // 3, 3), dtype=np.float32) | |
| start = 0 | |
| for group, minimum_size in zip(groups, GROUP_SIZES): | |
| if group is not None: | |
| length = min(len(group.names), len(result) - start) | |
| result[start:start + length] = group.values[:length] | |
| result[start:start + length, :2] -= offset | |
| start += max(minimum_size, len(group.names)) | |
| else: | |
| start += minimum_size | |
| if start >= len(result): | |
| break | |
| return result.reshape(FEATURES) | |
| def _resample(groups, serialized, start, end): | |
| count = end - start | |
| if count == 60: | |
| return serialized[start:end] | |
| result = np.empty((60, FEATURES), dtype=np.float32) | |
| for row in range(60): | |
| position = row / 59 * (count - 1) | |
| lower = int(position) | |
| upper = min(lower + 1, count - 1) | |
| if lower == upper: | |
| result[row] = serialized[start + lower] | |
| continue | |
| fraction = np.float32(position - lower) | |
| result[row] = _serialize([ | |
| _interpolate(a, b, fraction) for a, b in zip(groups[start + lower], groups[start + upper]) | |
| ]) | |
| return result | |
| def _position(time): | |
| estimate = int(time * 12) | |
| if estimate / 12 > time: | |
| return estimate - 1 | |
| if (estimate + 1) / 12 <= time: | |
| return estimate + 1 | |
| return estimate | |
| class _PoseSequence: | |
| def __init__(self, frames): | |
| self.times = [float(frame["timestamp"]) for frame in frames] | |
| if (not self.times or any(not np.isfinite(t) or t < 0 for t in self.times) | |
| or any(a > b for a, b in zip(self.times, self.times[1:]))): | |
| raise ValueError("Expected saved poses with finite chronological timestamps") | |
| self.groups = [tuple(_Group.decode(frame.get(key)) for key in GROUPS) for frame in frames] | |
| if any(groups[0] is None for groups in self.groups): | |
| raise ValueError("Missing body group in saved pose") | |
| self.rows = np.asarray([_serialize(groups) for groups in self.groups]) | |
| self.usable = np.isfinite(self.rows).all(axis=1) & (self.rows[:, 2::3] > 0).any(axis=1) | |
| self.next_frame = 0 | |
| self.last_rolling_time = -np.inf | |
| def rolling(self, time): | |
| start = bisect_right(self.times, time - 2) | |
| end = bisect_right(self.times, time, lo=start) | |
| if end - start < 2 or self.times[end - 1] <= self.last_rolling_time: | |
| return None | |
| self.last_rolling_time = self.times[end - 1] | |
| return _resample(self.groups, self.rows, start, end) | |
| def causal(self, time): | |
| horizon = _position(time) / 12 | |
| positions, values = [], [] | |
| while self.next_frame < len(self.times) and self.times[self.next_frame] <= horizon: | |
| first = self.next_frame | |
| frame_time = self.times[first] | |
| position = _position(frame_time) | |
| if position / 12 < frame_time: | |
| position += 1 | |
| grid_time = position / 12 | |
| self.next_frame = bisect_right(self.times, grid_time, lo=first) | |
| for i in range(self.next_frame - 1, first - 1, -1): | |
| if grid_time - self.times[i] > 0.25: | |
| break | |
| if self.usable[i]: | |
| positions.append(position) | |
| values.append(self.rows[i]) | |
| break | |
| return np.asarray(positions, dtype=np.int32), np.asarray(values, dtype=np.float32) | |
| class Transformer: | |
| """Load immutable model weights; each replay creates its own temporal state.""" | |
| def __init__(self, weights_path, vocabulary, arm): | |
| if arm not in WEIGHTS_SHA256: | |
| raise ValueError("Unknown Transformer mode") | |
| self.arm = arm | |
| self.weights = _weights(weights_path, arm) | |
| self.labels, self.vectors, self.vector_norms = _vocabulary(vocabulary) | |
| def _linear(self, x, name): | |
| x = x @ self.weights[name + ".weight"].T | |
| bias = self.weights.get(name + ".bias") | |
| return x if bias is None else x + bias | |
| def _norm(self, x, name): | |
| mean = x.mean(axis=-1, keepdims=True) | |
| centered = x - mean | |
| variance = (centered * centered).mean(axis=-1, keepdims=True) | |
| return centered / np.sqrt(variance + np.float32(1e-5)) * self.weights[name + ".weight"] + self.weights[name + ".bias"] | |
| def _embed(self, rows, positions, cache=None): | |
| x = self._norm(self._linear(rows, "inputProjection"), "inputNorm") | |
| phase = positions.astype(np.float32)[:, None] * FREQUENCIES | |
| x += np.concatenate((np.sin(phase), np.cos(phase)), axis=-1) | |
| if cache is not None: | |
| old_positions, old_layers = cache | |
| key_positions = np.concatenate((old_positions, positions)) | |
| recent = positions[-1] - (CONTEXT - 1) | |
| retain = int(np.searchsorted(key_positions, recent)) | |
| mask = (key_positions[None, :] <= positions[:, None]) & (key_positions[None, :] >= positions[:, None] - (CONTEXT - 1)) | |
| retained = [] | |
| for block in range(4): | |
| name = f"encoderLayers.{block}" | |
| z = self._norm(x, name + ".norm1") | |
| attention = name + ".attention." | |
| queries, keys, values = [ | |
| self._linear(z, attention + layer).reshape(len(x), HEADS, -1).transpose(1, 0, 2) | |
| for layer in ("query_proj", "key_proj", "value_proj") | |
| ] | |
| if cache is not None and old_layers: | |
| old_keys, old_values = old_layers[block] | |
| keys = np.concatenate((old_keys, keys), axis=1) | |
| values = np.concatenate((old_values, values), axis=1) | |
| scores = queries @ keys.swapaxes(-1, -2) * np.float32(1 / 8) | |
| if cache is not None: | |
| scores = np.where(mask, scores, np.float32(-np.inf)) | |
| retained.append((keys[:, retain:].copy(), values[:, retain:].copy())) | |
| joined = (_softmax(scores) @ values).transpose(1, 0, 2).reshape(len(x), HIDDEN) | |
| x += self._linear(joined, attention + "out_proj") * self.weights[name + ".gamma1"] | |
| z = self._linear(self._norm(x, name + ".norm2"), name + ".mlp.layers.0") | |
| z = _gelu(z) | |
| x += self._linear(z, name + ".mlp.layers.3") * self.weights[name + ".gamma2"] | |
| x = self._norm(x, "outputNorm") | |
| if cache is None: | |
| keys = self._linear(x, "pool.keyProj") | |
| values = self._linear(x, "pool.valueProj") | |
| scores = self.weights["pool.query"].reshape(1, HIDDEN) @ keys.T * np.float32(1 / 16) | |
| x = _softmax(scores) @ values | |
| next_cache = None | |
| else: | |
| next_cache = key_positions[retain:].copy(), retained | |
| x = self._linear(x, "outputProjection") | |
| x /= np.sqrt((x * x).sum(axis=-1, keepdims=True) + np.float32(1e-6)) | |
| if not np.isfinite(x).all(): | |
| raise ValueError("Transformer produced non-finite output") | |
| return x, next_cache | |
| def _predict(self, embedding): | |
| norms = np.linalg.norm(embedding) * self.vector_norms | |
| scores = np.divide(self.vectors @ embedding, norms, out=np.zeros_like(norms), where=norms > 0) | |
| best = int(scores.argmax()) | |
| return self.labels[best], float(max(0, scores[best]) * 100) | |
| def replay(self, frames, duration): | |
| """Yield scheduled predictions and the first latched confirmation.""" | |
| duration = float(duration) | |
| if not np.isfinite(duration) or not 0 < duration <= 120: | |
| raise ValueError("Expected a finite clip duration up to 120 seconds") | |
| poses = _PoseSequence(frames) | |
| rolling_positions = np.arange(60, dtype=np.int32) | |
| cache = np.empty(0, dtype=np.int32), [] | |
| previous = confirmed = confirmed_at = None | |
| streak = 0 | |
| index = 2 | |
| while index * TICK_SECONDS < duration: | |
| time = index * TICK_SECONDS | |
| label = score = None | |
| if self.arm == "rolling": | |
| rows = poses.rolling(time) | |
| if rows is not None: | |
| output, _ = self._embed(rows, rolling_positions) | |
| label, score = self._predict(output[-1]) | |
| else: | |
| positions, rows = poses.causal(time) | |
| for start in range(0, len(positions), CONTEXT): | |
| stop = start + CONTEXT | |
| output, cache = self._embed(rows[start:stop], positions[start:stop], cache) | |
| if len(positions): | |
| label, score = self._predict(output[-1]) | |
| if label is None: | |
| streak = 0 | |
| else: | |
| streak = streak + 1 if label == previous else 1 | |
| if confirmed is None and streak == 3: | |
| confirmed, confirmed_at = label, time | |
| previous = label | |
| yield {"index": index, "time": time, "label": label, "score": score, | |
| "confirmationLabel": confirmed, "confirmationTime": confirmed_at} | |
| index += 1 | |