"""CPU replay of the two frozen Apple Vision INCLUDE Transformer checkpoints.""" from bisect import bisect_right from dataclasses import dataclass import hashlib import json from pathlib import Path import struct import numpy as np WEIGHTS_SHA256 = { "rolling": "5df06195356eb8b61aff409d556cebcfb81accfe50fce23a3fb8dbb532a8e3d6", "causal": "6ed79c31aec5c232141aa9cfccf2fa7eec5265ace50fce3a73a8b75471e658f8", } VOCABULARY_SHA256 = "caaf0d329a7de21ffc170a19e539d1fe197e9ef7344f8bf57ea752812de43963" FEATURES = 180 HIDDEN = 256 HEADS = 4 EMBEDDING = 384 CONTEXT = 48 TICK_SECONDS = 0.25 GROUPS = ("body", "leftHand", "rightHand") GROUP_SIZES = (18, 21, 21) FREQUENCIES = np.exp(np.arange(0, HIDDEN, 2, dtype=np.float32) * np.float32(-np.log(10_000) / HIDDEN)) def _weights(path, arm): path = Path(path) if not 0 < path.stat().st_size <= 20_000_000: raise ValueError("Unexpected checkpoint length") data = path.read_bytes() if hashlib.sha256(data).hexdigest() != WEIGHTS_SHA256[arm]: raise ValueError(f"Unexpected {arm} checkpoint digest") header_size, = struct.unpack_from("= len(result): break return result.reshape(FEATURES) def _resample(groups, serialized, start, end): count = end - start if count == 60: return serialized[start:end] result = np.empty((60, FEATURES), dtype=np.float32) for row in range(60): position = row / 59 * (count - 1) lower = int(position) upper = min(lower + 1, count - 1) if lower == upper: result[row] = serialized[start + lower] continue fraction = np.float32(position - lower) result[row] = _serialize([ _interpolate(a, b, fraction) for a, b in zip(groups[start + lower], groups[start + upper]) ]) return result def _position(time): estimate = int(time * 12) if estimate / 12 > time: return estimate - 1 if (estimate + 1) / 12 <= time: return estimate + 1 return estimate class _PoseSequence: def __init__(self, frames): self.times = [float(frame["timestamp"]) for frame in frames] if (not self.times or any(not np.isfinite(t) or t < 0 for t in self.times) or any(a > b for a, b in zip(self.times, self.times[1:]))): raise ValueError("Expected saved poses with finite chronological timestamps") self.groups = [tuple(_Group.decode(frame.get(key)) for key in GROUPS) for frame in frames] if any(groups[0] is None for groups in self.groups): raise ValueError("Missing body group in saved pose") self.rows = np.asarray([_serialize(groups) for groups in self.groups]) self.usable = np.isfinite(self.rows).all(axis=1) & (self.rows[:, 2::3] > 0).any(axis=1) self.next_frame = 0 self.last_rolling_time = -np.inf def rolling(self, time): start = bisect_right(self.times, time - 2) end = bisect_right(self.times, time, lo=start) if end - start < 2 or self.times[end - 1] <= self.last_rolling_time: return None self.last_rolling_time = self.times[end - 1] return _resample(self.groups, self.rows, start, end) def causal(self, time): horizon = _position(time) / 12 positions, values = [], [] while self.next_frame < len(self.times) and self.times[self.next_frame] <= horizon: first = self.next_frame frame_time = self.times[first] position = _position(frame_time) if position / 12 < frame_time: position += 1 grid_time = position / 12 self.next_frame = bisect_right(self.times, grid_time, lo=first) for i in range(self.next_frame - 1, first - 1, -1): if grid_time - self.times[i] > 0.25: break if self.usable[i]: positions.append(position) values.append(self.rows[i]) break return np.asarray(positions, dtype=np.int32), np.asarray(values, dtype=np.float32) class Transformer: """Load immutable model weights; each replay creates its own temporal state.""" def __init__(self, weights_path, vocabulary, arm): if arm not in WEIGHTS_SHA256: raise ValueError("Unknown Transformer mode") self.arm = arm self.weights = _weights(weights_path, arm) self.labels, self.vectors, self.vector_norms = _vocabulary(vocabulary) def _linear(self, x, name): x = x @ self.weights[name + ".weight"].T bias = self.weights.get(name + ".bias") return x if bias is None else x + bias def _norm(self, x, name): mean = x.mean(axis=-1, keepdims=True) centered = x - mean variance = (centered * centered).mean(axis=-1, keepdims=True) return centered / np.sqrt(variance + np.float32(1e-5)) * self.weights[name + ".weight"] + self.weights[name + ".bias"] def _embed(self, rows, positions, cache=None): x = self._norm(self._linear(rows, "inputProjection"), "inputNorm") phase = positions.astype(np.float32)[:, None] * FREQUENCIES x += np.concatenate((np.sin(phase), np.cos(phase)), axis=-1) if cache is not None: old_positions, old_layers = cache key_positions = np.concatenate((old_positions, positions)) recent = positions[-1] - (CONTEXT - 1) retain = int(np.searchsorted(key_positions, recent)) mask = (key_positions[None, :] <= positions[:, None]) & (key_positions[None, :] >= positions[:, None] - (CONTEXT - 1)) retained = [] for block in range(4): name = f"encoderLayers.{block}" z = self._norm(x, name + ".norm1") attention = name + ".attention." queries, keys, values = [ self._linear(z, attention + layer).reshape(len(x), HEADS, -1).transpose(1, 0, 2) for layer in ("query_proj", "key_proj", "value_proj") ] if cache is not None and old_layers: old_keys, old_values = old_layers[block] keys = np.concatenate((old_keys, keys), axis=1) values = np.concatenate((old_values, values), axis=1) scores = queries @ keys.swapaxes(-1, -2) * np.float32(1 / 8) if cache is not None: scores = np.where(mask, scores, np.float32(-np.inf)) retained.append((keys[:, retain:].copy(), values[:, retain:].copy())) joined = (_softmax(scores) @ values).transpose(1, 0, 2).reshape(len(x), HIDDEN) x += self._linear(joined, attention + "out_proj") * self.weights[name + ".gamma1"] z = self._linear(self._norm(x, name + ".norm2"), name + ".mlp.layers.0") z = _gelu(z) x += self._linear(z, name + ".mlp.layers.3") * self.weights[name + ".gamma2"] x = self._norm(x, "outputNorm") if cache is None: keys = self._linear(x, "pool.keyProj") values = self._linear(x, "pool.valueProj") scores = self.weights["pool.query"].reshape(1, HIDDEN) @ keys.T * np.float32(1 / 16) x = _softmax(scores) @ values next_cache = None else: next_cache = key_positions[retain:].copy(), retained x = self._linear(x, "outputProjection") x /= np.sqrt((x * x).sum(axis=-1, keepdims=True) + np.float32(1e-6)) if not np.isfinite(x).all(): raise ValueError("Transformer produced non-finite output") return x, next_cache def _predict(self, embedding): norms = np.linalg.norm(embedding) * self.vector_norms scores = np.divide(self.vectors @ embedding, norms, out=np.zeros_like(norms), where=norms > 0) best = int(scores.argmax()) return self.labels[best], float(max(0, scores[best]) * 100) def replay(self, frames, duration): """Yield scheduled predictions and the first latched confirmation.""" duration = float(duration) if not np.isfinite(duration) or not 0 < duration <= 120: raise ValueError("Expected a finite clip duration up to 120 seconds") poses = _PoseSequence(frames) rolling_positions = np.arange(60, dtype=np.int32) cache = np.empty(0, dtype=np.int32), [] previous = confirmed = confirmed_at = None streak = 0 index = 2 while index * TICK_SECONDS < duration: time = index * TICK_SECONDS label = score = None if self.arm == "rolling": rows = poses.rolling(time) if rows is not None: output, _ = self._embed(rows, rolling_positions) label, score = self._predict(output[-1]) else: positions, rows = poses.causal(time) for start in range(0, len(positions), CONTEXT): stop = start + CONTEXT output, cache = self._embed(rows[start:stop], positions[start:stop], cache) if len(positions): label, score = self._predict(output[-1]) if label is None: streak = 0 else: streak = streak + 1 if label == previous else 1 if confirmed is None and streak == 3: confirmed, confirmed_at = label, time previous = label yield {"index": index, "time": time, "label": label, "score": score, "confirmationLabel": confirmed, "confirmationTime": confirmed_at} index += 1