Add MPS fast path for Apple Silicon (bucketed static shapes + compiled LM)

#3
.gitattributes CHANGED
@@ -46,3 +46,7 @@ assets/plays_snake.gif filter=lfs diff=lfs merge=lfs -text
46
  assets/training_distillation.png filter=lfs diff=lfs merge=lfs -text
47
  assets/progress.png filter=lfs diff=lfs merge=lfs -text
48
  assets/training_thinking.png filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
46
  assets/training_distillation.png filter=lfs diff=lfs merge=lfs -text
47
  assets/progress.png filter=lfs diff=lfs merge=lfs -text
48
  assets/training_thinking.png filter=lfs diff=lfs merge=lfs -text
49
+ assets/community/chart_speed_accuracy_size.png filter=lfs diff=lfs merge=lfs -text
50
+ assets/community/chart_resolution_adaptive.png filter=lfs diff=lfs merge=lfs -text
51
+ assets/community/chart_stage_timing.png filter=lfs diff=lfs merge=lfs -text
52
+ assets/community/chart_inductor_cache.png filter=lfs diff=lfs merge=lfs -text
assets/community/chart_inductor_cache.png ADDED

Git LFS Details

  • SHA256: f392e44a10b836d3d052f97b559d3bfd01f022933404c99ec9149406926ee6b9
  • Pointer size: 130 Bytes
  • Size of remote file: 50 kB
assets/community/chart_resolution_adaptive.png ADDED

Git LFS Details

  • SHA256: 2310b2db833e1845a19abe933a0c05715a1cbbea1b35dc1610ebec9faae64ae1
  • Pointer size: 130 Bytes
  • Size of remote file: 81 kB
assets/community/chart_speed_accuracy_size.png ADDED

Git LFS Details

  • SHA256: 2d4ed9d257bc525ce9b1c5786bd2dc7f2a75c188c4b646f1403c5bb3aa0f0bfd
  • Pointer size: 131 Bytes
  • Size of remote file: 103 kB
assets/community/chart_stage_timing.png ADDED

Git LFS Details

  • SHA256: f3cf7221c239b3f6b0c483354f840494403b238e518196026ee9563b300fd015
  • Pointer size: 130 Bytes
  • Size of remote file: 47.6 kB
thx02/model.py CHANGED
@@ -32,6 +32,7 @@ from .types import Request, Response
32
 
33
  if TYPE_CHECKING:
34
  from .fast import FastPath
 
35
 
36
  #: Default per-sequence token limit. Longer requests are split into question chunks (each with the full state).
37
  DEFAULT_MAX_LENGTH = 262_144 # backbone limit (max_position_embeddings); training used sequences up to about 32k tokens
@@ -112,7 +113,7 @@ class THXDecisionModel:
112
  checkpoint: Path,
113
  *,
114
  device: str = "cuda",
115
- fast: bool | FastOptions = False,
116
  max_length: int | None = DEFAULT_MAX_LENGTH,
117
  max_pixels: int | None = None,
118
  batch_tokens: int = 32768,
@@ -160,11 +161,17 @@ class THXDecisionModel:
160
  self.backbone.requires_grad_(False).eval().to(self.device)
161
  self.readout.requires_grad_(False).eval().to(self.device)
162
 
163
- self.fast: FastPath | None = None
164
  if fast:
165
- from .fast import FastPath
 
 
166
 
167
- self.fast = FastPath(self, fast if isinstance(fast, FastOptions) else FastOptions())
 
 
 
 
168
 
169
  @classmethod
170
  def from_pretrained(
@@ -172,7 +179,7 @@ class THXDecisionModel:
172
  path_or_repo: str | Path,
173
  *,
174
  device: str = "cuda",
175
- fast: bool | FastOptions = False,
176
  max_length: int | None = DEFAULT_MAX_LENGTH,
177
  max_pixels: int | None = None,
178
  revision: str | None = None,
@@ -182,7 +189,8 @@ class THXDecisionModel:
182
  """Load a local checkpoint directory or a Hugging Face repository.
183
 
184
  fast: ``False`` for the reference path (padded batches, exact evaluated computation); ``True`` or a
185
- ``FastOptions`` for the packed path (same results up to bf16 noise, several times faster).
 
186
  max_length: per-sequence token limit; longer requests are split into question chunks, see the README.
187
  max_pixels: image pixel cap (default: the checkpoint's ``max_pixels``, else 262,144).
188
  """
 
32
 
33
  if TYPE_CHECKING:
34
  from .fast import FastPath
35
+ from .mps_fast import MPSFastPath
36
 
37
  #: Default per-sequence token limit. Longer requests are split into question chunks (each with the full state).
38
  DEFAULT_MAX_LENGTH = 262_144 # backbone limit (max_position_embeddings); training used sequences up to about 32k tokens
 
113
  checkpoint: Path,
114
  *,
115
  device: str = "cuda",
116
+ fast: bool | FastOptions | Any = False,
117
  max_length: int | None = DEFAULT_MAX_LENGTH,
118
  max_pixels: int | None = None,
119
  batch_tokens: int = 32768,
 
161
  self.backbone.requires_grad_(False).eval().to(self.device)
162
  self.readout.requires_grad_(False).eval().to(self.device)
163
 
164
+ self.fast: FastPath | MPSFastPath | None = None
165
  if fast:
166
+ if self.device.type == "mps":
167
+ # No CUDA: bucketed static shapes + a compiled language model (no FLA, no CUDA graphs).
168
+ from .mps_fast import MPSFastOptions, MPSFastPath
169
 
170
+ self.fast = MPSFastPath(self, fast if isinstance(fast, MPSFastOptions) else MPSFastOptions())
171
+ else:
172
+ from .fast import FastPath
173
+
174
+ self.fast = FastPath(self, fast if isinstance(fast, FastOptions) else FastOptions())
175
 
176
  @classmethod
177
  def from_pretrained(
 
179
  path_or_repo: str | Path,
180
  *,
181
  device: str = "cuda",
182
+ fast: bool | FastOptions | Any = False,
183
  max_length: int | None = DEFAULT_MAX_LENGTH,
184
  max_pixels: int | None = None,
185
  revision: str | None = None,
 
189
  """Load a local checkpoint directory or a Hugging Face repository.
190
 
191
  fast: ``False`` for the reference path (padded batches, exact evaluated computation); ``True`` or a
192
+ ``FastOptions`` for the packed CUDA path, or a ``MPSFastOptions`` / ``"mps"`` for the bucketed
193
+ MPS path (same results up to bf16 noise, several times faster).
194
  max_length: per-sequence token limit; longer requests are split into question chunks, see the README.
195
  max_pixels: image pixel cap (default: the checkpoint's ``max_pixels``, else 262,144).
196
  """
thx02/mps_fast.py ADDED
@@ -0,0 +1,217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MPS fast path: bucketed static shapes with a compiled language model.
2
+
3
+ The packaged fast path (:mod:`thx02.fast`) needs CUDA: varlen FlashAttention, ``fla``'s chunked
4
+ gated delta rule and CUDA graphs. On Apple Silicon only the reference path runs, and its gated-delta
5
+ linear-attention layers fall back to the sequential PyTorch implementation, which is the main
6
+ bottleneck.
7
+
8
+ This path takes the same shape as the idea: decision requests are a single forward pass with no
9
+ generation loop, so a few length buckets plus ``torch.compile`` give Inductor stable graphs on MPS.
10
+
11
+ What is compiled, and what is not
12
+ Only ``backbone.language_model`` is compiled. Compiling the full backbone hits an Inductor Metal
13
+ codegen bug on image inputs (``no 'buffer' resource location available for 'error_buf'``), and the
14
+ vision tower is the cheap, cacheable half anyway. Embedding lookup, image-feature scatter and the
15
+ M-RoPE positions run eagerly at the request's natural length; the LM then runs over a fixed
16
+ ``(1, bucket)`` shape with right padding (masked, never read out).
17
+
18
+ Right padding is safe here because the checkpoint's full-attention layers are noncausal with a
19
+ padding mask and its linear-attention layers are causal: pad tokens to the right cannot influence
20
+ real tokens in either layer type.
21
+
22
+ Batching
23
+ One chunk per forward pass (batch = 1). ``torch.compile(dynamic=False)`` specializes per shape, so
24
+ a compiled batch dimension would need one graph per (batch, bucket) pair. MPS is a single device;
25
+ the speedup comes from compile, not from batching. Requests are still answered in input order and
26
+ the token usage reported is the natural length, not the bucket.
27
+
28
+ Over-length chunks
29
+ A chunk longer than the largest bucket runs eagerly at its natural length (no new graph, no
30
+ recompile). Raise ``buckets`` to cover longer inputs, or accept the eager fallback for the rare
31
+ long request.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import time
37
+ from dataclasses import dataclass, field
38
+ from typing import Any, Iterator
39
+
40
+ import torch
41
+
42
+ from .protocol import Encoded, Request
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class MPSFastOptions:
47
+ """Settings of the MPS fast path.
48
+
49
+ buckets: static sequence lengths to compile for. A chunk is right-padded up to the smallest bucket
50
+ that fits; longer chunks run eagerly. The default steps by ~1.25x from 128 to 1024, which
51
+ covers short questions as well as one 768 px image (~719 tokens) with at most 25% padding.
52
+ Power-of-two buckets are a trap: padding a 606-token request to 768 costs more than
53
+ compile saves, and the fast path ends up no faster than the reference path. The small
54
+ buckets matter just as much: a 90-token request padded to 256 is *slower* than the
55
+ reference path, while a 128 bucket makes it 1.36x faster.
56
+ compile: ``False`` keeps the same bucketed/eager-input layout without ``torch.compile``; useful as
57
+ an A/B baseline for the padding scheme alone.
58
+ dynamic: pass ``dynamic=True`` to ``torch.compile``. Slower to compile and historically fragile
59
+ with the gated-delta fallback on Metal; ``False`` (one graph per bucket) is the supported
60
+ default.
61
+ """
62
+
63
+ buckets: tuple[int, ...] = (128, 192, 256, 320, 384, 448, 512, 640, 768, 1024)
64
+ compile: bool = True
65
+ dynamic: bool = False
66
+
67
+
68
+ @dataclass
69
+ class MPSFastStats:
70
+ """Counters, for validation and logging (never used for control flow)."""
71
+
72
+ compile_s: float = 0.0
73
+ graphs: int = 0
74
+ bucketed: int = 0
75
+ eager: int = 0
76
+ per_bucket: dict[int, int] = field(default_factory=dict)
77
+
78
+
79
+ class MPSFastPath:
80
+ """Bucketed, LM-compiled replacement for ``FastPath`` on MPS.
81
+
82
+ Implements the same contract as ``FastPath``: ``run(requests)`` yields
83
+ ``(request, {question_key: logits}, chunks)`` and ``forward(chunks)`` returns one list of
84
+ per-question logit rows per chunk. ``warmup()`` compiles every bucket up front.
85
+ """
86
+
87
+ def __init__(self, decider: Any, options: MPSFastOptions | None = None) -> None:
88
+ if decider.device.type != "mps":
89
+ raise ValueError("The MPS fast path needs a Metal (MPS) device")
90
+ self.decider = decider
91
+ self.options = options or MPSFastOptions()
92
+ self.backbone = decider.backbone
93
+ self.lm = self.backbone.language_model
94
+ self.buckets = tuple(sorted({int(b) for b in self.options.buckets if int(b) > 0}))
95
+ if not self.buckets:
96
+ raise ValueError("The MPS fast path needs at least one positive bucket")
97
+ self.stats = MPSFastStats()
98
+ self._compiled: Any = None
99
+ self._seen_buckets: set[int] = set()
100
+ if self.options.compile:
101
+ torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 64)
102
+
103
+ # ------------------------------------------------------------------ setup
104
+
105
+ def reset(self) -> None:
106
+ """Drop compiled graphs (e.g. after weights were swapped for a quantized copy)."""
107
+ self._compiled = None
108
+ self._seen_buckets.clear()
109
+ self.stats.graphs = 0
110
+ self.stats.compile_s = 0.0
111
+
112
+ def _fn(self) -> Any:
113
+ if not self.options.compile:
114
+ return self.lm
115
+ if self._compiled is None:
116
+ self._compiled = torch.compile(self.lm, dynamic=self.options.dynamic)
117
+ return self._compiled
118
+
119
+ def warmup(self) -> None:
120
+ """Compile and run every bucket once, so the first real request does not pay for it."""
121
+ encoder = self.decider.encoder
122
+ request = {"state": "warmup", "questions": {"q": {"type": "noul", "instructions": "Warm up?"}}}
123
+ chunk = encoder.plan(request, None)[0]
124
+ for bucket in self.buckets:
125
+ self._forward_padded(chunk, bucket)
126
+ torch.mps.synchronize()
127
+
128
+ # -------------------------------------------------------------- scheduling
129
+
130
+ def run(self, requests: list[Request]) -> Iterator[tuple[Request, dict[str, list[float]], list[Encoded]]]:
131
+ """Yield (request, logits per question, chunks) in input order."""
132
+ encoder = self.decider.encoder
133
+ for request in requests:
134
+ encoder.validate(request)
135
+ plan = encoder.plan(request, self.decider.max_length)
136
+ results: dict[str, list[float]] = {}
137
+ for chunk, rows in zip(plan, self.forward(plan), strict=True):
138
+ results.update(zip(chunk.keys, rows, strict=True))
139
+ yield request, {key: results[key] for key in request["questions"]}, plan
140
+
141
+ # -------------------------------------------------------------- forward
142
+
143
+ def forward(self, chunks: list[Encoded]) -> list[list[list[float]]]:
144
+ return [self._forward_one(chunk) for chunk in chunks]
145
+
146
+ def _forward_one(self, chunk: Encoded) -> list[list[float]]:
147
+ bucket = next((size for size in self.buckets if size >= chunk.length), None)
148
+ if bucket is None: # over-length: eager at natural length, no new graph
149
+ self.stats.eager += 1
150
+ return self._run(chunk, None)
151
+ self.stats.bucketed += 1
152
+ self.stats.per_bucket[bucket] = self.stats.per_bucket.get(bucket, 0) + 1
153
+ return self._run(chunk, bucket)
154
+
155
+ def _forward_padded(self, chunk: Encoded, bucket: int) -> list[list[float]]:
156
+ return self._run(chunk, bucket)
157
+
158
+ @torch.inference_mode()
159
+ def _run(self, chunk: Encoded, bucket: int | None) -> list[list[float]]:
160
+ device = self.decider.device
161
+ embeds, positions, mask = self._inputs(chunk, bucket)
162
+ fn = self._fn() if bucket is not None else self.lm
163
+ first = bucket is not None and self.options.compile and bucket not in self._seen_buckets
164
+ started = time.perf_counter() if first else 0.0
165
+ hidden = fn(inputs_embeds=embeds, attention_mask=mask, position_ids=positions, use_cache=False)
166
+ if first: # the call above compiled and ran; the synchronize makes the timing honest
167
+ torch.mps.synchronize()
168
+ self._seen_buckets.add(bucket)
169
+ self.stats.graphs += 1
170
+ self.stats.compile_s += time.perf_counter() - started
171
+ markers = chunk.markers.to(device)
172
+ logits = self.decider.readout(hidden.last_hidden_state[0, markers]).float().cpu()
173
+ out, cursor = [], 0
174
+ for count in chunk.counts:
175
+ out.append(logits[cursor, :count].tolist())
176
+ cursor += 1
177
+ return out
178
+
179
+ def _inputs(self, chunk: Encoded, bucket: int | None) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
180
+ """Eager embeddings + vision scatter + M-RoPE positions, then right-pad to the bucket."""
181
+ device = self.decider.device
182
+ ids = chunk.input_ids.to(device)[None]
183
+ embeds = self.backbone.get_input_embeddings()(ids)
184
+ extra: dict[str, Any] = {}
185
+ if chunk.pixel_values is not None:
186
+ grid = chunk.image_grid_thw.to(device)
187
+ features = self.backbone.get_image_features(
188
+ chunk.pixel_values.to(device), grid, return_dict=True
189
+ ).pooler_output
190
+ features = torch.cat(features, dim=0).to(embeds.dtype)
191
+ image_mask, _ = self.backbone.get_placeholder_mask(ids, inputs_embeds=embeds, image_features=features)
192
+ embeds = embeds.masked_scatter(image_mask, features)
193
+ extra["image_grid_thw"] = grid
194
+ extra["mm_token_type_ids"] = chunk.mm_token_type_ids.to(device)[None]
195
+ mask = torch.ones((1, chunk.length), dtype=torch.long, device=device)
196
+ positions = self.backbone.compute_3d_position_ids(
197
+ input_ids=ids, inputs_embeds=embeds, attention_mask=mask, **extra
198
+ )
199
+ if positions is None: # text-only: the model's own 1-D positions, broadcast over the 3 M-RoPE axes
200
+ positions = torch.arange(chunk.length, device=device).view(1, 1, -1).expand(3, 1, -1)
201
+ if bucket is None or bucket == chunk.length:
202
+ return embeds, positions, mask
203
+ return self._pad(embeds, positions, mask, bucket)
204
+
205
+ @staticmethod
206
+ def _pad(
207
+ embeds: torch.Tensor, positions: torch.Tensor, mask: torch.Tensor, bucket: int
208
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
209
+ """Right-pad to a static length. Padding is masked and never read out, so zeros are enough."""
210
+ length = embeds.shape[1]
211
+ e = torch.zeros((1, bucket, embeds.shape[-1]), dtype=embeds.dtype, device=embeds.device)
212
+ e[:, :length] = embeds
213
+ m = torch.zeros((1, bucket), dtype=mask.dtype, device=mask.device)
214
+ m[:, :length] = mask
215
+ p = torch.zeros((*positions.shape[:-1], bucket), dtype=positions.dtype, device=positions.device)
216
+ p[..., :length] = positions
217
+ return e, p, m