Add MPS fast path for Apple Silicon (bucketed static shapes + compiled LM)
#3
by pylotlight - opened
.gitattributes
CHANGED
|
@@ -46,3 +46,7 @@ assets/plays_snake.gif filter=lfs diff=lfs merge=lfs -text
|
|
| 46 |
assets/training_distillation.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
assets/progress.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
assets/training_thinking.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
assets/training_distillation.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
assets/progress.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
assets/training_thinking.png filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
assets/community/chart_speed_accuracy_size.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
assets/community/chart_resolution_adaptive.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
assets/community/chart_stage_timing.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
assets/community/chart_inductor_cache.png filter=lfs diff=lfs merge=lfs -text
|
assets/community/chart_inductor_cache.png
ADDED
|
Git LFS Details
|
assets/community/chart_resolution_adaptive.png
ADDED
|
Git LFS Details
|
assets/community/chart_speed_accuracy_size.png
ADDED
|
Git LFS Details
|
assets/community/chart_stage_timing.png
ADDED
|
Git LFS Details
|
thx02/model.py
CHANGED
|
@@ -32,6 +32,7 @@ from .types import Request, Response
|
|
| 32 |
|
| 33 |
if TYPE_CHECKING:
|
| 34 |
from .fast import FastPath
|
|
|
|
| 35 |
|
| 36 |
#: Default per-sequence token limit. Longer requests are split into question chunks (each with the full state).
|
| 37 |
DEFAULT_MAX_LENGTH = 262_144 # backbone limit (max_position_embeddings); training used sequences up to about 32k tokens
|
|
@@ -112,7 +113,7 @@ class THXDecisionModel:
|
|
| 112 |
checkpoint: Path,
|
| 113 |
*,
|
| 114 |
device: str = "cuda",
|
| 115 |
-
fast: bool | FastOptions = False,
|
| 116 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 117 |
max_pixels: int | None = None,
|
| 118 |
batch_tokens: int = 32768,
|
|
@@ -160,11 +161,17 @@ class THXDecisionModel:
|
|
| 160 |
self.backbone.requires_grad_(False).eval().to(self.device)
|
| 161 |
self.readout.requires_grad_(False).eval().to(self.device)
|
| 162 |
|
| 163 |
-
self.fast: FastPath | None = None
|
| 164 |
if fast:
|
| 165 |
-
|
|
|
|
|
|
|
| 166 |
|
| 167 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
|
| 169 |
@classmethod
|
| 170 |
def from_pretrained(
|
|
@@ -172,7 +179,7 @@ class THXDecisionModel:
|
|
| 172 |
path_or_repo: str | Path,
|
| 173 |
*,
|
| 174 |
device: str = "cuda",
|
| 175 |
-
fast: bool | FastOptions = False,
|
| 176 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 177 |
max_pixels: int | None = None,
|
| 178 |
revision: str | None = None,
|
|
@@ -182,7 +189,8 @@ class THXDecisionModel:
|
|
| 182 |
"""Load a local checkpoint directory or a Hugging Face repository.
|
| 183 |
|
| 184 |
fast: ``False`` for the reference path (padded batches, exact evaluated computation); ``True`` or a
|
| 185 |
-
``FastOptions`` for the packed path
|
|
|
|
| 186 |
max_length: per-sequence token limit; longer requests are split into question chunks, see the README.
|
| 187 |
max_pixels: image pixel cap (default: the checkpoint's ``max_pixels``, else 262,144).
|
| 188 |
"""
|
|
|
|
| 32 |
|
| 33 |
if TYPE_CHECKING:
|
| 34 |
from .fast import FastPath
|
| 35 |
+
from .mps_fast import MPSFastPath
|
| 36 |
|
| 37 |
#: Default per-sequence token limit. Longer requests are split into question chunks (each with the full state).
|
| 38 |
DEFAULT_MAX_LENGTH = 262_144 # backbone limit (max_position_embeddings); training used sequences up to about 32k tokens
|
|
|
|
| 113 |
checkpoint: Path,
|
| 114 |
*,
|
| 115 |
device: str = "cuda",
|
| 116 |
+
fast: bool | FastOptions | Any = False,
|
| 117 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 118 |
max_pixels: int | None = None,
|
| 119 |
batch_tokens: int = 32768,
|
|
|
|
| 161 |
self.backbone.requires_grad_(False).eval().to(self.device)
|
| 162 |
self.readout.requires_grad_(False).eval().to(self.device)
|
| 163 |
|
| 164 |
+
self.fast: FastPath | MPSFastPath | None = None
|
| 165 |
if fast:
|
| 166 |
+
if self.device.type == "mps":
|
| 167 |
+
# No CUDA: bucketed static shapes + a compiled language model (no FLA, no CUDA graphs).
|
| 168 |
+
from .mps_fast import MPSFastOptions, MPSFastPath
|
| 169 |
|
| 170 |
+
self.fast = MPSFastPath(self, fast if isinstance(fast, MPSFastOptions) else MPSFastOptions())
|
| 171 |
+
else:
|
| 172 |
+
from .fast import FastPath
|
| 173 |
+
|
| 174 |
+
self.fast = FastPath(self, fast if isinstance(fast, FastOptions) else FastOptions())
|
| 175 |
|
| 176 |
@classmethod
|
| 177 |
def from_pretrained(
|
|
|
|
| 179 |
path_or_repo: str | Path,
|
| 180 |
*,
|
| 181 |
device: str = "cuda",
|
| 182 |
+
fast: bool | FastOptions | Any = False,
|
| 183 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 184 |
max_pixels: int | None = None,
|
| 185 |
revision: str | None = None,
|
|
|
|
| 189 |
"""Load a local checkpoint directory or a Hugging Face repository.
|
| 190 |
|
| 191 |
fast: ``False`` for the reference path (padded batches, exact evaluated computation); ``True`` or a
|
| 192 |
+
``FastOptions`` for the packed CUDA path, or a ``MPSFastOptions`` / ``"mps"`` for the bucketed
|
| 193 |
+
MPS path (same results up to bf16 noise, several times faster).
|
| 194 |
max_length: per-sequence token limit; longer requests are split into question chunks, see the README.
|
| 195 |
max_pixels: image pixel cap (default: the checkpoint's ``max_pixels``, else 262,144).
|
| 196 |
"""
|
thx02/mps_fast.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MPS fast path: bucketed static shapes with a compiled language model.
|
| 2 |
+
|
| 3 |
+
The packaged fast path (:mod:`thx02.fast`) needs CUDA: varlen FlashAttention, ``fla``'s chunked
|
| 4 |
+
gated delta rule and CUDA graphs. On Apple Silicon only the reference path runs, and its gated-delta
|
| 5 |
+
linear-attention layers fall back to the sequential PyTorch implementation, which is the main
|
| 6 |
+
bottleneck.
|
| 7 |
+
|
| 8 |
+
This path takes the same shape as the idea: decision requests are a single forward pass with no
|
| 9 |
+
generation loop, so a few length buckets plus ``torch.compile`` give Inductor stable graphs on MPS.
|
| 10 |
+
|
| 11 |
+
What is compiled, and what is not
|
| 12 |
+
Only ``backbone.language_model`` is compiled. Compiling the full backbone hits an Inductor Metal
|
| 13 |
+
codegen bug on image inputs (``no 'buffer' resource location available for 'error_buf'``), and the
|
| 14 |
+
vision tower is the cheap, cacheable half anyway. Embedding lookup, image-feature scatter and the
|
| 15 |
+
M-RoPE positions run eagerly at the request's natural length; the LM then runs over a fixed
|
| 16 |
+
``(1, bucket)`` shape with right padding (masked, never read out).
|
| 17 |
+
|
| 18 |
+
Right padding is safe here because the checkpoint's full-attention layers are noncausal with a
|
| 19 |
+
padding mask and its linear-attention layers are causal: pad tokens to the right cannot influence
|
| 20 |
+
real tokens in either layer type.
|
| 21 |
+
|
| 22 |
+
Batching
|
| 23 |
+
One chunk per forward pass (batch = 1). ``torch.compile(dynamic=False)`` specializes per shape, so
|
| 24 |
+
a compiled batch dimension would need one graph per (batch, bucket) pair. MPS is a single device;
|
| 25 |
+
the speedup comes from compile, not from batching. Requests are still answered in input order and
|
| 26 |
+
the token usage reported is the natural length, not the bucket.
|
| 27 |
+
|
| 28 |
+
Over-length chunks
|
| 29 |
+
A chunk longer than the largest bucket runs eagerly at its natural length (no new graph, no
|
| 30 |
+
recompile). Raise ``buckets`` to cover longer inputs, or accept the eager fallback for the rare
|
| 31 |
+
long request.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
+
import time
|
| 37 |
+
from dataclasses import dataclass, field
|
| 38 |
+
from typing import Any, Iterator
|
| 39 |
+
|
| 40 |
+
import torch
|
| 41 |
+
|
| 42 |
+
from .protocol import Encoded, Request
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass(frozen=True)
|
| 46 |
+
class MPSFastOptions:
|
| 47 |
+
"""Settings of the MPS fast path.
|
| 48 |
+
|
| 49 |
+
buckets: static sequence lengths to compile for. A chunk is right-padded up to the smallest bucket
|
| 50 |
+
that fits; longer chunks run eagerly. The default steps by ~1.25x from 128 to 1024, which
|
| 51 |
+
covers short questions as well as one 768 px image (~719 tokens) with at most 25% padding.
|
| 52 |
+
Power-of-two buckets are a trap: padding a 606-token request to 768 costs more than
|
| 53 |
+
compile saves, and the fast path ends up no faster than the reference path. The small
|
| 54 |
+
buckets matter just as much: a 90-token request padded to 256 is *slower* than the
|
| 55 |
+
reference path, while a 128 bucket makes it 1.36x faster.
|
| 56 |
+
compile: ``False`` keeps the same bucketed/eager-input layout without ``torch.compile``; useful as
|
| 57 |
+
an A/B baseline for the padding scheme alone.
|
| 58 |
+
dynamic: pass ``dynamic=True`` to ``torch.compile``. Slower to compile and historically fragile
|
| 59 |
+
with the gated-delta fallback on Metal; ``False`` (one graph per bucket) is the supported
|
| 60 |
+
default.
|
| 61 |
+
"""
|
| 62 |
+
|
| 63 |
+
buckets: tuple[int, ...] = (128, 192, 256, 320, 384, 448, 512, 640, 768, 1024)
|
| 64 |
+
compile: bool = True
|
| 65 |
+
dynamic: bool = False
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
@dataclass
|
| 69 |
+
class MPSFastStats:
|
| 70 |
+
"""Counters, for validation and logging (never used for control flow)."""
|
| 71 |
+
|
| 72 |
+
compile_s: float = 0.0
|
| 73 |
+
graphs: int = 0
|
| 74 |
+
bucketed: int = 0
|
| 75 |
+
eager: int = 0
|
| 76 |
+
per_bucket: dict[int, int] = field(default_factory=dict)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
class MPSFastPath:
|
| 80 |
+
"""Bucketed, LM-compiled replacement for ``FastPath`` on MPS.
|
| 81 |
+
|
| 82 |
+
Implements the same contract as ``FastPath``: ``run(requests)`` yields
|
| 83 |
+
``(request, {question_key: logits}, chunks)`` and ``forward(chunks)`` returns one list of
|
| 84 |
+
per-question logit rows per chunk. ``warmup()`` compiles every bucket up front.
|
| 85 |
+
"""
|
| 86 |
+
|
| 87 |
+
def __init__(self, decider: Any, options: MPSFastOptions | None = None) -> None:
|
| 88 |
+
if decider.device.type != "mps":
|
| 89 |
+
raise ValueError("The MPS fast path needs a Metal (MPS) device")
|
| 90 |
+
self.decider = decider
|
| 91 |
+
self.options = options or MPSFastOptions()
|
| 92 |
+
self.backbone = decider.backbone
|
| 93 |
+
self.lm = self.backbone.language_model
|
| 94 |
+
self.buckets = tuple(sorted({int(b) for b in self.options.buckets if int(b) > 0}))
|
| 95 |
+
if not self.buckets:
|
| 96 |
+
raise ValueError("The MPS fast path needs at least one positive bucket")
|
| 97 |
+
self.stats = MPSFastStats()
|
| 98 |
+
self._compiled: Any = None
|
| 99 |
+
self._seen_buckets: set[int] = set()
|
| 100 |
+
if self.options.compile:
|
| 101 |
+
torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 64)
|
| 102 |
+
|
| 103 |
+
# ------------------------------------------------------------------ setup
|
| 104 |
+
|
| 105 |
+
def reset(self) -> None:
|
| 106 |
+
"""Drop compiled graphs (e.g. after weights were swapped for a quantized copy)."""
|
| 107 |
+
self._compiled = None
|
| 108 |
+
self._seen_buckets.clear()
|
| 109 |
+
self.stats.graphs = 0
|
| 110 |
+
self.stats.compile_s = 0.0
|
| 111 |
+
|
| 112 |
+
def _fn(self) -> Any:
|
| 113 |
+
if not self.options.compile:
|
| 114 |
+
return self.lm
|
| 115 |
+
if self._compiled is None:
|
| 116 |
+
self._compiled = torch.compile(self.lm, dynamic=self.options.dynamic)
|
| 117 |
+
return self._compiled
|
| 118 |
+
|
| 119 |
+
def warmup(self) -> None:
|
| 120 |
+
"""Compile and run every bucket once, so the first real request does not pay for it."""
|
| 121 |
+
encoder = self.decider.encoder
|
| 122 |
+
request = {"state": "warmup", "questions": {"q": {"type": "noul", "instructions": "Warm up?"}}}
|
| 123 |
+
chunk = encoder.plan(request, None)[0]
|
| 124 |
+
for bucket in self.buckets:
|
| 125 |
+
self._forward_padded(chunk, bucket)
|
| 126 |
+
torch.mps.synchronize()
|
| 127 |
+
|
| 128 |
+
# -------------------------------------------------------------- scheduling
|
| 129 |
+
|
| 130 |
+
def run(self, requests: list[Request]) -> Iterator[tuple[Request, dict[str, list[float]], list[Encoded]]]:
|
| 131 |
+
"""Yield (request, logits per question, chunks) in input order."""
|
| 132 |
+
encoder = self.decider.encoder
|
| 133 |
+
for request in requests:
|
| 134 |
+
encoder.validate(request)
|
| 135 |
+
plan = encoder.plan(request, self.decider.max_length)
|
| 136 |
+
results: dict[str, list[float]] = {}
|
| 137 |
+
for chunk, rows in zip(plan, self.forward(plan), strict=True):
|
| 138 |
+
results.update(zip(chunk.keys, rows, strict=True))
|
| 139 |
+
yield request, {key: results[key] for key in request["questions"]}, plan
|
| 140 |
+
|
| 141 |
+
# -------------------------------------------------------------- forward
|
| 142 |
+
|
| 143 |
+
def forward(self, chunks: list[Encoded]) -> list[list[list[float]]]:
|
| 144 |
+
return [self._forward_one(chunk) for chunk in chunks]
|
| 145 |
+
|
| 146 |
+
def _forward_one(self, chunk: Encoded) -> list[list[float]]:
|
| 147 |
+
bucket = next((size for size in self.buckets if size >= chunk.length), None)
|
| 148 |
+
if bucket is None: # over-length: eager at natural length, no new graph
|
| 149 |
+
self.stats.eager += 1
|
| 150 |
+
return self._run(chunk, None)
|
| 151 |
+
self.stats.bucketed += 1
|
| 152 |
+
self.stats.per_bucket[bucket] = self.stats.per_bucket.get(bucket, 0) + 1
|
| 153 |
+
return self._run(chunk, bucket)
|
| 154 |
+
|
| 155 |
+
def _forward_padded(self, chunk: Encoded, bucket: int) -> list[list[float]]:
|
| 156 |
+
return self._run(chunk, bucket)
|
| 157 |
+
|
| 158 |
+
@torch.inference_mode()
|
| 159 |
+
def _run(self, chunk: Encoded, bucket: int | None) -> list[list[float]]:
|
| 160 |
+
device = self.decider.device
|
| 161 |
+
embeds, positions, mask = self._inputs(chunk, bucket)
|
| 162 |
+
fn = self._fn() if bucket is not None else self.lm
|
| 163 |
+
first = bucket is not None and self.options.compile and bucket not in self._seen_buckets
|
| 164 |
+
started = time.perf_counter() if first else 0.0
|
| 165 |
+
hidden = fn(inputs_embeds=embeds, attention_mask=mask, position_ids=positions, use_cache=False)
|
| 166 |
+
if first: # the call above compiled and ran; the synchronize makes the timing honest
|
| 167 |
+
torch.mps.synchronize()
|
| 168 |
+
self._seen_buckets.add(bucket)
|
| 169 |
+
self.stats.graphs += 1
|
| 170 |
+
self.stats.compile_s += time.perf_counter() - started
|
| 171 |
+
markers = chunk.markers.to(device)
|
| 172 |
+
logits = self.decider.readout(hidden.last_hidden_state[0, markers]).float().cpu()
|
| 173 |
+
out, cursor = [], 0
|
| 174 |
+
for count in chunk.counts:
|
| 175 |
+
out.append(logits[cursor, :count].tolist())
|
| 176 |
+
cursor += 1
|
| 177 |
+
return out
|
| 178 |
+
|
| 179 |
+
def _inputs(self, chunk: Encoded, bucket: int | None) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 180 |
+
"""Eager embeddings + vision scatter + M-RoPE positions, then right-pad to the bucket."""
|
| 181 |
+
device = self.decider.device
|
| 182 |
+
ids = chunk.input_ids.to(device)[None]
|
| 183 |
+
embeds = self.backbone.get_input_embeddings()(ids)
|
| 184 |
+
extra: dict[str, Any] = {}
|
| 185 |
+
if chunk.pixel_values is not None:
|
| 186 |
+
grid = chunk.image_grid_thw.to(device)
|
| 187 |
+
features = self.backbone.get_image_features(
|
| 188 |
+
chunk.pixel_values.to(device), grid, return_dict=True
|
| 189 |
+
).pooler_output
|
| 190 |
+
features = torch.cat(features, dim=0).to(embeds.dtype)
|
| 191 |
+
image_mask, _ = self.backbone.get_placeholder_mask(ids, inputs_embeds=embeds, image_features=features)
|
| 192 |
+
embeds = embeds.masked_scatter(image_mask, features)
|
| 193 |
+
extra["image_grid_thw"] = grid
|
| 194 |
+
extra["mm_token_type_ids"] = chunk.mm_token_type_ids.to(device)[None]
|
| 195 |
+
mask = torch.ones((1, chunk.length), dtype=torch.long, device=device)
|
| 196 |
+
positions = self.backbone.compute_3d_position_ids(
|
| 197 |
+
input_ids=ids, inputs_embeds=embeds, attention_mask=mask, **extra
|
| 198 |
+
)
|
| 199 |
+
if positions is None: # text-only: the model's own 1-D positions, broadcast over the 3 M-RoPE axes
|
| 200 |
+
positions = torch.arange(chunk.length, device=device).view(1, 1, -1).expand(3, 1, -1)
|
| 201 |
+
if bucket is None or bucket == chunk.length:
|
| 202 |
+
return embeds, positions, mask
|
| 203 |
+
return self._pad(embeds, positions, mask, bucket)
|
| 204 |
+
|
| 205 |
+
@staticmethod
|
| 206 |
+
def _pad(
|
| 207 |
+
embeds: torch.Tensor, positions: torch.Tensor, mask: torch.Tensor, bucket: int
|
| 208 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 209 |
+
"""Right-pad to a static length. Padding is masked and never read out, so zeros are enough."""
|
| 210 |
+
length = embeds.shape[1]
|
| 211 |
+
e = torch.zeros((1, bucket, embeds.shape[-1]), dtype=embeds.dtype, device=embeds.device)
|
| 212 |
+
e[:, :length] = embeds
|
| 213 |
+
m = torch.zeros((1, bucket), dtype=mask.dtype, device=mask.device)
|
| 214 |
+
m[:, :length] = mask
|
| 215 |
+
p = torch.zeros((*positions.shape[:-1], bucket), dtype=positions.dtype, device=positions.device)
|
| 216 |
+
p[..., :length] = positions
|
| 217 |
+
return e, p, m
|