Sync Ripple generation, training, and cache fixes

#1
Files changed (3) hide show
  1. PATCH.diff +619 -58
  2. README.md +19 -7
  3. modeling_ricursive.py +422 -49
PATCH.diff CHANGED
@@ -1,6 +1,6 @@
1
  --- a/modeling_ricursive.py
2
  +++ b/modeling_ricursive.py
3
- @@ -6,6 +6,25 @@
4
  - Gated attention: elementwise sigmoid gate on attention output
5
  - Mixed local/global layers: local = sliding window + RoPE, global = full attention + no RoPE
6
  - Per-head Q/K RMSNorm
@@ -23,10 +23,55 @@
23
  + window cannot bind.
24
  +* `RicursiveForCausalLM` inherits `GenerationMixin` explicitly; transformers 5
25
  + no longer injects it into remote-code classes.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  """
27
 
 
 
28
  from typing import Optional, Tuple, Union
29
- @@ -20,9 +39,20 @@
 
 
30
  CausalLMOutputWithPast,
31
  )
32
 
@@ -36,18 +81,52 @@
36
  + from transformers.generation import GenerationMixin
37
  +
38
  from .configuration_ricursive import RicursiveConfig
39
-
40
-
41
  +# Set when the transformers-5.x `generate()` fixes above are present natively.
42
  +# Downstream loaders that monkeypatch older checkpoints check this marker and
43
  +# stand down instead of re-installing the equivalent runtime patch.
44
- +RICURSIVE_TF5_COMPAT = "2026-11-05"
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  +
 
 
46
  +
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  class RicursiveRMSNorm(nn.Module):
48
- def __init__(self, hidden_size: int, eps: float = 1e-6):
49
- super().__init__()
50
- @@ -140,13 +170,37 @@
 
 
 
51
  if self.layer_type != "sliding_attention":
52
  return attention_mask
53
 
@@ -57,7 +136,13 @@
57
  + # shared mask produced by `_update_causal_mask` can be longer than this
58
  + # layer's key tensor. Keep only the keys this layer actually holds.
59
  + if attention_mask is not None and attention_mask.shape[-1] > kv_len:
60
- + attention_mask = attention_mask[..., -kv_len:]
 
 
 
 
 
 
61
  +
62
  if cache_position is not None:
63
  - query_positions = cache_position.to(device=device)
@@ -73,7 +158,14 @@
73
  + # unchanged (already causal) mask or, without one, plain causal
74
  + # attention -- which SDPA applies through `is_causal`.
75
  + if attention_mask is None:
76
- + return None
 
 
 
 
 
 
 
77
  + if q_len == 1:
78
  + return attention_mask
79
  +
@@ -87,7 +179,302 @@
87
  allowed = (
88
  (key_positions[None, :] <= query_positions[:, None])
89
  & (key_positions[None, :] >= query_positions[:, None] - (window - 1))
90
- @@ -466,43 +520,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  dtype, device = input_tensor.dtype, input_tensor.device
92
  sequence_length = input_tensor.shape[1]
93
 
@@ -101,11 +488,12 @@
101
  + query_positions = torch.arange(sequence_length, device=device)
102
  else:
103
  - target_length = sequence_length
104
- + query_positions = cache_position.to(device=device).reshape(-1)
105
-
106
  - causal_mask = torch.full(
107
  - (sequence_length, target_length),
108
  - fill_value=torch.finfo(dtype).min,
 
 
109
  + # Size the key axis from `cache_position`, which is exactly what the
110
  + # attention layer uses to size its key tensor.
111
  + # `past_key_values.get_max_cache_shape()` was removed in transformers 5,
@@ -135,6 +523,23 @@
135
  - causal_mask = torch.triu(causal_mask, diagonal=target_length - sequence_length + 1)
136
  -
137
  - causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
  + # `0.0` for visible keys (not `1.0`, which would bias every score) and the
139
  + # dtype minimum elsewhere. Building the mask from `allowed` also keeps a
140
  + # single-token decode step from coming out fully masked.
@@ -152,25 +557,39 @@
152
  + return int(past_key_values[0][0].shape[-2])
153
  + except Exception: # pragma: no cover - unknown cache implementations
154
  + return 0
155
-
156
- - if attention_mask.dim() == 2:
157
- - mask_length = attention_mask.shape[-1]
158
- - padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(
159
- - causal_mask.dtype
160
- - ).masked_fill(attention_mask[:, None, None, :] == 0, torch.finfo(dtype).min)
161
- - causal_mask = (
162
- - torch.cat([padding_mask, causal_mask[:, :, :, mask_length:]], dim=-1)
163
- - if mask_length < target_length
164
- - else padding_mask
165
- - )
166
- -
167
- - return causal_mask
168
-
169
- -
170
- -class RicursiveForCausalLM(RicursivePreTrainedModel):
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  +class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
172
- _tied_weights_keys = []
173
-
 
174
  + # transformers 5 no longer injects `GenerationMixin` into remote-code classes,
175
  + # so inherit it explicitly above. Without it `generate` is missing entirely and
176
  + # `PreTrainedModel.__init__` never builds `generation_config`.
@@ -178,20 +597,123 @@
178
  + # This attribute marks the native transformers-5 compatibility fixes so that
179
  + # downstream loaders can skip their runtime monkeypatch.
180
  + _ricursive_tf5_native_fix = RICURSIVE_TF5_COMPAT
181
- +
182
  def __init__(self, config: RicursiveConfig):
183
  super().__init__(config)
184
- self.model = RicursiveModel(config)
185
- @@ -583,20 +660,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  inputs_embeds: Optional[torch.FloatTensor] = None,
187
  cache_position: Optional[torch.LongTensor] = None,
188
  use_cache: Optional[bool] = None,
189
  + position_ids: Optional[torch.LongTensor] = None,
 
 
190
  **kwargs,
191
  ):
192
  - if past_key_values is not None:
193
  - if input_ids.shape[1] != 1:
194
  - input_ids = input_ids[:, cache_position]
 
 
 
 
 
 
 
 
 
 
 
 
195
  + # transformers 5.x does not move remote-code inputs onto the model device for
196
  + # this model, so a plain CPU `input_ids` reaches the embedding lookup and
197
  + # raises. Its own generation helper performs this move, so mirror it here.
@@ -211,46 +733,74 @@
211
  +
212
  + past_length = _cache_seq_length(past_key_values)
213
  +
 
 
 
 
 
 
214
  + if cache_position is None:
215
- + # With a non-empty cache and the full sequence supplied, only the trailing
216
- + # (uncached) tokens are new. Otherwise every token in `input_ids` is new.
217
- + if past_length > 0 and input_ids.shape[1] > 1:
218
- + new_len = input_ids.shape[1] - past_length
219
- + cache_position = torch.arange(
220
- + past_length, past_length + new_len, device=input_ids.device
221
- + )
222
- + else:
223
- + cache_position = torch.arange(
224
- + past_length, past_length + input_ids.shape[1], device=input_ids.device
225
- + )
 
 
 
 
 
226
  +
227
  + # Only slice when the cache is warm and `input_ids` still carries cached
228
  + # tokens. Slicing unconditionally turned a `None` cache position into
229
  + # `input_ids[:, None]` and injected a bogus axis.
230
- + sliced = past_length > 0 and input_ids.shape[1] > cache_position.numel()
 
231
  + if sliced:
232
- + input_ids = input_ids[:, cache_position]
233
-
234
- - if inputs_embeds is not None and cache_position[0] == 0:
235
- + if inputs_embeds is not None and cache_position.numel() and int(cache_position[0]) == 0:
236
- model_inputs = {"inputs_embeds": inputs_embeds}
237
- else:
238
- model_inputs = {"input_ids": input_ids}
239
-
 
 
 
 
 
 
 
 
 
240
  + # Derive per-row positions from the padding mask. Using `cache_position` for
241
  + # every row is only correct when nothing is padded: with left padding the real
242
  + # tokens of a short row sit at higher indices, so a shared position vector
243
  + # shifts its RoPE phases and the row decodes to garbage.
244
  + if position_ids is not None:
245
- + if sliced:
246
- + position_ids = position_ids[:, cache_position]
 
 
 
 
 
247
  + elif attention_mask is not None and attention_mask.dim() == 2:
248
  + position_ids = attention_mask.long().cumsum(-1) - 1
249
  + position_ids = position_ids.masked_fill(attention_mask == 0, 0)
250
- + position_ids = position_ids[:, -input_ids.shape[1]:]
 
 
251
  + else:
252
  + position_ids = cache_position.unsqueeze(0)
253
- +
254
  model_inputs.update(
255
  {
256
  - "position_ids": cache_position.unsqueeze(0) if cache_position is not None else None,
@@ -258,3 +808,14 @@
258
  "past_key_values": past_key_values,
259
  "use_cache": use_cache,
260
  "attention_mask": attention_mask,
 
 
 
 
 
 
 
 
 
 
 
 
1
  --- a/modeling_ricursive.py
2
  +++ b/modeling_ricursive.py
3
+ @@ -6,8 +6,70 @@
4
  - Gated attention: elementwise sigmoid gate on attention output
5
  - Mixed local/global layers: local = sliding window + RoPE, global = full attention + no RoPE
6
  - Per-head Q/K RMSNorm
 
23
  + window cannot bind.
24
  +* `RicursiveForCausalLM` inherits `GenerationMixin` explicitly; transformers 5
25
  + no longer injects it into remote-code classes.
26
+ +
27
+ +DFlash2 inference fixes (2026-11-06)
28
+ +-----------------------------------
29
+ +* `RicursiveAttention.forward` builds an end-anchored causal mask for a
30
+ + multi-token verify block running against a warm cache. SDPA's implicit mask
31
+ + is top-left aligned, so with `q_len < kv_len` it hid valid context.
32
+ + Preallocated caches (`StaticCache`) return their whole capacity from
33
+ + `update()`, so the mask is now built from the layer's *actual* cache length
34
+ + and query positions; the unused tail is trimmed before attention.
35
+ +* `RicursiveForCausalLM.forward` honours `logits_to_keep` by slicing the LM
36
+ + head *input*, so the full `[batch, seq, vocab]` projection is never
37
+ + materialised; `prepare_inputs_for_generation` requests one row. Labeled
38
+ + calls keep the full sequence so the shifted causal-LM loss stays defined,
39
+ + then trim only the returned logits to the requested width.
40
+ +* `RicursiveMLP.forward` chunks the row dimension
41
+ + (`RIPPLE_MLP_CHUNK`, default 2048, `0` disables) to bound the
42
+ + prefill activation peak. Every op is row-wise, so the result is exact.
43
+ +
44
+ +Cache-alignment fixes from review (2026-11-06)
45
+ +----------------------------------------------
46
+ +* A mask longer than the key tensor is now aligned to the absolute positions
47
+ + the layer actually holds, `[total_kv - kv_len, total_kv)`, instead of to the
48
+ + tail of the mask. With a mask sized to the allocated cache *capacity*
49
+ + (`StaticCache`), the old `[-kv_len:]` slice selected unwritten slots after
50
+ + sliding-window eviction and silently returned wrong attention.
51
+ +* `RicursiveAttention.forward` forwards `cache_position` to
52
+ + `past_key_value.update()`. A transformers 4.x `StaticCache` otherwise falls
53
+ + back to `arange(q_len)` and the first decode step overwrites the prompt
54
+ + instead of appending after it.
55
+ +
56
+ +Model contract fixes (2026-11-09)
57
+ +-------------------------------
58
+ +* Generation supports embeddings-only prompts, continuation buffers, explicit
59
+ + position advancement, short masks, and requested diagnostic outputs.
60
+ +* Attention preserves caller 4D masks, reports requested weights without changing
61
+ + SDPA outputs, and trims unused static capacity before expanding KV heads.
62
+ +* Default caches follow the hybrid layer configuration on Transformers 5;
63
+ + Transformers 4 keeps a plain, crop-compatible full cache. Training
64
+ + implements gradient checkpointing and computes causal-LM loss in FP32.
65
+ +* Loss honors accumulation-wide token counts; TP reshapes use local head counts.
66
+ +* Tied embeddings support Transformers 4/5, and window masks preserve exclusions.
67
  """
68
 
69
+ +import os
70
+ +from functools import partial
71
  from typing import Optional, Tuple, Union
72
+
73
+ import torch
74
+ @@ -20,7 +82,49 @@
75
  CausalLMOutputWithPast,
76
  )
77
 
 
81
  + from transformers.generation import GenerationMixin
82
  +
83
  from .configuration_ricursive import RicursiveConfig
84
+ +
85
+ +
86
  +# Set when the transformers-5.x `generate()` fixes above are present natively.
87
  +# Downstream loaders that monkeypatch older checkpoints check this marker and
88
  +# stand down instead of re-installing the equivalent runtime patch.
89
+ +RICURSIVE_TF5_COMPAT = "2026-11-09.1"
90
+ +
91
+ +
92
+ +def _transformers_major_version() -> int:
93
+ + """Return the installed Transformers major version, or 0 when unknown."""
94
+ + try:
95
+ + from transformers import __version__ as version
96
+ + except ImportError: # pragma: no cover - Transformers is a hard dependency
97
+ + return 0
98
+ + try:
99
+ + return int(version.split(".", 1)[0])
100
+ + except (AttributeError, ValueError):
101
+ + return 0
102
+ +
103
  +
104
+ +def _new_default_cache(config: RicursiveConfig) -> DynamicCache:
105
+ + """Create the default KV cache for a forward pass.
106
  +
107
+ + Transformers 5 builds a hybrid cache from ``config``: sliding layers keep
108
+ + only their window while full layers grow, which bounds long-context decode
109
+ + memory. Transformers 4 also accepts ``config`` but its
110
+ + ``DynamicSlidingWindowLayer`` cannot be cropped after it has seen more
111
+ + tokens than the window, so assisted decoding and any cache rollback raise
112
+ + ``ValueError: Cannot crop a DynamicSlidingWindowLayer``. Keep the plain
113
+ + full-attention cache on 4.x, matching the pre-config default.
114
+ + """
115
+ + if _transformers_major_version() >= 5:
116
+ + try:
117
+ + return DynamicCache(config=config)
118
+ + except TypeError: # A 5.x build without configured hybrid caches.
119
+ + pass
120
+ + return DynamicCache()
121
+
122
+
123
  class RicursiveRMSNorm(nn.Module):
124
+ @@ -136,17 +240,55 @@
125
+ dtype: torch.dtype,
126
+ device: torch.device,
127
+ cache_position: Optional[torch.LongTensor],
128
+ + total_kv: Optional[int] = None,
129
+ ) -> Optional[torch.Tensor]:
130
  if self.layer_type != "sliding_attention":
131
  return attention_mask
132
 
 
136
  + # shared mask produced by `_update_causal_mask` can be longer than this
137
  + # layer's key tensor. Keep only the keys this layer actually holds.
138
  + if attention_mask is not None and attention_mask.shape[-1] > kv_len:
139
+ + mask_len = attention_mask.shape[-1]
140
+ + if total_kv is not None:
141
+ + # The retained keys sit at `[total_kv - kv_len, total_kv)`.
142
+ + start = max(0, min(total_kv - kv_len, mask_len - kv_len))
143
+ + attention_mask = attention_mask[..., start : start + kv_len]
144
+ + else:
145
+ + attention_mask = attention_mask[..., -kv_len:]
146
  +
147
  if cache_position is not None:
148
  - query_positions = cache_position.to(device=device)
 
158
  + # unchanged (already causal) mask or, without one, plain causal
159
  + # attention -- which SDPA applies through `is_causal`.
160
  + if attention_mask is None:
161
+ + # SDPA's implicit causal mask is top-left aligned. It only
162
+ + # matches this query block when it is the whole sequence
163
+ + # (`q_len == kv_len`) or a single decode position
164
+ + # (`q_len == 1`). A multi-token block verified against a warm
165
+ + # cache sits at the *end* of the key span, so fall through and
166
+ + # build the end-anchored mask below.
167
+ + if q_len == 1 or q_len == kv_len:
168
+ + return None
169
  + if q_len == 1:
170
  + return attention_mask
171
  +
 
179
  allowed = (
180
  (key_positions[None, :] <= query_positions[:, None])
181
  & (key_positions[None, :] >= query_positions[:, None] - (window - 1))
182
+ @@ -156,7 +298,8 @@
183
+ return allowed
184
+ if attention_mask.dtype == torch.bool:
185
+ return attention_mask & allowed
186
+ - return attention_mask.masked_fill(~allowed, torch.finfo(dtype).min)
187
+ + # Finite minima would reopen an existing -inf exclusion on an empty row.
188
+ + return attention_mask.masked_fill(~allowed, float("-inf"))
189
+
190
+ def forward(
191
+ self,
192
+ @@ -185,10 +328,10 @@
193
+ )
194
+ v = self.v_proj(value_states)
195
+
196
+ - # Reshape to (B, T, n_heads, head_dim) then transpose to (B, n_heads, T, head_dim)
197
+ - q = q.view(bsz, q_len, self.num_heads, self.head_dim)
198
+ - k = k.view(bsz, q_len, self.num_kv_heads, self.head_dim)
199
+ - v = v.view(bsz, q_len, self.num_kv_heads, self.head_dim)
200
+ + # Infer rank-local head counts from projections after tensor parallel sharding.
201
+ + q = q.view(bsz, q_len, -1, self.head_dim)
202
+ + k = k.view(bsz, q_len, -1, self.head_dim)
203
+ + v = v.view(bsz, q_len, -1, self.head_dim)
204
+
205
+ # Per-head Q/K norm (applied before transpose, on head_dim)
206
+ q = self.q_norm(q)
207
+ @@ -205,12 +348,62 @@
208
+ q, k = apply_rotary_pos_emb(q, k, cos, sin)
209
+
210
+ # KV cache
211
+ + total_kv = None
212
+ if past_key_value is not None:
213
+ - k, v = past_key_value.update(k, v, self.layer_idx)
214
+ -
215
+ - # GQA: expand KV heads
216
+ - k = repeat_kv(k, self.num_kv_groups)
217
+ - v = repeat_kv(v, self.num_kv_groups)
218
+ + # Forward the absolute positions of the tokens being written. A
219
+ + # transformers 4.x `StaticCache` falls back to `arange(q_len)` when
220
+ + # `cache_position` is absent, so the first decode step would write
221
+ + # at slots 0..q_len-1 and overwrite the prompt instead of appending
222
+ + # after it. `DynamicCache`/5.x caches ignore the extra kwarg.
223
+ + cache_kwargs = (
224
+ + {"cache_position": cache_position} if cache_position is not None else None
225
+ + )
226
+ + if cache_kwargs is not None:
227
+ + try:
228
+ + k, v = past_key_value.update(k, v, self.layer_idx, cache_kwargs)
229
+ + except TypeError: # cache without a `cache_kwargs` parameter
230
+ + k, v = past_key_value.update(k, v, self.layer_idx)
231
+ + else:
232
+ + k, v = past_key_value.update(k, v, self.layer_idx)
233
+ + # Number of keys this layer has actually written. A preallocated
234
+ + # (StaticCache) layer returns its full capacity from `update()`,
235
+ + # including slots that have not been filled yet; an evicting
236
+ + # (sliding-window) layer reports more than the trailing window it
237
+ + # returns. The mask must use the real positions, not the tensor size.
238
+ + total_kv = _cache_layer_seq_length(past_key_value, self.layer_idx)
239
+ +
240
+ + # Drop the unwritten tail of a preallocated cache. Without this a
241
+ + # six-token prefill into a sixteen-slot StaticCache looks like a
242
+ + # ten-token prefix, and the end-anchored verify mask below would let
243
+ + # each query attend slots belonging to a different absolute position.
244
+ + if total_kv is not None and 0 < total_kv < k.shape[2]:
245
+ + k = k[:, :, :total_kv]
246
+ + v = v[:, :, :total_kv]
247
+ +
248
+ + # Expand only written slots, not the entire capacity of a static cache.
249
+ + num_kv_groups = q.shape[1] // k.shape[1]
250
+ + k = repeat_kv(k, num_kv_groups)
251
+ + v = repeat_kv(v, num_kv_groups)
252
+ +
253
+ + # Align an incoming mask that is longer than the keys this layer holds.
254
+ + # The shared mask is built over absolute key positions
255
+ + # `0..target_length-1`; keep the slice that matches the keys present.
256
+ + #
257
+ + # The keys returned by `update()` occupy the absolute positions
258
+ + # `[total_kv - kv_len, total_kv)`. Anchoring on the tensor tail
259
+ + # (`[-kv_len:]`) is only correct when the mask length equals
260
+ + # `total_kv`; with a mask sized to the *allocated cache capacity* it
261
+ + # selects the unwritten tail instead of the retained window and
262
+ + # silently produces wrong attention.
263
+ + if attention_mask is not None and attention_mask.shape[-1] > k.shape[2]:
264
+ + kv_len_now = k.shape[2]
265
+ + mask_len = attention_mask.shape[-1]
266
+ + if total_kv is not None:
267
+ + start = max(0, min(total_kv - kv_len_now, mask_len - kv_len_now))
268
+ + attention_mask = attention_mask[..., start : start + kv_len_now]
269
+ + else:
270
+ + # Unknown cache: the keys are the trailing window.
271
+ + attention_mask = attention_mask[..., -kv_len_now:]
272
+
273
+ attention_mask = self._apply_sliding_window_mask(
274
+ attention_mask=attention_mask,
275
+ @@ -219,14 +412,59 @@
276
+ dtype=q.dtype,
277
+ device=q.device,
278
+ cache_position=cache_position,
279
+ - )
280
+ + total_kv=total_kv,
281
+ + )
282
+ +
283
+ + if attention_mask is None and q.shape[2] > 1 and k.shape[2] > q.shape[2]:
284
+ + # Multi-token verify block against a warm cache. SDPA's implicit
285
+ + # causal mask is top-left aligned, so when `q_len < kv_len` it hides
286
+ + # every key before the block. Build the mask from the actual query
287
+ + # positions (and the absolute positions of the returned keys) rather
288
+ + # than assuming the key tensor starts at position 0.
289
+ + if cache_position is not None and cache_position.numel() == q.shape[2]:
290
+ + query_positions = cache_position.to(
291
+ + device=q.device, dtype=torch.long
292
+ + ).reshape(-1)
293
+ + else:
294
+ + query_positions = torch.arange(
295
+ + k.shape[2] - q.shape[2], k.shape[2], device=q.device
296
+ + )
297
+ + if total_kv is not None and total_kv > k.shape[2]:
298
+ + # Evicting cache: the returned keys are the trailing window that
299
+ + # ends at the newest query, not positions 0..kv_len-1.
300
+ + last_query = int(query_positions[-1].item())
301
+ + key_positions = torch.arange(
302
+ + last_query - k.shape[2] + 1, last_query + 1, device=q.device
303
+ + )
304
+ + else:
305
+ + key_positions = torch.arange(k.shape[2], device=q.device)
306
+ + attention_mask = (
307
+ + key_positions[None, :] <= query_positions[:, None]
308
+ + )[None, None]
309
+
310
+ # Attention (use is_causal when no explicit mask for correct causal behavior)
311
+ # Cast to plain bool to avoid SymBool issues with torch.compile
312
+ is_causal = bool(attention_mask is None and q.shape[2] > 1)
313
+ + # Keep the SDPA result even when returning diagnostic weights. Rebuilding
314
+ + # the output from rounded BF16 probabilities changes logits and token IDs.
315
+ attn_output = F.scaled_dot_product_attention(
316
+ q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
317
+ )
318
+ + attn_weights = None
319
+ + if output_attentions:
320
+ + # SDPA does not expose weights; compute them separately only on request.
321
+ + weight_keys = k
322
+ + with torch.autocast(device_type=q.device.type, enabled=False):
323
+ + scores = torch.matmul(q.float(), weight_keys.float().transpose(-1, -2)) * self.head_dim**-0.5
324
+ + if is_causal:
325
+ + causal = torch.ones(q_len, weight_keys.shape[2], dtype=torch.bool, device=q.device).tril()
326
+ + scores = scores.masked_fill(~causal, float("-inf"))
327
+ + if attention_mask is not None:
328
+ + if attention_mask.dtype == torch.bool:
329
+ + scores = scores.masked_fill(~attention_mask, float("-inf"))
330
+ + else:
331
+ + scores = scores + attention_mask.float()
332
+ + attn_weights = F.softmax(scores, dim=-1).nan_to_num().to(q.dtype)
333
+
334
+ # (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
335
+ attn_output = attn_output.transpose(1, 2).contiguous()
336
+ @@ -244,7 +482,15 @@
337
+ # Output projection
338
+ attn_output = self.o_proj(attn_output)
339
+
340
+ - return attn_output, None, past_key_value
341
+ + return attn_output, attn_weights, past_key_value
342
+ +
343
+ +
344
+ +def _mlp_chunk_size() -> int:
345
+ + """Row budget for the chunked MLP forward. `0` (or negative) disables it."""
346
+ + try:
347
+ + return int(os.environ.get("RIPPLE_MLP_CHUNK", "2048"))
348
+ + except ValueError:
349
+ + return 2048
350
+
351
+
352
+ class RicursiveMLP(nn.Module):
353
+ @@ -255,13 +501,41 @@
354
+ self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
355
+ self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
356
+
357
+ - def forward(self, x: torch.Tensor) -> torch.Tensor:
358
+ + def _forward_rows(self, x: torch.Tensor) -> torch.Tensor:
359
+ gate = self.gate_proj(x)
360
+ up = self.up_proj(x)
361
+ if self.clamp_threshold:
362
+ gate = gate.clamp(min=None, max=self.clamp_threshold)
363
+ up = up.clamp(min=-self.clamp_threshold, max=self.clamp_threshold)
364
+ return self.down_proj(F.silu(gate) * up)
365
+ +
366
+ + def forward(self, x: torch.Tensor) -> torch.Tensor:
367
+ + # Ripple's MLP expands hidden_size -> intermediate_size (5120 -> 40960
368
+ + # for the 65B checkpoint), so a long prefill materialises `gate`, `up`
369
+ + # and `silu(gate) * up` at once. Every op here is row-wise, so splitting
370
+ + # the row dimension is exact: each row still uses the same kernels with
371
+ + # the same reduction order, and the pieces are concatenated back. Only
372
+ + # the transient peak shrinks. `RIPPLE_MLP_CHUNK` overrides the
373
+ + # row budget; `0` disables chunking.
374
+ + chunk = _mlp_chunk_size()
375
+ + if chunk <= 0 or not torch.is_tensor(x) or x.dim() not in (2, 3):
376
+ + return self._forward_rows(x)
377
+ +
378
+ + shape = x.shape
379
+ + hidden = shape[-1]
380
+ + rows = 1
381
+ + for size in shape[:-1]:
382
+ + rows *= size
383
+ + if rows <= chunk:
384
+ + return self._forward_rows(x)
385
+ +
386
+ + flat = x.reshape(rows, hidden)
387
+ + pieces = [
388
+ + self._forward_rows(flat[start : start + chunk])
389
+ + for start in range(0, rows, chunk)
390
+ + ]
391
+ + out = torch.cat(pieces, dim=0) if len(pieces) > 1 else pieces[0]
392
+ + return out.reshape(*shape[:-1], out.shape[-1])
393
+
394
+
395
+ class RicursiveDecoderLayer(nn.Module):
396
+ @@ -341,6 +615,7 @@
397
+ [RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
398
+ )
399
+ self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
400
+ + self.gradient_checkpointing = False
401
+ self.post_init()
402
+
403
+ def forward(
404
+ @@ -368,6 +643,13 @@
405
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
406
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
407
+
408
+ + checkpointing = self.gradient_checkpointing and self.training
409
+ + if checkpointing:
410
+ + if past_key_values is not None:
411
+ + raise ValueError("Gradient checkpointing does not support a supplied KV cache")
412
+ + # Recomputed layers must not append to a cache for a second time.
413
+ + use_cache = False
414
+ +
415
+ if inputs_embeds is None:
416
+ inputs_embeds = self.embed_tokens(input_ids)
417
+
418
+ @@ -381,7 +663,7 @@
419
+ inputs_embeds = inputs_embeds * embed_scale
420
+
421
+ if use_cache and past_key_values is None:
422
+ - past_key_values = DynamicCache()
423
+ + past_key_values = _new_default_cache(self.config)
424
+
425
+ if cache_position is None:
426
+ past_seen_tokens = (
427
+ @@ -410,17 +692,26 @@
428
+ if output_hidden_states:
429
+ all_hidden_states += (hidden_states,)
430
+
431
+ - layer_outputs = layer(
432
+ - hidden_states,
433
+ - attention_mask=causal_mask,
434
+ - position_ids=position_ids,
435
+ - past_key_value=past_key_values,
436
+ - output_attentions=output_attentions,
437
+ - use_cache=use_cache,
438
+ - cache_position=cache_position,
439
+ - value_residual=value_residual,
440
+ - **kwargs,
441
+ - )
442
+ + if checkpointing:
443
+ + # Pass the value-residual seed as an input too, so gradients
444
+ + # through the embedding path survive reentrant checkpointing.
445
+ + layer_outputs = self._gradient_checkpointing_func(
446
+ + partial(layer.__call__, **kwargs), hidden_states, causal_mask,
447
+ + position_ids, None, output_attentions, False, cache_position,
448
+ + value_residual,
449
+ + )
450
+ + else:
451
+ + layer_outputs = layer(
452
+ + hidden_states,
453
+ + attention_mask=causal_mask,
454
+ + position_ids=position_ids,
455
+ + past_key_value=past_key_values,
456
+ + output_attentions=output_attentions,
457
+ + use_cache=use_cache,
458
+ + cache_position=cache_position,
459
+ + value_residual=value_residual,
460
+ + **kwargs,
461
+ + )
462
+ hidden_states = layer_outputs[0]
463
+
464
+ if output_attentions:
465
+ @@ -459,6 +750,11 @@
466
+ # from float min mask values with large activations.
467
+ if attention_mask is None:
468
+ return None
469
+ + if attention_mask.dim() == 4:
470
+ + # A caller-provided 4D mask already defines visibility (or additive
471
+ + # bias). Preserve it; each local layer still applies its window.
472
+ + dtype = torch.bool if attention_mask.dtype == torch.bool else input_tensor.dtype
473
+ + return attention_mask.to(device=input_tensor.device, dtype=dtype)
474
+ if torch.all(attention_mask == 1):
475
+ return None
476
+
477
+ @@ -466,42 +762,93 @@
478
  dtype, device = input_tensor.dtype, input_tensor.device
479
  sequence_length = input_tensor.shape[1]
480
 
 
488
  + query_positions = torch.arange(sequence_length, device=device)
489
  else:
490
  - target_length = sequence_length
491
+ -
 
492
  - causal_mask = torch.full(
493
  - (sequence_length, target_length),
494
  - fill_value=torch.finfo(dtype).min,
495
+ + query_positions = cache_position.to(device=device).reshape(-1)
496
+ +
497
  + # Size the key axis from `cache_position`, which is exactly what the
498
  + # attention layer uses to size its key tensor.
499
  + # `past_key_values.get_max_cache_shape()` was removed in transformers 5,
 
523
  - causal_mask = torch.triu(causal_mask, diagonal=target_length - sequence_length + 1)
524
  -
525
  - causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1)
526
+ -
527
+ - if attention_mask.dim() == 2:
528
+ - mask_length = attention_mask.shape[-1]
529
+ - padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(
530
+ - causal_mask.dtype
531
+ - ).masked_fill(attention_mask[:, None, None, :] == 0, torch.finfo(dtype).min)
532
+ - causal_mask = (
533
+ - torch.cat([padding_mask, causal_mask[:, :, :, mask_length:]], dim=-1)
534
+ - if mask_length < target_length
535
+ - else padding_mask
536
+ - )
537
+ -
538
+ - return causal_mask
539
+ -
540
+ -
541
+ -class RicursiveForCausalLM(RicursivePreTrainedModel):
542
+ - _tied_weights_keys = []
543
  + # `0.0` for visible keys (not `1.0`, which would bias every score) and the
544
  + # dtype minimum elsewhere. Building the mask from `allowed` also keeps a
545
  + # single-token decode step from coming out fully masked.
 
557
  + return int(past_key_values[0][0].shape[-2])
558
  + except Exception: # pragma: no cover - unknown cache implementations
559
  + return 0
560
+ +
561
+ +
562
+ +def _cache_layer_seq_length(
563
+ + past_key_values: Optional[Cache], layer_idx: int
564
+ +) -> Optional[int]:
565
+ + """Actual number of keys written for `layer_idx`, or `None` if unknown.
566
+ +
567
+ + `Cache.get_seq_length(layer_idx)` counts *written* tokens. For a
568
+ + preallocated static layer that is smaller than the tensor `update()`
569
+ + returns until the cache fills; for an evicting sliding-window layer it is
570
+ + larger, because the layer only returns a trailing window.
571
+ + """
572
+ + if past_key_values is None:
573
+ + return None
574
+ + getter = getattr(past_key_values, "get_seq_length", None)
575
+ + if not callable(getter):
576
+ + return None
577
+ + try:
578
+ + return int(getter(layer_idx))
579
+ + except TypeError: # transformers 4.x cache without a layer argument
580
+ + pass
581
+ + except Exception: # pragma: no cover - unknown cache implementations
582
+ + return None
583
+ + try:
584
+ + return int(getter())
585
+ + except Exception: # pragma: no cover - unknown cache implementations
586
+ + return None
587
+ +
588
+ +
589
  +class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
590
+ + # Transformers 5 reads this mapping; Transformers 4 iterates its target keys.
591
+ + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
592
+ +
593
  + # transformers 5 no longer injects `GenerationMixin` into remote-code classes,
594
  + # so inherit it explicitly above. Without it `generate` is missing entirely and
595
  + # `PreTrainedModel.__init__` never builds `generation_config`.
 
597
  + # This attribute marks the native transformers-5 compatibility fixes so that
598
  + # downstream loaders can skip their runtime monkeypatch.
599
  + _ricursive_tf5_native_fix = RICURSIVE_TF5_COMPAT
600
+
601
  def __init__(self, config: RicursiveConfig):
602
  super().__init__(config)
603
+ @@ -534,6 +881,8 @@
604
+ output_hidden_states: Optional[bool] = None,
605
+ return_dict: Optional[bool] = None,
606
+ cache_position: Optional[torch.LongTensor] = None,
607
+ + logits_to_keep: Optional[int] = None,
608
+ + num_items_in_batch: Optional[torch.Tensor] = None,
609
+ **kwargs,
610
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
611
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
612
+ @@ -552,16 +901,41 @@
613
+ **kwargs,
614
+ )
615
+ hidden_states = outputs[0]
616
+ + # Honour `logits_to_keep` by slicing the LM head *input*. Projecting the
617
+ + # full `[batch, seq, vocab]` tensor only to discard all but the last row
618
+ + # costs several GiB of transient memory at prefill batch sizes.
619
+ + #
620
+ + # When `labels` are supplied the full sequence must be kept: the causal-LM
621
+ + # loss shifts logits and labels by one position, so trimming both to the
622
+ + # same short window leaves the shift empty (and the loss NaN). Labeled
623
+ + # calls therefore project the full sequence to get a defined loss, then
624
+ + # trim only the *returned* logits so the output shape still honours
625
+ + # `logits_to_keep`. The generation path, which never passes labels, keeps
626
+ + # the memory win from slicing the LM head input.
627
+ + keep = (
628
+ + int(logits_to_keep)
629
+ + if logits_to_keep is not None and logits_to_keep > 0
630
+ + else 0
631
+ + )
632
+ + if keep and labels is None and hidden_states.shape[1] > keep:
633
+ + hidden_states = hidden_states[:, -keep:, :]
634
+ logits = self.lm_head(hidden_states)
635
+
636
+ loss = None
637
+ if labels is not None:
638
+ - shift_logits = logits[..., :-1, :].contiguous()
639
+ - shift_labels = labels[..., 1:].contiguous()
640
+ + shift_logits = logits[..., :-1, :].float().contiguous()
641
+ + shift_labels = labels[..., 1:].to(logits.device).contiguous()
642
+ loss = F.cross_entropy(
643
+ shift_logits.view(-1, shift_logits.size(-1)),
644
+ shift_labels.view(-1),
645
+ + reduction="sum" if num_items_in_batch is not None else "mean",
646
+ )
647
+ + if num_items_in_batch is not None:
648
+ + # Trainer supplies the token count across all accumulated microbatches.
649
+ + loss = loss / torch.as_tensor(num_items_in_batch, device=loss.device)
650
+ + if keep:
651
+ + # Loss used the full sequence; only the returned tensor is trimmed.
652
+ + logits = logits[..., -keep:, :]
653
+
654
+ if not return_dict:
655
+ output = (logits,) + outputs[1:]
656
+ @@ -574,6 +948,36 @@
657
+ hidden_states=outputs.hidden_states,
658
+ attentions=outputs.attentions,
659
+ )
660
+ +
661
+ + def _update_model_kwargs_for_generation(
662
+ + self, outputs, model_kwargs, is_encoder_decoder=False, num_new_tokens=1
663
+ + ):
664
+ + """Advance explicit positions when the generation backend leaves them unchanged."""
665
+ + position_ids = model_kwargs.get("position_ids")
666
+ + cache_position = model_kwargs.get("cache_position")
667
+ + model_kwargs = super()._update_model_kwargs_for_generation(
668
+ + outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder,
669
+ + num_new_tokens=num_new_tokens,
670
+ + )
671
+ + # Transformers 4 leaves positions unchanged; Transformers 5 already extends them.
672
+ + if position_ids is not None and model_kwargs.get("position_ids") is position_ids:
673
+ + offsets = torch.arange(
674
+ + 1, num_new_tokens + 1, device=position_ids.device, dtype=position_ids.dtype
675
+ + )
676
+ + model_kwargs["position_ids"] = torch.cat(
677
+ + [position_ids, position_ids[:, -1:] + offsets], dim=-1
678
+ + )
679
+ + # Transformers 5 no longer maintains cache_position. Keep an explicitly
680
+ + # supplied vector in sync, without advancing it twice on Transformers 4.
681
+ + if cache_position is not None and model_kwargs.get("cache_position") is cache_position:
682
+ + if model_kwargs.get("use_cache", True):
683
+ + model_kwargs["cache_position"] = cache_position[-1:] + num_new_tokens
684
+ + else:
685
+ + offsets = torch.arange(
686
+ + 1, num_new_tokens + 1, device=cache_position.device, dtype=cache_position.dtype
687
+ + )
688
+ + model_kwargs["cache_position"] = torch.cat([cache_position, cache_position[-1:] + offsets])
689
+ + return model_kwargs
690
+
691
+ def prepare_inputs_for_generation(
692
+ self,
693
+ @@ -583,24 +987,120 @@
694
  inputs_embeds: Optional[torch.FloatTensor] = None,
695
  cache_position: Optional[torch.LongTensor] = None,
696
  use_cache: Optional[bool] = None,
697
  + position_ids: Optional[torch.LongTensor] = None,
698
+ + next_sequence_length: Optional[int] = None,
699
+ + is_first_iteration: Optional[bool] = None,
700
  **kwargs,
701
  ):
702
  - if past_key_values is not None:
703
  - if input_ids.shape[1] != 1:
704
  - input_ids = input_ids[:, cache_position]
705
+ -
706
+ - if inputs_embeds is not None and cache_position[0] == 0:
707
+ - model_inputs = {"inputs_embeds": inputs_embeds}
708
+ + """Prepare a full prompt or a new-only block for cached generation.
709
+ +
710
+ + With a warm cache and no explicit ``cache_position``, inputs longer
711
+ + than the cached prefix are full prompts; shorter or equal-length inputs
712
+ + are new-only blocks. For a longer new-only block, pass
713
+ + ``torch.arange(past_length, past_length + input_ids.shape[1])`` as
714
+ + ``cache_position``. Attention masks may be padded to cache capacity and
715
+ + therefore cannot identify which input layout the caller supplied.
716
+ + """
717
  + # transformers 5.x does not move remote-code inputs onto the model device for
718
  + # this model, so a plain CPU `input_ids` reaches the embedding lookup and
719
  + # raises. Its own generation helper performs this move, so mirror it here.
 
733
  +
734
  + past_length = _cache_seq_length(past_key_values)
735
  +
736
+ + use_embeds = inputs_embeds is not None and (
737
+ + is_first_iteration or past_length == 0 or input_ids is None or input_ids.numel() == 0
738
+ + )
739
+ + input_tensor = inputs_embeds if use_embeds else input_ids
740
+ + input_length = input_tensor.shape[1]
741
+ +
742
  + if cache_position is None:
743
+ + # Inputs no longer than the cache are treated as new-only blocks.
744
+ + # Longer inputs default to full prompts containing the cached prefix.
745
+ + # Mask width/content cannot distinguish a longer new-only block from
746
+ + # a full prompt with a capacity-sized mask (even an all-visible one).
747
+ + # Callers supplying longer new-only blocks must pass cache_position.
748
+ + full_prompt = past_length > 0 and input_length > past_length
749
+ + new_len = (
750
+ + next_sequence_length if next_sequence_length is not None
751
+ + else input_length - past_length if full_prompt else input_length
752
+ + )
753
+ + cache_position = torch.arange(
754
+ + past_length, past_length + new_len, device=input_tensor.device
755
+ + )
756
+ else:
757
+ - model_inputs = {"input_ids": input_ids}
758
+ + cache_position = cache_position.to(input_tensor.device)
759
  +
760
  + # Only slice when the cache is warm and `input_ids` still carries cached
761
  + # tokens. Slicing unconditionally turned a `None` cache position into
762
  + # `input_ids[:, None]` and injected a bogus axis.
763
+ + new_len = cache_position.numel()
764
+ + sliced = past_length > 0 and input_length > new_len
765
  + if sliced:
766
+ + # During continuation, input_ids may contain only the new prompt
767
+ + # and generated tokens. Absolute cache offsets cannot index that
768
+ + # shorter buffer. Both layouts end with the unprocessed tokens.
769
+ + input_tensor = input_tensor[:, -new_len:] if new_len else input_tensor[:, :0]
770
+ +
771
+ + if use_embeds:
772
+ + model_inputs = {"inputs_embeds": input_tensor}
773
+ + else:
774
+ + model_inputs = {"input_ids": input_tensor}
775
+ +
776
+ + sequence_length = input_tensor.shape[1]
777
+ + if attention_mask is not None and attention_mask.dim() == 2 and new_len:
778
+ + required_length = int(cache_position[-1]) + 1
779
+ + if attention_mask.shape[-1] < required_length:
780
+ + # A current-block mask omits an implicitly visible cached prefix.
781
+ + attention_mask = F.pad(attention_mask, (required_length - attention_mask.shape[-1], 0), value=1)
782
+ +
783
  + # Derive per-row positions from the padding mask. Using `cache_position` for
784
  + # every row is only correct when nothing is padded: with left padding the real
785
  + # tokens of a short row sit at higher indices, so a shared position vector
786
  + # shifts its RoPE phases and the row decodes to garbage.
787
  + if position_ids is not None:
788
+ + # Callers may supply positions for the full prompt/capacity, or may
789
+ + # already have selected just the positions for the current block.
790
+ + if position_ids.shape[-1] != sequence_length:
791
+ + if new_len and position_ids.shape[-1] > int(cache_position[-1]):
792
+ + position_ids = position_ids[:, cache_position]
793
+ + else:
794
+ + position_ids = position_ids[:, -sequence_length:] if sequence_length else position_ids[:, :0]
795
  + elif attention_mask is not None and attention_mask.dim() == 2:
796
  + position_ids = attention_mask.long().cumsum(-1) - 1
797
  + position_ids = position_ids.masked_fill(attention_mask == 0, 0)
798
+ + # The mask may include unused cache capacity after the current block.
799
+ + # Select its actual cache slots, not the final entries of that mask.
800
+ + position_ids = position_ids[:, cache_position]
801
  + else:
802
  + position_ids = cache_position.unsqueeze(0)
803
+
804
  model_inputs.update(
805
  {
806
  - "position_ids": cache_position.unsqueeze(0) if cache_position is not None else None,
 
808
  "past_key_values": past_key_values,
809
  "use_cache": use_cache,
810
  "attention_mask": attention_mask,
811
+ "cache_position": cache_position,
812
+ }
813
+ )
814
+ + # Generation output flags and model-specific forward options must reach
815
+ + # forward. Labels are never generation inputs.
816
+ + model_inputs.update({key: value for key, value in kwargs.items() if key != "labels"})
817
+ + if use_cache is not False:
818
+ + # Generation only reads the final position. Requesting one row keeps
819
+ + # the 100k-way LM head from projecting the whole prompt.
820
+ + model_inputs.setdefault("logits_to_keep", 1)
821
+ return model_inputs
README.md CHANGED
@@ -14,9 +14,9 @@ Drop-in replacement for `modeling_ricursive.py` from
14
  [`ricursiveAI/Ripple-65B-Preview`](https://huggingface.co/ricursiveAI/Ripple-65B-Preview).
15
 
16
  The upstream custom code was written against the Transformers 4.x cache API.
17
- Under Transformers 5.x, `generate()` fails on the very first prefill, and even
18
- after that single line is fixed, padded batches decode incorrectly and
19
- generations longer than `sliding_window` crash.
20
 
21
  This repository distributes the fix only. It contains **no weights**; use it
22
  with the original checkpoint.
@@ -46,7 +46,7 @@ model = AutoModelForCausalLM.from_pretrained(
46
  )
47
  ```
48
 
49
- ## What is broken upstream
50
 
51
  1. **Spurious axis on prefill.** `prepare_inputs_for_generation` does
52
  `input_ids[:, cache_position]` whenever `input_ids.shape[1] != 1`.
@@ -95,18 +95,30 @@ model = AutoModelForCausalLM.from_pretrained(
95
  mask, and moves `input_ids` / `attention_mask` / `inputs_embeds` /
96
  `position_ids` onto the model device.
97
  * `_update_causal_mask` sizes the key axis from `cache_position`, emits `0.0`
98
- for visible keys and `finfo.min` elsewhere, and never returns an all-masked
99
- single-token decode row.
100
  * `_apply_sliding_window_mask` trims the shared mask to the keys the layer
101
  holds, anchors key positions at the first key the cache still holds, and is
102
  skipped when the window cannot bind.
103
  * `RicursiveForCausalLM` inherits `GenerationMixin` explicitly.
 
 
 
 
 
 
 
 
 
 
 
 
104
 
105
  The patch is backward compatible with Transformers 4.x.
106
 
107
  ## Validation
108
 
109
- * `generate()` works under Transformers 5.22.1.
110
  * Cached and uncached generation are bit-identical, including past
111
  `sliding_window` (the eviction path).
112
  * Left-padded batches decode bit-identically to the same rows generated
 
14
  [`ricursiveAI/Ripple-65B-Preview`](https://huggingface.co/ricursiveAI/Ripple-65B-Preview).
15
 
16
  The upstream custom code was written against the Transformers 4.x cache API.
17
+ Under Transformers 5.x, `generate()` failed on the very first prefill, and even
18
+ after that single line was fixed, padded batches decoded incorrectly and
19
+ generations longer than `sliding_window` crashed.
20
 
21
  This repository distributes the fix only. It contains **no weights**; use it
22
  with the original checkpoint.
 
46
  )
47
  ```
48
 
49
+ ## What was broken upstream
50
 
51
  1. **Spurious axis on prefill.** `prepare_inputs_for_generation` does
52
  `input_ids[:, cache_position]` whenever `input_ids.shape[1] != 1`.
 
95
  mask, and moves `input_ids` / `attention_mask` / `inputs_embeds` /
96
  `position_ids` onto the model device.
97
  * `_update_causal_mask` sizes the key axis from `cache_position`, emits `0.0`
98
+ for visible keys and `finfo.min` elsewhere, and preserves visible keys during
99
+ single-token decode.
100
  * `_apply_sliding_window_mask` trims the shared mask to the keys the layer
101
  holds, anchors key positions at the first key the cache still holds, and is
102
  skipped when the window cannot bind.
103
  * `RicursiveForCausalLM` inherits `GenerationMixin` explicitly.
104
+ * Cached generation also handles new-token blocks, continuation buffers,
105
+ embeddings-only prompts, and explicit position advancement.
106
+ * Attention forwards static-cache write positions, trims unwritten capacity,
107
+ preserves caller 4D masks, and returns requested attention weights without
108
+ changing the SDPA output. Tensor-parallel heads use local projection widths.
109
+ * `logits_to_keep` limits the LM-head input during inference, and MLP row
110
+ chunking bounds prefill activation memory. Labeled calls retain full-sequence
111
+ FP32 loss and honor accumulation-wide token counts.
112
+ * Training implements gradient checkpointing with value-residual gradients;
113
+ tied embeddings work with Transformers 4 and 5.
114
+ * Default caches use hybrid layers on Transformers 5 and a crop-compatible
115
+ full cache on Transformers 4, preserving assisted decoding after the window.
116
 
117
  The patch is backward compatible with Transformers 4.x.
118
 
119
  ## Validation
120
 
121
+ * `generate()` works under Transformers 5.22.2.
122
  * Cached and uncached generation are bit-identical, including past
123
  `sliding_window` (the eviction path).
124
  * Left-padded batches decode bit-identically to the same rows generated
modeling_ricursive.py CHANGED
@@ -25,8 +25,51 @@ transformers 4.x cache API. The fixes are applied inline below:
25
  window cannot bind.
26
  * `RicursiveForCausalLM` inherits `GenerationMixin` explicitly; transformers 5
27
  no longer injects it into remote-code classes.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  """
29
 
 
 
30
  from typing import Optional, Tuple, Union
31
 
32
  import torch
@@ -50,7 +93,38 @@ from .configuration_ricursive import RicursiveConfig
50
  # Set when the transformers-5.x `generate()` fixes above are present natively.
51
  # Downstream loaders that monkeypatch older checkpoints check this marker and
52
  # stand down instead of re-installing the equivalent runtime patch.
53
- RICURSIVE_TF5_COMPAT = "2026-11-05"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
 
56
  class RicursiveRMSNorm(nn.Module):
@@ -166,6 +240,7 @@ class RicursiveAttention(nn.Module):
166
  dtype: torch.dtype,
167
  device: torch.device,
168
  cache_position: Optional[torch.LongTensor],
 
169
  ) -> Optional[torch.Tensor]:
170
  if self.layer_type != "sliding_attention":
171
  return attention_mask
@@ -175,7 +250,13 @@ class RicursiveAttention(nn.Module):
175
  # shared mask produced by `_update_causal_mask` can be longer than this
176
  # layer's key tensor. Keep only the keys this layer actually holds.
177
  if attention_mask is not None and attention_mask.shape[-1] > kv_len:
178
- attention_mask = attention_mask[..., -kv_len:]
 
 
 
 
 
 
179
 
180
  if cache_position is not None:
181
  query_positions = cache_position.to(device=device).reshape(-1)
@@ -190,7 +271,14 @@ class RicursiveAttention(nn.Module):
190
  # unchanged (already causal) mask or, without one, plain causal
191
  # attention -- which SDPA applies through `is_causal`.
192
  if attention_mask is None:
193
- return None
 
 
 
 
 
 
 
194
  if q_len == 1:
195
  return attention_mask
196
 
@@ -210,7 +298,8 @@ class RicursiveAttention(nn.Module):
210
  return allowed
211
  if attention_mask.dtype == torch.bool:
212
  return attention_mask & allowed
213
- return attention_mask.masked_fill(~allowed, torch.finfo(dtype).min)
 
214
 
215
  def forward(
216
  self,
@@ -239,10 +328,10 @@ class RicursiveAttention(nn.Module):
239
  )
240
  v = self.v_proj(value_states)
241
 
242
- # Reshape to (B, T, n_heads, head_dim) then transpose to (B, n_heads, T, head_dim)
243
- q = q.view(bsz, q_len, self.num_heads, self.head_dim)
244
- k = k.view(bsz, q_len, self.num_kv_heads, self.head_dim)
245
- v = v.view(bsz, q_len, self.num_kv_heads, self.head_dim)
246
 
247
  # Per-head Q/K norm (applied before transpose, on head_dim)
248
  q = self.q_norm(q)
@@ -259,12 +348,62 @@ class RicursiveAttention(nn.Module):
259
  q, k = apply_rotary_pos_emb(q, k, cos, sin)
260
 
261
  # KV cache
 
262
  if past_key_value is not None:
263
- k, v = past_key_value.update(k, v, self.layer_idx)
264
-
265
- # GQA: expand KV heads
266
- k = repeat_kv(k, self.num_kv_groups)
267
- v = repeat_kv(v, self.num_kv_groups)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
 
269
  attention_mask = self._apply_sliding_window_mask(
270
  attention_mask=attention_mask,
@@ -273,14 +412,59 @@ class RicursiveAttention(nn.Module):
273
  dtype=q.dtype,
274
  device=q.device,
275
  cache_position=cache_position,
 
276
  )
277
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
278
  # Attention (use is_causal when no explicit mask for correct causal behavior)
279
  # Cast to plain bool to avoid SymBool issues with torch.compile
280
  is_causal = bool(attention_mask is None and q.shape[2] > 1)
 
 
281
  attn_output = F.scaled_dot_product_attention(
282
  q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
283
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
284
 
285
  # (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
286
  attn_output = attn_output.transpose(1, 2).contiguous()
@@ -298,7 +482,15 @@ class RicursiveAttention(nn.Module):
298
  # Output projection
299
  attn_output = self.o_proj(attn_output)
300
 
301
- return attn_output, None, past_key_value
 
 
 
 
 
 
 
 
302
 
303
 
304
  class RicursiveMLP(nn.Module):
@@ -309,7 +501,7 @@ class RicursiveMLP(nn.Module):
309
  self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
310
  self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
311
 
312
- def forward(self, x: torch.Tensor) -> torch.Tensor:
313
  gate = self.gate_proj(x)
314
  up = self.up_proj(x)
315
  if self.clamp_threshold:
@@ -317,6 +509,34 @@ class RicursiveMLP(nn.Module):
317
  up = up.clamp(min=-self.clamp_threshold, max=self.clamp_threshold)
318
  return self.down_proj(F.silu(gate) * up)
319
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
 
321
  class RicursiveDecoderLayer(nn.Module):
322
  """PeriNorm decoder layer with 4 layer norms."""
@@ -395,6 +615,7 @@ class RicursiveModel(RicursivePreTrainedModel):
395
  [RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
396
  )
397
  self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
 
398
  self.post_init()
399
 
400
  def forward(
@@ -422,6 +643,13 @@ class RicursiveModel(RicursivePreTrainedModel):
422
  use_cache = use_cache if use_cache is not None else self.config.use_cache
423
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
424
 
 
 
 
 
 
 
 
425
  if inputs_embeds is None:
426
  inputs_embeds = self.embed_tokens(input_ids)
427
 
@@ -435,7 +663,7 @@ class RicursiveModel(RicursivePreTrainedModel):
435
  inputs_embeds = inputs_embeds * embed_scale
436
 
437
  if use_cache and past_key_values is None:
438
- past_key_values = DynamicCache()
439
 
440
  if cache_position is None:
441
  past_seen_tokens = (
@@ -464,17 +692,26 @@ class RicursiveModel(RicursivePreTrainedModel):
464
  if output_hidden_states:
465
  all_hidden_states += (hidden_states,)
466
 
467
- layer_outputs = layer(
468
- hidden_states,
469
- attention_mask=causal_mask,
470
- position_ids=position_ids,
471
- past_key_value=past_key_values,
472
- output_attentions=output_attentions,
473
- use_cache=use_cache,
474
- cache_position=cache_position,
475
- value_residual=value_residual,
476
- **kwargs,
477
- )
 
 
 
 
 
 
 
 
 
478
  hidden_states = layer_outputs[0]
479
 
480
  if output_attentions:
@@ -513,6 +750,11 @@ class RicursiveModel(RicursivePreTrainedModel):
513
  # from float min mask values with large activations.
514
  if attention_mask is None:
515
  return None
 
 
 
 
 
516
  if torch.all(attention_mask == 1):
517
  return None
518
 
@@ -569,8 +811,36 @@ def _cache_seq_length(past_key_values: Optional[Cache]) -> int:
569
  return 0
570
 
571
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
572
  class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
573
- _tied_weights_keys = []
 
574
 
575
  # transformers 5 no longer injects `GenerationMixin` into remote-code classes,
576
  # so inherit it explicitly above. Without it `generate` is missing entirely and
@@ -611,6 +881,8 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
611
  output_hidden_states: Optional[bool] = None,
612
  return_dict: Optional[bool] = None,
613
  cache_position: Optional[torch.LongTensor] = None,
 
 
614
  **kwargs,
615
  ) -> Union[Tuple, CausalLMOutputWithPast]:
616
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -629,16 +901,41 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
629
  **kwargs,
630
  )
631
  hidden_states = outputs[0]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
632
  logits = self.lm_head(hidden_states)
633
 
634
  loss = None
635
  if labels is not None:
636
- shift_logits = logits[..., :-1, :].contiguous()
637
- shift_labels = labels[..., 1:].contiguous()
638
  loss = F.cross_entropy(
639
  shift_logits.view(-1, shift_logits.size(-1)),
640
  shift_labels.view(-1),
 
641
  )
 
 
 
 
 
 
642
 
643
  if not return_dict:
644
  output = (logits,) + outputs[1:]
@@ -652,6 +949,36 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
652
  attentions=outputs.attentions,
653
  )
654
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
655
  def prepare_inputs_for_generation(
656
  self,
657
  input_ids: torch.LongTensor,
@@ -661,8 +988,19 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
661
  cache_position: Optional[torch.LongTensor] = None,
662
  use_cache: Optional[bool] = None,
663
  position_ids: Optional[torch.LongTensor] = None,
 
 
664
  **kwargs,
665
  ):
 
 
 
 
 
 
 
 
 
666
  # transformers 5.x does not move remote-code inputs onto the model device for
667
  # this model, so a plain CPU `input_ids` reaches the embedding lookup and
668
  # raises. Its own generation helper performs this move, so mirror it here.
@@ -682,42 +1020,70 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
682
 
683
  past_length = _cache_seq_length(past_key_values)
684
 
 
 
 
 
 
 
685
  if cache_position is None:
686
- # With a non-empty cache and the full sequence supplied, only the trailing
687
- # (uncached) tokens are new. Otherwise every token in `input_ids` is new.
688
- if past_length > 0 and input_ids.shape[1] > 1:
689
- new_len = input_ids.shape[1] - past_length
690
- cache_position = torch.arange(
691
- past_length, past_length + new_len, device=input_ids.device
692
- )
693
- else:
694
- cache_position = torch.arange(
695
- past_length, past_length + input_ids.shape[1], device=input_ids.device
696
- )
 
 
 
 
697
 
698
  # Only slice when the cache is warm and `input_ids` still carries cached
699
  # tokens. Slicing unconditionally turned a `None` cache position into
700
  # `input_ids[:, None]` and injected a bogus axis.
701
- sliced = past_length > 0 and input_ids.shape[1] > cache_position.numel()
 
702
  if sliced:
703
- input_ids = input_ids[:, cache_position]
 
 
 
704
 
705
- if inputs_embeds is not None and cache_position.numel() and int(cache_position[0]) == 0:
706
- model_inputs = {"inputs_embeds": inputs_embeds}
707
  else:
708
- model_inputs = {"input_ids": input_ids}
 
 
 
 
 
 
 
709
 
710
  # Derive per-row positions from the padding mask. Using `cache_position` for
711
  # every row is only correct when nothing is padded: with left padding the real
712
  # tokens of a short row sit at higher indices, so a shared position vector
713
  # shifts its RoPE phases and the row decodes to garbage.
714
  if position_ids is not None:
715
- if sliced:
716
- position_ids = position_ids[:, cache_position]
 
 
 
 
 
717
  elif attention_mask is not None and attention_mask.dim() == 2:
718
  position_ids = attention_mask.long().cumsum(-1) - 1
719
  position_ids = position_ids.masked_fill(attention_mask == 0, 0)
720
- position_ids = position_ids[:, -input_ids.shape[1]:]
 
 
721
  else:
722
  position_ids = cache_position.unsqueeze(0)
723
 
@@ -730,4 +1096,11 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
730
  "cache_position": cache_position,
731
  }
732
  )
 
 
 
 
 
 
 
733
  return model_inputs
 
25
  window cannot bind.
26
  * `RicursiveForCausalLM` inherits `GenerationMixin` explicitly; transformers 5
27
  no longer injects it into remote-code classes.
28
+
29
+ DFlash2 inference fixes (2026-11-06)
30
+ -----------------------------------
31
+ * `RicursiveAttention.forward` builds an end-anchored causal mask for a
32
+ multi-token verify block running against a warm cache. SDPA's implicit mask
33
+ is top-left aligned, so with `q_len < kv_len` it hid valid context.
34
+ Preallocated caches (`StaticCache`) return their whole capacity from
35
+ `update()`, so the mask is now built from the layer's *actual* cache length
36
+ and query positions; the unused tail is trimmed before attention.
37
+ * `RicursiveForCausalLM.forward` honours `logits_to_keep` by slicing the LM
38
+ head *input*, so the full `[batch, seq, vocab]` projection is never
39
+ materialised; `prepare_inputs_for_generation` requests one row. Labeled
40
+ calls keep the full sequence so the shifted causal-LM loss stays defined,
41
+ then trim only the returned logits to the requested width.
42
+ * `RicursiveMLP.forward` chunks the row dimension
43
+ (`RIPPLE_MLP_CHUNK`, default 2048, `0` disables) to bound the
44
+ prefill activation peak. Every op is row-wise, so the result is exact.
45
+
46
+ Cache-alignment fixes from review (2026-11-06)
47
+ ----------------------------------------------
48
+ * A mask longer than the key tensor is now aligned to the absolute positions
49
+ the layer actually holds, `[total_kv - kv_len, total_kv)`, instead of to the
50
+ tail of the mask. With a mask sized to the allocated cache *capacity*
51
+ (`StaticCache`), the old `[-kv_len:]` slice selected unwritten slots after
52
+ sliding-window eviction and silently returned wrong attention.
53
+ * `RicursiveAttention.forward` forwards `cache_position` to
54
+ `past_key_value.update()`. A transformers 4.x `StaticCache` otherwise falls
55
+ back to `arange(q_len)` and the first decode step overwrites the prompt
56
+ instead of appending after it.
57
+
58
+ Model contract fixes (2026-11-09)
59
+ -------------------------------
60
+ * Generation supports embeddings-only prompts, continuation buffers, explicit
61
+ position advancement, short masks, and requested diagnostic outputs.
62
+ * Attention preserves caller 4D masks, reports requested weights without changing
63
+ SDPA outputs, and trims unused static capacity before expanding KV heads.
64
+ * Default caches follow the hybrid layer configuration on Transformers 5;
65
+ Transformers 4 keeps a plain, crop-compatible full cache. Training
66
+ implements gradient checkpointing and computes causal-LM loss in FP32.
67
+ * Loss honors accumulation-wide token counts; TP reshapes use local head counts.
68
+ * Tied embeddings support Transformers 4/5, and window masks preserve exclusions.
69
  """
70
 
71
+ import os
72
+ from functools import partial
73
  from typing import Optional, Tuple, Union
74
 
75
  import torch
 
93
  # Set when the transformers-5.x `generate()` fixes above are present natively.
94
  # Downstream loaders that monkeypatch older checkpoints check this marker and
95
  # stand down instead of re-installing the equivalent runtime patch.
96
+ RICURSIVE_TF5_COMPAT = "2026-11-09.1"
97
+
98
+
99
+ def _transformers_major_version() -> int:
100
+ """Return the installed Transformers major version, or 0 when unknown."""
101
+ try:
102
+ from transformers import __version__ as version
103
+ except ImportError: # pragma: no cover - Transformers is a hard dependency
104
+ return 0
105
+ try:
106
+ return int(version.split(".", 1)[0])
107
+ except (AttributeError, ValueError):
108
+ return 0
109
+
110
+
111
+ def _new_default_cache(config: RicursiveConfig) -> DynamicCache:
112
+ """Create the default KV cache for a forward pass.
113
+
114
+ Transformers 5 builds a hybrid cache from ``config``: sliding layers keep
115
+ only their window while full layers grow, which bounds long-context decode
116
+ memory. Transformers 4 also accepts ``config`` but its
117
+ ``DynamicSlidingWindowLayer`` cannot be cropped after it has seen more
118
+ tokens than the window, so assisted decoding and any cache rollback raise
119
+ ``ValueError: Cannot crop a DynamicSlidingWindowLayer``. Keep the plain
120
+ full-attention cache on 4.x, matching the pre-config default.
121
+ """
122
+ if _transformers_major_version() >= 5:
123
+ try:
124
+ return DynamicCache(config=config)
125
+ except TypeError: # A 5.x build without configured hybrid caches.
126
+ pass
127
+ return DynamicCache()
128
 
129
 
130
  class RicursiveRMSNorm(nn.Module):
 
240
  dtype: torch.dtype,
241
  device: torch.device,
242
  cache_position: Optional[torch.LongTensor],
243
+ total_kv: Optional[int] = None,
244
  ) -> Optional[torch.Tensor]:
245
  if self.layer_type != "sliding_attention":
246
  return attention_mask
 
250
  # shared mask produced by `_update_causal_mask` can be longer than this
251
  # layer's key tensor. Keep only the keys this layer actually holds.
252
  if attention_mask is not None and attention_mask.shape[-1] > kv_len:
253
+ mask_len = attention_mask.shape[-1]
254
+ if total_kv is not None:
255
+ # The retained keys sit at `[total_kv - kv_len, total_kv)`.
256
+ start = max(0, min(total_kv - kv_len, mask_len - kv_len))
257
+ attention_mask = attention_mask[..., start : start + kv_len]
258
+ else:
259
+ attention_mask = attention_mask[..., -kv_len:]
260
 
261
  if cache_position is not None:
262
  query_positions = cache_position.to(device=device).reshape(-1)
 
271
  # unchanged (already causal) mask or, without one, plain causal
272
  # attention -- which SDPA applies through `is_causal`.
273
  if attention_mask is None:
274
+ # SDPA's implicit causal mask is top-left aligned. It only
275
+ # matches this query block when it is the whole sequence
276
+ # (`q_len == kv_len`) or a single decode position
277
+ # (`q_len == 1`). A multi-token block verified against a warm
278
+ # cache sits at the *end* of the key span, so fall through and
279
+ # build the end-anchored mask below.
280
+ if q_len == 1 or q_len == kv_len:
281
+ return None
282
  if q_len == 1:
283
  return attention_mask
284
 
 
298
  return allowed
299
  if attention_mask.dtype == torch.bool:
300
  return attention_mask & allowed
301
+ # Finite minima would reopen an existing -inf exclusion on an empty row.
302
+ return attention_mask.masked_fill(~allowed, float("-inf"))
303
 
304
  def forward(
305
  self,
 
328
  )
329
  v = self.v_proj(value_states)
330
 
331
+ # Infer rank-local head counts from projections after tensor parallel sharding.
332
+ q = q.view(bsz, q_len, -1, self.head_dim)
333
+ k = k.view(bsz, q_len, -1, self.head_dim)
334
+ v = v.view(bsz, q_len, -1, self.head_dim)
335
 
336
  # Per-head Q/K norm (applied before transpose, on head_dim)
337
  q = self.q_norm(q)
 
348
  q, k = apply_rotary_pos_emb(q, k, cos, sin)
349
 
350
  # KV cache
351
+ total_kv = None
352
  if past_key_value is not None:
353
+ # Forward the absolute positions of the tokens being written. A
354
+ # transformers 4.x `StaticCache` falls back to `arange(q_len)` when
355
+ # `cache_position` is absent, so the first decode step would write
356
+ # at slots 0..q_len-1 and overwrite the prompt instead of appending
357
+ # after it. `DynamicCache`/5.x caches ignore the extra kwarg.
358
+ cache_kwargs = (
359
+ {"cache_position": cache_position} if cache_position is not None else None
360
+ )
361
+ if cache_kwargs is not None:
362
+ try:
363
+ k, v = past_key_value.update(k, v, self.layer_idx, cache_kwargs)
364
+ except TypeError: # cache without a `cache_kwargs` parameter
365
+ k, v = past_key_value.update(k, v, self.layer_idx)
366
+ else:
367
+ k, v = past_key_value.update(k, v, self.layer_idx)
368
+ # Number of keys this layer has actually written. A preallocated
369
+ # (StaticCache) layer returns its full capacity from `update()`,
370
+ # including slots that have not been filled yet; an evicting
371
+ # (sliding-window) layer reports more than the trailing window it
372
+ # returns. The mask must use the real positions, not the tensor size.
373
+ total_kv = _cache_layer_seq_length(past_key_value, self.layer_idx)
374
+
375
+ # Drop the unwritten tail of a preallocated cache. Without this a
376
+ # six-token prefill into a sixteen-slot StaticCache looks like a
377
+ # ten-token prefix, and the end-anchored verify mask below would let
378
+ # each query attend slots belonging to a different absolute position.
379
+ if total_kv is not None and 0 < total_kv < k.shape[2]:
380
+ k = k[:, :, :total_kv]
381
+ v = v[:, :, :total_kv]
382
+
383
+ # Expand only written slots, not the entire capacity of a static cache.
384
+ num_kv_groups = q.shape[1] // k.shape[1]
385
+ k = repeat_kv(k, num_kv_groups)
386
+ v = repeat_kv(v, num_kv_groups)
387
+
388
+ # Align an incoming mask that is longer than the keys this layer holds.
389
+ # The shared mask is built over absolute key positions
390
+ # `0..target_length-1`; keep the slice that matches the keys present.
391
+ #
392
+ # The keys returned by `update()` occupy the absolute positions
393
+ # `[total_kv - kv_len, total_kv)`. Anchoring on the tensor tail
394
+ # (`[-kv_len:]`) is only correct when the mask length equals
395
+ # `total_kv`; with a mask sized to the *allocated cache capacity* it
396
+ # selects the unwritten tail instead of the retained window and
397
+ # silently produces wrong attention.
398
+ if attention_mask is not None and attention_mask.shape[-1] > k.shape[2]:
399
+ kv_len_now = k.shape[2]
400
+ mask_len = attention_mask.shape[-1]
401
+ if total_kv is not None:
402
+ start = max(0, min(total_kv - kv_len_now, mask_len - kv_len_now))
403
+ attention_mask = attention_mask[..., start : start + kv_len_now]
404
+ else:
405
+ # Unknown cache: the keys are the trailing window.
406
+ attention_mask = attention_mask[..., -kv_len_now:]
407
 
408
  attention_mask = self._apply_sliding_window_mask(
409
  attention_mask=attention_mask,
 
412
  dtype=q.dtype,
413
  device=q.device,
414
  cache_position=cache_position,
415
+ total_kv=total_kv,
416
  )
417
 
418
+ if attention_mask is None and q.shape[2] > 1 and k.shape[2] > q.shape[2]:
419
+ # Multi-token verify block against a warm cache. SDPA's implicit
420
+ # causal mask is top-left aligned, so when `q_len < kv_len` it hides
421
+ # every key before the block. Build the mask from the actual query
422
+ # positions (and the absolute positions of the returned keys) rather
423
+ # than assuming the key tensor starts at position 0.
424
+ if cache_position is not None and cache_position.numel() == q.shape[2]:
425
+ query_positions = cache_position.to(
426
+ device=q.device, dtype=torch.long
427
+ ).reshape(-1)
428
+ else:
429
+ query_positions = torch.arange(
430
+ k.shape[2] - q.shape[2], k.shape[2], device=q.device
431
+ )
432
+ if total_kv is not None and total_kv > k.shape[2]:
433
+ # Evicting cache: the returned keys are the trailing window that
434
+ # ends at the newest query, not positions 0..kv_len-1.
435
+ last_query = int(query_positions[-1].item())
436
+ key_positions = torch.arange(
437
+ last_query - k.shape[2] + 1, last_query + 1, device=q.device
438
+ )
439
+ else:
440
+ key_positions = torch.arange(k.shape[2], device=q.device)
441
+ attention_mask = (
442
+ key_positions[None, :] <= query_positions[:, None]
443
+ )[None, None]
444
+
445
  # Attention (use is_causal when no explicit mask for correct causal behavior)
446
  # Cast to plain bool to avoid SymBool issues with torch.compile
447
  is_causal = bool(attention_mask is None and q.shape[2] > 1)
448
+ # Keep the SDPA result even when returning diagnostic weights. Rebuilding
449
+ # the output from rounded BF16 probabilities changes logits and token IDs.
450
  attn_output = F.scaled_dot_product_attention(
451
  q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
452
  )
453
+ attn_weights = None
454
+ if output_attentions:
455
+ # SDPA does not expose weights; compute them separately only on request.
456
+ weight_keys = k
457
+ with torch.autocast(device_type=q.device.type, enabled=False):
458
+ scores = torch.matmul(q.float(), weight_keys.float().transpose(-1, -2)) * self.head_dim**-0.5
459
+ if is_causal:
460
+ causal = torch.ones(q_len, weight_keys.shape[2], dtype=torch.bool, device=q.device).tril()
461
+ scores = scores.masked_fill(~causal, float("-inf"))
462
+ if attention_mask is not None:
463
+ if attention_mask.dtype == torch.bool:
464
+ scores = scores.masked_fill(~attention_mask, float("-inf"))
465
+ else:
466
+ scores = scores + attention_mask.float()
467
+ attn_weights = F.softmax(scores, dim=-1).nan_to_num().to(q.dtype)
468
 
469
  # (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
470
  attn_output = attn_output.transpose(1, 2).contiguous()
 
482
  # Output projection
483
  attn_output = self.o_proj(attn_output)
484
 
485
+ return attn_output, attn_weights, past_key_value
486
+
487
+
488
+ def _mlp_chunk_size() -> int:
489
+ """Row budget for the chunked MLP forward. `0` (or negative) disables it."""
490
+ try:
491
+ return int(os.environ.get("RIPPLE_MLP_CHUNK", "2048"))
492
+ except ValueError:
493
+ return 2048
494
 
495
 
496
  class RicursiveMLP(nn.Module):
 
501
  self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
502
  self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
503
 
504
+ def _forward_rows(self, x: torch.Tensor) -> torch.Tensor:
505
  gate = self.gate_proj(x)
506
  up = self.up_proj(x)
507
  if self.clamp_threshold:
 
509
  up = up.clamp(min=-self.clamp_threshold, max=self.clamp_threshold)
510
  return self.down_proj(F.silu(gate) * up)
511
 
512
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
513
+ # Ripple's MLP expands hidden_size -> intermediate_size (5120 -> 40960
514
+ # for the 65B checkpoint), so a long prefill materialises `gate`, `up`
515
+ # and `silu(gate) * up` at once. Every op here is row-wise, so splitting
516
+ # the row dimension is exact: each row still uses the same kernels with
517
+ # the same reduction order, and the pieces are concatenated back. Only
518
+ # the transient peak shrinks. `RIPPLE_MLP_CHUNK` overrides the
519
+ # row budget; `0` disables chunking.
520
+ chunk = _mlp_chunk_size()
521
+ if chunk <= 0 or not torch.is_tensor(x) or x.dim() not in (2, 3):
522
+ return self._forward_rows(x)
523
+
524
+ shape = x.shape
525
+ hidden = shape[-1]
526
+ rows = 1
527
+ for size in shape[:-1]:
528
+ rows *= size
529
+ if rows <= chunk:
530
+ return self._forward_rows(x)
531
+
532
+ flat = x.reshape(rows, hidden)
533
+ pieces = [
534
+ self._forward_rows(flat[start : start + chunk])
535
+ for start in range(0, rows, chunk)
536
+ ]
537
+ out = torch.cat(pieces, dim=0) if len(pieces) > 1 else pieces[0]
538
+ return out.reshape(*shape[:-1], out.shape[-1])
539
+
540
 
541
  class RicursiveDecoderLayer(nn.Module):
542
  """PeriNorm decoder layer with 4 layer norms."""
 
615
  [RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
616
  )
617
  self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
618
+ self.gradient_checkpointing = False
619
  self.post_init()
620
 
621
  def forward(
 
643
  use_cache = use_cache if use_cache is not None else self.config.use_cache
644
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
645
 
646
+ checkpointing = self.gradient_checkpointing and self.training
647
+ if checkpointing:
648
+ if past_key_values is not None:
649
+ raise ValueError("Gradient checkpointing does not support a supplied KV cache")
650
+ # Recomputed layers must not append to a cache for a second time.
651
+ use_cache = False
652
+
653
  if inputs_embeds is None:
654
  inputs_embeds = self.embed_tokens(input_ids)
655
 
 
663
  inputs_embeds = inputs_embeds * embed_scale
664
 
665
  if use_cache and past_key_values is None:
666
+ past_key_values = _new_default_cache(self.config)
667
 
668
  if cache_position is None:
669
  past_seen_tokens = (
 
692
  if output_hidden_states:
693
  all_hidden_states += (hidden_states,)
694
 
695
+ if checkpointing:
696
+ # Pass the value-residual seed as an input too, so gradients
697
+ # through the embedding path survive reentrant checkpointing.
698
+ layer_outputs = self._gradient_checkpointing_func(
699
+ partial(layer.__call__, **kwargs), hidden_states, causal_mask,
700
+ position_ids, None, output_attentions, False, cache_position,
701
+ value_residual,
702
+ )
703
+ else:
704
+ layer_outputs = layer(
705
+ hidden_states,
706
+ attention_mask=causal_mask,
707
+ position_ids=position_ids,
708
+ past_key_value=past_key_values,
709
+ output_attentions=output_attentions,
710
+ use_cache=use_cache,
711
+ cache_position=cache_position,
712
+ value_residual=value_residual,
713
+ **kwargs,
714
+ )
715
  hidden_states = layer_outputs[0]
716
 
717
  if output_attentions:
 
750
  # from float min mask values with large activations.
751
  if attention_mask is None:
752
  return None
753
+ if attention_mask.dim() == 4:
754
+ # A caller-provided 4D mask already defines visibility (or additive
755
+ # bias). Preserve it; each local layer still applies its window.
756
+ dtype = torch.bool if attention_mask.dtype == torch.bool else input_tensor.dtype
757
+ return attention_mask.to(device=input_tensor.device, dtype=dtype)
758
  if torch.all(attention_mask == 1):
759
  return None
760
 
 
811
  return 0
812
 
813
 
814
+ def _cache_layer_seq_length(
815
+ past_key_values: Optional[Cache], layer_idx: int
816
+ ) -> Optional[int]:
817
+ """Actual number of keys written for `layer_idx`, or `None` if unknown.
818
+
819
+ `Cache.get_seq_length(layer_idx)` counts *written* tokens. For a
820
+ preallocated static layer that is smaller than the tensor `update()`
821
+ returns until the cache fills; for an evicting sliding-window layer it is
822
+ larger, because the layer only returns a trailing window.
823
+ """
824
+ if past_key_values is None:
825
+ return None
826
+ getter = getattr(past_key_values, "get_seq_length", None)
827
+ if not callable(getter):
828
+ return None
829
+ try:
830
+ return int(getter(layer_idx))
831
+ except TypeError: # transformers 4.x cache without a layer argument
832
+ pass
833
+ except Exception: # pragma: no cover - unknown cache implementations
834
+ return None
835
+ try:
836
+ return int(getter())
837
+ except Exception: # pragma: no cover - unknown cache implementations
838
+ return None
839
+
840
+
841
  class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
842
+ # Transformers 5 reads this mapping; Transformers 4 iterates its target keys.
843
+ _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
844
 
845
  # transformers 5 no longer injects `GenerationMixin` into remote-code classes,
846
  # so inherit it explicitly above. Without it `generate` is missing entirely and
 
881
  output_hidden_states: Optional[bool] = None,
882
  return_dict: Optional[bool] = None,
883
  cache_position: Optional[torch.LongTensor] = None,
884
+ logits_to_keep: Optional[int] = None,
885
+ num_items_in_batch: Optional[torch.Tensor] = None,
886
  **kwargs,
887
  ) -> Union[Tuple, CausalLMOutputWithPast]:
888
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
901
  **kwargs,
902
  )
903
  hidden_states = outputs[0]
904
+ # Honour `logits_to_keep` by slicing the LM head *input*. Projecting the
905
+ # full `[batch, seq, vocab]` tensor only to discard all but the last row
906
+ # costs several GiB of transient memory at prefill batch sizes.
907
+ #
908
+ # When `labels` are supplied the full sequence must be kept: the causal-LM
909
+ # loss shifts logits and labels by one position, so trimming both to the
910
+ # same short window leaves the shift empty (and the loss NaN). Labeled
911
+ # calls therefore project the full sequence to get a defined loss, then
912
+ # trim only the *returned* logits so the output shape still honours
913
+ # `logits_to_keep`. The generation path, which never passes labels, keeps
914
+ # the memory win from slicing the LM head input.
915
+ keep = (
916
+ int(logits_to_keep)
917
+ if logits_to_keep is not None and logits_to_keep > 0
918
+ else 0
919
+ )
920
+ if keep and labels is None and hidden_states.shape[1] > keep:
921
+ hidden_states = hidden_states[:, -keep:, :]
922
  logits = self.lm_head(hidden_states)
923
 
924
  loss = None
925
  if labels is not None:
926
+ shift_logits = logits[..., :-1, :].float().contiguous()
927
+ shift_labels = labels[..., 1:].to(logits.device).contiguous()
928
  loss = F.cross_entropy(
929
  shift_logits.view(-1, shift_logits.size(-1)),
930
  shift_labels.view(-1),
931
+ reduction="sum" if num_items_in_batch is not None else "mean",
932
  )
933
+ if num_items_in_batch is not None:
934
+ # Trainer supplies the token count across all accumulated microbatches.
935
+ loss = loss / torch.as_tensor(num_items_in_batch, device=loss.device)
936
+ if keep:
937
+ # Loss used the full sequence; only the returned tensor is trimmed.
938
+ logits = logits[..., -keep:, :]
939
 
940
  if not return_dict:
941
  output = (logits,) + outputs[1:]
 
949
  attentions=outputs.attentions,
950
  )
951
 
952
+ def _update_model_kwargs_for_generation(
953
+ self, outputs, model_kwargs, is_encoder_decoder=False, num_new_tokens=1
954
+ ):
955
+ """Advance explicit positions when the generation backend leaves them unchanged."""
956
+ position_ids = model_kwargs.get("position_ids")
957
+ cache_position = model_kwargs.get("cache_position")
958
+ model_kwargs = super()._update_model_kwargs_for_generation(
959
+ outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder,
960
+ num_new_tokens=num_new_tokens,
961
+ )
962
+ # Transformers 4 leaves positions unchanged; Transformers 5 already extends them.
963
+ if position_ids is not None and model_kwargs.get("position_ids") is position_ids:
964
+ offsets = torch.arange(
965
+ 1, num_new_tokens + 1, device=position_ids.device, dtype=position_ids.dtype
966
+ )
967
+ model_kwargs["position_ids"] = torch.cat(
968
+ [position_ids, position_ids[:, -1:] + offsets], dim=-1
969
+ )
970
+ # Transformers 5 no longer maintains cache_position. Keep an explicitly
971
+ # supplied vector in sync, without advancing it twice on Transformers 4.
972
+ if cache_position is not None and model_kwargs.get("cache_position") is cache_position:
973
+ if model_kwargs.get("use_cache", True):
974
+ model_kwargs["cache_position"] = cache_position[-1:] + num_new_tokens
975
+ else:
976
+ offsets = torch.arange(
977
+ 1, num_new_tokens + 1, device=cache_position.device, dtype=cache_position.dtype
978
+ )
979
+ model_kwargs["cache_position"] = torch.cat([cache_position, cache_position[-1:] + offsets])
980
+ return model_kwargs
981
+
982
  def prepare_inputs_for_generation(
983
  self,
984
  input_ids: torch.LongTensor,
 
988
  cache_position: Optional[torch.LongTensor] = None,
989
  use_cache: Optional[bool] = None,
990
  position_ids: Optional[torch.LongTensor] = None,
991
+ next_sequence_length: Optional[int] = None,
992
+ is_first_iteration: Optional[bool] = None,
993
  **kwargs,
994
  ):
995
+ """Prepare a full prompt or a new-only block for cached generation.
996
+
997
+ With a warm cache and no explicit ``cache_position``, inputs longer
998
+ than the cached prefix are full prompts; shorter or equal-length inputs
999
+ are new-only blocks. For a longer new-only block, pass
1000
+ ``torch.arange(past_length, past_length + input_ids.shape[1])`` as
1001
+ ``cache_position``. Attention masks may be padded to cache capacity and
1002
+ therefore cannot identify which input layout the caller supplied.
1003
+ """
1004
  # transformers 5.x does not move remote-code inputs onto the model device for
1005
  # this model, so a plain CPU `input_ids` reaches the embedding lookup and
1006
  # raises. Its own generation helper performs this move, so mirror it here.
 
1020
 
1021
  past_length = _cache_seq_length(past_key_values)
1022
 
1023
+ use_embeds = inputs_embeds is not None and (
1024
+ is_first_iteration or past_length == 0 or input_ids is None or input_ids.numel() == 0
1025
+ )
1026
+ input_tensor = inputs_embeds if use_embeds else input_ids
1027
+ input_length = input_tensor.shape[1]
1028
+
1029
  if cache_position is None:
1030
+ # Inputs no longer than the cache are treated as new-only blocks.
1031
+ # Longer inputs default to full prompts containing the cached prefix.
1032
+ # Mask width/content cannot distinguish a longer new-only block from
1033
+ # a full prompt with a capacity-sized mask (even an all-visible one).
1034
+ # Callers supplying longer new-only blocks must pass cache_position.
1035
+ full_prompt = past_length > 0 and input_length > past_length
1036
+ new_len = (
1037
+ next_sequence_length if next_sequence_length is not None
1038
+ else input_length - past_length if full_prompt else input_length
1039
+ )
1040
+ cache_position = torch.arange(
1041
+ past_length, past_length + new_len, device=input_tensor.device
1042
+ )
1043
+ else:
1044
+ cache_position = cache_position.to(input_tensor.device)
1045
 
1046
  # Only slice when the cache is warm and `input_ids` still carries cached
1047
  # tokens. Slicing unconditionally turned a `None` cache position into
1048
  # `input_ids[:, None]` and injected a bogus axis.
1049
+ new_len = cache_position.numel()
1050
+ sliced = past_length > 0 and input_length > new_len
1051
  if sliced:
1052
+ # During continuation, input_ids may contain only the new prompt
1053
+ # and generated tokens. Absolute cache offsets cannot index that
1054
+ # shorter buffer. Both layouts end with the unprocessed tokens.
1055
+ input_tensor = input_tensor[:, -new_len:] if new_len else input_tensor[:, :0]
1056
 
1057
+ if use_embeds:
1058
+ model_inputs = {"inputs_embeds": input_tensor}
1059
  else:
1060
+ model_inputs = {"input_ids": input_tensor}
1061
+
1062
+ sequence_length = input_tensor.shape[1]
1063
+ if attention_mask is not None and attention_mask.dim() == 2 and new_len:
1064
+ required_length = int(cache_position[-1]) + 1
1065
+ if attention_mask.shape[-1] < required_length:
1066
+ # A current-block mask omits an implicitly visible cached prefix.
1067
+ attention_mask = F.pad(attention_mask, (required_length - attention_mask.shape[-1], 0), value=1)
1068
 
1069
  # Derive per-row positions from the padding mask. Using `cache_position` for
1070
  # every row is only correct when nothing is padded: with left padding the real
1071
  # tokens of a short row sit at higher indices, so a shared position vector
1072
  # shifts its RoPE phases and the row decodes to garbage.
1073
  if position_ids is not None:
1074
+ # Callers may supply positions for the full prompt/capacity, or may
1075
+ # already have selected just the positions for the current block.
1076
+ if position_ids.shape[-1] != sequence_length:
1077
+ if new_len and position_ids.shape[-1] > int(cache_position[-1]):
1078
+ position_ids = position_ids[:, cache_position]
1079
+ else:
1080
+ position_ids = position_ids[:, -sequence_length:] if sequence_length else position_ids[:, :0]
1081
  elif attention_mask is not None and attention_mask.dim() == 2:
1082
  position_ids = attention_mask.long().cumsum(-1) - 1
1083
  position_ids = position_ids.masked_fill(attention_mask == 0, 0)
1084
+ # The mask may include unused cache capacity after the current block.
1085
+ # Select its actual cache slots, not the final entries of that mask.
1086
+ position_ids = position_ids[:, cache_position]
1087
  else:
1088
  position_ids = cache_position.unsqueeze(0)
1089
 
 
1096
  "cache_position": cache_position,
1097
  }
1098
  )
1099
+ # Generation output flags and model-specific forward options must reach
1100
+ # forward. Labels are never generation inputs.
1101
+ model_inputs.update({key: value for key, value in kwargs.items() if key != "labels"})
1102
+ if use_cache is not False:
1103
+ # Generation only reads the final position. Requesting one row keeps
1104
+ # the 100k-way LM head from projecting the whole prompt.
1105
+ model_inputs.setdefault("logits_to_keep", 1)
1106
  return model_inputs