Fix generation + training

#11
by Qubitium - opened
Files changed (1) hide show
  1. modeling_ricursive.py +192 -46
modeling_ricursive.py CHANGED
@@ -54,9 +54,24 @@ Cache-alignment fixes from review (2026-11-06)
54
  `past_key_value.update()`. A transformers 4.x `StaticCache` otherwise falls
55
  back to `arange(q_len)` and the first decode step overwrites the prompt
56
  instead of appending after it.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  """
58
 
59
  import os
 
60
  from typing import Optional, Tuple, Union
61
 
62
  import torch
@@ -80,7 +95,38 @@ from .configuration_ricursive import RicursiveConfig
80
  # Set when the transformers-5.x `generate()` fixes above are present natively.
81
  # Downstream loaders that monkeypatch older checkpoints check this marker and
82
  # stand down instead of re-installing the equivalent runtime patch.
83
- RICURSIVE_TF5_COMPAT = "2026-11-05"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
 
85
 
86
  class RicursiveRMSNorm(nn.Module):
@@ -254,7 +300,8 @@ class RicursiveAttention(nn.Module):
254
  return allowed
255
  if attention_mask.dtype == torch.bool:
256
  return attention_mask & allowed
257
- return attention_mask.masked_fill(~allowed, torch.finfo(dtype).min)
 
258
 
259
  def forward(
260
  self,
@@ -283,10 +330,10 @@ class RicursiveAttention(nn.Module):
283
  )
284
  v = self.v_proj(value_states)
285
 
286
- # Reshape to (B, T, n_heads, head_dim) then transpose to (B, n_heads, T, head_dim)
287
- q = q.view(bsz, q_len, self.num_heads, self.head_dim)
288
- k = k.view(bsz, q_len, self.num_kv_heads, self.head_dim)
289
- v = v.view(bsz, q_len, self.num_kv_heads, self.head_dim)
290
 
291
  # Per-head Q/K norm (applied before transpose, on head_dim)
292
  q = self.q_norm(q)
@@ -327,10 +374,6 @@ class RicursiveAttention(nn.Module):
327
  # returns. The mask must use the real positions, not the tensor size.
328
  total_kv = _cache_layer_seq_length(past_key_value, self.layer_idx)
329
 
330
- # GQA: expand KV heads
331
- k = repeat_kv(k, self.num_kv_groups)
332
- v = repeat_kv(v, self.num_kv_groups)
333
-
334
  # Drop the unwritten tail of a preallocated cache. Without this a
335
  # six-token prefill into a sixteen-slot StaticCache looks like a
336
  # ten-token prefix, and the end-anchored verify mask below would let
@@ -339,6 +382,11 @@ class RicursiveAttention(nn.Module):
339
  k = k[:, :, :total_kv]
340
  v = v[:, :, :total_kv]
341
 
 
 
 
 
 
342
  # Align an incoming mask that is longer than the keys this layer holds.
343
  # The shared mask is built over absolute key positions
344
  # `0..target_length-1`; keep the slice that matches the keys present.
@@ -399,9 +447,26 @@ class RicursiveAttention(nn.Module):
399
  # Attention (use is_causal when no explicit mask for correct causal behavior)
400
  # Cast to plain bool to avoid SymBool issues with torch.compile
401
  is_causal = bool(attention_mask is None and q.shape[2] > 1)
 
 
402
  attn_output = F.scaled_dot_product_attention(
403
  q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
404
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
405
 
406
  # (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
407
  attn_output = attn_output.transpose(1, 2).contiguous()
@@ -419,7 +484,7 @@ class RicursiveAttention(nn.Module):
419
  # Output projection
420
  attn_output = self.o_proj(attn_output)
421
 
422
- return attn_output, None, past_key_value
423
 
424
 
425
  def _mlp_chunk_size() -> int:
@@ -552,6 +617,7 @@ class RicursiveModel(RicursivePreTrainedModel):
552
  [RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
553
  )
554
  self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
 
555
  self.post_init()
556
 
557
  def forward(
@@ -579,6 +645,13 @@ class RicursiveModel(RicursivePreTrainedModel):
579
  use_cache = use_cache if use_cache is not None else self.config.use_cache
580
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
581
 
 
 
 
 
 
 
 
582
  if inputs_embeds is None:
583
  inputs_embeds = self.embed_tokens(input_ids)
584
 
@@ -592,7 +665,7 @@ class RicursiveModel(RicursivePreTrainedModel):
592
  inputs_embeds = inputs_embeds * embed_scale
593
 
594
  if use_cache and past_key_values is None:
595
- past_key_values = DynamicCache()
596
 
597
  if cache_position is None:
598
  past_seen_tokens = (
@@ -621,17 +694,26 @@ class RicursiveModel(RicursivePreTrainedModel):
621
  if output_hidden_states:
622
  all_hidden_states += (hidden_states,)
623
 
624
- layer_outputs = layer(
625
- hidden_states,
626
- attention_mask=causal_mask,
627
- position_ids=position_ids,
628
- past_key_value=past_key_values,
629
- output_attentions=output_attentions,
630
- use_cache=use_cache,
631
- cache_position=cache_position,
632
- value_residual=value_residual,
633
- **kwargs,
634
- )
 
 
 
 
 
 
 
 
 
635
  hidden_states = layer_outputs[0]
636
 
637
  if output_attentions:
@@ -670,6 +752,11 @@ class RicursiveModel(RicursivePreTrainedModel):
670
  # from float min mask values with large activations.
671
  if attention_mask is None:
672
  return None
 
 
 
 
 
673
  if torch.all(attention_mask == 1):
674
  return None
675
 
@@ -754,7 +841,8 @@ def _cache_layer_seq_length(
754
 
755
 
756
  class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
757
- _tied_weights_keys = []
 
758
 
759
  # transformers 5 no longer injects `GenerationMixin` into remote-code classes,
760
  # so inherit it explicitly above. Without it `generate` is missing entirely and
@@ -796,6 +884,7 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
796
  return_dict: Optional[bool] = None,
797
  cache_position: Optional[torch.LongTensor] = None,
798
  logits_to_keep: Optional[int] = None,
 
799
  **kwargs,
800
  ) -> Union[Tuple, CausalLMOutputWithPast]:
801
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -836,12 +925,16 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
836
 
837
  loss = None
838
  if labels is not None:
839
- shift_logits = logits[..., :-1, :].contiguous()
840
- shift_labels = labels[..., 1:].contiguous()
841
  loss = F.cross_entropy(
842
  shift_logits.view(-1, shift_logits.size(-1)),
843
  shift_labels.view(-1),
 
844
  )
 
 
 
845
  if keep:
846
  # Loss used the full sequence; only the returned tensor is trimmed.
847
  logits = logits[..., -keep:, :]
@@ -863,6 +956,7 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
863
  ):
864
  """Advance explicit positions when the generation backend leaves them unchanged."""
865
  position_ids = model_kwargs.get("position_ids")
 
866
  model_kwargs = super()._update_model_kwargs_for_generation(
867
  outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder,
868
  num_new_tokens=num_new_tokens,
@@ -875,6 +969,16 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
875
  model_kwargs["position_ids"] = torch.cat(
876
  [position_ids, position_ids[:, -1:] + offsets], dim=-1
877
  )
 
 
 
 
 
 
 
 
 
 
878
  return model_kwargs
879
 
880
  def prepare_inputs_for_generation(
@@ -886,8 +990,19 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
886
  cache_position: Optional[torch.LongTensor] = None,
887
  use_cache: Optional[bool] = None,
888
  position_ids: Optional[torch.LongTensor] = None,
 
 
889
  **kwargs,
890
  ):
 
 
 
 
 
 
 
 
 
891
  # transformers 5.x does not move remote-code inputs onto the model device for
892
  # this model, so a plain CPU `input_ids` reaches the embedding lookup and
893
  # raises. Its own generation helper performs this move, so mirror it here.
@@ -907,42 +1022,70 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
907
 
908
  past_length = _cache_seq_length(past_key_values)
909
 
 
 
 
 
 
 
910
  if cache_position is None:
911
- # With a non-empty cache and the full sequence supplied, only the trailing
912
- # (uncached) tokens are new. Otherwise every token in `input_ids` is new.
913
- if past_length > 0 and input_ids.shape[1] > 1:
914
- new_len = input_ids.shape[1] - past_length
915
- cache_position = torch.arange(
916
- past_length, past_length + new_len, device=input_ids.device
917
- )
918
- else:
919
- cache_position = torch.arange(
920
- past_length, past_length + input_ids.shape[1], device=input_ids.device
921
- )
 
 
 
 
922
 
923
  # Only slice when the cache is warm and `input_ids` still carries cached
924
  # tokens. Slicing unconditionally turned a `None` cache position into
925
  # `input_ids[:, None]` and injected a bogus axis.
926
- sliced = past_length > 0 and input_ids.shape[1] > cache_position.numel()
 
927
  if sliced:
928
- input_ids = input_ids[:, cache_position]
 
 
 
929
 
930
- if inputs_embeds is not None and cache_position.numel() and int(cache_position[0]) == 0:
931
- model_inputs = {"inputs_embeds": inputs_embeds}
932
  else:
933
- model_inputs = {"input_ids": input_ids}
 
 
 
 
 
 
 
934
 
935
  # Derive per-row positions from the padding mask. Using `cache_position` for
936
  # every row is only correct when nothing is padded: with left padding the real
937
  # tokens of a short row sit at higher indices, so a shared position vector
938
  # shifts its RoPE phases and the row decodes to garbage.
939
  if position_ids is not None:
940
- if sliced:
941
- position_ids = position_ids[:, cache_position]
 
 
 
 
 
942
  elif attention_mask is not None and attention_mask.dim() == 2:
943
  position_ids = attention_mask.long().cumsum(-1) - 1
944
  position_ids = position_ids.masked_fill(attention_mask == 0, 0)
945
- position_ids = position_ids[:, -input_ids.shape[1]:]
 
 
946
  else:
947
  position_ids = cache_position.unsqueeze(0)
948
 
@@ -955,8 +1098,11 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
955
  "cache_position": cache_position,
956
  }
957
  )
 
 
 
958
  if use_cache is not False:
959
  # Generation only reads the final position. Requesting one row keeps
960
  # the 100k-way LM head from projecting the whole prompt.
961
- model_inputs["logits_to_keep"] = 1
962
  return model_inputs
 
54
  `past_key_value.update()`. A transformers 4.x `StaticCache` otherwise falls
55
  back to `arange(q_len)` and the first decode step overwrites the prompt
56
  instead of appending after it.
57
+
58
+ Model contract fixes (2026-11-09)
59
+ -------------------------------
60
+ * Generation supports embeddings-only prompts, continuation buffers, explicit
61
+ position advancement, short masks, and requested diagnostic outputs.
62
+ * Attention preserves caller 4D masks, reports requested weights without changing
63
+ SDPA outputs, and trims unused static capacity before expanding KV heads.
64
+ * Default caches follow the hybrid layer configuration on Transformers 5;
65
+ Transformers 4 keeps a plain, crop-compatible full cache. Training
66
+ implements gradient checkpointing and computes causal-LM loss in FP32.
67
+ * Loss honors accumulation-wide token counts; TP reshapes use local head counts.
68
+ * Tied embeddings support Transformers 4/5, and window masks preserve exclusions.
69
+ * Regression tests are maintained in GPT-QModel, not this model repository:
70
+ https://github.com/ModelCloud/GPTQModel/tree/main
71
  """
72
 
73
  import os
74
+ from functools import partial
75
  from typing import Optional, Tuple, Union
76
 
77
  import torch
 
95
  # Set when the transformers-5.x `generate()` fixes above are present natively.
96
  # Downstream loaders that monkeypatch older checkpoints check this marker and
97
  # stand down instead of re-installing the equivalent runtime patch.
98
+ RICURSIVE_TF5_COMPAT = "2026-11-09.1"
99
+
100
+
101
+ def _transformers_major_version() -> int:
102
+ """Return the installed Transformers major version, or 0 when unknown."""
103
+ try:
104
+ from transformers import __version__ as version
105
+ except ImportError: # pragma: no cover - Transformers is a hard dependency
106
+ return 0
107
+ try:
108
+ return int(version.split(".", 1)[0])
109
+ except (AttributeError, ValueError):
110
+ return 0
111
+
112
+
113
+ def _new_default_cache(config: RicursiveConfig) -> DynamicCache:
114
+ """Create the default KV cache for a forward pass.
115
+
116
+ Transformers 5 builds a hybrid cache from ``config``: sliding layers keep
117
+ only their window while full layers grow, which bounds long-context decode
118
+ memory. Transformers 4 also accepts ``config`` but its
119
+ ``DynamicSlidingWindowLayer`` cannot be cropped after it has seen more
120
+ tokens than the window, so assisted decoding and any cache rollback raise
121
+ ``ValueError: Cannot crop a DynamicSlidingWindowLayer``. Keep the plain
122
+ full-attention cache on 4.x, matching the pre-config default.
123
+ """
124
+ if _transformers_major_version() >= 5:
125
+ try:
126
+ return DynamicCache(config=config)
127
+ except TypeError: # A 5.x build without configured hybrid caches.
128
+ pass
129
+ return DynamicCache()
130
 
131
 
132
  class RicursiveRMSNorm(nn.Module):
 
300
  return allowed
301
  if attention_mask.dtype == torch.bool:
302
  return attention_mask & allowed
303
+ # Finite minima would reopen an existing -inf exclusion on an empty row.
304
+ return attention_mask.masked_fill(~allowed, float("-inf"))
305
 
306
  def forward(
307
  self,
 
330
  )
331
  v = self.v_proj(value_states)
332
 
333
+ # Infer rank-local head counts from projections after tensor parallel sharding.
334
+ q = q.view(bsz, q_len, -1, self.head_dim)
335
+ k = k.view(bsz, q_len, -1, self.head_dim)
336
+ v = v.view(bsz, q_len, -1, self.head_dim)
337
 
338
  # Per-head Q/K norm (applied before transpose, on head_dim)
339
  q = self.q_norm(q)
 
374
  # returns. The mask must use the real positions, not the tensor size.
375
  total_kv = _cache_layer_seq_length(past_key_value, self.layer_idx)
376
 
 
 
 
 
377
  # Drop the unwritten tail of a preallocated cache. Without this a
378
  # six-token prefill into a sixteen-slot StaticCache looks like a
379
  # ten-token prefix, and the end-anchored verify mask below would let
 
382
  k = k[:, :, :total_kv]
383
  v = v[:, :, :total_kv]
384
 
385
+ # Expand only written slots, not the entire capacity of a static cache.
386
+ num_kv_groups = q.shape[1] // k.shape[1]
387
+ k = repeat_kv(k, num_kv_groups)
388
+ v = repeat_kv(v, num_kv_groups)
389
+
390
  # Align an incoming mask that is longer than the keys this layer holds.
391
  # The shared mask is built over absolute key positions
392
  # `0..target_length-1`; keep the slice that matches the keys present.
 
447
  # Attention (use is_causal when no explicit mask for correct causal behavior)
448
  # Cast to plain bool to avoid SymBool issues with torch.compile
449
  is_causal = bool(attention_mask is None and q.shape[2] > 1)
450
+ # Keep the SDPA result even when returning diagnostic weights. Rebuilding
451
+ # the output from rounded BF16 probabilities changes logits and token IDs.
452
  attn_output = F.scaled_dot_product_attention(
453
  q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
454
  )
455
+ attn_weights = None
456
+ if output_attentions:
457
+ # SDPA does not expose weights; compute them separately only on request.
458
+ weight_keys = k
459
+ with torch.autocast(device_type=q.device.type, enabled=False):
460
+ scores = torch.matmul(q.float(), weight_keys.float().transpose(-1, -2)) * self.head_dim**-0.5
461
+ if is_causal:
462
+ causal = torch.ones(q_len, weight_keys.shape[2], dtype=torch.bool, device=q.device).tril()
463
+ scores = scores.masked_fill(~causal, float("-inf"))
464
+ if attention_mask is not None:
465
+ if attention_mask.dtype == torch.bool:
466
+ scores = scores.masked_fill(~attention_mask, float("-inf"))
467
+ else:
468
+ scores = scores + attention_mask.float()
469
+ attn_weights = F.softmax(scores, dim=-1).nan_to_num().to(q.dtype)
470
 
471
  # (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
472
  attn_output = attn_output.transpose(1, 2).contiguous()
 
484
  # Output projection
485
  attn_output = self.o_proj(attn_output)
486
 
487
+ return attn_output, attn_weights, past_key_value
488
 
489
 
490
  def _mlp_chunk_size() -> int:
 
617
  [RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
618
  )
619
  self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
620
+ self.gradient_checkpointing = False
621
  self.post_init()
622
 
623
  def forward(
 
645
  use_cache = use_cache if use_cache is not None else self.config.use_cache
646
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
647
 
648
+ checkpointing = self.gradient_checkpointing and self.training
649
+ if checkpointing:
650
+ if past_key_values is not None:
651
+ raise ValueError("Gradient checkpointing does not support a supplied KV cache")
652
+ # Recomputed layers must not append to a cache for a second time.
653
+ use_cache = False
654
+
655
  if inputs_embeds is None:
656
  inputs_embeds = self.embed_tokens(input_ids)
657
 
 
665
  inputs_embeds = inputs_embeds * embed_scale
666
 
667
  if use_cache and past_key_values is None:
668
+ past_key_values = _new_default_cache(self.config)
669
 
670
  if cache_position is None:
671
  past_seen_tokens = (
 
694
  if output_hidden_states:
695
  all_hidden_states += (hidden_states,)
696
 
697
+ if checkpointing:
698
+ # Pass the value-residual seed as an input too, so gradients
699
+ # through the embedding path survive reentrant checkpointing.
700
+ layer_outputs = self._gradient_checkpointing_func(
701
+ partial(layer.__call__, **kwargs), hidden_states, causal_mask,
702
+ position_ids, None, output_attentions, False, cache_position,
703
+ value_residual,
704
+ )
705
+ else:
706
+ layer_outputs = layer(
707
+ hidden_states,
708
+ attention_mask=causal_mask,
709
+ position_ids=position_ids,
710
+ past_key_value=past_key_values,
711
+ output_attentions=output_attentions,
712
+ use_cache=use_cache,
713
+ cache_position=cache_position,
714
+ value_residual=value_residual,
715
+ **kwargs,
716
+ )
717
  hidden_states = layer_outputs[0]
718
 
719
  if output_attentions:
 
752
  # from float min mask values with large activations.
753
  if attention_mask is None:
754
  return None
755
+ if attention_mask.dim() == 4:
756
+ # A caller-provided 4D mask already defines visibility (or additive
757
+ # bias). Preserve it; each local layer still applies its window.
758
+ dtype = torch.bool if attention_mask.dtype == torch.bool else input_tensor.dtype
759
+ return attention_mask.to(device=input_tensor.device, dtype=dtype)
760
  if torch.all(attention_mask == 1):
761
  return None
762
 
 
841
 
842
 
843
  class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
844
+ # Transformers 5 reads this mapping; Transformers 4 iterates its target keys.
845
+ _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
846
 
847
  # transformers 5 no longer injects `GenerationMixin` into remote-code classes,
848
  # so inherit it explicitly above. Without it `generate` is missing entirely and
 
884
  return_dict: Optional[bool] = None,
885
  cache_position: Optional[torch.LongTensor] = None,
886
  logits_to_keep: Optional[int] = None,
887
+ num_items_in_batch: Optional[torch.Tensor] = None,
888
  **kwargs,
889
  ) -> Union[Tuple, CausalLMOutputWithPast]:
890
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
925
 
926
  loss = None
927
  if labels is not None:
928
+ shift_logits = logits[..., :-1, :].float().contiguous()
929
+ shift_labels = labels[..., 1:].to(logits.device).contiguous()
930
  loss = F.cross_entropy(
931
  shift_logits.view(-1, shift_logits.size(-1)),
932
  shift_labels.view(-1),
933
+ reduction="sum" if num_items_in_batch is not None else "mean",
934
  )
935
+ if num_items_in_batch is not None:
936
+ # Trainer supplies the token count across all accumulated microbatches.
937
+ loss = loss / torch.as_tensor(num_items_in_batch, device=loss.device)
938
  if keep:
939
  # Loss used the full sequence; only the returned tensor is trimmed.
940
  logits = logits[..., -keep:, :]
 
956
  ):
957
  """Advance explicit positions when the generation backend leaves them unchanged."""
958
  position_ids = model_kwargs.get("position_ids")
959
+ cache_position = model_kwargs.get("cache_position")
960
  model_kwargs = super()._update_model_kwargs_for_generation(
961
  outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder,
962
  num_new_tokens=num_new_tokens,
 
969
  model_kwargs["position_ids"] = torch.cat(
970
  [position_ids, position_ids[:, -1:] + offsets], dim=-1
971
  )
972
+ # Transformers 5 no longer maintains cache_position. Keep an explicitly
973
+ # supplied vector in sync, without advancing it twice on Transformers 4.
974
+ if cache_position is not None and model_kwargs.get("cache_position") is cache_position:
975
+ if model_kwargs.get("use_cache", True):
976
+ model_kwargs["cache_position"] = cache_position[-1:] + num_new_tokens
977
+ else:
978
+ offsets = torch.arange(
979
+ 1, num_new_tokens + 1, device=cache_position.device, dtype=cache_position.dtype
980
+ )
981
+ model_kwargs["cache_position"] = torch.cat([cache_position, cache_position[-1:] + offsets])
982
  return model_kwargs
983
 
984
  def prepare_inputs_for_generation(
 
990
  cache_position: Optional[torch.LongTensor] = None,
991
  use_cache: Optional[bool] = None,
992
  position_ids: Optional[torch.LongTensor] = None,
993
+ next_sequence_length: Optional[int] = None,
994
+ is_first_iteration: Optional[bool] = None,
995
  **kwargs,
996
  ):
997
+ """Prepare a full prompt or a new-only block for cached generation.
998
+
999
+ With a warm cache and no explicit ``cache_position``, inputs longer
1000
+ than the cached prefix are full prompts; shorter or equal-length inputs
1001
+ are new-only blocks. For a longer new-only block, pass
1002
+ ``torch.arange(past_length, past_length + input_ids.shape[1])`` as
1003
+ ``cache_position``. Attention masks may be padded to cache capacity and
1004
+ therefore cannot identify which input layout the caller supplied.
1005
+ """
1006
  # transformers 5.x does not move remote-code inputs onto the model device for
1007
  # this model, so a plain CPU `input_ids` reaches the embedding lookup and
1008
  # raises. Its own generation helper performs this move, so mirror it here.
 
1022
 
1023
  past_length = _cache_seq_length(past_key_values)
1024
 
1025
+ use_embeds = inputs_embeds is not None and (
1026
+ is_first_iteration or past_length == 0 or input_ids is None or input_ids.numel() == 0
1027
+ )
1028
+ input_tensor = inputs_embeds if use_embeds else input_ids
1029
+ input_length = input_tensor.shape[1]
1030
+
1031
  if cache_position is None:
1032
+ # Inputs no longer than the cache are treated as new-only blocks.
1033
+ # Longer inputs default to full prompts containing the cached prefix.
1034
+ # Mask width/content cannot distinguish a longer new-only block from
1035
+ # a full prompt with a capacity-sized mask (even an all-visible one).
1036
+ # Callers supplying longer new-only blocks must pass cache_position.
1037
+ full_prompt = past_length > 0 and input_length > past_length
1038
+ new_len = (
1039
+ next_sequence_length if next_sequence_length is not None
1040
+ else input_length - past_length if full_prompt else input_length
1041
+ )
1042
+ cache_position = torch.arange(
1043
+ past_length, past_length + new_len, device=input_tensor.device
1044
+ )
1045
+ else:
1046
+ cache_position = cache_position.to(input_tensor.device)
1047
 
1048
  # Only slice when the cache is warm and `input_ids` still carries cached
1049
  # tokens. Slicing unconditionally turned a `None` cache position into
1050
  # `input_ids[:, None]` and injected a bogus axis.
1051
+ new_len = cache_position.numel()
1052
+ sliced = past_length > 0 and input_length > new_len
1053
  if sliced:
1054
+ # During continuation, input_ids may contain only the new prompt
1055
+ # and generated tokens. Absolute cache offsets cannot index that
1056
+ # shorter buffer. Both layouts end with the unprocessed tokens.
1057
+ input_tensor = input_tensor[:, -new_len:] if new_len else input_tensor[:, :0]
1058
 
1059
+ if use_embeds:
1060
+ model_inputs = {"inputs_embeds": input_tensor}
1061
  else:
1062
+ model_inputs = {"input_ids": input_tensor}
1063
+
1064
+ sequence_length = input_tensor.shape[1]
1065
+ if attention_mask is not None and attention_mask.dim() == 2 and new_len:
1066
+ required_length = int(cache_position[-1]) + 1
1067
+ if attention_mask.shape[-1] < required_length:
1068
+ # A current-block mask omits an implicitly visible cached prefix.
1069
+ attention_mask = F.pad(attention_mask, (required_length - attention_mask.shape[-1], 0), value=1)
1070
 
1071
  # Derive per-row positions from the padding mask. Using `cache_position` for
1072
  # every row is only correct when nothing is padded: with left padding the real
1073
  # tokens of a short row sit at higher indices, so a shared position vector
1074
  # shifts its RoPE phases and the row decodes to garbage.
1075
  if position_ids is not None:
1076
+ # Callers may supply positions for the full prompt/capacity, or may
1077
+ # already have selected just the positions for the current block.
1078
+ if position_ids.shape[-1] != sequence_length:
1079
+ if new_len and position_ids.shape[-1] > int(cache_position[-1]):
1080
+ position_ids = position_ids[:, cache_position]
1081
+ else:
1082
+ position_ids = position_ids[:, -sequence_length:] if sequence_length else position_ids[:, :0]
1083
  elif attention_mask is not None and attention_mask.dim() == 2:
1084
  position_ids = attention_mask.long().cumsum(-1) - 1
1085
  position_ids = position_ids.masked_fill(attention_mask == 0, 0)
1086
+ # The mask may include unused cache capacity after the current block.
1087
+ # Select its actual cache slots, not the final entries of that mask.
1088
+ position_ids = position_ids[:, cache_position]
1089
  else:
1090
  position_ids = cache_position.unsqueeze(0)
1091
 
 
1098
  "cache_position": cache_position,
1099
  }
1100
  )
1101
+ # Generation output flags and model-specific forward options must reach
1102
+ # forward. Labels are never generation inputs.
1103
+ model_inputs.update({key: value for key, value in kwargs.items() if key != "labels"})
1104
  if use_cache is not False:
1105
  # Generation only reads the final position. Requesting one row keeps
1106
  # the 100k-way LM head from projecting the whole prompt.
1107
+ model_inputs.setdefault("logits_to_keep", 1)
1108
  return model_inputs