Fix generation + training
#11
by Qubitium - opened
- modeling_ricursive.py +192 -46
modeling_ricursive.py
CHANGED
|
@@ -54,9 +54,24 @@ Cache-alignment fixes from review (2026-11-06)
|
|
| 54 |
`past_key_value.update()`. A transformers 4.x `StaticCache` otherwise falls
|
| 55 |
back to `arange(q_len)` and the first decode step overwrites the prompt
|
| 56 |
instead of appending after it.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
"""
|
| 58 |
|
| 59 |
import os
|
|
|
|
| 60 |
from typing import Optional, Tuple, Union
|
| 61 |
|
| 62 |
import torch
|
|
@@ -80,7 +95,38 @@ from .configuration_ricursive import RicursiveConfig
|
|
| 80 |
# Set when the transformers-5.x `generate()` fixes above are present natively.
|
| 81 |
# Downstream loaders that monkeypatch older checkpoints check this marker and
|
| 82 |
# stand down instead of re-installing the equivalent runtime patch.
|
| 83 |
-
RICURSIVE_TF5_COMPAT = "2026-11-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
|
| 85 |
|
| 86 |
class RicursiveRMSNorm(nn.Module):
|
|
@@ -254,7 +300,8 @@ class RicursiveAttention(nn.Module):
|
|
| 254 |
return allowed
|
| 255 |
if attention_mask.dtype == torch.bool:
|
| 256 |
return attention_mask & allowed
|
| 257 |
-
|
|
|
|
| 258 |
|
| 259 |
def forward(
|
| 260 |
self,
|
|
@@ -283,10 +330,10 @@ class RicursiveAttention(nn.Module):
|
|
| 283 |
)
|
| 284 |
v = self.v_proj(value_states)
|
| 285 |
|
| 286 |
-
#
|
| 287 |
-
q = q.view(bsz, q_len,
|
| 288 |
-
k = k.view(bsz, q_len,
|
| 289 |
-
v = v.view(bsz, q_len,
|
| 290 |
|
| 291 |
# Per-head Q/K norm (applied before transpose, on head_dim)
|
| 292 |
q = self.q_norm(q)
|
|
@@ -327,10 +374,6 @@ class RicursiveAttention(nn.Module):
|
|
| 327 |
# returns. The mask must use the real positions, not the tensor size.
|
| 328 |
total_kv = _cache_layer_seq_length(past_key_value, self.layer_idx)
|
| 329 |
|
| 330 |
-
# GQA: expand KV heads
|
| 331 |
-
k = repeat_kv(k, self.num_kv_groups)
|
| 332 |
-
v = repeat_kv(v, self.num_kv_groups)
|
| 333 |
-
|
| 334 |
# Drop the unwritten tail of a preallocated cache. Without this a
|
| 335 |
# six-token prefill into a sixteen-slot StaticCache looks like a
|
| 336 |
# ten-token prefix, and the end-anchored verify mask below would let
|
|
@@ -339,6 +382,11 @@ class RicursiveAttention(nn.Module):
|
|
| 339 |
k = k[:, :, :total_kv]
|
| 340 |
v = v[:, :, :total_kv]
|
| 341 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
# Align an incoming mask that is longer than the keys this layer holds.
|
| 343 |
# The shared mask is built over absolute key positions
|
| 344 |
# `0..target_length-1`; keep the slice that matches the keys present.
|
|
@@ -399,9 +447,26 @@ class RicursiveAttention(nn.Module):
|
|
| 399 |
# Attention (use is_causal when no explicit mask for correct causal behavior)
|
| 400 |
# Cast to plain bool to avoid SymBool issues with torch.compile
|
| 401 |
is_causal = bool(attention_mask is None and q.shape[2] > 1)
|
|
|
|
|
|
|
| 402 |
attn_output = F.scaled_dot_product_attention(
|
| 403 |
q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
|
| 404 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 405 |
|
| 406 |
# (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
|
| 407 |
attn_output = attn_output.transpose(1, 2).contiguous()
|
|
@@ -419,7 +484,7 @@ class RicursiveAttention(nn.Module):
|
|
| 419 |
# Output projection
|
| 420 |
attn_output = self.o_proj(attn_output)
|
| 421 |
|
| 422 |
-
return attn_output,
|
| 423 |
|
| 424 |
|
| 425 |
def _mlp_chunk_size() -> int:
|
|
@@ -552,6 +617,7 @@ class RicursiveModel(RicursivePreTrainedModel):
|
|
| 552 |
[RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
|
| 553 |
)
|
| 554 |
self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
|
|
|
| 555 |
self.post_init()
|
| 556 |
|
| 557 |
def forward(
|
|
@@ -579,6 +645,13 @@ class RicursiveModel(RicursivePreTrainedModel):
|
|
| 579 |
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
| 580 |
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 581 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 582 |
if inputs_embeds is None:
|
| 583 |
inputs_embeds = self.embed_tokens(input_ids)
|
| 584 |
|
|
@@ -592,7 +665,7 @@ class RicursiveModel(RicursivePreTrainedModel):
|
|
| 592 |
inputs_embeds = inputs_embeds * embed_scale
|
| 593 |
|
| 594 |
if use_cache and past_key_values is None:
|
| 595 |
-
past_key_values =
|
| 596 |
|
| 597 |
if cache_position is None:
|
| 598 |
past_seen_tokens = (
|
|
@@ -621,17 +694,26 @@ class RicursiveModel(RicursivePreTrainedModel):
|
|
| 621 |
if output_hidden_states:
|
| 622 |
all_hidden_states += (hidden_states,)
|
| 623 |
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
hidden_states = layer_outputs[0]
|
| 636 |
|
| 637 |
if output_attentions:
|
|
@@ -670,6 +752,11 @@ class RicursiveModel(RicursivePreTrainedModel):
|
|
| 670 |
# from float min mask values with large activations.
|
| 671 |
if attention_mask is None:
|
| 672 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 673 |
if torch.all(attention_mask == 1):
|
| 674 |
return None
|
| 675 |
|
|
@@ -754,7 +841,8 @@ def _cache_layer_seq_length(
|
|
| 754 |
|
| 755 |
|
| 756 |
class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
| 757 |
-
|
|
|
|
| 758 |
|
| 759 |
# transformers 5 no longer injects `GenerationMixin` into remote-code classes,
|
| 760 |
# so inherit it explicitly above. Without it `generate` is missing entirely and
|
|
@@ -796,6 +884,7 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 796 |
return_dict: Optional[bool] = None,
|
| 797 |
cache_position: Optional[torch.LongTensor] = None,
|
| 798 |
logits_to_keep: Optional[int] = None,
|
|
|
|
| 799 |
**kwargs,
|
| 800 |
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 801 |
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
|
@@ -836,12 +925,16 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 836 |
|
| 837 |
loss = None
|
| 838 |
if labels is not None:
|
| 839 |
-
shift_logits = logits[..., :-1, :].contiguous()
|
| 840 |
-
shift_labels = labels[..., 1:].contiguous()
|
| 841 |
loss = F.cross_entropy(
|
| 842 |
shift_logits.view(-1, shift_logits.size(-1)),
|
| 843 |
shift_labels.view(-1),
|
|
|
|
| 844 |
)
|
|
|
|
|
|
|
|
|
|
| 845 |
if keep:
|
| 846 |
# Loss used the full sequence; only the returned tensor is trimmed.
|
| 847 |
logits = logits[..., -keep:, :]
|
|
@@ -863,6 +956,7 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 863 |
):
|
| 864 |
"""Advance explicit positions when the generation backend leaves them unchanged."""
|
| 865 |
position_ids = model_kwargs.get("position_ids")
|
|
|
|
| 866 |
model_kwargs = super()._update_model_kwargs_for_generation(
|
| 867 |
outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder,
|
| 868 |
num_new_tokens=num_new_tokens,
|
|
@@ -875,6 +969,16 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 875 |
model_kwargs["position_ids"] = torch.cat(
|
| 876 |
[position_ids, position_ids[:, -1:] + offsets], dim=-1
|
| 877 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 878 |
return model_kwargs
|
| 879 |
|
| 880 |
def prepare_inputs_for_generation(
|
|
@@ -886,8 +990,19 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 886 |
cache_position: Optional[torch.LongTensor] = None,
|
| 887 |
use_cache: Optional[bool] = None,
|
| 888 |
position_ids: Optional[torch.LongTensor] = None,
|
|
|
|
|
|
|
| 889 |
**kwargs,
|
| 890 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 891 |
# transformers 5.x does not move remote-code inputs onto the model device for
|
| 892 |
# this model, so a plain CPU `input_ids` reaches the embedding lookup and
|
| 893 |
# raises. Its own generation helper performs this move, so mirror it here.
|
|
@@ -907,42 +1022,70 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 907 |
|
| 908 |
past_length = _cache_seq_length(past_key_values)
|
| 909 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 910 |
if cache_position is None:
|
| 911 |
-
#
|
| 912 |
-
#
|
| 913 |
-
|
| 914 |
-
|
| 915 |
-
|
| 916 |
-
|
| 917 |
-
|
| 918 |
-
|
| 919 |
-
|
| 920 |
-
|
| 921 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 922 |
|
| 923 |
# Only slice when the cache is warm and `input_ids` still carries cached
|
| 924 |
# tokens. Slicing unconditionally turned a `None` cache position into
|
| 925 |
# `input_ids[:, None]` and injected a bogus axis.
|
| 926 |
-
|
|
|
|
| 927 |
if sliced:
|
| 928 |
-
|
|
|
|
|
|
|
|
|
|
| 929 |
|
| 930 |
-
if
|
| 931 |
-
model_inputs = {"inputs_embeds":
|
| 932 |
else:
|
| 933 |
-
model_inputs = {"input_ids":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 934 |
|
| 935 |
# Derive per-row positions from the padding mask. Using `cache_position` for
|
| 936 |
# every row is only correct when nothing is padded: with left padding the real
|
| 937 |
# tokens of a short row sit at higher indices, so a shared position vector
|
| 938 |
# shifts its RoPE phases and the row decodes to garbage.
|
| 939 |
if position_ids is not None:
|
| 940 |
-
|
| 941 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 942 |
elif attention_mask is not None and attention_mask.dim() == 2:
|
| 943 |
position_ids = attention_mask.long().cumsum(-1) - 1
|
| 944 |
position_ids = position_ids.masked_fill(attention_mask == 0, 0)
|
| 945 |
-
|
|
|
|
|
|
|
| 946 |
else:
|
| 947 |
position_ids = cache_position.unsqueeze(0)
|
| 948 |
|
|
@@ -955,8 +1098,11 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 955 |
"cache_position": cache_position,
|
| 956 |
}
|
| 957 |
)
|
|
|
|
|
|
|
|
|
|
| 958 |
if use_cache is not False:
|
| 959 |
# Generation only reads the final position. Requesting one row keeps
|
| 960 |
# the 100k-way LM head from projecting the whole prompt.
|
| 961 |
-
model_inputs
|
| 962 |
return model_inputs
|
|
|
|
| 54 |
`past_key_value.update()`. A transformers 4.x `StaticCache` otherwise falls
|
| 55 |
back to `arange(q_len)` and the first decode step overwrites the prompt
|
| 56 |
instead of appending after it.
|
| 57 |
+
|
| 58 |
+
Model contract fixes (2026-11-09)
|
| 59 |
+
-------------------------------
|
| 60 |
+
* Generation supports embeddings-only prompts, continuation buffers, explicit
|
| 61 |
+
position advancement, short masks, and requested diagnostic outputs.
|
| 62 |
+
* Attention preserves caller 4D masks, reports requested weights without changing
|
| 63 |
+
SDPA outputs, and trims unused static capacity before expanding KV heads.
|
| 64 |
+
* Default caches follow the hybrid layer configuration on Transformers 5;
|
| 65 |
+
Transformers 4 keeps a plain, crop-compatible full cache. Training
|
| 66 |
+
implements gradient checkpointing and computes causal-LM loss in FP32.
|
| 67 |
+
* Loss honors accumulation-wide token counts; TP reshapes use local head counts.
|
| 68 |
+
* Tied embeddings support Transformers 4/5, and window masks preserve exclusions.
|
| 69 |
+
* Regression tests are maintained in GPT-QModel, not this model repository:
|
| 70 |
+
https://github.com/ModelCloud/GPTQModel/tree/main
|
| 71 |
"""
|
| 72 |
|
| 73 |
import os
|
| 74 |
+
from functools import partial
|
| 75 |
from typing import Optional, Tuple, Union
|
| 76 |
|
| 77 |
import torch
|
|
|
|
| 95 |
# Set when the transformers-5.x `generate()` fixes above are present natively.
|
| 96 |
# Downstream loaders that monkeypatch older checkpoints check this marker and
|
| 97 |
# stand down instead of re-installing the equivalent runtime patch.
|
| 98 |
+
RICURSIVE_TF5_COMPAT = "2026-11-09.1"
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _transformers_major_version() -> int:
|
| 102 |
+
"""Return the installed Transformers major version, or 0 when unknown."""
|
| 103 |
+
try:
|
| 104 |
+
from transformers import __version__ as version
|
| 105 |
+
except ImportError: # pragma: no cover - Transformers is a hard dependency
|
| 106 |
+
return 0
|
| 107 |
+
try:
|
| 108 |
+
return int(version.split(".", 1)[0])
|
| 109 |
+
except (AttributeError, ValueError):
|
| 110 |
+
return 0
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _new_default_cache(config: RicursiveConfig) -> DynamicCache:
|
| 114 |
+
"""Create the default KV cache for a forward pass.
|
| 115 |
+
|
| 116 |
+
Transformers 5 builds a hybrid cache from ``config``: sliding layers keep
|
| 117 |
+
only their window while full layers grow, which bounds long-context decode
|
| 118 |
+
memory. Transformers 4 also accepts ``config`` but its
|
| 119 |
+
``DynamicSlidingWindowLayer`` cannot be cropped after it has seen more
|
| 120 |
+
tokens than the window, so assisted decoding and any cache rollback raise
|
| 121 |
+
``ValueError: Cannot crop a DynamicSlidingWindowLayer``. Keep the plain
|
| 122 |
+
full-attention cache on 4.x, matching the pre-config default.
|
| 123 |
+
"""
|
| 124 |
+
if _transformers_major_version() >= 5:
|
| 125 |
+
try:
|
| 126 |
+
return DynamicCache(config=config)
|
| 127 |
+
except TypeError: # A 5.x build without configured hybrid caches.
|
| 128 |
+
pass
|
| 129 |
+
return DynamicCache()
|
| 130 |
|
| 131 |
|
| 132 |
class RicursiveRMSNorm(nn.Module):
|
|
|
|
| 300 |
return allowed
|
| 301 |
if attention_mask.dtype == torch.bool:
|
| 302 |
return attention_mask & allowed
|
| 303 |
+
# Finite minima would reopen an existing -inf exclusion on an empty row.
|
| 304 |
+
return attention_mask.masked_fill(~allowed, float("-inf"))
|
| 305 |
|
| 306 |
def forward(
|
| 307 |
self,
|
|
|
|
| 330 |
)
|
| 331 |
v = self.v_proj(value_states)
|
| 332 |
|
| 333 |
+
# Infer rank-local head counts from projections after tensor parallel sharding.
|
| 334 |
+
q = q.view(bsz, q_len, -1, self.head_dim)
|
| 335 |
+
k = k.view(bsz, q_len, -1, self.head_dim)
|
| 336 |
+
v = v.view(bsz, q_len, -1, self.head_dim)
|
| 337 |
|
| 338 |
# Per-head Q/K norm (applied before transpose, on head_dim)
|
| 339 |
q = self.q_norm(q)
|
|
|
|
| 374 |
# returns. The mask must use the real positions, not the tensor size.
|
| 375 |
total_kv = _cache_layer_seq_length(past_key_value, self.layer_idx)
|
| 376 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 377 |
# Drop the unwritten tail of a preallocated cache. Without this a
|
| 378 |
# six-token prefill into a sixteen-slot StaticCache looks like a
|
| 379 |
# ten-token prefix, and the end-anchored verify mask below would let
|
|
|
|
| 382 |
k = k[:, :, :total_kv]
|
| 383 |
v = v[:, :, :total_kv]
|
| 384 |
|
| 385 |
+
# Expand only written slots, not the entire capacity of a static cache.
|
| 386 |
+
num_kv_groups = q.shape[1] // k.shape[1]
|
| 387 |
+
k = repeat_kv(k, num_kv_groups)
|
| 388 |
+
v = repeat_kv(v, num_kv_groups)
|
| 389 |
+
|
| 390 |
# Align an incoming mask that is longer than the keys this layer holds.
|
| 391 |
# The shared mask is built over absolute key positions
|
| 392 |
# `0..target_length-1`; keep the slice that matches the keys present.
|
|
|
|
| 447 |
# Attention (use is_causal when no explicit mask for correct causal behavior)
|
| 448 |
# Cast to plain bool to avoid SymBool issues with torch.compile
|
| 449 |
is_causal = bool(attention_mask is None and q.shape[2] > 1)
|
| 450 |
+
# Keep the SDPA result even when returning diagnostic weights. Rebuilding
|
| 451 |
+
# the output from rounded BF16 probabilities changes logits and token IDs.
|
| 452 |
attn_output = F.scaled_dot_product_attention(
|
| 453 |
q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
|
| 454 |
)
|
| 455 |
+
attn_weights = None
|
| 456 |
+
if output_attentions:
|
| 457 |
+
# SDPA does not expose weights; compute them separately only on request.
|
| 458 |
+
weight_keys = k
|
| 459 |
+
with torch.autocast(device_type=q.device.type, enabled=False):
|
| 460 |
+
scores = torch.matmul(q.float(), weight_keys.float().transpose(-1, -2)) * self.head_dim**-0.5
|
| 461 |
+
if is_causal:
|
| 462 |
+
causal = torch.ones(q_len, weight_keys.shape[2], dtype=torch.bool, device=q.device).tril()
|
| 463 |
+
scores = scores.masked_fill(~causal, float("-inf"))
|
| 464 |
+
if attention_mask is not None:
|
| 465 |
+
if attention_mask.dtype == torch.bool:
|
| 466 |
+
scores = scores.masked_fill(~attention_mask, float("-inf"))
|
| 467 |
+
else:
|
| 468 |
+
scores = scores + attention_mask.float()
|
| 469 |
+
attn_weights = F.softmax(scores, dim=-1).nan_to_num().to(q.dtype)
|
| 470 |
|
| 471 |
# (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
|
| 472 |
attn_output = attn_output.transpose(1, 2).contiguous()
|
|
|
|
| 484 |
# Output projection
|
| 485 |
attn_output = self.o_proj(attn_output)
|
| 486 |
|
| 487 |
+
return attn_output, attn_weights, past_key_value
|
| 488 |
|
| 489 |
|
| 490 |
def _mlp_chunk_size() -> int:
|
|
|
|
| 617 |
[RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
|
| 618 |
)
|
| 619 |
self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 620 |
+
self.gradient_checkpointing = False
|
| 621 |
self.post_init()
|
| 622 |
|
| 623 |
def forward(
|
|
|
|
| 645 |
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
| 646 |
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 647 |
|
| 648 |
+
checkpointing = self.gradient_checkpointing and self.training
|
| 649 |
+
if checkpointing:
|
| 650 |
+
if past_key_values is not None:
|
| 651 |
+
raise ValueError("Gradient checkpointing does not support a supplied KV cache")
|
| 652 |
+
# Recomputed layers must not append to a cache for a second time.
|
| 653 |
+
use_cache = False
|
| 654 |
+
|
| 655 |
if inputs_embeds is None:
|
| 656 |
inputs_embeds = self.embed_tokens(input_ids)
|
| 657 |
|
|
|
|
| 665 |
inputs_embeds = inputs_embeds * embed_scale
|
| 666 |
|
| 667 |
if use_cache and past_key_values is None:
|
| 668 |
+
past_key_values = _new_default_cache(self.config)
|
| 669 |
|
| 670 |
if cache_position is None:
|
| 671 |
past_seen_tokens = (
|
|
|
|
| 694 |
if output_hidden_states:
|
| 695 |
all_hidden_states += (hidden_states,)
|
| 696 |
|
| 697 |
+
if checkpointing:
|
| 698 |
+
# Pass the value-residual seed as an input too, so gradients
|
| 699 |
+
# through the embedding path survive reentrant checkpointing.
|
| 700 |
+
layer_outputs = self._gradient_checkpointing_func(
|
| 701 |
+
partial(layer.__call__, **kwargs), hidden_states, causal_mask,
|
| 702 |
+
position_ids, None, output_attentions, False, cache_position,
|
| 703 |
+
value_residual,
|
| 704 |
+
)
|
| 705 |
+
else:
|
| 706 |
+
layer_outputs = layer(
|
| 707 |
+
hidden_states,
|
| 708 |
+
attention_mask=causal_mask,
|
| 709 |
+
position_ids=position_ids,
|
| 710 |
+
past_key_value=past_key_values,
|
| 711 |
+
output_attentions=output_attentions,
|
| 712 |
+
use_cache=use_cache,
|
| 713 |
+
cache_position=cache_position,
|
| 714 |
+
value_residual=value_residual,
|
| 715 |
+
**kwargs,
|
| 716 |
+
)
|
| 717 |
hidden_states = layer_outputs[0]
|
| 718 |
|
| 719 |
if output_attentions:
|
|
|
|
| 752 |
# from float min mask values with large activations.
|
| 753 |
if attention_mask is None:
|
| 754 |
return None
|
| 755 |
+
if attention_mask.dim() == 4:
|
| 756 |
+
# A caller-provided 4D mask already defines visibility (or additive
|
| 757 |
+
# bias). Preserve it; each local layer still applies its window.
|
| 758 |
+
dtype = torch.bool if attention_mask.dtype == torch.bool else input_tensor.dtype
|
| 759 |
+
return attention_mask.to(device=input_tensor.device, dtype=dtype)
|
| 760 |
if torch.all(attention_mask == 1):
|
| 761 |
return None
|
| 762 |
|
|
|
|
| 841 |
|
| 842 |
|
| 843 |
class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
| 844 |
+
# Transformers 5 reads this mapping; Transformers 4 iterates its target keys.
|
| 845 |
+
_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
|
| 846 |
|
| 847 |
# transformers 5 no longer injects `GenerationMixin` into remote-code classes,
|
| 848 |
# so inherit it explicitly above. Without it `generate` is missing entirely and
|
|
|
|
| 884 |
return_dict: Optional[bool] = None,
|
| 885 |
cache_position: Optional[torch.LongTensor] = None,
|
| 886 |
logits_to_keep: Optional[int] = None,
|
| 887 |
+
num_items_in_batch: Optional[torch.Tensor] = None,
|
| 888 |
**kwargs,
|
| 889 |
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 890 |
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
|
|
|
| 925 |
|
| 926 |
loss = None
|
| 927 |
if labels is not None:
|
| 928 |
+
shift_logits = logits[..., :-1, :].float().contiguous()
|
| 929 |
+
shift_labels = labels[..., 1:].to(logits.device).contiguous()
|
| 930 |
loss = F.cross_entropy(
|
| 931 |
shift_logits.view(-1, shift_logits.size(-1)),
|
| 932 |
shift_labels.view(-1),
|
| 933 |
+
reduction="sum" if num_items_in_batch is not None else "mean",
|
| 934 |
)
|
| 935 |
+
if num_items_in_batch is not None:
|
| 936 |
+
# Trainer supplies the token count across all accumulated microbatches.
|
| 937 |
+
loss = loss / torch.as_tensor(num_items_in_batch, device=loss.device)
|
| 938 |
if keep:
|
| 939 |
# Loss used the full sequence; only the returned tensor is trimmed.
|
| 940 |
logits = logits[..., -keep:, :]
|
|
|
|
| 956 |
):
|
| 957 |
"""Advance explicit positions when the generation backend leaves them unchanged."""
|
| 958 |
position_ids = model_kwargs.get("position_ids")
|
| 959 |
+
cache_position = model_kwargs.get("cache_position")
|
| 960 |
model_kwargs = super()._update_model_kwargs_for_generation(
|
| 961 |
outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder,
|
| 962 |
num_new_tokens=num_new_tokens,
|
|
|
|
| 969 |
model_kwargs["position_ids"] = torch.cat(
|
| 970 |
[position_ids, position_ids[:, -1:] + offsets], dim=-1
|
| 971 |
)
|
| 972 |
+
# Transformers 5 no longer maintains cache_position. Keep an explicitly
|
| 973 |
+
# supplied vector in sync, without advancing it twice on Transformers 4.
|
| 974 |
+
if cache_position is not None and model_kwargs.get("cache_position") is cache_position:
|
| 975 |
+
if model_kwargs.get("use_cache", True):
|
| 976 |
+
model_kwargs["cache_position"] = cache_position[-1:] + num_new_tokens
|
| 977 |
+
else:
|
| 978 |
+
offsets = torch.arange(
|
| 979 |
+
1, num_new_tokens + 1, device=cache_position.device, dtype=cache_position.dtype
|
| 980 |
+
)
|
| 981 |
+
model_kwargs["cache_position"] = torch.cat([cache_position, cache_position[-1:] + offsets])
|
| 982 |
return model_kwargs
|
| 983 |
|
| 984 |
def prepare_inputs_for_generation(
|
|
|
|
| 990 |
cache_position: Optional[torch.LongTensor] = None,
|
| 991 |
use_cache: Optional[bool] = None,
|
| 992 |
position_ids: Optional[torch.LongTensor] = None,
|
| 993 |
+
next_sequence_length: Optional[int] = None,
|
| 994 |
+
is_first_iteration: Optional[bool] = None,
|
| 995 |
**kwargs,
|
| 996 |
):
|
| 997 |
+
"""Prepare a full prompt or a new-only block for cached generation.
|
| 998 |
+
|
| 999 |
+
With a warm cache and no explicit ``cache_position``, inputs longer
|
| 1000 |
+
than the cached prefix are full prompts; shorter or equal-length inputs
|
| 1001 |
+
are new-only blocks. For a longer new-only block, pass
|
| 1002 |
+
``torch.arange(past_length, past_length + input_ids.shape[1])`` as
|
| 1003 |
+
``cache_position``. Attention masks may be padded to cache capacity and
|
| 1004 |
+
therefore cannot identify which input layout the caller supplied.
|
| 1005 |
+
"""
|
| 1006 |
# transformers 5.x does not move remote-code inputs onto the model device for
|
| 1007 |
# this model, so a plain CPU `input_ids` reaches the embedding lookup and
|
| 1008 |
# raises. Its own generation helper performs this move, so mirror it here.
|
|
|
|
| 1022 |
|
| 1023 |
past_length = _cache_seq_length(past_key_values)
|
| 1024 |
|
| 1025 |
+
use_embeds = inputs_embeds is not None and (
|
| 1026 |
+
is_first_iteration or past_length == 0 or input_ids is None or input_ids.numel() == 0
|
| 1027 |
+
)
|
| 1028 |
+
input_tensor = inputs_embeds if use_embeds else input_ids
|
| 1029 |
+
input_length = input_tensor.shape[1]
|
| 1030 |
+
|
| 1031 |
if cache_position is None:
|
| 1032 |
+
# Inputs no longer than the cache are treated as new-only blocks.
|
| 1033 |
+
# Longer inputs default to full prompts containing the cached prefix.
|
| 1034 |
+
# Mask width/content cannot distinguish a longer new-only block from
|
| 1035 |
+
# a full prompt with a capacity-sized mask (even an all-visible one).
|
| 1036 |
+
# Callers supplying longer new-only blocks must pass cache_position.
|
| 1037 |
+
full_prompt = past_length > 0 and input_length > past_length
|
| 1038 |
+
new_len = (
|
| 1039 |
+
next_sequence_length if next_sequence_length is not None
|
| 1040 |
+
else input_length - past_length if full_prompt else input_length
|
| 1041 |
+
)
|
| 1042 |
+
cache_position = torch.arange(
|
| 1043 |
+
past_length, past_length + new_len, device=input_tensor.device
|
| 1044 |
+
)
|
| 1045 |
+
else:
|
| 1046 |
+
cache_position = cache_position.to(input_tensor.device)
|
| 1047 |
|
| 1048 |
# Only slice when the cache is warm and `input_ids` still carries cached
|
| 1049 |
# tokens. Slicing unconditionally turned a `None` cache position into
|
| 1050 |
# `input_ids[:, None]` and injected a bogus axis.
|
| 1051 |
+
new_len = cache_position.numel()
|
| 1052 |
+
sliced = past_length > 0 and input_length > new_len
|
| 1053 |
if sliced:
|
| 1054 |
+
# During continuation, input_ids may contain only the new prompt
|
| 1055 |
+
# and generated tokens. Absolute cache offsets cannot index that
|
| 1056 |
+
# shorter buffer. Both layouts end with the unprocessed tokens.
|
| 1057 |
+
input_tensor = input_tensor[:, -new_len:] if new_len else input_tensor[:, :0]
|
| 1058 |
|
| 1059 |
+
if use_embeds:
|
| 1060 |
+
model_inputs = {"inputs_embeds": input_tensor}
|
| 1061 |
else:
|
| 1062 |
+
model_inputs = {"input_ids": input_tensor}
|
| 1063 |
+
|
| 1064 |
+
sequence_length = input_tensor.shape[1]
|
| 1065 |
+
if attention_mask is not None and attention_mask.dim() == 2 and new_len:
|
| 1066 |
+
required_length = int(cache_position[-1]) + 1
|
| 1067 |
+
if attention_mask.shape[-1] < required_length:
|
| 1068 |
+
# A current-block mask omits an implicitly visible cached prefix.
|
| 1069 |
+
attention_mask = F.pad(attention_mask, (required_length - attention_mask.shape[-1], 0), value=1)
|
| 1070 |
|
| 1071 |
# Derive per-row positions from the padding mask. Using `cache_position` for
|
| 1072 |
# every row is only correct when nothing is padded: with left padding the real
|
| 1073 |
# tokens of a short row sit at higher indices, so a shared position vector
|
| 1074 |
# shifts its RoPE phases and the row decodes to garbage.
|
| 1075 |
if position_ids is not None:
|
| 1076 |
+
# Callers may supply positions for the full prompt/capacity, or may
|
| 1077 |
+
# already have selected just the positions for the current block.
|
| 1078 |
+
if position_ids.shape[-1] != sequence_length:
|
| 1079 |
+
if new_len and position_ids.shape[-1] > int(cache_position[-1]):
|
| 1080 |
+
position_ids = position_ids[:, cache_position]
|
| 1081 |
+
else:
|
| 1082 |
+
position_ids = position_ids[:, -sequence_length:] if sequence_length else position_ids[:, :0]
|
| 1083 |
elif attention_mask is not None and attention_mask.dim() == 2:
|
| 1084 |
position_ids = attention_mask.long().cumsum(-1) - 1
|
| 1085 |
position_ids = position_ids.masked_fill(attention_mask == 0, 0)
|
| 1086 |
+
# The mask may include unused cache capacity after the current block.
|
| 1087 |
+
# Select its actual cache slots, not the final entries of that mask.
|
| 1088 |
+
position_ids = position_ids[:, cache_position]
|
| 1089 |
else:
|
| 1090 |
position_ids = cache_position.unsqueeze(0)
|
| 1091 |
|
|
|
|
| 1098 |
"cache_position": cache_position,
|
| 1099 |
}
|
| 1100 |
)
|
| 1101 |
+
# Generation output flags and model-specific forward options must reach
|
| 1102 |
+
# forward. Labels are never generation inputs.
|
| 1103 |
+
model_inputs.update({key: value for key, value in kwargs.items() if key != "labels"})
|
| 1104 |
if use_cache is not False:
|
| 1105 |
# Generation only reads the final position. Requesting one row keeps
|
| 1106 |
# the 100k-way LM head from projecting the whole prompt.
|
| 1107 |
+
model_inputs.setdefault("logits_to_keep", 1)
|
| 1108 |
return model_inputs
|