Keep a crop-compatible default cache on Transformers 4
Browse filesTransformers 4.57 accepts DynamicCache(config=...) and builds hybrid
sliding-window layers. Those layers cannot be cropped once they have seen
more tokens than the window, so assisted decoding fails with
"ValueError: Cannot crop a DynamicSlidingWindowLayer" when generation reuses
a cache returned by a direct forward. The constructor succeeds, so the
existing TypeError fallback never ran.
Only build the configured hybrid cache on Transformers 5, where rollback is
supported. Transformers 4 keeps the plain full-attention DynamicCache used
before the hybrid default, which preserves cropping and rollback.
Verified with a five-layer tiny model (four sliding, one full,
sliding_window=8): the reported prompt-lookup reproduction now matches plain
greedy generation on 4.57.6 and still uses the hybrid cache on 5.22.1.
- modeling_ricursive.py +35 -6
|
@@ -61,8 +61,9 @@ Model contract fixes (2026-11-09)
|
|
| 61 |
position advancement, short masks, and requested diagnostic outputs.
|
| 62 |
* Attention preserves caller 4D masks, reports requested weights without changing
|
| 63 |
SDPA outputs, and trims unused static capacity before expanding KV heads.
|
| 64 |
-
* Default caches follow the hybrid layer configuration
|
| 65 |
-
|
|
|
|
| 66 |
* Loss honors accumulation-wide token counts; TP reshapes use local head counts.
|
| 67 |
* Tied embeddings support Transformers 4/5, and window masks preserve exclusions.
|
| 68 |
* Regression tests are maintained in GPT-QModel, not this model repository:
|
|
@@ -97,6 +98,37 @@ from .configuration_ricursive import RicursiveConfig
|
|
| 97 |
RICURSIVE_TF5_COMPAT = "2026-11-09.1"
|
| 98 |
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
class RicursiveRMSNorm(nn.Module):
|
| 101 |
def __init__(self, hidden_size: int, eps: float = 1e-6):
|
| 102 |
super().__init__()
|
|
@@ -633,10 +665,7 @@ class RicursiveModel(RicursivePreTrainedModel):
|
|
| 633 |
inputs_embeds = inputs_embeds * embed_scale
|
| 634 |
|
| 635 |
if use_cache and past_key_values is None:
|
| 636 |
-
|
| 637 |
-
past_key_values = DynamicCache(config=self.config)
|
| 638 |
-
except TypeError: # Older Transformers without configured hybrid caches.
|
| 639 |
-
past_key_values = DynamicCache()
|
| 640 |
|
| 641 |
if cache_position is None:
|
| 642 |
past_seen_tokens = (
|
|
|
|
| 61 |
position advancement, short masks, and requested diagnostic outputs.
|
| 62 |
* Attention preserves caller 4D masks, reports requested weights without changing
|
| 63 |
SDPA outputs, and trims unused static capacity before expanding KV heads.
|
| 64 |
+
* Default caches follow the hybrid layer configuration on Transformers 5;
|
| 65 |
+
Transformers 4 keeps a plain, crop-compatible full cache. Training
|
| 66 |
+
implements gradient checkpointing and computes causal-LM loss in FP32.
|
| 67 |
* Loss honors accumulation-wide token counts; TP reshapes use local head counts.
|
| 68 |
* Tied embeddings support Transformers 4/5, and window masks preserve exclusions.
|
| 69 |
* Regression tests are maintained in GPT-QModel, not this model repository:
|
|
|
|
| 98 |
RICURSIVE_TF5_COMPAT = "2026-11-09.1"
|
| 99 |
|
| 100 |
|
| 101 |
+
def _transformers_major_version() -> int:
|
| 102 |
+
"""Return the installed Transformers major version, or 0 when unknown."""
|
| 103 |
+
try:
|
| 104 |
+
from transformers import __version__ as version
|
| 105 |
+
except ImportError: # pragma: no cover - Transformers is a hard dependency
|
| 106 |
+
return 0
|
| 107 |
+
try:
|
| 108 |
+
return int(version.split(".", 1)[0])
|
| 109 |
+
except (AttributeError, ValueError):
|
| 110 |
+
return 0
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _new_default_cache(config: RicursiveConfig) -> DynamicCache:
|
| 114 |
+
"""Create the default KV cache for a forward pass.
|
| 115 |
+
|
| 116 |
+
Transformers 5 builds a hybrid cache from ``config``: sliding layers keep
|
| 117 |
+
only their window while full layers grow, which bounds long-context decode
|
| 118 |
+
memory. Transformers 4 also accepts ``config`` but its
|
| 119 |
+
``DynamicSlidingWindowLayer`` cannot be cropped after it has seen more
|
| 120 |
+
tokens than the window, so assisted decoding and any cache rollback raise
|
| 121 |
+
``ValueError: Cannot crop a DynamicSlidingWindowLayer``. Keep the plain
|
| 122 |
+
full-attention cache on 4.x, matching the pre-config default.
|
| 123 |
+
"""
|
| 124 |
+
if _transformers_major_version() >= 5:
|
| 125 |
+
try:
|
| 126 |
+
return DynamicCache(config=config)
|
| 127 |
+
except TypeError: # A 5.x build without configured hybrid caches.
|
| 128 |
+
pass
|
| 129 |
+
return DynamicCache()
|
| 130 |
+
|
| 131 |
+
|
| 132 |
class RicursiveRMSNorm(nn.Module):
|
| 133 |
def __init__(self, hidden_size: int, eps: float = 1e-6):
|
| 134 |
super().__init__()
|
|
|
|
| 665 |
inputs_embeds = inputs_embeds * embed_scale
|
| 666 |
|
| 667 |
if use_cache and past_key_values is None:
|
| 668 |
+
past_key_values = _new_default_cache(self.config)
|
|
|
|
|
|
|
|
|
|
| 669 |
|
| 670 |
if cache_position is None:
|
| 671 |
past_seen_tokens = (
|