Fix cache in UniversalTransformerCache

#7
Files changed (1) hide show
  1. modeling_ouro.py +9 -8
modeling_ouro.py CHANGED
@@ -146,21 +146,22 @@ class UniversalTransformerCache(Cache):
146
  ) -> tuple[torch.Tensor, torch.Tensor]:
147
  if layer_idx < 0:
148
  raise ValueError(f"layer_idx must be non-negative, got {layer_idx}")
149
-
150
  if self.max_cache_size is not None and layer_idx >= self.max_cache_size:
151
- raise IndexError(
152
- f"Cache index {layer_idx} exceeds configured max_cache_size={self.max_cache_size}. "
153
- "Check total_ut_steps and num_hidden_layers."
154
- )
155
-
 
156
  # Expand cache storage so the requested index is available.
157
  while len(self.key_cache) <= layer_idx:
158
  self.key_cache.append(None)
159
  self.value_cache.append(None)
160
-
161
  cached_key = self.key_cache[layer_idx]
162
  cached_value = self.value_cache[layer_idx]
163
-
164
  if cached_key is None:
165
  self.key_cache[layer_idx] = key_states
166
  self.value_cache[layer_idx] = value_states
 
146
  ) -> tuple[torch.Tensor, torch.Tensor]:
147
  if layer_idx < 0:
148
  raise ValueError(f"layer_idx must be non-negative, got {layer_idx}")
149
+
150
  if self.max_cache_size is not None and layer_idx >= self.max_cache_size:
151
+ self.max_cache_size = layer_idx + 1
152
+
153
+ while len(self.key_cache) <= layer_idx:
154
+ self.key_cache.append(None)
155
+ self.value_cache.append(None)
156
+
157
  # Expand cache storage so the requested index is available.
158
  while len(self.key_cache) <= layer_idx:
159
  self.key_cache.append(None)
160
  self.value_cache.append(None)
161
+
162
  cached_key = self.key_cache[layer_idx]
163
  cached_value = self.value_cache[layer_idx]
164
+
165
  if cached_key is None:
166
  self.key_cache[layer_idx] = key_states
167
  self.value_cache[layer_idx] = value_states