feat(transformers): progress on update transformers to 4.55.0

generation.py

@@ -444,7 +444,7 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
                     proc.set_begin_index(decoder_input_ids.shape[-1])
 
             # 6.8 Run generate with fallback
-            seek_sequences, seek_outputs, should_skip, do_condition_on_prev_tokens = self.generate_with_fallback(
+            seek_sequences, seek_outputs, should_skip, do_condition_on_prev_tokens, _model_output_type = self.generate_with_fallback(
                 segment_input=segment_input,
                 decoder_input_ids=decoder_input_ids,
                 cur_bsz=cur_bsz,
@@ -827,7 +827,7 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
         num_beams = beam_scorer.num_beams
 
         batch_beam_size, cur_len = input_ids.shape
-        model_kwargs = self._get_initial_cache_position(input_ids, model_kwargs)
+        model_kwargs = self._get_initial_cache_position(input_ids.shape[-1], input_ids.device, model_kwargs)
 
         if num_beams * batch_size != batch_beam_size:
             raise ValueError(
@@ -890,7 +890,8 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
                 )
                 outputs_per_sub_batch = [
                     self(
-                        **inputs_per_sub_batch,
+                        **{k: v for k, v in inputs_per_sub_batch.items()
+                           if k not in ("output_attentions", "output_hidden_states")},
                         return_dict=True,
                         output_attentions=output_attentions,
                         output_hidden_states=output_hidden_states,
@@ -901,8 +902,10 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
                 outputs = stack_model_outputs(outputs_per_sub_batch)
 
             else:  # Unchanged original behavior
+                _beam_model_inputs = {k: v for k, v in model_inputs.items()
+                                      if k not in ("output_attentions", "output_hidden_states")}
                 outputs = self(
-                    **model_inputs,
+                    **_beam_model_inputs,
                     return_dict=True,
                     output_attentions=output_attentions,
                     output_hidden_states=output_hidden_states,
@@ -1129,13 +1132,16 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
         batch_size = input_ids.shape[0]
         this_peer_finished = False
         unfinished_sequences = torch.ones(batch_size, dtype=torch.long, device=input_ids.device)
-        model_kwargs = self._get_initial_cache_position(input_ids, model_kwargs)
+        model_kwargs = self._get_initial_cache_position(input_ids.shape[-1], input_ids.device, model_kwargs)
 
         while self._has_unfinished_sequences(this_peer_finished, synced_gpus, device=input_ids.device):
             # prepare model inputs
             model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
 
             # forward pass to get next token
+            # Pop keys that we pass explicitly to avoid duplicate keyword argument error
+            model_inputs.pop("output_attentions", None)
+            model_inputs.pop("output_hidden_states", None)
             outputs = self(
                 **model_inputs,
                 return_dict=True,
@@ -1330,7 +1336,7 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
             )
         self.stno_mask_seek = None
 
-        return seek_sequences, seek_outputs, should_skip, do_condition_on_prev_tokens
+        return seek_sequences, seek_outputs, should_skip, do_condition_on_prev_tokens, _model_output_type
 
     def _retrieve_init_tokens(self, input_features, batch_size, generation_config, config, num_segment_frames, kwargs):
         def replace_or_add(lst: List[int], num: int, itr: Iterator[int]):
@@ -1731,6 +1737,8 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
             prev_idx,
             idx,
             return_token_timestamps,
+            time_precision_features=None,
+            decoder_input_ids=None,
     ):
         # find the predicted "end of segment" predictions of Whisper
         # "end of segment" predictions occur whenever Whisper predicts a timestamp token
@@ -1822,7 +1830,8 @@ class DiCoWGenerationMixin(WhisperForConditionalGeneration):
 
         return segments, segment_offset
 
-    def _postprocess_outputs(self, seek_outputs, decoder_input_ids, return_token_timestamps, generation_config):
+    def _postprocess_outputs(self, seek_outputs, decoder_input_ids, return_token_timestamps, generation_config,
+                             is_shortform=False, seek=None, batch_idx_map=None):
         # remove all previously passed decoder input ids
         if isinstance(seek_outputs, torch.Tensor):
             seek_outputs = seek_outputs[:, decoder_input_ids.shape[-1]:]

modeling_dicow.py

@@ -240,6 +240,7 @@ class DiCoWForConditionalGeneration(DiCoWGenerationMixin, WhisperForConditionalG
             output_hidden_states: Optional[bool] = None,
             return_dict: Optional[bool] = None,
             is_valid: Optional[bool] = None,
+            cache_position: Optional[torch.LongTensor] = None,
     ) -> Union[Tuple[torch.Tensor], Seq2SeqLMOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):