transformer version clip fix

external/arc2face/models.py

@@ -32,6 +32,7 @@ except ImportError:
 class CLIPTextModelWrapper(CLIPTextModel):
     # Adapted from https://github.com/huggingface/transformers/blob/v4.34.1/src/transformers/models/clip/modeling_clip.py#L812
     # Modified to accept precomputed token embeddings "input_token_embs" as input or calculate them from input_ids and return them.
+    # Supports both transformers <=4.46 (self.text_model sub-attribute) and >=4.47 (flat structure, no text_model).
     def forward(
         self,
         input_ids: Optional[torch.Tensor] = None,
@@ -44,16 +45,16 @@ class CLIPTextModelWrapper(CLIPTextModel):
         return_token_embs: Optional[bool] = False,
     ) -> Union[Tuple, torch.Tensor, BaseModelOutputWithPooling]:
 
+        # In transformers <=4.46 the transformer lives in self.text_model;
+        # in >=4.47 it was inlined directly onto CLIPTextModel (flat structure).
+        tm = getattr(self, 'text_model', self)
+
         if return_token_embs:
-            return self.text_model.embeddings.token_embedding(input_ids)
+            return tm.embeddings.token_embedding(input_ids)
 
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        output_attentions = output_attentions if output_attentions is not None else self.text_model.config.output_attentions
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.text_model.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.text_model.config.use_return_dict
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
 
         if input_ids is None:
             raise ValueError("You have to specify input_ids")
@@ -61,17 +62,13 @@ class CLIPTextModelWrapper(CLIPTextModel):
         input_shape = input_ids.size()
         input_ids = input_ids.view(-1, input_shape[-1])
 
-        hidden_states = self.text_model.embeddings(input_ids=input_ids, position_ids=position_ids, inputs_embeds=input_token_embs)
+        hidden_states = tm.embeddings(input_ids=input_ids, position_ids=position_ids, inputs_embeds=input_token_embs)
 
-        # CLIP's text model uses causal mask, prepare it here.
-        # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324
         causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
-        # expand attention_mask
         if attention_mask is not None:
-            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
             attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
 
-        encoder_outputs = self.text_model.encoder(
+        encoder_outputs = tm.encoder(
             inputs_embeds=hidden_states,
             attention_mask=attention_mask,
             causal_attention_mask=causal_attention_mask,
@@ -81,9 +78,10 @@ class CLIPTextModelWrapper(CLIPTextModel):
         )
 
         last_hidden_state = encoder_outputs[0]
-        last_hidden_state = self.text_model.final_layer_norm(last_hidden_state)
+        last_hidden_state = tm.final_layer_norm(last_hidden_state)
 
-        if self.text_model.eos_token_id == 2:
+        eos_token_id = getattr(tm, 'eos_token_id', self.config.eos_token_id)
+        if eos_token_id == 2:
             pooled_output = last_hidden_state[
                 torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
                 input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),
@@ -91,7 +89,7 @@ class CLIPTextModelWrapper(CLIPTextModel):
         else:
             pooled_output = last_hidden_state[
                 torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
-                (input_ids.to(dtype=torch.int, device=last_hidden_state.device) == self.text_model.eos_token_id)
+                (input_ids.to(dtype=torch.int, device=last_hidden_state.device) == eos_token_id)
                 .int()
                 .argmax(dim=-1),
             ]