add support for sequence classification

config.json

@@ -1,13 +1,14 @@
 {
-  "_commit_hash": "58597eee6783b5d4405df333573fe4b4368bce29",
+  "_name_or_path": "babylm/git-2024",
   "architectures": [
     "GitForCausalLM"
   ],
+  "attention_probs_dropout_prob": 0.1,
   "auto_map": {
     "AutoConfig": "configuration_git.GitConfig",
-    "AutoModelForCausalLM": "modeling_git.GitForCausalLM"
+    "AutoModelForCausalLM": "modeling_git.GitForCausalLM",
+    "AutoModelForSequenceClassification": "modeling_git.GitForSequenceClassification"
   },
-  "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 101,
   "classifier_dropout": null,
   "eos_token_id": 102,
@@ -24,11 +25,11 @@
   "num_image_with_embedding": null,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
-  "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": null,
+  "transformers_version": "4.26.0",
   "use_cache": true,
   "vision_config": {
+    "_commit_hash": null,
     "_name_or_path": "",
     "add_cross_attention": false,
     "architectures": null,

modeling_git.py

@@ -7,7 +7,7 @@ import ipdb
 import os
 import torch
 from torch import nn
-from torch.nn import CrossEntropyLoss
+from torch.nn import CrossEntropyLoss, BCEWithLogitsLoss, MSELoss
 from itertools import product
 import numpy as np
 import transformers.models.git.modeling_git as modeling_git
@@ -15,6 +15,7 @@ import transformers.models.vit.modeling_vit as modeling_vit
 from transformers.models.opt.modeling_opt import OPTConfig
 import transformers.models.opt.modeling_opt as hg_opt
 import transformers.models.clip.modeling_clip as modeling_clip
+from transformers.modeling_outputs import SequenceClassifierOutputWithPast
 
 
 class GitForCausalLM(modeling_git.GitForCausalLM):
@@ -98,3 +99,124 @@ class GitForCausalLM(modeling_git.GitForCausalLM):
             hidden_states=outputs.hidden_states,
             attentions=outputs.attentions,
         )
+
+
+class GitForSequenceClassification(modeling_git.GitPreTrainedModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.num_labels = self.config.num_labels
+        self.classifier = nn.Linear(
+                self.config.hidden_size,
+                self.config.num_labels,
+                bias=False)
+        self.post_init()
+        self.git = modeling_git.GitModel(self.config)
+
+        del self.git.image_encoder
+        self.git.image_encoder = ViTModel.from_pretrained('facebook/dino-vitb16')
+        dino_cfg = self.git.image_encoder.config
+        config = self.git.config
+        config.vision_config.hidden_size = dino_cfg.hidden_size
+
+        del self.git.visual_projection
+        self.git.visual_projection = modeling_git.GitProjection(config)
+        num_tks = (dino_cfg.image_size // dino_cfg.patch_size) ** 2 + 1
+        self.git.encoder.layer[0].attention.self.image_patch_tokens = num_tks
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        *args, **kwargs) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.git(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            pixel_values=pixel_values,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            *args, **kwargs)
+        
+        hidden_states = outputs[0]
+        logits = self.classifier(hidden_states)
+
+        if input_ids is not None:
+            batch_size, sequence_length = input_ids.shape[:2]
+        else:
+            batch_size, sequence_length = inputs_embeds.shape[:2]
+
+        if self.config.pad_token_id is None:
+            sequence_lengths = -1
+        else:
+            if input_ids is not None:
+                # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+                sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+                sequence_lengths = sequence_lengths % input_ids.shape[-1]
+                sequence_lengths = sequence_lengths.to(logits.device)
+            else:
+                sequence_lengths = -1
+                # logger.warning(
+                #     f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be "
+                #     "unexpected if using padding tokens in conjunction with `inputs_embeds.`"
+                # )
+
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+
+        loss = None
+        if labels is not None:
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(pooled_logits, labels)
+
+        if not return_dict:
+            output = (pooled_logits,) + outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )