Upload modeling_git.py with huggingface_hub

modeling_git.py

@@ -99,3 +99,58 @@ class GitForCausalLM(modeling_git.GitForCausalLM):
             hidden_states=outputs.hidden_states,
             attentions=outputs.attentions,
         )
+
+class GitModel(modeling_git.GitForCausalLM):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        del self.output
+        self.post_init()
+
+        del self.git.image_encoder
+        self.git.image_encoder = ViTModel.from_pretrained('facebook/dino-vitb16')
+        dino_cfg = self.git.image_encoder.config
+        config = self.git.config
+        config.vision_config.hidden_size = dino_cfg.hidden_size
+
+        del self.git.visual_projection
+        self.git.visual_projection = modeling_git.GitProjection(config)
+        num_tks = (dino_cfg.image_size // dino_cfg.patch_size) ** 2 + 1
+        self.git.encoder.layer[0].attention.self.image_patch_tokens = num_tks
+    
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        past_key_values: Optional[List[torch.Tensor]] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple[torch.Tensor], modeling_git.CausalLMOutputWithPast]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        if labels is not None:
+            use_cache = False
+
+        outputs = self.git(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            pixel_values=pixel_values,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        return outputs
+