Update modeling_gpt2vision.py

modeling_gpt2vision.py

@@ -7,13 +7,30 @@ from .vision_encoder import VisionEncoder
 from .configuration_gpt2vision import GPT2VisionConfig
 from .modeling_gpt2 import GPT2LMHeadModel
 
+IMAGE_TOKEN = "<image>"
+ANSWER_EOS = "<|endoftext|>"
+
+def resize_token_embeds(model_name="openai-community/gpt2"):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    new_tokens = {
+        "additional_special_tokens": [IMAGE_TOKEN]
+    }
+    tokenizer.add_special_tokens(new_tokens)
+    return tokenizer
+
+tokenizer = resize_token_embeds()
+
 class GPT2Vision(PreTrainedModel):
     config_class = GPT2VisionConfig
 
     def __init__(self, config):
         super().__init__(config)
         self.vision_encoder = VisionEncoder()
-
+        self.language_model.resize_token_embeddings(len(tokenizer))
+        self.tokenizer = tokenizer
+        tokenizer.pad_token = tokenizer.eos_token
+        self.image_token_id = self.tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
+        
         if isinstance(config.gpt2_config, dict):
             gpt2_config = GPT2Config(**config.gpt2_config)
         else: