update wemm

configuration_wemm.py

@@ -63,8 +63,3 @@ class WeMMConfig(PretrainedConfig):
             self.tokenizer_config = tokenizer_config
         
         super().__init__(**kwargs)
-
-if __name__=="__main__":
-    wemm_config_path = "/mnt/csp/mmvision/home/feipengma/projects/wemm_evaluation/WeMM/config.json"
-    wemm_config = WeMMConfig.from_pretrained(wemm_config_path)
-    print(wemm_config.connector_config)

connector.py

@@ -668,15 +668,6 @@ class Idefics2Qformer(nn.Module):
                     False,
                     False,
                     use_reentrant=True)
-            #layer_outputs = perceiver_layer(
-            #    compressed_context,
-            #    context,
-            #    attention_mask=attention_mask,
-            #    position_ids=None,
-            #    past_key_value=None,
-            #    output_attentions=False,
-            #    use_cache=False,
-            #)
             compressed_context = layer_outputs[0]
             #all_latents.append(compressed_context)
 
@@ -710,7 +701,7 @@ class Idefics2Connector(PreTrainedModel):
         return reshaped_image_hidden_states
     
     @classmethod
-    def from_pretrained(self, config_path="/mnt/csp/mmvision/home/arrayyang/idefics2-8b/idefics2_connector"):
+    def from_pretrained(self, config_path):
         config = Idefics2ConnectorConfig.from_pretrained(f'{config_path}/config.json')
         cls = Idefics2Connector(config=config)
         

image_processor.py

@@ -636,7 +636,7 @@ class Idefics2ImageProcessor(BaseImageProcessor):
         return BatchFeature(data=data, tensor_type=return_tensors)
     
     @classmethod
-    def from_pretrained(self, config_path="/mnt/csp/mmvision/home/arrayyang/idefics2-8b/idefics2_image_processor"):
+    def from_pretrained(self, config_path):
         with open(f'{config_path}/config.json', "r", encoding="utf-8") as f:
             config = json.load(f)
         

vision_model.py

@@ -715,14 +715,3 @@ class Idefics2VisionTransformer(PreTrainedModel):
             hidden_states=encoder_outputs.hidden_states,
             attentions=encoder_outputs.attentions,
         )
-    """
-    @classmethod
-    def from_pretrained(self, config_path="/mnt/csp/mmvision/home/arrayyang/idefics2-8b/idefics2_vision_model"):
-        config = Idefics2VisionConfig.from_pretrained(f'{config_path}/config.json')
-        cls = Idefics2VisionTransformer(config=config)
-        
-        state_dict = torch.load(f'{config_path}/vision_model.pth', map_location='cpu')
-        ret = cls.load_state_dict(state_dict, strict=False)
-        print("Loading idefics2 Vision Model: {}".format(config_path))
-        return cls
-    """