Upload 2 files

configuration_centurio.py

@@ -37,7 +37,7 @@ class CenturioConfig(PretrainedConfig):
         ignore_index=-100,
         image_token_index=32000,
         adapter_type="multiscale-pool",
-        adapter_config=None,
+        adapter_config=dict(),
         **kwargs,
     ):
         self.ignore_index = ignore_index

modeling_centurio.py

@@ -74,7 +74,7 @@ class LlavaMultiModalAdapter(nn.Module):
 class WindowMLPProjector(nn.Module):
     def __init__(self, config: LlavaConfig):
         super().__init__()
-        self.multi_scale = getattr(config, "adapter_multi_scale", 2)
+        self.multi_scale = config.adapter_config.get("multi_scale", 2) #config.adapter_config.get("multi_scale")
         self.linear_1 = nn.Linear(config.image_hidden_size, config.text_config.hidden_size, bias=True)
         self.act = ACT2FN["gelu"]
         self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True)
@@ -93,7 +93,7 @@ class WindowMLPProjector(nn.Module):
 class WindowPoolProjector(nn.Module):
     def __init__(self, config: LlavaConfig):
         super().__init__()
-        self.multi_scale = getattr(config, "adapter_multi_scale", 2)
+        self.multi_scale = config.adapter_config.get("multi_scale", 2) #config.adapter_config.get("multi_scale")
         self.pool = nn.AdaptiveAvgPool2d(getattr(config, "adapter_pool", 8))
         self.linear_1 = nn.Linear(config.image_hidden_size, config.text_config.hidden_size, bias=True)
         self.act = ACT2FN["gelu"]
@@ -119,7 +119,7 @@ class WindowPoolProjector(nn.Module):
 class WindowShuffelProjector(nn.Module):
     def __init__(self, config: LlavaConfig):
         super().__init__()
-        self.multi_scale = getattr(config, "adapter_multi_scale", 2)
+        self.multi_scale = config.adapter_config.get("multi_scale", 2) #config.adapter_config.get("multi_scale")
         self.scale_factor = getattr(config, "adapter_pool", 2)
         self.pixel_unshuffel = nn.PixelUnshuffle(self.scale_factor)
         self.linear_1 = nn.Linear(config.image_hidden_size*(self.scale_factor**2), config.text_config.hidden_size, bias=True)
@@ -148,7 +148,7 @@ class MultiscalePoolProjector(nn.Module):
     def __init__(self, config: LlavaConfig):
         super().__init__()
 
-        self.multi_scale = getattr(config, "adapter_multi_scale", 2)
+        self.multi_scale = config.adapter_config.get("multi_scale", 2) #getattr(config.adapter_config, "adapter_multi_scale", 2)
         self.pool = nn.AvgPool2d(self.multi_scale)
         self.linear_1 = nn.Linear(config.image_hidden_size*2, config.text_config.hidden_size, bias=True)
         self.act = ACT2FN["gelu"]
@@ -181,7 +181,7 @@ class MultiscaleShuffleProjector(nn.Module):
     def __init__(self, config):
         super().__init__()
 
-        self.multi_scale = getattr(config, "adapter_multi_scale", 2)
+        self.multi_scale = config.adapter_config.get("multi_scale", 2) #config.adapter_config.get("multi_scale")
         self.shuffle = nn.PixelUnshuffle(self.multi_scale)
 
         inc, ouc = config.image_hidden_size*(1+self.multi_scale**2), config.text_config.hidden_size
@@ -447,7 +447,8 @@ class CenturioForConditionalGeneration(LlavaPreTrainedModel):
         self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1
         self.post_init()
 
-
+    def tie_weights(self):
+        return self.language_model.tie_weights()
 
     def get_input_embeddings(self):
         return self.language_model.get_input_embeddings()