Update files for transformers integration

config.json

@@ -1,5 +1,4 @@
 {
-  "model_type": "tipsv2",
   "architectures": [
     "TIPSv2Model"
   ],
@@ -7,18 +6,70 @@
     "AutoConfig": "configuration_tips.TIPSv2Config",
     "AutoModel": "modeling_tips.TIPSv2Model"
   },
-  "patch_size": 14,
-  "img_size": 448,
-  "init_values": 1.0,
-  "num_register_tokens": 1,
-  "vocab_size": 32000,
-  "max_len": 64,
-  "vision_fn": "vit_base",
-  "embed_dim": 768,
-  "text_hidden_size": 768,
-  "text_mlp_dim": 3072,
-  "text_num_heads": 12,
-  "text_num_layers": 12,
-  "ffn_layer": "mlp",
-  "temperature": 0.005065968260169029
-}
+  "model_type": "tipsv2",
+  "temperature_init_value": 0.005065968260169029,
+  "text_config": {
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "eos_token_id": null,
+    "hidden_act": "relu",
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-05,
+    "max_position_embeddings": 64,
+    "model_type": "tipsv2_text_model",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 0,
+    "pooling_epsilon": 1e-08,
+    "scale_sqrt_depth": true,
+    "vocab_size": 32000
+  },
+  "transformers_version": "5.10.0.dev0",
+  "vision_config": {
+    "apply_layernorm": true,
+    "attention_probs_dropout_prob": 0.0,
+    "drop_path_rate": 0.0,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 768,
+    "image_size": 448,
+    "initializer_range": 0.02,
+    "interpolate_antialias": true,
+    "interpolate_offset": 0.0,
+    "layer_norm_eps": 1e-06,
+    "layerscale_value": 1.0,
+    "mlp_ratio": 4,
+    "model_type": "tipsv2_vision_model",
+    "num_attention_heads": 12,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_register_tokens": 1,
+    "out_features": [
+      "stage12"
+    ],
+    "out_indices": [
+      12
+    ],
+    "patch_size": 14,
+    "qkv_bias": true,
+    "reshape_hidden_states": true,
+    "stage_names": [
+      "stem",
+      "stage1",
+      "stage2",
+      "stage3",
+      "stage4",
+      "stage5",
+      "stage6",
+      "stage7",
+      "stage8",
+      "stage9",
+      "stage10",
+      "stage11",
+      "stage12"
+    ],
+    "use_swiglu_ffn": false
+  }
+}

configuration_tips.py

@@ -3,6 +3,14 @@
 from transformers import PretrainedConfig
 
 
+_VISION_FN_BY_GEOMETRY = {
+    (768, 12): "vit_base",
+    (1024, 24): "vit_large",
+    (1152, 27): "vit_so400m",
+    (1536, 40): "vit_giant2",
+}
+
+
 class TIPSv2Config(PretrainedConfig):
     """Configuration for TIPSv2 vision-language model."""
 
@@ -10,37 +18,27 @@ class TIPSv2Config(PretrainedConfig):
 
     def __init__(
         self,
-        # Vision encoder
-        vision_fn="vit_base",
-        embed_dim=768,
-        patch_size=14,
-        img_size=448,
-        ffn_layer="mlp",
-        init_values=1.0,
-        num_register_tokens=1,
-        # Text encoder
-        text_hidden_size=768,
-        text_mlp_dim=3072,
-        text_num_heads=12,
-        text_num_layers=12,
-        vocab_size=32000,
-        max_len=64,
-        # Contrastive
-        temperature=0.01,
+        vision_config=None,
+        text_config=None,
+        temperature_init_value=0.01,
         **kwargs,
     ):
         super().__init__(**kwargs)
-        self.vision_fn = vision_fn
-        self.embed_dim = embed_dim
-        self.patch_size = patch_size
-        self.img_size = img_size
-        self.ffn_layer = ffn_layer
-        self.init_values = init_values
-        self.num_register_tokens = num_register_tokens
-        self.text_hidden_size = text_hidden_size
-        self.text_mlp_dim = text_mlp_dim
-        self.text_num_heads = text_num_heads
-        self.text_num_layers = text_num_layers
-        self.vocab_size = vocab_size
-        self.max_len = max_len
-        self.temperature = temperature
+        vision_config = vision_config or {}
+        text_config = text_config or {}
+        hidden_size = vision_config.get("hidden_size", 768)
+        num_hidden_layers = vision_config.get("num_hidden_layers", 12)
+        self.vision_fn = _VISION_FN_BY_GEOMETRY[(hidden_size, num_hidden_layers)]
+        self.embed_dim = hidden_size
+        self.patch_size = vision_config.get("patch_size", 14)
+        self.img_size = vision_config.get("image_size", 448)
+        self.ffn_layer = "swiglu" if vision_config.get("use_swiglu_ffn", False) else "mlp"
+        self.init_values = vision_config.get("layerscale_value", 1.0)
+        self.num_register_tokens = vision_config.get("num_register_tokens", 1)
+        self.text_hidden_size = text_config.get("hidden_size", 768)
+        self.text_mlp_dim = text_config.get("intermediate_size", 3072)
+        self.text_num_heads = text_config.get("num_attention_heads", 12)
+        self.text_num_layers = text_config.get("num_hidden_layers", 12)
+        self.vocab_size = text_config.get("vocab_size", 32000)
+        self.max_len = text_config.get("max_position_embeddings", 64)
+        self.temperature = temperature_init_value

processor_config.json

@@ -0,0 +1,16 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": false,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_processor_type": "Tipsv2ImageProcessor",
+    "resample": 2,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "height": 448,
+      "width": 448
+    }
+  },
+  "processor_class": "Tipsv2Processor"
+}

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "backend": "tokenizers",
+  "bos_token": null,
+  "do_lower_case": true,
+  "eos_token": null,
+  "model_max_length": 64,
+  "pad_token": "<pad>",
+  "processor_class": "Tipsv2Processor",
+  "token_type_ids_pattern": "all_zeros",
+  "tokenizer_class": "Tipsv2Tokenizer",
+  "unk_token": "<unk>"
+}