Upload 6 files

config.json

@@ -1,34 +1,43 @@
 {
-  "model_type": "keep",
-  "vision_config": {
-    "model_type": "vit",
-    "img_size": 224,
-    "patch_size": 16,
-    "hidden_size": 1024,
-    "num_heads": 16,
-    "num_layers": 24,
-    "mlp_ratio": 4.0,
-    "qkv_bias": true,
-    "drop_rate": 0.0,
-    "attn_drop_rate": 0.0,
-    "init_values": 1e-5,
-    "num_classes": 0,
-    "dynamic_img_size": true
+  "architectures": [
+    "KEEPModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "modeling_keep.KEEPConfig",
+    "AutoModel": "modeling_keep.KEEPModel"
   },
+  "model_type": "keep",
+  "projection_dim": 768,
   "text_config": {
-    "model_type": "bert",
-    "vocab_size": 30522,
-    "hidden_size": 768,
-    "num_hidden_layers": 12,
-    "num_attention_heads": 12,
-    "intermediate_size": 3072,
+    "attention_probs_dropout_prob": 0.1,
     "hidden_act": "gelu",
     "hidden_dropout_prob": 0.1,
-    "attention_probs_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
     "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
     "type_vocab_size": 2,
-    "initializer_range": 0.02,
-    "layer_norm_eps": 1e-12
+    "vocab_size": 30522
   },
-  "projection_dim": 768
-}
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.3",
+  "vision_config": {
+    "attn_drop_rate": 0.0,
+    "drop_rate": 0.0,
+    "dynamic_img_size": true,
+    "hidden_size": 1024,
+    "img_size": 224,
+    "init_values": 1e-05,
+    "mlp_ratio": 4.0,
+    "model_type": "vit",
+    "num_classes": 0,
+    "num_heads": 16,
+    "num_layers": 24,
+    "patch_size": 16,
+    "qkv_bias": true
+  }
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8ed817b279417d3c67a842477fcae056212eda4e24350c860fe2ee70d9623fc
-size 1656902036
+oid sha256:82f610d5359aca67b5fd5d841009f26db430ae78d0693743589c0a727b0a146d
+size 1656902084

modeling_keep.py

@@ -6,48 +6,61 @@ import numpy
 from torchvision import transforms
 from PIL import Image
 
+class RenameLayerScale(nn.Module):
+    def __init__(
+            self,
+            dim: int,
+            init_values: float = 1e-5,
+            inplace: bool = False,
+    ) -> None:
+        super().__init__()
+        self.inplace = inplace
+        self.weight = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x.mul_(self.weight) if self.inplace else x * self.weight
+
+timm.models.vision_transformer.LayerScale = RenameLayerScale
+
 class KEEPConfig(PretrainedConfig):
-    model_type = "keep"  # 标记模型类型
+    model_type = "keep"  # 
 
     def __init__(
         self,
-        vision_config=None,  # Vision Encoder 的配置
-        text_config=None,    # Text Encoder 的配置
-        projection_dim=768,  # 投影维度，默认为 768
+        vision_config=None,  # Vision Encoder
+        text_config=None,    # Text Encoder
+        projection_dim=768,  
         **kwargs,
     ):
         super().__init__(**kwargs)
         self.vision_config = vision_config
         self.text_config = text_config
         self.projection_dim = projection_dim
-        
 
 class KEEPModel(PreTrainedModel):
-    config_class = KEEPConfig  # 绑定到自定义配置类
+    config_class = KEEPConfig  # 
 
     def __init__(self, config):
         super().__init__(config)
 
-        # Vision Encoder (基于 timm 的 ViT)
+        # Vision Encoder 
         vision_config = config.vision_config
         self.visual = timm.create_model(
             "vit_large_patch16_224",
             pretrained=False,
-            img_size=vision_config.get("img_size", 224),
-            patch_size=vision_config.get("patch_size", 16),
-            init_values=vision_config.get("init_values", 1e-5),
-            num_classes=vision_config.get("num_classes", 0),
-            dynamic_img_size=vision_config.get("dynamic_img_size", True),
+            img_size=vision_config["img_size"],
+            patch_size=vision_config["patch_size"],
+            init_values=vision_config["init_values"],
+            num_classes=vision_config["num_classes"],
         )
 
-        # 线性投影层，将 Vision Encoder 的输出投影到 768 维
         self.visual_head = nn.Sequential(
                     nn.Linear(self.visual.num_features, config.projection_dim),
                     nn.GELU(),
                     nn.Linear(config.projection_dim, config.projection_dim)
                 )
 
-        # Text Encoder (基于 PubMedBERT)
+        # Text Encoder
         text_config =  BertConfig(**config.text_config)
         self.text = BertModel(text_config)
         
@@ -69,8 +82,7 @@ class KEEPModel(PreTrainedModel):
         
         text_features = self.encode_text(text_inputs)
 
-        # 返回两个独立的特征
         return {
-            "vision_features": vision_features,  # 视觉特征
-            "text_features": text_features       # 文本特征
+            "vision_features": vision_features,  
+            "text_features": text_features       
         }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19a9ef805fcde4f1a255892ed755f960214fb19da59e87d2fc0de49d4683946b
+oid sha256:526a677bf714388d2485a45f5c372505a9874d56a86645b154e2d46ab60d87ca
 size 1657016149