import os from siglip_encoder import SigLipVisionTower import torch import torch.nn as nn import re def build_vision_tower(vision_tower_cfg, **kwargs): vision_tower = getattr(vision_tower_cfg, "mm_vision_tower", getattr(vision_tower_cfg, "vision_tower", None)) is_absolute_path_exists = os.path.exists(vision_tower) use_s2 = getattr(vision_tower_cfg, "s2", False) if "siglip" in vision_tower: return SigLipVisionTower(vision_tower, vision_tower_cfg=vision_tower_cfg, **kwargs) raise ValueError(f"Unknown vision tower: {vision_tower}") def build_vision_resampler(confg, **kwargs): ''' act as place holder, useless in our model ''' print(f'No use of vision_resampler') class IdentityMap(nn.Module): def __init__(self): super().__init__() def forward(self, x, *args, **kwargs): return x @property def config(self): return {"mm_projector_type": "identity"} class SimpleResBlock(nn.Module): def __init__(self, channels): super().__init__() self.pre_norm = nn.LayerNorm(channels) self.proj = nn.Sequential(nn.Linear(channels, channels), nn.GELU(), nn.Linear(channels, channels)) def forward(self, x): x = self.pre_norm(x) return x + self.proj(x) def build_vision_projector(config, delay_load=False, **kwargs): projector_type = getattr(config, "mm_projector_type", "linear") if projector_type == "linear": ### this is default return nn.Linear(config.mm_hidden_size, config.hidden_size) if projector_type == "pooler": return PoolerProjector(config, kwargs["vision_cfg"]) mlp_gelu_match = re.match(r"^mlp(\d+)x_gelu$", projector_type) if mlp_gelu_match: ###this is default mlp_depth = int(mlp_gelu_match.group(1)) ###mlx2x_gelu ----> 2 projector modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)] ### 4096 - 4096 - gelu - 4096 for _ in range(1, mlp_depth): modules.append(nn.GELU()) modules.append(nn.Linear(config.hidden_size, config.hidden_size)) return nn.Sequential(*modules) mlp_gelu_resnet_match = re.match(r"^mlp(\d+)x_res(\d+)x_gelu$", projector_type) if mlp_gelu_resnet_match: mlp_depth = int(mlp_gelu_resnet_match.group(1)) res_depth = int(mlp_gelu_resnet_match.group(2)) modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)] for _ in range(1, mlp_depth): modules.append(nn.GELU()) modules.append(nn.Linear(config.hidden_size, config.hidden_size)) for _ in range(res_depth): modules.append(SimpleResBlock(config.hidden_size)) return nn.Sequential(*modules) if projector_type == "identity": return IdentityMap() raise ValueError(f"Unknown projector type: {projector_type}") class PoolerProjector(nn.Module): def __init__(self, config, vision_cfg): super().__init__() self._config = config self.hw = vision_cfg.image_size // vision_cfg.patch_size self.conv_pool = nn.Conv2d(config.mm_hidden_size, config.hidden_size, kernel_size=2, stride=2) self.proj = nn.Sequential( nn.GELU(), nn.Linear(config.hidden_size, config.hidden_size), ) def forward(self, x, *args, **kwargs): height = width = self.hw assert height * width == x.shape[1] x = x.view(x.shape[0], height, width, -1).permute(0, 3, 1, 2) x = self.conv_pool(x) x = x.flatten(2).transpose(1, 2) x = self.proj(x) return x @property def config(self): return {"mm_projector_type": "pooler"}