""" SAR-specific adapter layers for CLIP. Adds lightweight adapter modules to improve SAR modality handling without modifying the base CLIP weights. """ import torch import torch.nn as nn import torch.nn.functional as F from typing import Optional class SARAdapter(nn.Module): """ Lightweight adapter for SAR imagery. Applies learnable transformations to bridge the domain gap between optical and SAR imagery. Uses: 1. Channel projection (2ch SAR → 3ch RGB-like) 2. Learnable scaling to match CLIP input distribution 3. Optional speckle noise reduction """ def __init__( self, in_channels: int = 2, out_channels: int = 3, hidden_dim: int = 64, dropout: float = 0.1 ): super().__init__() # Channel projection: 2ch (VV, VH) → 3ch (RGB-like) self.channel_proj = nn.Sequential( nn.Conv2d(in_channels, hidden_dim, kernel_size=1, bias=False), nn.BatchNorm2d(hidden_dim), nn.GELU(), nn.Conv2d(hidden_dim, out_channels, kernel_size=1, bias=False), nn.BatchNorm2d(out_channels), ) # Learnable scaling per channel self.channel_scale = nn.Parameter(torch.ones(out_channels)) self.channel_bias = nn.Parameter(torch.zeros(out_channels)) # Optional speckle reduction self.speckle_reduction = nn.Sequential( nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, groups=out_channels), nn.Sigmoid(), ) self.dropout = nn.Dropout2d(dropout) self._init_weights() def _init_weights(self): """Initialize weights with small values.""" for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0) def forward(self, x: torch.Tensor, apply_speckle: bool = True) -> torch.Tensor: """ Forward pass. Args: x: SAR image tensor, shape (B, 2, H, W) with VV, VH channels apply_speckle: Whether to apply speckle reduction Returns: Projected tensor, shape (B, 3, H, W) """ # Channel projection x = self.channel_proj(x) # Apply speckle reduction if apply_speckle: mask = self.speckle_reduction(x) x = x * mask # Apply learnable scaling x = x * self.channel_scale.view(1, -1, 1, 1) + self.channel_bias.view(1, -1, 1, 1) x = self.dropout(x) return x class SARCLIPWrapper(nn.Module): """ Wraps a CLIP model with SAR adapter. Handles the preprocessing pipeline for SAR imagery: 1. Log-scale transformation 2. Speckle reduction 3. Channel projection via SARAdapter """ def __init__( self, clip_model: nn.Module, adapter: Optional[SARAdapter] = None, device: Optional[str] = None ): super().__init__() self.clip_model = clip_model self.adapter = adapter or SARAdapter() self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") self.adapter.to(self.device) def log_scale(self, x: torch.Tensor) -> torch.Tensor: """Apply log-scale transformation to SAR amplitude data.""" return torch.log1p(x) def preprocess_sar(self, x: torch.Tensor) -> torch.Tensor: """ Preprocess SAR imagery for CLIP. Args: x: Raw SAR tensor, shape (B, 2, H, W) Returns: Preprocessed tensor, shape (B, 3, H, W) """ # Log-scale x = self.log_scale(x) # Normalize to [0, 1] range x = x - x.min(dim=-1, keepdim=True)[0].min(dim=-2, keepdim=True)[0] x = x / (x.max(dim=-1, keepdim=True)[0].max(dim=-2, keepdim=True)[0] + 1e-8) # Apply adapter x = self.adapter(x, apply_speckle=True) return x def forward(self, x: torch.Tensor) -> torch.Tensor: """ Forward pass through SAR adapter then CLIP. Args: x: SAR image tensor, shape (B, 2, H, W) Returns: CLIP embedding, shape (B, embed_dim) """ x = self.preprocess_sar(x) return self.clip_model(x) def create_sar_adapter_for_clip( clip_model: nn.Module, in_channels: int = 2, hidden_dim: int = 64 ) -> SARCLIPWrapper: """ Convenience function to create SAR adapter for existing CLIP model. Args: clip_model: Existing CLIP model in_channels: Number of SAR channels (default: 2 for VV/VH) hidden_dim: Hidden dimension in adapter Returns: SARCLIPWrapper with adapter attached """ adapter = SARAdapter( in_channels=in_channels, out_channels=3, hidden_dim=hidden_dim ) return SARCLIPWrapper(clip_model, adapter) # Self-check if __name__ == "__main__": print("Testing SARAdapter...") # Test adapter adapter = SARAdapter(in_channels=2, out_channels=3) # Dummy SAR input (2 channels: VV, VH) x = torch.randn(2, 2, 224, 224) # Forward pass out = adapter(x) print(f"Input shape: {x.shape}") print(f"Output shape: {out.shape}") # Verify output is 3 channels assert out.shape[1] == 3, f"Expected 3 channels, got {out.shape[1]}" # Test log scale wrapper = SARCLIPWrapper.__new__(SARCLIPWrapper) wrapper.adapter = adapter x_log = wrapper.log_scale(x.abs()) # abs() because SAR can have negative values print(f"Log-scaled shape: {x_log.shape}") print(f"Log-scaled range: [{x_log.min():.4f}, {x_log.max():.4f}]") print("\nSARAdapter test passed!")