Add files using upload-large-folder tool

config.json

@@ -0,0 +1,33 @@
+{
+  "ape": false,
+  "architectures": [
+    "RSPSwinForImageClassification"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_rsp.RSPSwinConfig",
+    "AutoModelForImageClassification": "modeling_rsp.RSPSwinForImageClassification"
+  },
+  "depths": [
+    2,
+    2,
+    6,
+    2
+  ],
+  "embed_dim": 96,
+  "image_size": 224,
+  "mlp_ratio": 4.0,
+  "model_type": "rsp_swin",
+  "num_channels": 3,
+  "num_heads": [
+    3,
+    6,
+    12,
+    24
+  ],
+  "num_labels": 51,
+  "patch_norm": true,
+  "patch_size": 4,
+  "qkv_bias": true,
+  "source_checkpoint": "/mnt/data/projects/model_hubs/raw/rsp-swin-t-ckpt.pth",
+  "window_size": 7
+}

configuration_rsp.py

@@ -0,0 +1,99 @@
+"""Configuration classes for RSP models compatible with transformers"""
+
+from transformers import PretrainedConfig
+
+
+class RSPResNetConfig(PretrainedConfig):
+    """Configuration for RSP ResNet models"""
+    
+    model_type = "rsp_resnet"
+    
+    def __init__(
+        self,
+        block="Bottleneck",
+        layers=[3, 4, 6, 3],
+        image_size=224,
+        num_channels=3,
+        num_labels=51,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.block = block
+        self.layers = layers
+        self.image_size = image_size
+        self.num_channels = num_channels
+        self.num_labels = num_labels
+
+
+class RSPSwinConfig(PretrainedConfig):
+    """Configuration for RSP Swin Transformer models"""
+    
+    model_type = "rsp_swin"
+    
+    def __init__(
+        self,
+        image_size=224,
+        patch_size=4,
+        num_channels=3,
+        embed_dim=96,
+        depths=[2, 2, 6, 2],
+        num_heads=[3, 6, 12, 24],
+        window_size=7,
+        mlp_ratio=4.0,
+        qkv_bias=True,
+        ape=False,
+        patch_norm=True,
+        num_labels=51,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.image_size = image_size
+        self.patch_size = patch_size
+        self.num_channels = num_channels
+        self.embed_dim = embed_dim
+        self.depths = depths
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.mlp_ratio = mlp_ratio
+        self.qkv_bias = qkv_bias
+        self.ape = ape
+        self.patch_norm = patch_norm
+        self.num_labels = num_labels
+
+
+class RSPViTAEConfig(PretrainedConfig):
+    """Configuration for RSP ViTAE models"""
+    
+    model_type = "rsp_vitae"
+    
+    def __init__(
+        self,
+        image_size=224,
+        num_channels=3,
+        stages=4,
+        embed_dims=[64, 64, 128, 256],
+        token_dims=[64, 128, 256, 512],
+        downsample_ratios=[4, 2, 2, 2],
+        NC_depth=[2, 2, 8, 2],
+        NC_heads=[1, 2, 4, 8],
+        RC_heads=[1, 1, 2, 4],
+        NC_group=[1, 32, 64, 128],
+        RC_group=[1, 16, 32, 64],
+        mlp_ratio=4.0,
+        num_labels=51,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.image_size = image_size
+        self.num_channels = num_channels
+        self.stages = stages
+        self.embed_dims = embed_dims
+        self.token_dims = token_dims
+        self.downsample_ratios = downsample_ratios
+        self.NC_depth = NC_depth
+        self.NC_heads = NC_heads
+        self.RC_heads = RC_heads
+        self.NC_group = NC_group
+        self.RC_group = RC_group
+        self.mlp_ratio = mlp_ratio
+        self.num_labels = num_labels

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:175ce1246be89814fbd96872eb438367fd2099e01816fab1305d8563fdedf17c
+size 111367532

modeling_rsp.py

@@ -0,0 +1,266 @@
+"""Model classes for RSP models compatible with transformers"""
+
+import sys
+import os
+from pathlib import Path
+import torch
+import torch.nn as nn
+from transformers import PreTrainedModel
+from safetensors.torch import load_file
+
+# Import local modular model
+from modular_swin import SwinTransformer
+
+# Import other models from sibling directories if needed
+_parent_dir = Path(__file__).parent.parent
+import importlib.util
+
+# Import ResNet from RSP-ResNet-50
+_resnet_path = _parent_dir / "RSP-ResNet-50" / "modular_resnet.py"
+if _resnet_path.exists():
+    spec = importlib.util.spec_from_file_location("modular_resnet_resnet", _resnet_path)
+    resnet_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(resnet_module)
+    ResNet = resnet_module.ResNet
+    Bottleneck = resnet_module.Bottleneck
+else:
+    ResNet = None
+    Bottleneck = None
+
+# Import ViTAE from RSP-ViTAEv2-S
+_vitae_path = _parent_dir / "RSP-ViTAEv2-S" / "modular_vitae_window_noshift.py"
+if _vitae_path.exists():
+    spec = importlib.util.spec_from_file_location("modular_vitae_window_noshift_vitae", _vitae_path)
+    vitae_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(vitae_module)
+    ViTAE_Window_NoShift_12_basic_stages4_14 = vitae_module.ViTAE_Window_NoShift_12_basic_stages4_14
+else:
+    ViTAE_Window_NoShift_12_basic_stages4_14 = None
+
+# Import configuration - handle both relative and absolute imports
+try:
+    from configuration_rsp import RSPResNetConfig, RSPSwinConfig, RSPViTAEConfig
+except ImportError:
+    # Fallback: import from same directory
+    import importlib.util
+    config_path = Path(__file__).parent / "configuration_rsp.py"
+    spec = importlib.util.spec_from_file_location("configuration_rsp", config_path)
+    config_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(config_module)
+    RSPResNetConfig = config_module.RSPResNetConfig
+    RSPSwinConfig = config_module.RSPSwinConfig
+    RSPViTAEConfig = config_module.RSPViTAEConfig
+
+
+class RSPResNetForImageClassification(PreTrainedModel):
+    """RSP ResNet model for image classification"""
+    
+    config_class = RSPResNetConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        
+        # Build ResNet model from config
+        block = Bottleneck if config.block == "Bottleneck" else None
+        if block is None:
+            raise ValueError(f"Unsupported block type: {config.block}")
+        
+        self.model = ResNet(
+            block=block,
+            layers=config.layers,
+            num_classes=config.num_labels
+        )
+    
+    def forward(self, pixel_values=None, labels=None, **kwargs):
+        """
+        Args:
+            pixel_values: Input images (B, C, H, W)
+            labels: Optional labels for loss computation
+        """
+        if pixel_values is None:
+            raise ValueError("pixel_values must be provided")
+        
+        logits = self.model(pixel_values)
+        
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1))
+        
+        return {
+            "logits": logits,
+            "loss": loss
+        } if loss is not None else {"logits": logits}
+    
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        """Load model from pretrained checkpoint"""
+        config = kwargs.pop("config", None)
+        if config is None:
+            config = RSPResNetConfig.from_pretrained(pretrained_model_name_or_path)
+        
+        model = cls(config)
+        
+        # Load weights from safetensors
+        model_path = Path(pretrained_model_name_or_path)
+        safetensors_path = model_path / "model.safetensors"
+        
+        if safetensors_path.exists():
+            state_dict = load_file(str(safetensors_path))
+            # Remove 'model.' prefix if present
+            state_dict_clean = {}
+            for k, v in state_dict.items():
+                if k.startswith("model."):
+                    state_dict_clean[k[6:]] = v
+                else:
+                    state_dict_clean[k] = v
+            model.model.load_state_dict(state_dict_clean, strict=False)
+        else:
+            raise FileNotFoundError(f"Model weights not found at {safetensors_path}")
+        
+        return model
+
+
+class RSPSwinForImageClassification(PreTrainedModel):
+    """RSP Swin Transformer model for image classification"""
+    
+    config_class = RSPSwinConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        
+        # Build SwinTransformer model from config
+        self.model = SwinTransformer(
+            img_size=config.image_size,
+            patch_size=config.patch_size,
+            in_chans=config.num_channels,
+            num_classes=config.num_labels,
+            embed_dim=config.embed_dim,
+            depths=config.depths,
+            num_heads=config.num_heads,
+            window_size=config.window_size,
+            mlp_ratio=config.mlp_ratio,
+            qkv_bias=config.qkv_bias,
+            ape=config.ape,
+            patch_norm=config.patch_norm,
+        )
+    
+    def forward(self, pixel_values=None, labels=None, **kwargs):
+        """
+        Args:
+            pixel_values: Input images (B, C, H, W)
+            labels: Optional labels for loss computation
+        """
+        if pixel_values is None:
+            raise ValueError("pixel_values must be provided")
+        
+        logits = self.model(pixel_values)
+        
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1))
+        
+        return {
+            "logits": logits,
+            "loss": loss
+        } if loss is not None else {"logits": logits}
+    
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        """Load model from pretrained checkpoint"""
+        config = kwargs.pop("config", None)
+        if config is None:
+            config = RSPSwinConfig.from_pretrained(pretrained_model_name_or_path)
+        
+        model = cls(config)
+        
+        # Load weights from safetensors
+        model_path = Path(pretrained_model_name_or_path)
+        safetensors_path = model_path / "model.safetensors"
+        
+        if safetensors_path.exists():
+            state_dict = load_file(str(safetensors_path))
+            # Remove 'model.' prefix if present
+            state_dict_clean = {}
+            for k, v in state_dict.items():
+                if k.startswith("model."):
+                    state_dict_clean[k[6:]] = v
+                else:
+                    state_dict_clean[k] = v
+            model.model.load_state_dict(state_dict_clean, strict=False)
+        else:
+            raise FileNotFoundError(f"Model weights not found at {safetensors_path}")
+        
+        return model
+
+
+class RSPViTAEForImageClassification(PreTrainedModel):
+    """RSP ViTAE model for image classification"""
+    
+    config_class = RSPViTAEConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        
+        # Build ViTAE model from config
+        # Note: ViTAE_Window_NoShift_12_basic_stages4_14 already sets most parameters as defaults:
+        # - stages=4, embed_dims=[64, 64, 128, 256], token_dims=[64, 128, 256, 512]
+        # - downsample_ratios=[4, 2, 2, 2], NC_depth=[2, 2, 8, 2], etc.
+        # We only pass parameters that need to be overridden (img_size, num_classes)
+        # The function accepts **kwargs, so we can pass window_size if needed
+        self.model = ViTAE_Window_NoShift_12_basic_stages4_14(
+            pretrained=False,
+            img_size=config.image_size,
+            num_classes=config.num_labels,
+            window_size=7,
+        )
+    
+    def forward(self, pixel_values=None, labels=None, **kwargs):
+        """
+        Args:
+            pixel_values: Input images (B, C, H, W)
+            labels: Optional labels for loss computation
+        """
+        if pixel_values is None:
+            raise ValueError("pixel_values must be provided")
+        
+        logits = self.model(pixel_values)
+        
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1))
+        
+        return {
+            "logits": logits,
+            "loss": loss
+        } if loss is not None else {"logits": logits}
+    
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        """Load model from pretrained checkpoint"""
+        config = kwargs.pop("config", None)
+        if config is None:
+            config = RSPViTAEConfig.from_pretrained(pretrained_model_name_or_path)
+        
+        model = cls(config)
+        
+        # Load weights from safetensors
+        model_path = Path(pretrained_model_name_or_path)
+        safetensors_path = model_path / "model.safetensors"
+        
+        if safetensors_path.exists():
+            state_dict = load_file(str(safetensors_path))
+            # Remove 'model.' prefix if present
+            state_dict_clean = {}
+            for k, v in state_dict.items():
+                if k.startswith("model."):
+                    state_dict_clean[k[6:]] = v
+                else:
+                    state_dict_clean[k] = v
+            model.model.load_state_dict(state_dict_clean, strict=False)
+        else:
+            raise FileNotFoundError(f"Model weights not found at {safetensors_path}")
+        
+        return model

modular_swin.py

@@ -0,0 +1,650 @@
+# --------------------------------------------------------
+# Swin Transformer
+# Copyright (c) 2021 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Ze Liu
+# --------------------------------------------------------
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint as checkpoint
+
+# Use transformers equivalents instead of timm
+from transformers.models.swin.modeling_swin import SwinDropPath as DropPath
+from transformers import initialization as init
+
+# Simple to_2tuple replacement (no external dependency needed)
+def to_2tuple(x):
+    """Convert input to 2-tuple if not already a tuple."""
+    return x if isinstance(x, tuple) else (x, x)
+
+# Use transformers trunc_normal_ for initialization
+trunc_normal_ = init.trunc_normal_
+
+
+class Mlp(nn.Module):
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+def window_partition(x, window_size):
+    """
+    Args:
+        x: (B, H, W, C)
+        window_size (int): window size
+
+    Returns:
+        windows: (num_windows*B, window_size, window_size, C)
+    """
+    # 按windows分块
+    B, H, W, C = x.shape
+    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows
+
+
+def window_reverse(windows, window_size, H, W):
+    """
+    Args:
+        windows: (num_windows*B, window_size, window_size, C)
+        window_size (int): Window size
+        H (int): Height of image
+        W (int): Width of image
+
+    Returns:
+        x: (B, H, W, C)
+    """
+    # 将windows形式变成2D特征形式
+    B = int(windows.shape[0] / (H * W / window_size / window_size))
+    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
+    return x
+
+
+class WindowAttention(nn.Module):
+    r""" Window based multi-head self attention (W-MSA) module with relative position bias.
+    It supports both of shifted and non-shifted window.
+
+    Args:
+        dim (int): Number of input channels.
+        window_size (tuple[int]): The height and width of the window.
+        num_heads (int): Number of attention heads.
+        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set
+        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
+        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
+    """
+
+    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):
+
+        super().__init__()
+        self.dim = dim
+        self.window_size = window_size  # Wh, Ww
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+
+        # define a parameter table of relative position bias
+        # 相对位置表, 对于每个head, 大小为(2wh-1)*(2ww-1)
+        # 大于索引的最大值(2wh-2)*(2ww-1), 可以保证寻址
+        self.relative_position_bias_table = nn.Parameter(
+            torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads))  # 2*Wh-1 * 2*Ww-1, nH
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = torch.arange(self.window_size[0])
+        coords_w = torch.arange(self.window_size[1])
+        # coods: (2, windwows_size, windows_size)
+        # 在一个windows size的窗口内，生成每个位置的行列号
+        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+        # 窗口内每个位置的行列号展平
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        # 窗口内每个位置的的行列号与窗口内所有位置的行列号的差值
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        # 差值范围从[1-ws, ws-1]->[0, 2ws-2]
+        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += self.window_size[1] - 1
+        # 行号差值范围[0, 2wh-2] -> [0, (2wh-2)(ww-1)]
+        # 列号差值范围仍然为[0, 2ww-2]
+        # 乘法操作是为了区分沿主对���线对称的像素，此类像素在将行列号转换成一维偏移时值相同
+        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
+        # 获得相对位置索引，尺寸为(ws*ws, ws*ws)
+        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        self.register_buffer("relative_position_index", relative_position_index)
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+        init.trunc_normal_(self.relative_position_bias_table, std=0.02)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x, mask=None):
+        """
+        Args:
+            x: input features with shape of (num_windows*B, N, C)
+            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
+        """
+        B_, N, C = x.shape
+        # N是一个窗口内的token数
+        # qkv: B, N, 3, H, C/H->3, B, H, N, C/H
+        qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        # q/k/v: B, H, N, C', 其中C' = C/H
+        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
+
+        q = q * self.scale
+        attn = (q @ k.transpose(-2, -1)) # B,H,N,N
+
+        # 索引表： (2wh-1)*(2ww-1), H
+        # 相对索引：wh*ww, wh*ww， 索引范围[0,(2wh-2)*(2ww-1)]
+        # 取出相对位置向量 (wh*ww, wh*ww, H)
+        relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH
+        # 相对位置向量尺寸变换 (wh*ww, wh*ww, H) -> (H, wh*ww, wh*ww)
+        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+        attn = attn + relative_position_bias.unsqueeze(0)
+
+        if mask is not None:
+            nW = mask.shape[0]
+            # nw是窗口的数量
+            # 加了-100之后。softmax生成的权值很小，相当于这部分被忽略掉
+            # mask: nW, ws*ws, ws*ws -> 1, nW, 1, ws*ws, ws*ws
+            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
+            attn = attn.view(-1, self.num_heads, N, N)
+            attn = self.softmax(attn)
+        else:
+            attn = self.softmax(attn)
+
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+    def extra_repr(self) -> str:
+        return f'dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}'
+
+    def flops(self, N):
+        # calculate flops for 1 window with token length of N
+        flops = 0
+        # qkv = self.qkv(x)
+        flops += N * self.dim * 3 * self.dim
+        # attn = (q @ k.transpose(-2, -1))
+        flops += self.num_heads * N * (self.dim // self.num_heads) * N
+        #  x = (attn @ v)
+        flops += self.num_heads * N * N * (self.dim // self.num_heads)
+        # x = self.proj(x)
+        flops += N * self.dim * self.dim
+        return flops
+
+
+class SwinTransformerBlock(nn.Module):
+    r""" Swin Transformer Block.
+
+    Args:
+        dim (int): Number of input channels.
+        input_resolution (tuple[int]): Input resulotion.
+        num_heads (int): Number of attention heads.
+        window_size (int): Window size.
+        shift_size (int): Shift size for SW-MSA.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float, optional): Stochastic depth rate. Default: 0.0
+        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+
+    def __init__(self, dim, input_resolution, num_heads, window_size=7, shift_size=0,
+                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0.,
+                 act_layer=nn.GELU, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.input_resolution = input_resolution
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.shift_size = shift_size # 图片平移的距离
+        self.mlp_ratio = mlp_ratio
+        # 特征大小小于窗口，就不分窗口了，也不平移图片了
+        if min(self.input_resolution) <= self.window_size:
+            # if window size is larger than input resolution, we don't partition windows
+            self.shift_size = 0
+            self.window_size = min(self.input_resolution)
+        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
+
+        self.norm1 = norm_layer(dim)
+        self.attn = WindowAttention(
+            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,
+            qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+
+        # 每个block随机drop
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+        if self.shift_size > 0:
+            # calculate attention mask for SW-MSA
+            H, W = self.input_resolution
+            img_mask = torch.zeros((1, H, W, 1))  # 1 H W 1
+            # 将图像分成9份，并分别打上标号， 相当于模拟出要SW-MSA算attention值时候的情形
+            # 因为图片循环平移以后，有的窗口包括了不连续的图片，所以得上mask
+            h_slices = (slice(0, -self.window_size),
+                        slice(-self.window_size, -self.shift_size),
+                        slice(-self.shift_size, None))
+            w_slices = (slice(0, -self.window_size),
+                        slice(-self.window_size, -self.shift_size),
+                        slice(-self.shift_size, None))
+            cnt = 0
+            for h in h_slices:
+                for w in w_slices:
+                    img_mask[:, h, w, :] = cnt
+                    cnt += 1
+            # mask逐windows设置，N，ws, ws，N是windows的个数
+            # 1, H, W, 1 -> 1, nh，ws, nw, ws, 1 -> 1*nh*nw，ws, ws, 1
+            mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1
+            # 1*nh*nw，ws, ws, 1 -> 1*nh*nw，ws*ws, 每个窗口一个掩模
+            mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
+            # N, 1, ws*ws - N, ws*ws, 1
+            # N，ws*ws, ws*ws 能得到窗口中每个位置与其它位置标号的差，只要不是0都不能算到attention中
+            attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
+            # 通过设置合理的mask，让Shifted Window Attention在与Window Attention相同的窗口个数下，达到等价的计算结果
+            # 在计算Attention的时候，让具有相同indexQK进行计算，而忽略不同indexQK计算结果。
+            # 因为图片滚动出现了无关区域，无关区域就是标号不为0的区域
+            # 那么无关区域标记为-100，attention的时候就不用关注了
+            attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
+        else:
+            attn_mask = None
+
+        self.register_buffer("attn_mask", attn_mask)
+
+    def forward(self, x):
+        H, W = self.input_resolution
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+
+        shortcut = x
+        x = self.norm1(x)
+        x = x.view(B, H, W, C)
+
+        # cyclic shift
+        if self.shift_size > 0:
+            # SWMSA并非平移窗口，而是滚动平移图片，这样就实现了不同区域的交互
+            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
+        else:
+            shifted_x = x
+
+        # partition windows
+        # B, H, W, C -> B, nh，ws, nw, ws, C -> B*nh*nw，ws, ws, C
+        x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
+        # B*nh*nw，ws, ws, C -> B*nh*nw，l, C
+        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C
+
+        # W-MSA/SW-MSA
+        # 在windows内部求self attention
+        # B*nh*nw，l, C -> B*nh*nw，l, C
+        attn_windows = self.attn(x_windows, mask=self.attn_mask)  # nW*B, window_size*window_size, C
+
+        # merge windows
+        # B*nh*nw，l, C -> B*nh*nw，ws, ws, C
+        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
+        # B*nh*nw，ws, ws, C -> B，nh*ws, nw*ws, C
+        shifted_x = window_reverse(attn_windows, self.window_size, H, W)  # B H' W' C
+
+        # reverse cyclic shift
+        # 
+        if self.shift_size > 0:
+            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
+        else:
+            x = shifted_x
+
+        # 2D -> 1D特征
+        x = x.view(B, H * W, C)
+
+        # FFN
+        x = shortcut + self.drop_path(x)
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+
+        return x
+
+    def extra_repr(self) -> str:
+        return f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " \
+               f"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}"
+
+    def flops(self):
+        flops = 0
+        H, W = self.input_resolution
+        # norm1
+        flops += self.dim * H * W
+        # W-MSA/SW-MSA
+        nW = H * W / self.window_size / self.window_size
+        flops += nW * self.attn.flops(self.window_size * self.window_size)
+        # mlp
+        flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio
+        # norm2
+        flops += self.dim * H * W
+        return flops
+
+
+class PatchMerging(nn.Module):
+    r""" Patch Merging Layer.
+
+    Args:
+        input_resolution (tuple[int]): Resolution of input feature.
+        dim (int): Number of input channels.
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+
+    def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.dim = dim
+        # 按照原文，由4C变2C
+        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
+        self.norm = norm_layer(4 * dim)
+
+    def forward(self, x):
+        """
+        x: B, H*W, C
+        """
+        H, W = self.input_resolution
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+        assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even."
+
+        x = x.view(B, H, W, C)
+
+        # 把4个patch的token合成一个，pixel-shuffle的思想
+        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
+        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
+        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
+        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
+        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
+        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
+
+        x = self.norm(x)
+        x = self.reduction(x)
+
+        return x
+
+    def extra_repr(self) -> str:
+        return f"input_resolution={self.input_resolution}, dim={self.dim}"
+
+    def flops(self):
+        H, W = self.input_resolution
+        flops = H * W * self.dim
+        flops += (H // 2) * (W // 2) * 4 * self.dim * 2 * self.dim
+        return flops
+
+
+class BasicLayer(nn.Module):
+    """ A basic Swin Transformer layer for one stage.
+
+    Args:
+        dim (int): Number of input channels.
+        input_resolution (tuple[int]): Input resolution.
+        depth (int): Number of blocks.
+        num_heads (int): Number of attention heads.
+        window_size (int): Local window size.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
+        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
+        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+    """
+
+    def __init__(self, dim, input_resolution, depth, num_heads, window_size,
+                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False):
+
+        super().__init__()
+        self.dim = dim
+        self.input_resolution = input_resolution
+        self.depth = depth
+        self.use_checkpoint = use_checkpoint
+
+        # build blocks
+        self.blocks = nn.ModuleList([
+            SwinTransformerBlock(dim=dim, input_resolution=input_resolution,
+                                 num_heads=num_heads, window_size=window_size,
+                                 # idx为偶数的block采用WMSA, idx为奇数的block采用SWMSA
+                                 shift_size=0 if (i % 2 == 0) else window_size // 2,
+                                 mlp_ratio=mlp_ratio,
+                                 qkv_bias=qkv_bias, qk_scale=qk_scale,
+                                 drop=drop, attn_drop=attn_drop,
+                                 drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
+                                 norm_layer=norm_layer)
+            for i in range(depth)])
+
+        # patch merging layer
+        if downsample is not None:
+            self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer)
+        else:
+            self.downsample = None
+
+    def forward(self, x):
+        for blk in self.blocks:
+            if self.use_checkpoint:
+                x = checkpoint.checkpoint(blk, x)
+            else:
+                x = blk(x)
+        if self.downsample is not None:
+            x = self.downsample(x)
+        return x
+
+    def extra_repr(self) -> str:
+        return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}"
+
+    def flops(self):
+        flops = 0
+        for blk in self.blocks:
+            flops += blk.flops()
+        if self.downsample is not None:
+            flops += self.downsample.flops()
+        return flops
+
+
+class PatchEmbed(nn.Module):
+    r""" Image to Patch Embedding
+
+    Args:
+        img_size (int): Image size.  Default: 224.
+        patch_size (int): Patch token size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        norm_layer (nn.Module, optional): Normalization layer. Default: None
+    """
+
+    def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
+        super().__init__()
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]]
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.patches_resolution = patches_resolution # embedding后的特征的分辨率
+        self.num_patches = patches_resolution[0] * patches_resolution[1] # embedding后的token数，其实就是分辨率乘积
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        # 直接通过大小与步长均为patch_size的卷积核实现patch的embedding
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+        if norm_layer is not None:
+            self.norm = norm_layer(embed_dim)
+        else:
+            self.norm = None
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        # FIXME look at relaxing size constraints
+        assert H == self.img_size[0] and W == self.img_size[1], \
+            f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
+        x = self.proj(x).flatten(2).transpose(1, 2)  # B Ph*Pw C
+        if self.norm is not None:
+            x = self.norm(x)
+        return x
+
+    def flops(self):
+        Ho, Wo = self.patches_resolution
+        flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
+        if self.norm is not None:
+            flops += Ho * Wo * self.embed_dim
+        return flops
+
+
+class SwinTransformer(nn.Module):
+    r""" Swin Transformer
+        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
+          https://arxiv.org/pdf/2103.14030
+
+    Args:
+        img_size (int | tuple(int)): Input image size. Default 224
+        patch_size (int | tuple(int)): Patch size. Default: 4
+        in_chans (int): Number of input image channels. Default: 3
+        num_classes (int): Number of classes for classification head. Default: 1000
+        embed_dim (int): Patch embedding dimension. Default: 96
+        depths (tuple(int)): Depth of each Swin Transformer layer.
+        num_heads (tuple(int)): Number of attention heads in different layers.
+        window_size (int): Window size. Default: 7
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4
+        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. Default: None
+        drop_rate (float): Dropout rate. Default: 0
+        attn_drop_rate (float): Attention dropout rate. Default: 0
+        drop_path_rate (float): Stochastic depth rate. Default: 0.1
+        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
+        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False
+        patch_norm (bool): If True, add normalization after patch embedding. Default: True
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False
+    """
+
+    def __init__(self, img_size=224, patch_size=4, in_chans=3, num_classes=1000,
+                 embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24],
+                 window_size=7, mlp_ratio=4., qkv_bias=True, qk_scale=None,
+                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1,
+                 norm_layer=nn.LayerNorm, ape=False, patch_norm=True,
+                 use_checkpoint=False, **kwargs):
+        super().__init__()
+
+        self.num_classes = num_classes
+        self.num_layers = len(depths)
+        self.embed_dim = embed_dim
+        self.ape = ape
+        self.patch_norm = patch_norm
+        self.num_features = int(embed_dim * 2 ** (self.num_layers - 1))
+        self.mlp_ratio = mlp_ratio
+
+        # split image into non-overlapping patches
+        self.patch_embed = PatchEmbed(
+            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim,
+            norm_layer=norm_layer if self.patch_norm else None)
+        num_patches = self.patch_embed.num_patches
+        patches_resolution = self.patch_embed.patches_resolution
+        self.patches_resolution = patches_resolution
+
+        # absolute position embedding
+        if self.ape:
+            # 1, Ph*Pw, C
+            self.absolute_pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim))
+            init.trunc_normal_(self.absolute_pos_embed, std=0.02)
+
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        # stochastic depth
+        # 生成长度为block总数，从0到 drop_path_rate的等差数列
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # stochastic depth decay rule
+
+        # build layers
+        self.layers = nn.ModuleList()
+        for i_layer in range(self.num_layers):
+            layer = BasicLayer(dim=int(embed_dim * 2 ** i_layer),
+                               input_resolution=(patches_resolution[0] // (2 ** i_layer),
+                                                 patches_resolution[1] // (2 ** i_layer)),
+                               depth=depths[i_layer],
+                               num_heads=num_heads[i_layer],
+                               window_size=window_size,
+                               mlp_ratio=self.mlp_ratio,
+                               qkv_bias=qkv_bias, qk_scale=qk_scale,
+                               drop=drop_rate, attn_drop=attn_drop_rate,
+                               # 每个stage各个block的drop都不一样，越往后drop越多
+                               drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
+                               norm_layer=norm_layer,
+                               # 通过合并patch的embedding来实现降采样
+                               # 前三个stage后边有，最后一个stage后边没有
+                               downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
+                               use_checkpoint=use_checkpoint)
+            self.layers.append(layer)
+
+        self.norm = norm_layer(self.num_features)
+        self.avgpool = nn.AdaptiveAvgPool1d(1)
+        self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity()
+
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            init.trunc_normal_(m.weight, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            init.constant_(m.bias, 0)
+            init.constant_(m.weight, 1.0)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {'absolute_pos_embed'}
+
+    @torch.jit.ignore
+    def no_weight_decay_keywords(self):
+        return {'relative_position_bias_table'}
+
+    def forward_features(self, x):
+        x = self.patch_embed(x) # B Ph*Pw C 
+        if self.ape:
+            x = x + self.absolute_pos_embed
+        x = self.pos_drop(x)
+
+        for layer in self.layers:
+            x = layer(x)
+
+        x = self.norm(x)  # B L C
+        x = self.avgpool(x.transpose(1, 2))  # B C 1
+        x = torch.flatten(x, 1)
+        return x
+
+    def forward(self, x):
+        x = self.forward_features(x)
+        x = self.head(x)
+        return x
+
+    def flops(self):
+        flops = 0
+        flops += self.patch_embed.flops()
+        for i, layer in enumerate(self.layers):
+            flops += layer.flops()
+        flops += self.num_features * self.patches_resolution[0] * self.patches_resolution[1] // (2 ** self.num_layers)
+        flops += self.num_features * self.num_classes
+        return flops