demo

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/*.png filter=lfs diff=lfs merge=lfs -text
+examples/*.jpg filter=lfs diff=lfs merge=lfs -text
+examples/*.jpeg filter=lfs diff=lfs merge=lfs -text
+examples/*.bmp filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,150 @@
+
+import os
+import io
+import cv2
+import gradio as gr
+import numpy as np
+import torch
+import spaces
+from PIL import Image
+from functools import lru_cache
+from huggingface_hub import hf_hub_download, snapshot_download
+from torchvision.transforms.functional import normalize
+import glob
+
+
+from restormerRFR_arch import RestormerRFR
+from dino_feature_extractor import DinoFeatureModule
+
+WEIGHT_REPO_ID = "233zzl/RAM_plus_plus"
+WEIGHT_FILENAME = "7task/RestormerRFR.pth" 
+MODEL_NAME = "RestormerRFR"
+
+def get_device():
+    return torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def warmup():
+  
+    hf_hub_download(
+        repo_id=WEIGHT_REPO_ID,
+        filename=WEIGHT_FILENAME,
+        repo_type="model",
+        revision="main"
+    )
+    snapshot_download(
+        repo_id="facebook/dinov2-giant",
+        repo_type="model",
+        revision="main"
+    )
+
+
+def build_model():
+    model = RestormerRFR(
+        inp_channels=3,
+        out_channels=3,
+        dim=48,
+        num_blocks=[4, 6, 6, 8],
+        num_refinement_blocks=4,
+        heads=[1, 2, 4, 8],
+        ffn_expansion_factor=2.66,
+        bias=False,
+        LayerNorm_type="WithBias",
+        finetune_type=None,
+        img_size=128,
+    )
+    return model
+
+@lru_cache(maxsize=1)
+def get_dino_extractor(device):
+    extractor = DinoFeatureModule().to(device).eval()
+    return extractor
+
+@lru_cache(maxsize=1)
+def get_model_and_device():
+    device = get_device()
+    model = build_model()
+
+   
+    weight_path = hf_hub_download(
+        repo_id=WEIGHT_REPO_ID,
+        filename=WEIGHT_FILENAME,
+        
+    )
+   
+    ckpt = torch.load(weight_path, map_location="cpu")
+    keyname = "params" if "params" in ckpt else None
+    if keyname is not None:
+        model.load_state_dict(ckpt[keyname], strict=False)
+    else:
+        model.load_state_dict(ckpt, strict=False)
+
+    model.eval().to(device)
+    return model, device
+
+
+@spaces.GPU(duration=120)
+def restore_image(pil_img: Image.Image) -> Image.Image:
+    """
+    输入一张图片，输出复原后的图片（与 RAM++ RestormerRFR + DINO 特征推理一致）
+    """
+    model, device = get_model_and_device()
+    dino_extractor = get_dino_extractor(device)
+
+
+    img_bgr = cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR).astype(np.float32) / 255.0
+    img = torch.from_numpy(np.transpose(img_bgr[:, :, [2, 1, 0]], (2, 0, 1))).float()  # (3,H,W), RGB
+    img = img.unsqueeze(0).to(device)  # (1,3,H,W)
+
+
+    mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+    std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+    normalize(img, mean, std, inplace=True)
+
+    with torch.no_grad():
+        dino_features = dino_extractor(img)
+        output = model(img, dino_features)
+
+
+    output = normalize(output, -1 * mean / std, 1 / std)
+    output = output.data.squeeze().float().cpu().clamp_(0, 1).numpy()  # (3,H,W)
+    output = np.transpose(output[[2, 1, 0], :, :], (1, 2, 0))  # (H,W,RGB)
+    output = (output * 255.0).round().astype(np.uint8)
+    out_pil = Image.fromarray(output, mode="RGB")
+    return out_pil
+
+
+DESCRIPTION = """
+# RAM++ Demo 
+"""
+
+with gr.Blocks(title="RAM++ ZeroGPU Demo") as demo:
+    gr.Markdown(DESCRIPTION)
+
+    with gr.Row():
+        with gr.Column():
+            inp = gr.Image(type="pil", label="load picture（JPEG/PNG）")
+            btn = gr.Button("Run (ZeroGPU)")
+        with gr.Column():
+            out = gr.Image(type="pil", label="output")
+
+    ex_files = []
+    for ext in ("*.png", "*.jpg", "*.jpeg", "*.bmp"):
+        ex_files.extend(glob.glob(os.path.join("examples", ext)))
+    ex_files = sorted(ex_files)
+    if ex_files:
+        gr.Examples(examples=ex_files, inputs=inp, label="exampls）")
+
+    btn.click(restore_image, inputs=inp, outputs=out, api_name="run")
+
+    gr.Markdown("""
+**Tips**
+- If the queue is long or you hit the quota, please try again later, or upgrade to Pro for a higher ZeroGPU quota and priority.
+""")
+
+    demo.load(fn=warmup, inputs=None, outputs=None)
+
+
+if __name__ == "__main__":
+   
+    demo.launch()

dino_feature_extractor.py

@@ -0,0 +1,138 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numbers
+import numpy as np
+import os
+from transformers import AutoImageProcessor, AutoModel
+import math
+class DinoFeatureModule(nn.Module):
+    def __init__(self, model_id: str = "facebook/dinov2-giant"):
+        super(DinoFeatureModule, self).__init__()
+        dtype = torch.float32
+        self.dino = AutoModel.from_pretrained(
+            model_id,
+            torch_dtype=dtype
+        )
+        
+      
+        self.dino.eval()
+        for param in self.dino.parameters():
+            param.requires_grad = False
+        
+       
+        frozen = all(not p.requires_grad for p in self.dino.parameters())
+        assert frozen, "DINOv2 model parameters are not completely frozen!"
+        
+        
+        self.shallow_dim = 1536  
+        self.mid_dim = 1536      
+        self.deep_dim = 1536     
+        
+    def get_dino_features(self, x):
+        with torch.no_grad(): 
+            outputs = self.dino(x, output_hidden_states=True)
+            hidden_states = outputs.hidden_states
+           
+            _, _, H, W = x.shape
+            aspect_ratio = W / H
+          
+            shallow_feat1 = hidden_states[7]  
+            shallow_feat2 = hidden_states[15]
+            mid_feat1 = hidden_states[20]
+            mid_feat2 = hidden_states[22]
+            deep_feat1 = hidden_states[33]
+            deep_feat2 = hidden_states[39]
+
+            def reshape_features(feat):
+                feat = feat[:, 1:, :]
+                B, N, C = feat.shape
+                
+                h = int(math.sqrt(N / aspect_ratio))
+                w = int(N / h)
+                
+               
+                if(aspect_ratio > 1):
+                    if h * w > N:
+                        h -= 1
+                        w = N // h
+                    if h * w < N:
+                        h += 1
+                        w = N // h
+                else: 
+                    if h * w > N:
+                        w -= 1
+                        h = N // w
+                    if h * w < N:
+                        w += 1
+                        h = N // w
+                
+                
+                assert h * w == N, f"Dimensions mismatch: {h}*{w} != {N}"
+                
+                
+                feat = feat.reshape(B, h, w, C).permute(0, 3, 1, 2)
+                return feat
+
+          
+            shallow_feat1 = reshape_features(shallow_feat1).float()
+            mid_feat1 = reshape_features(mid_feat1).float()
+            deep_feat1 = reshape_features(deep_feat1).float()
+            shallow_feat2 = reshape_features(shallow_feat2).float()
+            mid_feat2 = reshape_features(mid_feat2).float()
+            deep_feat2 = reshape_features(deep_feat2).float()
+
+            return shallow_feat1, mid_feat1, deep_feat1, shallow_feat2, mid_feat2, deep_feat2
+    
+    def check_image_size(self, x):
+        _, _, h, w = x.size()
+        pad_size = 16
+        mod_pad_h = (pad_size - h % pad_size) % pad_size
+        mod_pad_w = (pad_size - w % pad_size) % pad_size
+        x = F.pad(x, (0, mod_pad_w, 0, mod_pad_h), 'reflect')
+        return x
+
+    def forward(self, inp_img):
+        
+        device = inp_img.device
+        
+        mean = torch.tensor([0.485, 0.456, 0.406], device=device).view(1, 3, 1, 1)
+        std = torch.tensor([0.229, 0.224, 0.225], device=device).view(1, 3, 1, 1)
+
+
+        denormalized_img = inp_img * std + mean
+        denormalized_img = self.check_image_size(denormalized_img)
+        h_denormalized, w_denormalized = denormalized_img.shape[2], denormalized_img.shape[3]
+        # To ensure minimal changes and maintain code generality, the image size is directly scaled here to guarantee spatial alignment.
+
+        target_h = (h_denormalized // 8) * 14  
+        target_w = (w_denormalized // 8) * 14  
+       
+        shortest_edge = min(target_h, target_w)
+        processor = AutoImageProcessor.from_pretrained(
+            model_id,
+            local_files_only=False,
+            do_rescale=False,
+            do_center_crop=False,  
+            use_fast=True,
+            size={"shortest_edge": shortest_edge}
+        )
+        
+        inputs = processor(
+            images=denormalized_img,  
+            return_tensors="pt"
+        ).to(device)
+            
+       
+        shallow_feat1, mid_feat1, deep_feat1, shallow_feat2, mid_feat2, deep_feat2 = self.get_dino_features(inputs['pixel_values'])
+        
+        dino_features = {
+            'shallow_feat1': shallow_feat1,
+            'mid_feat1': mid_feat1,
+            'deep_feat1': deep_feat1,
+            'shallow_feat2': shallow_feat2,
+            'mid_feat2': mid_feat2,
+            'deep_feat2': deep_feat2
+        }
+        
+        return dino_features

requirements.txt

@@ -0,0 +1,12 @@
+gradio>=4.0.0
+spaces>=0.28.3
+huggingface_hub>=0.23.0
+transformers>=4.41.0
+safetensors>=0.4.3
+numpy>=1.26.0
+Pillow>=10.0.0
+opencv-python-headless>=4.8.0.76
+einops>=0.7.0
+torch>=2.1.0
+torchvision>=0.16.0
+timm>=0.9.10

restormerRFR_arch.py

@@ -0,0 +1,408 @@
+# RAM++: Robust Representation Learning via Adaptive Mask for All-in-One Image Restoration
+# Zilong Zhang, Chujie Qin, Chunle Guo, Yong Zhang, Chao Xue, Ming-Ming Cheng and Chongyi Li
+# https://arxiv.org/abs/2509.12039
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numbers
+
+from einops import rearrange
+
+
+
+def to_3d(x):
+    return rearrange(x, 'b c h w -> b (h w) c')
+
+def to_4d(x,h,w):
+    return rearrange(x, 'b (h w) c -> b c h w',h=h,w=w)
+
+class BiasFree_LayerNorm(nn.Module):
+    def __init__(self, normalized_shape):
+        super(BiasFree_LayerNorm, self).__init__()
+        if isinstance(normalized_shape, numbers.Integral):
+            normalized_shape = (normalized_shape,)
+        normalized_shape = torch.Size(normalized_shape)
+
+        assert len(normalized_shape) == 1
+
+        self.weight = nn.Parameter(torch.ones(normalized_shape))
+        self.normalized_shape = normalized_shape
+
+    def forward(self, x):
+        sigma = x.var(-1, keepdim=True, unbiased=False)
+        return x / torch.sqrt(sigma+1e-5) * self.weight
+
+class WithBias_LayerNorm(nn.Module):
+    def __init__(self, normalized_shape):
+        super(WithBias_LayerNorm, self).__init__()
+        if isinstance(normalized_shape, numbers.Integral):
+            normalized_shape = (normalized_shape,)
+        normalized_shape = torch.Size(normalized_shape)
+
+        assert len(normalized_shape) == 1
+
+        self.weight = nn.Parameter(torch.ones(normalized_shape))
+        self.bias = nn.Parameter(torch.zeros(normalized_shape))
+        self.normalized_shape = normalized_shape
+
+    def forward(self, x):
+        mu = x.mean(-1, keepdim=True)
+        sigma = x.var(-1, keepdim=True, unbiased=False)
+        return (x - mu) / torch.sqrt(sigma+1e-5) * self.weight + self.bias
+
+
+class LayerNorm(nn.Module):
+    def __init__(self, dim, LayerNorm_type):
+        super(LayerNorm, self).__init__()
+        if LayerNorm_type =='BiasFree':
+            self.body = BiasFree_LayerNorm(dim)
+        else:
+            self.body = WithBias_LayerNorm(dim)
+
+    def forward(self, x):
+        h, w = x.shape[-2:]
+        return to_4d(self.body(to_3d(x)), h, w)
+
+
+
+##########################################################################
+## Gated-Dconv Feed-Forward Network (GDFN)
+class FeedForward(nn.Module):
+    def __init__(self, dim, ffn_expansion_factor, bias,finetune_type=None):
+        super(FeedForward, self).__init__()
+
+        hidden_features = int(dim*ffn_expansion_factor)
+
+        self.project_in = nn.Conv2d(dim, hidden_features*2, kernel_size=1, bias=bias)
+
+        self.dwconv = nn.Conv2d(hidden_features*2, hidden_features*2, kernel_size=3, stride=1, padding=1, groups=hidden_features*2, bias=bias)
+
+        self.project_out = nn.Conv2d(hidden_features, dim, kernel_size=1, bias=bias)
+                
+    def forward(self, x):
+        x = self.project_in(x)
+        x1, x2 = self.dwconv(x).chunk(2, dim=1)
+        x = F.gelu(x1) * x2
+        x = self.project_out(x)
+        
+        return x
+
+
+
+##########################################################################
+## Multi-DConv Head Transposed Self-Attention (MDTA)
+class Attention(nn.Module):
+    def __init__(self, dim, num_heads, bias):
+        super(Attention, self).__init__()
+        self.num_heads = num_heads
+        self.temperature = nn.Parameter(torch.ones(num_heads, 1, 1))
+
+        self.qkv = nn.Conv2d(dim, dim*3, kernel_size=1, bias=bias)
+        self.qkv_dwconv = nn.Conv2d(dim*3, dim*3, kernel_size=3, stride=1, padding=1, groups=dim*3, bias=bias)
+        self.project_out = nn.Conv2d(dim, dim, kernel_size=1, bias=bias)
+        
+
+    def forward(self, x):
+        b,c,h,w = x.shape
+
+        qkv = self.qkv_dwconv(self.qkv(x))
+        q,k,v = qkv.chunk(3, dim=1)   
+        
+        q = rearrange(q, 'b (head c) h w -> b head c (h w)', head=self.num_heads)
+        k = rearrange(k, 'b (head c) h w -> b head c (h w)', head=self.num_heads)
+        v = rearrange(v, 'b (head c) h w -> b head c (h w)', head=self.num_heads)
+
+        q = torch.nn.functional.normalize(q, dim=-1)
+        k = torch.nn.functional.normalize(k, dim=-1)
+
+        attn = (q @ k.transpose(-2, -1)) * self.temperature
+        attn = attn.softmax(dim=-1)
+
+        out = (attn @ v)
+        
+        out = rearrange(out, 'b head c (h w) -> b (head c) h w', head=self.num_heads, h=h, w=w)
+
+        out = self.project_out(out)
+        return out
+
+
+
+class resblock(nn.Module):
+    def __init__(self, dim):
+
+        super(resblock, self).__init__()
+        # self.norm = LayerNorm(dim, LayerNorm_type='BiasFree')
+
+        self.body = nn.Sequential(nn.Conv2d(dim, dim, kernel_size=3, stride=1, padding=1, bias=False),
+                                  nn.PReLU(dim),
+                                  nn.Conv2d(dim, dim, kernel_size=3, stride=1, padding=1, bias=False))
+
+    def forward(self, x):
+        res = self.body((x))
+        res += x
+        return res
+
+
+##########################################################################
+## Resizing modules
+class Downsample(nn.Module):
+    def __init__(self, n_feat):
+        super(Downsample, self).__init__()
+
+        self.body = nn.Sequential(nn.Conv2d(n_feat, n_feat//2, kernel_size=3, stride=1, padding=1, bias=False),
+                                  nn.PixelUnshuffle(2))
+
+    def forward(self, x):
+        return self.body(x)
+
+class Upsample(nn.Module):
+    def __init__(self, n_feat):
+        super(Upsample, self).__init__()
+
+        self.body = nn.Sequential(nn.Conv2d(n_feat, n_feat*2, kernel_size=3, stride=1, padding=1, bias=False),
+                                  nn.PixelShuffle(2))
+
+    def forward(self, x):
+        return self.body(x)
+
+
+##########################################################################
+## Transformer Block
+class TransformerBlock(nn.Module):
+    def __init__(self, dim, num_heads, ffn_expansion_factor, bias, LayerNorm_type,finetune_type=None):
+        super(TransformerBlock, self).__init__()
+
+        self.norm1 = LayerNorm(dim, LayerNorm_type)
+        self.attn = Attention(dim, num_heads, bias)
+        self.norm2 = LayerNorm(dim, LayerNorm_type)
+        self.ffn = FeedForward(dim, ffn_expansion_factor, bias,finetune_type)
+
+    def forward(self, x):
+        x = x + self.attn(self.norm1(x))
+        x = x + self.ffn(self.norm2(x))
+
+        return x
+
+
+
+##########################################################################
+## Overlapped image patch embedding with 3x3 Conv
+class OverlapPatchEmbed(nn.Module):
+    def __init__(self, in_c=3, embed_dim=48, bias=False):
+        super(OverlapPatchEmbed, self).__init__()
+
+        self.proj = nn.Conv2d(in_c, embed_dim, kernel_size=3, stride=1, padding=1, bias=bias)
+
+    def forward(self, x):
+        x = self.proj(x)
+
+        return x
+
+class TemperatureSoftmax(nn.Module):
+    def __init__(self, temperature):
+        super().__init__()
+        self.temperature = temperature
+
+    def forward(self, x):
+        return F.softmax(x / torch.clamp(self.temperature, min=1e-8), dim=1)
+
+class DinoFeatureFusion(nn.Module):
+    def __init__(self, dino_dim=1536):
+        super(DinoFeatureFusion, self).__init__()
+        
+        self.global_pool = nn.AdaptiveAvgPool2d(1)
+        self.temperature = nn.Parameter(torch.ones(1) * 1.0)
+        
+        self.gate_network = nn.Sequential(
+            nn.Linear(dino_dim * 2, dino_dim),
+            nn.PReLU(dino_dim),
+            nn.Linear(dino_dim, 512),
+            nn.PReLU(512),
+            nn.Linear(512, 2),
+            TemperatureSoftmax(self.temperature) 
+        )
+
+    def forward(self, dino_feat1, dino_feat2):
+        pooled_feat1 = self.global_pool(dino_feat1).squeeze(-1).squeeze(-1) 
+        pooled_feat2 = self.global_pool(dino_feat2).squeeze(-1).squeeze(-1) 
+        pooled_features = torch.cat([pooled_feat1, pooled_feat2], dim=1)  
+
+        weights = self.gate_network(pooled_features)  
+        weighted_feat1 = dino_feat1 * weights[:, 0:1].view(-1, 1, 1, 1)
+        weighted_feat2 = dino_feat2 * weights[:, 1:2].view(-1, 1, 1, 1)
+
+        fused_feat = weighted_feat1 + weighted_feat2  
+        return fused_feat
+        
+        
+
+
+
+
+class DRAdaptation(nn.Module):
+    def __init__(self, dino_dim=1536, restore_dim=48, scale_factor=14, size=128):      
+        super(DRAdaptation, self).__init__()
+        self.size = size  
+        self.restore_dim = restore_dim
+        self.adaptation = nn.Sequential(
+            nn.Conv2d(dino_dim, restore_dim*16, kernel_size=3, padding=1), #768
+            nn.PReLU(restore_dim*16),
+            nn.Conv2d(restore_dim*16, restore_dim*8, kernel_size=1),#384
+        )
+
+    def forward(self, dino_feat, restore_feat):
+        B, C, H, W = restore_feat.shape
+        
+        adapted_dino = self.adaptation(dino_feat)  
+
+        return adapted_dino
+
+
+
+##########################################################################
+##---------- D-R Fusion -----------------------
+class DinoRestoreFeatureFusion(nn.Module):
+    def __init__(self, dim, num_heads, bias):
+        super(DinoRestoreFeatureFusion, self).__init__()
+        self.reduce_chan = nn.Conv2d(dim*2, dim, kernel_size=1, bias=bias)
+    def forward(self, dino_feat, restore_feat):
+        x_fusion = self.reduce_chan(torch.cat([dino_feat, restore_feat], dim=1))
+        res = x_fusion + restore_feat
+        return res
+        
+    
+##---------- restormerRFR -----------------------
+class RestormerRFR(nn.Module):
+    def __init__(self, 
+        inp_channels=3, 
+        out_channels=3, 
+        dim = 48,
+        num_blocks = [4,6,6,8], 
+        num_refinement_blocks = 4,
+        heads = [1,2,4,8],
+        ffn_expansion_factor = 2.66,
+        bias = False,
+        LayerNorm_type = 'WithBias',   
+        finetune_type = None,
+        img_size = 128
+    ):
+
+        super(RestormerRFR, self).__init__()
+        
+        
+        self.patch_embed = OverlapPatchEmbed(inp_channels, dim)
+        
+        self.mask_token = torch.zeros(1, 3, img_size, img_size)
+        
+        self.dr_adaptation1 = DRAdaptation(dino_dim=1536, restore_dim=48, scale_factor=14, size=128)
+        self.dr_adaptation2 = DRAdaptation(dino_dim=1536, restore_dim=48, scale_factor=14, size=128)
+        self.dr_adaptation3 = DRAdaptation(dino_dim=1536, restore_dim=48, scale_factor=14, size=128)
+        self.dr_fusion1 = DinoRestoreFeatureFusion(dim=int(dim*2**3), num_heads=heads[3], bias=bias) 
+        self.dr_fusion2 = DinoRestoreFeatureFusion(dim=int(dim*2**2), num_heads=heads[2], bias=bias)
+        self.dr_fusion3 = DinoRestoreFeatureFusion(dim=int(dim*2**1), num_heads=heads[1], bias=bias)
+        self.up_4_3_dino1 = Upsample(int(dim*2**3))
+        self.up_4_3_dino2 = Upsample(int(dim*2**3))
+        self.up_3_2_dino = Upsample(int(dim*2**2))
+        self.dino_fusion_shallow = DinoFeatureFusion(dino_dim=1536)
+        self.dino_fusion_mid = DinoFeatureFusion(dino_dim=1536)
+        self.dino_fusion_deep = DinoFeatureFusion(dino_dim=1536)
+
+
+
+        self.encoder_level1 = nn.Sequential(*[TransformerBlock(dim=dim, num_heads=heads[0], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_blocks[0]-1 else None) for i in range(num_blocks[0])])
+        self.down1_2 = Downsample(dim) 
+        self.encoder_level2 = nn.Sequential(*[TransformerBlock(dim=int(dim*2**1), num_heads=heads[1], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_blocks[1]-1 else None) for i in range(num_blocks[1])])       
+        self.down2_3 = Downsample(int(dim*2**1)) 
+        self.encoder_level3 = nn.Sequential(*[TransformerBlock(dim=int(dim*2**2), num_heads=heads[2], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_blocks[2]-1 else None) for i in range(num_blocks[2])])
+        self.down3_4 = Downsample(int(dim*2**2)) 
+        self.latent = nn.Sequential(*[TransformerBlock(dim=int(dim*2**3), num_heads=heads[3], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_blocks[3]-1 else None) for i in range(num_blocks[3])])
+        
+        self.up4_3 = Upsample(int(dim*2**3)) 
+        self.reduce_chan_level3 = nn.Conv2d(int(dim*2**3), int(dim*2**2), kernel_size=1, bias=bias)
+        self.decoder_level3 = nn.Sequential(*[TransformerBlock(dim=int(dim*2**2), num_heads=heads[2], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_blocks[2]-1 else None) for i in range(num_blocks[2])])
+
+
+        self.up3_2 = Upsample(int(dim*2**2))
+        self.reduce_chan_level2 = nn.Conv2d(int(dim*2**2), int(dim*2**1), kernel_size=1, bias=bias)
+        self.decoder_level2 = nn.Sequential(*[TransformerBlock(dim=int(dim*2**1), num_heads=heads[1], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_blocks[1]-1 else None) for i in range(num_blocks[1])])
+        
+        self.up2_1 = Upsample(int(dim*2**1)) 
+        self.decoder_level1 = nn.Sequential(*[TransformerBlock(dim=int(dim*2**1), num_heads=heads[0], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_blocks[0]-1 else None) for i in range(num_blocks[0])])
+        self.refinement = nn.Sequential(*[TransformerBlock(dim=int(dim*2**1), num_heads=heads[0], ffn_expansion_factor=ffn_expansion_factor, bias=bias, LayerNorm_type=LayerNorm_type,finetune_type=finetune_type if i==num_refinement_blocks-1 else None) for i in range(num_refinement_blocks)])
+        self.output = nn.Conv2d(int(dim*2**1), out_channels, kernel_size=3, stride=1, padding=1, bias=bias)
+
+    
+    def check_image_size(self, x):
+        _, _, h, w = x.size()
+        pad_size = 16
+        mod_pad_h = (pad_size - h % pad_size) % pad_size
+        mod_pad_w = (pad_size - w % pad_size) % pad_size
+        x = F.pad(x, (0, mod_pad_w, 0, mod_pad_h), 'reflect')
+        return x
+    def forward(self, inp_img, dino_features =None ):
+        b,c,h,w = inp_img.shape
+
+        shallow_feat1, mid_feat1, deep_feat1, shallow_feat2, mid_feat2, deep_feat2 = dino_features.values()
+        inp_img = self.check_image_size(inp_img)
+
+        inp_enc_level1 = self.patch_embed(inp_img)
+
+        out_enc_level1 = self.encoder_level1(inp_enc_level1)
+        
+        inp_enc_level2 = self.down1_2(out_enc_level1)
+
+        out_enc_level2 = self.encoder_level2(inp_enc_level2)
+
+        inp_enc_level3 = self.down2_3(out_enc_level2)
+
+        out_enc_level3 = self.encoder_level3(inp_enc_level3) 
+
+        inp_enc_level4 = self.down3_4(out_enc_level3)        
+
+        latent = self.latent(inp_enc_level4)  
+        
+
+       
+        
+        shallow_feat = self.dino_fusion_shallow(shallow_feat1, shallow_feat2)
+        mid_feat = self.dino_fusion_mid(mid_feat1, mid_feat2)
+        deep_feat = self.dino_fusion_deep(deep_feat1, deep_feat2)
+
+        shallow_feat = self.dr_adaptation1(shallow_feat, latent) 
+        mid_feat = self.dr_adaptation2(mid_feat, latent)
+        deep_feat = self.dr_adaptation3(deep_feat, latent)
+
+        latent = self.dr_fusion1(dino_feat=deep_feat, restore_feat=latent) 
+        shallow_feat = self.up_4_3_dino1(shallow_feat)
+        mid_feat = self.up_4_3_dino2(mid_feat)
+
+        inp_dec_level3 = self.up4_3(latent) 
+        inp_dec_level3 = torch.cat([inp_dec_level3, out_enc_level3], 1)
+        inp_dec_level3 = self.reduce_chan_level3(inp_dec_level3)
+
+        out_dec_level3 = self.decoder_level3(inp_dec_level3)
+
+        out_dec_level3 = self.dr_fusion2(dino_feat=mid_feat, restore_feat=out_dec_level3) 
+        shallow_feat = self.up_3_2_dino(shallow_feat)
+        inp_dec_level2 = self.up3_2(out_dec_level3)
+        
+        inp_dec_level2 = torch.cat([inp_dec_level2, out_enc_level2], 1)
+        inp_dec_level2 = self.reduce_chan_level2(inp_dec_level2)
+
+        out_dec_level2 = self.decoder_level2(inp_dec_level2)  
+
+        out_dec_level2 = self.dr_fusion3(dino_feat=shallow_feat, restore_feat=out_dec_level2) 
+
+
+        inp_dec_level1 = self.up2_1(out_dec_level2)
+        inp_dec_level1 = torch.cat([inp_dec_level1, out_enc_level1], 1)
+        
+        out_dec_level1 = self.decoder_level1(inp_dec_level1)
+
+        out_dec_level1 = self.refinement(out_dec_level1)
+
+        out_dec_level1 = self.output(out_dec_level1)
+
+
+        return out_dec_level1[:,:,:h,:w]