Update files for transformers integration

Updates the repo to smoothly integrate with transformers.

PR: https://github.com/huggingface/transformers/pull/46347

Changes:
- Update `config.json` to match transformers structure
- Update config classes to handle new `config.json`
- Add processor and tokenizer configs
- Update DPT `model.safetensors` files to include DPT head + backbone weights
- Update DPT model code to handle extra weights in `model.safetensors` file

config.json

@@ -1,5 +1,4 @@
 {
-  "model_type": "tipsv2_dpt",
   "architectures": [
     "TIPSv2DPTModel"
   ],
@@ -7,24 +6,408 @@
     "AutoConfig": "configuration_dpt.TIPSv2DPTConfig",
     "AutoModel": "modeling_dpt.TIPSv2DPTModel"
   },
-  "channels": 256,
-  "readout_type": "project",
-  "num_depth_bins": 256,
-  "min_depth": 0.001,
+  "backbone_config": {
+    "apply_layernorm": true,
+    "attention_probs_dropout_prob": 0.0,
+    "drop_path_rate": 0.0,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1536,
+    "image_size": 448,
+    "initializer_range": 0.02,
+    "interpolate_antialias": true,
+    "interpolate_offset": 0.0,
+    "layer_norm_eps": 1e-06,
+    "layerscale_value": 1.0,
+    "mlp_ratio": 4,
+    "model_type": "tipsv2_vision_model",
+    "num_attention_heads": 24,
+    "num_channels": 3,
+    "num_hidden_layers": 40,
+    "num_register_tokens": 1,
+    "out_features": [
+      "stage10",
+      "stage20",
+      "stage30",
+      "stage40"
+    ],
+    "out_indices": [
+      10,
+      20,
+      30,
+      40
+    ],
+    "patch_size": 14,
+    "qkv_bias": true,
+    "reshape_hidden_states": false,
+    "stage_names": [
+      "stem",
+      "stage1",
+      "stage2",
+      "stage3",
+      "stage4",
+      "stage5",
+      "stage6",
+      "stage7",
+      "stage8",
+      "stage9",
+      "stage10",
+      "stage11",
+      "stage12",
+      "stage13",
+      "stage14",
+      "stage15",
+      "stage16",
+      "stage17",
+      "stage18",
+      "stage19",
+      "stage20",
+      "stage21",
+      "stage22",
+      "stage23",
+      "stage24",
+      "stage25",
+      "stage26",
+      "stage27",
+      "stage28",
+      "stage29",
+      "stage30",
+      "stage31",
+      "stage32",
+      "stage33",
+      "stage34",
+      "stage35",
+      "stage36",
+      "stage37",
+      "stage38",
+      "stage39",
+      "stage40"
+    ],
+    "use_swiglu_ffn": true
+  },
+  "depth_decoder_activation": "relu",
+  "fusion_hidden_size": 256,
+  "id2label": {
+    "0": "wall",
+    "1": "building",
+    "10": "cabinet",
+    "100": "poster",
+    "101": "stage",
+    "102": "van",
+    "103": "ship",
+    "104": "fountain",
+    "105": "conveyer belt",
+    "106": "canopy",
+    "107": "washer",
+    "108": "plaything",
+    "109": "swimming pool",
+    "11": "sidewalk",
+    "110": "stool",
+    "111": "barrel",
+    "112": "basket",
+    "113": "waterfall",
+    "114": "tent",
+    "115": "bag",
+    "116": "minibike",
+    "117": "cradle",
+    "118": "oven",
+    "119": "ball",
+    "12": "person",
+    "120": "food",
+    "121": "step",
+    "122": "tank",
+    "123": "trade name",
+    "124": "microwave",
+    "125": "pot",
+    "126": "animal",
+    "127": "bicycle",
+    "128": "lake",
+    "129": "dishwasher",
+    "13": "earth",
+    "130": "screen",
+    "131": "blanket",
+    "132": "sculpture",
+    "133": "hood",
+    "134": "sconce",
+    "135": "vase",
+    "136": "traffic light",
+    "137": "tray",
+    "138": "ashcan",
+    "139": "fan",
+    "14": "door",
+    "140": "pier",
+    "141": "crt screen",
+    "142": "plate",
+    "143": "monitor",
+    "144": "bulletin board",
+    "145": "shower",
+    "146": "radiator",
+    "147": "glass",
+    "148": "clock",
+    "149": "flag",
+    "15": "table",
+    "16": "mountain",
+    "17": "plant",
+    "18": "curtain",
+    "19": "chair",
+    "2": "sky",
+    "20": "car",
+    "21": "water",
+    "22": "painting",
+    "23": "sofa",
+    "24": "shelf",
+    "25": "house",
+    "26": "sea",
+    "27": "mirror",
+    "28": "rug",
+    "29": "field",
+    "3": "floor",
+    "30": "armchair",
+    "31": "seat",
+    "32": "fence",
+    "33": "desk",
+    "34": "rock",
+    "35": "wardrobe",
+    "36": "lamp",
+    "37": "bathtub",
+    "38": "railing",
+    "39": "cushion",
+    "4": "tree",
+    "40": "base",
+    "41": "box",
+    "42": "column",
+    "43": "signboard",
+    "44": "chest of drawers",
+    "45": "counter",
+    "46": "sand",
+    "47": "sink",
+    "48": "skyscraper",
+    "49": "fireplace",
+    "5": "ceiling",
+    "50": "refrigerator",
+    "51": "grandstand",
+    "52": "path",
+    "53": "stairs",
+    "54": "runway",
+    "55": "case",
+    "56": "pool table",
+    "57": "pillow",
+    "58": "screen door",
+    "59": "stairway",
+    "6": "road",
+    "60": "river",
+    "61": "bridge",
+    "62": "bookcase",
+    "63": "blind",
+    "64": "coffee table",
+    "65": "toilet",
+    "66": "flower",
+    "67": "book",
+    "68": "hill",
+    "69": "bench",
+    "7": "bed ",
+    "70": "countertop",
+    "71": "stove",
+    "72": "palm",
+    "73": "kitchen island",
+    "74": "computer",
+    "75": "swivel chair",
+    "76": "boat",
+    "77": "bar",
+    "78": "arcade machine",
+    "79": "hovel",
+    "8": "windowpane",
+    "80": "bus",
+    "81": "towel",
+    "82": "light",
+    "83": "truck",
+    "84": "tower",
+    "85": "chandelier",
+    "86": "awning",
+    "87": "streetlight",
+    "88": "booth",
+    "89": "television receiver",
+    "9": "grass",
+    "90": "airplane",
+    "91": "dirt track",
+    "92": "apparel",
+    "93": "pole",
+    "94": "land",
+    "95": "bannister",
+    "96": "escalator",
+    "97": "ottoman",
+    "98": "bottle",
+    "99": "buffet"
+  },
+  "label2id": {
+    "airplane": 90,
+    "animal": 126,
+    "apparel": 92,
+    "arcade machine": 78,
+    "armchair": 30,
+    "ashcan": 138,
+    "awning": 86,
+    "bag": 115,
+    "ball": 119,
+    "bannister": 95,
+    "bar": 77,
+    "barrel": 111,
+    "base": 40,
+    "basket": 112,
+    "bathtub": 37,
+    "bed ": 7,
+    "bench": 69,
+    "bicycle": 127,
+    "blanket": 131,
+    "blind": 63,
+    "boat": 76,
+    "book": 67,
+    "bookcase": 62,
+    "booth": 88,
+    "bottle": 98,
+    "box": 41,
+    "bridge": 61,
+    "buffet": 99,
+    "building": 1,
+    "bulletin board": 144,
+    "bus": 80,
+    "cabinet": 10,
+    "canopy": 106,
+    "car": 20,
+    "case": 55,
+    "ceiling": 5,
+    "chair": 19,
+    "chandelier": 85,
+    "chest of drawers": 44,
+    "clock": 148,
+    "coffee table": 64,
+    "column": 42,
+    "computer": 74,
+    "conveyer belt": 105,
+    "counter": 45,
+    "countertop": 70,
+    "cradle": 117,
+    "crt screen": 141,
+    "curtain": 18,
+    "cushion": 39,
+    "desk": 33,
+    "dirt track": 91,
+    "dishwasher": 129,
+    "door": 14,
+    "earth": 13,
+    "escalator": 96,
+    "fan": 139,
+    "fence": 32,
+    "field": 29,
+    "fireplace": 49,
+    "flag": 149,
+    "floor": 3,
+    "flower": 66,
+    "food": 120,
+    "fountain": 104,
+    "glass": 147,
+    "grandstand": 51,
+    "grass": 9,
+    "hill": 68,
+    "hood": 133,
+    "house": 25,
+    "hovel": 79,
+    "kitchen island": 73,
+    "lake": 128,
+    "lamp": 36,
+    "land": 94,
+    "light": 82,
+    "microwave": 124,
+    "minibike": 116,
+    "mirror": 27,
+    "monitor": 143,
+    "mountain": 16,
+    "ottoman": 97,
+    "oven": 118,
+    "painting": 22,
+    "palm": 72,
+    "path": 52,
+    "person": 12,
+    "pier": 140,
+    "pillow": 57,
+    "plant": 17,
+    "plate": 142,
+    "plaything": 108,
+    "pole": 93,
+    "pool table": 56,
+    "poster": 100,
+    "pot": 125,
+    "radiator": 146,
+    "railing": 38,
+    "refrigerator": 50,
+    "river": 60,
+    "road": 6,
+    "rock": 34,
+    "rug": 28,
+    "runway": 54,
+    "sand": 46,
+    "sconce": 134,
+    "screen": 130,
+    "screen door": 58,
+    "sculpture": 132,
+    "sea": 26,
+    "seat": 31,
+    "shelf": 24,
+    "ship": 103,
+    "shower": 145,
+    "sidewalk": 11,
+    "signboard": 43,
+    "sink": 47,
+    "sky": 2,
+    "skyscraper": 48,
+    "sofa": 23,
+    "stage": 101,
+    "stairs": 53,
+    "stairway": 59,
+    "step": 121,
+    "stool": 110,
+    "stove": 71,
+    "streetlight": 87,
+    "swimming pool": 109,
+    "swivel chair": 75,
+    "table": 15,
+    "tank": 122,
+    "television receiver": 89,
+    "tent": 114,
+    "toilet": 65,
+    "towel": 81,
+    "tower": 84,
+    "trade name": 123,
+    "traffic light": 136,
+    "tray": 137,
+    "tree": 4,
+    "truck": 83,
+    "van": 102,
+    "vase": 135,
+    "wall": 0,
+    "wardrobe": 35,
+    "washer": 107,
+    "water": 21,
+    "waterfall": 113,
+    "windowpane": 8
+  },
   "max_depth": 10.0,
-  "num_seg_classes": 150,
-  "backbone_repo": "google/tipsv2-g14",
-  "embed_dim": 1536,
-  "block_indices": [
-    9,
-    19,
-    29,
-    39
-  ],
-  "post_process_channels": [
+  "min_depth": 0.001,
+  "model_type": "tipsv2_dpt",
+  "neck_hidden_sizes": [
     192,
     384,
     768,
     1536
-  ]
-}
+  ],
+  "num_depth_bins": 256,
+  "readout_act": "gelu_pytorch_tanh",
+  "reassemble_factors": [
+    4,
+    2,
+    1,
+    0.5
+  ],
+  "semantic_loss_ignore_index": 255,
+  "transformers_version": "5.10.0.dev0"
+}

configuration_dpt.py

@@ -3,6 +3,14 @@
 from transformers import PretrainedConfig
 
 
+_BACKBONE_REPO_BY_HIDDEN_SIZE = {
+    768: "google/tipsv2-b14",
+    1024: "google/tipsv2-l14",
+    1152: "google/tipsv2-so400m14",
+    1536: "google/tipsv2-g14",
+}
+
+
 class TIPSv2DPTConfig(PretrainedConfig):
     """Configuration for TIPSv2 DPT dense prediction heads."""
 
@@ -10,26 +18,29 @@ class TIPSv2DPTConfig(PretrainedConfig):
 
     def __init__(
         self,
-        backbone_repo="google/tipsv2-l14",
-        embed_dim=1024,
-        channels=256,
-        post_process_channels=(128, 256, 512, 1024),
-        block_indices=(5, 11, 17, 23),
-        readout_type="project",
+        backbone_config=None,
+        neck_hidden_sizes=(96, 192, 384, 768),
+        fusion_hidden_size=256,
+        reassemble_factors=(4, 2, 1, 0.5),
+        readout_act="gelu_pytorch_tanh",
         num_depth_bins=256,
         min_depth=1e-3,
         max_depth=10.0,
-        num_seg_classes=150,
+        depth_decoder_activation="relu",
+        semantic_loss_ignore_index=255,
         **kwargs,
     ):
         super().__init__(**kwargs)
-        self.backbone_repo = backbone_repo
-        self.embed_dim = embed_dim
-        self.channels = channels
-        self.post_process_channels = list(post_process_channels)
-        self.block_indices = list(block_indices)
-        self.readout_type = readout_type
+        backbone_config = backbone_config or {}
+        hidden_size = backbone_config.get("hidden_size", 768)
+        out_indices = backbone_config.get("out_indices", (3, 6, 9, 12))
+        self.backbone_repo = _BACKBONE_REPO_BY_HIDDEN_SIZE[hidden_size]
+        self.embed_dim = hidden_size
+        self.channels = fusion_hidden_size
+        self.post_process_channels = list(neck_hidden_sizes)
+        self.block_indices = [out_index - 1 for out_index in out_indices]
+        self.readout_type = "project"
         self.num_depth_bins = num_depth_bins
         self.min_depth = min_depth
         self.max_depth = max_depth
-        self.num_seg_classes = num_seg_classes
+        self.num_seg_classes = self.num_labels

image_encoder.py

@@ -0,0 +1,1002 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Vision encoder implementation in PyTorch."""
+
+import functools
+import math
+import os
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
+import warnings
+import torch
+from torch import nn
+import torch.nn.functional as F
+import torch.utils.checkpoint
+
+
+class Mlp(nn.Module):
+  """Transformer MLP, following DINOv2 implementation."""
+
+  def __init__(
+      self,
+      in_features: int,
+      hidden_features: Optional[int] = None,
+      out_features: Optional[int] = None,
+      act_layer: Callable[..., nn.Module] = nn.GELU,
+      drop: float = 0.0,
+      bias: bool = True,
+  ) -> None:
+    super().__init__()
+    out_features = out_features or in_features
+    hidden_features = hidden_features or in_features
+    self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
+    self.act = act_layer()
+    self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
+    self.drop = nn.Dropout(drop)
+
+  def forward(self, x: torch.Tensor) -> torch.Tensor:
+    x = self.fc1(x)
+    x = self.act(x)
+    x = self.drop(x)
+    x = self.fc2(x)
+    x = self.drop(x)
+    return x
+
+
+def make_2tuple(x):
+  if isinstance(x, tuple):
+    assert len(x) == 2
+    return x
+
+  assert isinstance(x, int)
+  return (x, x)
+
+
+class PatchEmbed(nn.Module):
+  """2D image to patch embedding: (B,C,H,W) -> (B,N,D)."""
+
+  def __init__(
+      self,
+      img_size: Union[int, Tuple[int, int]] = 224,
+      patch_size: Union[int, Tuple[int, int]] = 16,
+      in_chans: int = 3,
+      embed_dim: int = 768,
+      norm_layer: Optional[Callable] = None,  # pylint: disable=g-bare-generic
+      flatten_embedding: bool = True,
+  ) -> None:
+    super().__init__()
+
+    image_hw = make_2tuple(img_size)
+    patch_hw = make_2tuple(patch_size)
+    patch_grid_size = (
+        image_hw[0] // patch_hw[0],
+        image_hw[1] // patch_hw[1],
+    )
+
+    self.img_size = image_hw
+    self.patch_size = patch_hw
+    self.patches_resolution = patch_grid_size
+    self.num_patches = patch_grid_size[0] * patch_grid_size[1]
+
+    self.in_chans = in_chans
+    self.embed_dim = embed_dim
+
+    self.flatten_embedding = flatten_embedding
+
+    self.proj = nn.Conv2d(
+        in_chans, embed_dim, kernel_size=patch_hw, stride=patch_hw
+    )
+    self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+  def forward(self, x: torch.Tensor) -> torch.Tensor:
+    _, _, h, w = x.shape
+    patch_h, patch_w = self.patch_size
+
+    assert (
+        h % patch_h == 0
+    ), f"Input image height {h} is not a multiple of patch height {patch_h}"
+    assert (
+        w % patch_w == 0
+    ), f"Input image width {w} is not a multiple of patch width: {patch_w}"
+
+    x = self.proj(x)  # B C H W
+    h, w = x.size(2), x.size(3)
+    x = x.flatten(2).transpose(1, 2)  # B HW C
+    x = self.norm(x)
+    if not self.flatten_embedding:
+      x = x.reshape(-1, h, w, self.embed_dim)  # B H W C
+    return x
+
+  def flops(self) -> float:
+    ho, wo = self.patches_resolution
+    flops = (
+        ho
+        * wo
+        * self.embed_dim
+        * self.in_chans
+        * (self.patch_size[0] * self.patch_size[1])
+    )
+    if self.norm is not None:
+      flops += ho * wo * self.embed_dim
+    return flops
+
+
+class SwiGLUFFN(nn.Module):
+  """SwiGLU FFN layer, following DINOv2 implementation."""
+
+  def __init__(
+      self,
+      in_features: int,
+      hidden_features: Optional[int] = None,
+      out_features: Optional[int] = None,
+      act_layer: Callable[..., nn.Module] = None,
+      drop: float = 0.0,
+      bias: bool = True,
+  ) -> None:
+    super().__init__()
+    out_features = out_features or in_features
+    hidden_features = hidden_features or in_features
+    self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
+    self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
+
+  def forward(self, x: torch.Tensor) -> torch.Tensor:
+    x12 = self.w12(x)
+    x1, x2 = x12.chunk(2, dim=-1)
+    hidden = F.silu(x1) * x2
+    return self.w3(hidden)
+
+
+XFORMERS_ENABLED = os.environ.get("XFORMERS_DISABLED") is None
+try:
+  if XFORMERS_ENABLED:
+    from xformers.ops import SwiGLU, memory_efficient_attention, unbind, fmha, scaled_index_add, index_select_cat  # pylint: disable=g-multiple-import, g-import-not-at-top
+
+    XFORMERS_AVAILABLE = True
+    warnings.warn("xFormers is available (SwiGLU)")
+  else:
+    warnings.warn("xFormers is disabled (SwiGLU)")
+    raise ImportError
+except ImportError:
+  SwiGLU = SwiGLUFFN
+  XFORMERS_AVAILABLE = False
+
+  warnings.warn("xFormers is not available (SwiGLU)")
+
+
+class SwiGLUFFNFused(SwiGLU):
+  """SwiGLU FFN layer, following DINOv2 implementation."""
+
+  def __init__(
+      self,
+      in_features: int,
+      hidden_features: Optional[int] = None,
+      out_features: Optional[int] = None,
+      act_layer: Callable[..., nn.Module] = None,  # pylint: disable=unused-argument
+      drop: float = 0.0,  # pylint: disable=unused-argument
+      bias: bool = True,
+  ) -> None:
+    out_features = out_features or in_features
+    hidden_features = hidden_features or in_features
+    hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
+    super().__init__(
+        in_features=in_features,
+        hidden_features=hidden_features,
+        out_features=out_features,
+        bias=bias,
+    )
+
+
+class Attention(nn.Module):
+  """Attention layer, following DINOv2 implementation."""
+
+  def __init__(
+      self,
+      dim: int,
+      num_heads: int = 8,
+      qkv_bias: bool = False,
+      proj_bias: bool = True,
+      attn_drop: float = 0.0,
+      proj_drop: float = 0.0,
+  ) -> None:
+    super().__init__()
+    self.num_heads = num_heads
+    head_dim = dim // num_heads
+    self.scale = head_dim**-0.5
+
+    self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+    self.attn_drop = nn.Dropout(attn_drop)
+    self.proj = nn.Linear(dim, dim, bias=proj_bias)
+    self.proj_drop = nn.Dropout(proj_drop)
+
+  def forward(self, x: torch.Tensor) -> torch.Tensor:
+    b_dim, n_dim, c_dim = x.shape
+    qkv = (
+        self.qkv(x)
+        .reshape(b_dim, n_dim, 3, self.num_heads, c_dim // self.num_heads)
+        .permute(2, 0, 3, 1, 4)
+    )
+
+    q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
+    attn = q @ k.transpose(-2, -1)
+
+    attn = attn.softmax(dim=-1)
+    attn = self.attn_drop(attn)
+
+    x = (attn @ v).transpose(1, 2).reshape(b_dim, n_dim, c_dim)
+    x = self.proj(x)
+    x = self.proj_drop(x)
+    return x
+
+
+class MemEffAttention(Attention):
+  """Memory Efficient Attention layer, following DINOv2 implementation."""
+
+  def forward(self, x: torch.Tensor, attn_bias=None) -> torch.Tensor:
+    if not XFORMERS_AVAILABLE:
+      if attn_bias is not None:
+        raise AssertionError("xFormers is required for using nested tensors")
+      return super().forward(x)
+
+    b_dim, n_dim, c_dim = x.shape
+    qkv = self.qkv(x).reshape(
+        b_dim, n_dim, 3, self.num_heads, c_dim // self.num_heads
+    )
+
+    q, k, v = unbind(qkv, 2)
+
+    x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+    x = x.reshape([b_dim, n_dim, c_dim])
+
+    x = self.proj(x)
+    x = self.proj_drop(x)
+    return x
+
+
+class LayerScale(nn.Module):
+  """Layer scale, following DINOv2 implementation."""
+
+  def __init__(
+      self,
+      dim: int,
+      init_values: Union[float, torch.Tensor] = 1e-5,
+      inplace: bool = False,
+  ) -> None:
+    super().__init__()
+    self.inplace = inplace
+    self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+  def forward(self, x: torch.Tensor) -> torch.Tensor:
+    return x.mul_(self.gamma) if self.inplace else x * self.gamma
+
+
+def drop_path_impl(x, drop_prob: float = 0.0, training: bool = False):
+  if drop_prob == 0.0 or not training:
+    return x
+  keep_prob = 1 - drop_prob
+  shape = (x.shape[0],) + (1,) * (
+      x.ndim - 1
+  )  # work with diff dim tensors, not just 2D ConvNets
+  random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+  if keep_prob > 0.0:
+    random_tensor.div_(keep_prob)
+  output = x * random_tensor
+  return output
+
+
+class DropPath(nn.Module):
+  """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
+
+  def __init__(self, drop_prob=None):
+    super(DropPath, self).__init__()
+    self.drop_prob = drop_prob
+
+  def forward(self, x):
+    return drop_path_impl(x, self.drop_prob, self.training)
+
+
+class Block(nn.Module):
+  """Transformer Block Implementation, following DINOv2 implementation."""
+
+  def __init__(
+      self,
+      dim: int,
+      num_heads: int,
+      mlp_ratio: float = 4.0,
+      qkv_bias: bool = False,
+      proj_bias: bool = True,
+      ffn_bias: bool = True,
+      drop: float = 0.0,
+      attn_drop: float = 0.0,
+      init_values=None,
+      drop_path: float = 0.0,
+      act_layer: Callable[..., nn.Module] = nn.GELU,
+      norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+      attn_class: Callable[..., nn.Module] = Attention,
+      ffn_layer: Callable[..., nn.Module] = Mlp,
+  ) -> None:
+    super().__init__()
+    self.norm1 = norm_layer(dim)
+    self.attn = attn_class(
+        dim,
+        num_heads=num_heads,
+        qkv_bias=qkv_bias,
+        proj_bias=proj_bias,
+        attn_drop=attn_drop,
+        proj_drop=drop,
+    )
+    self.ls1 = (
+        LayerScale(dim, init_values=init_values)
+        if init_values
+        else nn.Identity()
+    )
+    self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+    self.norm2 = norm_layer(dim)
+    mlp_hidden_dim = int(dim * mlp_ratio)
+    self.mlp = ffn_layer(
+        in_features=dim,
+        hidden_features=mlp_hidden_dim,
+        act_layer=act_layer,
+        drop=drop,
+        bias=ffn_bias,
+    )
+    self.ls2 = (
+        LayerScale(dim, init_values=init_values)
+        if init_values
+        else nn.Identity()
+    )
+    self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+    self.sample_drop_ratio = drop_path
+
+  def forward(self, x: torch.Tensor) -> torch.Tensor:
+    def attn_residual_func(x: torch.Tensor) -> torch.Tensor:
+      return self.ls1(self.attn(self.norm1(x)))
+
+    def ffn_residual_func(x: torch.Tensor) -> torch.Tensor:
+      return self.ls2(self.mlp(self.norm2(x)))
+
+    if self.training and self.sample_drop_ratio > 0.1:
+      # the overhead is compensated only for a drop path rate larger than 0.1
+      x = drop_add_residual_stochastic_depth(
+          x,
+          residual_func=attn_residual_func,
+          sample_drop_ratio=self.sample_drop_ratio,
+      )
+      x = drop_add_residual_stochastic_depth(
+          x,
+          residual_func=ffn_residual_func,
+          sample_drop_ratio=self.sample_drop_ratio,
+      )
+    elif self.training and self.sample_drop_ratio > 0.0:
+      x = x + self.drop_path1(attn_residual_func(x))
+      x = x + self.drop_path1(ffn_residual_func(x))
+    else:
+      x = x + attn_residual_func(x)
+      x = x + ffn_residual_func(x)
+    return x
+
+
+def drop_add_residual_stochastic_depth(
+    x: torch.Tensor,
+    residual_func: Callable[[torch.Tensor], torch.Tensor],
+    sample_drop_ratio: float = 0.0,
+) -> torch.Tensor:
+  """This function is taken from the original implementation in DINOv2 to implement stochastic depth in the image encoder."""
+  # 1) extract subset using permutation
+  b, _, _ = x.shape
+  sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+  brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+  x_subset = x[brange]
+
+  # 2) apply residual_func to get residual
+  residual = residual_func(x_subset)
+
+  x_flat = x.flatten(1)
+  residual = residual.flatten(1)
+
+  residual_scale_factor = b / sample_subset_size
+
+  # 3) add the residual
+  x_plus_residual = torch.index_add(
+      x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor
+  )
+  return x_plus_residual.view_as(x)
+
+
+def get_branges_scales(x, sample_drop_ratio=0.0):
+  b, _, _ = x.shape
+  sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+  brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+  residual_scale_factor = b / sample_subset_size
+  return brange, residual_scale_factor
+
+
+def add_residual(
+    x, brange, residual, residual_scale_factor, scaling_vector=None
+):
+  """Implement residual addition in the image encoder."""
+  if scaling_vector is None:
+    x_flat = x.flatten(1)
+    residual = residual.flatten(1)
+    x_plus_residual = torch.index_add(
+        x_flat,
+        0,
+        brange,
+        residual.to(dtype=x.dtype),
+        alpha=residual_scale_factor,
+    )
+  else:
+    x_plus_residual = scaled_index_add(
+        x,
+        brange,
+        residual.to(dtype=x.dtype),
+        scaling=scaling_vector,
+        alpha=residual_scale_factor,
+    )
+  return x_plus_residual
+
+
+attn_bias_cache: Dict[Tuple, Any] = {}  # pylint: disable=g-bare-generic
+
+
+def get_attn_bias_and_cat(x_list, branges=None):
+  """this will perform the index select, cat the tensors, and provide the attn_bias from cache."""
+  batch_sizes = (
+      [b.shape[0] for b in branges]
+      if branges is not None
+      else [x.shape[0] for x in x_list]
+  )
+  all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
+  if all_shapes not in attn_bias_cache.keys():
+    seqlens = []
+    for b, x in zip(batch_sizes, x_list):
+      for _ in range(b):
+        seqlens.append(x.shape[1])
+    attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
+    attn_bias._batch_sizes = batch_sizes  # pylint: disable=protected-access
+    attn_bias_cache[all_shapes] = attn_bias
+
+  if branges is not None:
+    cat_tensors = index_select_cat(
+        [x.flatten(1) for x in x_list], branges
+    ).view(1, -1, x_list[0].shape[-1])
+  else:
+    tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
+    cat_tensors = torch.cat(tensors_bs1, dim=1)
+
+  return attn_bias_cache[all_shapes], cat_tensors
+
+
+def drop_add_residual_stochastic_depth_list(
+    x_list: List[torch.Tensor],
+    residual_func: Callable[[torch.Tensor, Any], torch.Tensor],
+    sample_drop_ratio: float = 0.0,
+    scaling_vector=None,
+) -> torch.Tensor:
+  """Add residual to a list of tensors."""
+  # 1) generate random set of indices for dropping samples in the batch.
+  branges_scales = [
+      get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list
+  ]
+  branges = [s[0] for s in branges_scales]
+  residual_scale_factors = [s[1] for s in branges_scales]
+
+  # 2) get attention bias and index+concat the tensors.
+  attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
+
+  # 3) apply residual_func to get residual, and split the result.
+  residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias))  # type: ignore
+
+  outputs = []
+  for x, brange, residual, residual_scale_factor in zip(
+      x_list, branges, residual_list, residual_scale_factors
+  ):
+    outputs.append(
+        add_residual(
+            x, brange, residual, residual_scale_factor, scaling_vector
+        ).view_as(x)
+    )
+  return outputs
+
+
+class NestedTensorBlock(Block):
+  """Nested tensor block implementation."""
+
+  def forward_nested(self, x_list: List[torch.Tensor]) -> List[torch.Tensor]:
+    """x_list contains a list of tensors to nest together and run."""
+    assert isinstance(self.attn, MemEffAttention)
+
+    if self.training and self.sample_drop_ratio > 0.0:
+
+      def attn_residual_func(x: torch.Tensor, attn_bias=None) -> torch.Tensor:
+        return self.attn(self.norm1(x), attn_bias=attn_bias)
+
+      def ffn_residual_func(x: torch.Tensor, attn_bias=None) -> torch.Tensor:
+        del attn_bias
+        return self.mlp(self.norm2(x))
+
+      x_list = drop_add_residual_stochastic_depth_list(
+          x_list,
+          residual_func=attn_residual_func,
+          sample_drop_ratio=self.sample_drop_ratio,
+          scaling_vector=self.ls1.gamma
+          if isinstance(self.ls1, LayerScale)
+          else None,
+      )
+      x_list = drop_add_residual_stochastic_depth_list(
+          x_list,
+          residual_func=ffn_residual_func,
+          sample_drop_ratio=self.sample_drop_ratio,
+          scaling_vector=self.ls2.gamma
+          if isinstance(self.ls1, LayerScale)
+          else None,
+      )
+      return x_list
+    else:
+
+      def attn_residual_func(x: torch.Tensor, attn_bias=None) -> torch.Tensor:
+        return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
+
+      def ffn_residual_func(x: torch.Tensor, attn_bias=None) -> torch.Tensor:
+        del attn_bias
+        return self.ls2(self.mlp(self.norm2(x)))
+
+      attn_bias, x = get_attn_bias_and_cat(x_list)
+      x = x + attn_residual_func(x, attn_bias=attn_bias)
+      x = x + ffn_residual_func(x)
+      return attn_bias.split(x)
+
+  def forward(self, x):
+    if isinstance(x, torch.Tensor):
+      return super().forward(x)
+    elif isinstance(x, list):
+      if not XFORMERS_AVAILABLE:
+        raise AssertionError("xFormers is required for using nested tensors")
+      return self.forward_nested(x)
+    else:
+      raise AssertionError
+
+
+def named_apply(
+    fn: Callable,  # pylint: disable=g-bare-generic
+    module: nn.Module,
+    name="",
+    depth_first=True,
+    include_root=False,
+) -> nn.Module:
+  """Apply a function to a module and its children."""
+  if not depth_first and include_root:
+    fn(module=module, name=name)
+  for child_name, child_module in module.named_children():
+    child_name = ".".join((name, child_name)) if name else child_name
+    named_apply(
+        fn=fn,
+        module=child_module,
+        name=child_name,
+        depth_first=depth_first,
+        include_root=True,
+    )
+  if depth_first and include_root:
+    fn(module=module, name=name)
+  return module
+
+
+class BlockChunk(nn.ModuleList):
+
+  def forward(self, x):
+    for b in self:
+      x = b(x)
+    return x
+
+
+class VisionTransformer(nn.Module):
+  """Vision Transformer implementation."""
+
+  def __init__(
+      self,
+      img_size=224,
+      patch_size=16,
+      in_chans=3,
+      embed_dim=768,
+      depth=12,
+      num_heads=12,
+      mlp_ratio=4.0,
+      qkv_bias=True,
+      ffn_bias=True,
+      proj_bias=True,
+      drop_path_rate=0.0,
+      drop_path_uniform=False,
+      init_values=None,  # for layerscale: None or 0 => no layerscale
+      embed_layer=PatchEmbed,
+      act_layer=nn.GELU,
+      block_fn=Block,
+      ffn_layer="mlp",
+      block_chunks=1,
+      num_register_tokens=0,
+      interpolate_antialias=False,
+      interpolate_offset=0.1,
+  ):
+    """Defines the Vision Transformer model.
+
+    Args:
+      img_size (int, tuple): input image size
+      patch_size (int, tuple): patch size
+      in_chans (int): number of input channels
+      embed_dim (int): embedding dimension
+      depth (int): depth of transformer
+      num_heads (int): number of attention heads
+      mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+      qkv_bias (bool): enable bias for qkv if True
+      ffn_bias (bool): enable bias for ffn if True
+      proj_bias (bool): enable bias for proj in attn if True
+      drop_path_rate (float): stochastic depth rate
+      drop_path_uniform (bool): apply uniform drop rate across blocks
+      init_values (float): layer-scale init values
+      embed_layer (nn.Module): patch embedding layer
+      act_layer (nn.Module): MLP activation layer
+      block_fn (nn.Module): transformer block class
+      ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
+      block_chunks: (int) split block sequence into block_chunks units for FSDP
+        wrap
+      num_register_tokens: (int) number of extra cls tokens (so-called
+        "registers")
+      interpolate_antialias: (str) flag to apply anti-aliasing when
+        interpolating positional embeddings
+      interpolate_offset: (float) work-around offset to apply when interpolating
+        positional embeddings
+    """
+    super().__init__()
+    norm_layer = functools.partial(nn.LayerNorm, eps=1e-6)
+
+    self.num_features = self.embed_dim = (
+        embed_dim  # num_features for consistency with other models
+    )
+    self.num_tokens = 1
+    self.n_blocks = depth
+    self.num_heads = num_heads
+    self.patch_size = patch_size
+    self.num_register_tokens = num_register_tokens
+    self.interpolate_antialias = interpolate_antialias
+    self.interpolate_offset = interpolate_offset
+
+    self.patch_embed = embed_layer(
+        img_size=img_size,
+        patch_size=patch_size,
+        in_chans=in_chans,
+        embed_dim=embed_dim,
+    )
+    num_patches = self.patch_embed.num_patches
+
+    self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+    self.pos_embed = nn.Parameter(
+        torch.zeros(1, num_patches + self.num_tokens, embed_dim)
+    )
+    assert num_register_tokens >= 0
+    self.register_tokens = (
+        nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim))
+        if num_register_tokens
+        else None
+    )
+
+    if drop_path_uniform:
+      dpr = [drop_path_rate] * depth
+    else:
+      dpr = [
+          drop_path_rate * i / max(depth - 1, 1) for i in range(depth)
+      ]  # stochastic depth decay rule
+
+    if ffn_layer == "mlp":
+      ffn_layer = Mlp
+    elif ffn_layer == "swiglufused" or ffn_layer == "swiglu":
+      ffn_layer = SwiGLUFFNFused
+    else:
+      raise NotImplementedError
+
+    blocks_list = [
+        block_fn(
+            dim=embed_dim,
+            num_heads=num_heads,
+            mlp_ratio=mlp_ratio,
+            qkv_bias=qkv_bias,
+            proj_bias=proj_bias,
+            ffn_bias=ffn_bias,
+            drop_path=dpr[i],
+            norm_layer=norm_layer,
+            act_layer=act_layer,
+            ffn_layer=ffn_layer,
+            init_values=init_values,
+        )
+        for i in range(depth)
+    ]
+    if block_chunks > 0:
+      self.chunked_blocks = True
+      chunked_blocks = []
+      chunksize = depth // block_chunks
+      for i in range(0, depth, chunksize):
+        # this is to keep the block index consistent if we chunk the block list
+        chunked_blocks.append(
+            [nn.Identity()] * i + blocks_list[i : i + chunksize]
+        )
+      self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks])
+    else:
+      self.chunked_blocks = False
+      self.blocks = nn.ModuleList(blocks_list)
+
+    self.norm = norm_layer(embed_dim)
+    self.head = nn.Identity()
+
+    self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
+
+    self.init_weights()
+
+  def init_weights(self):
+    nn.init.trunc_normal_(self.pos_embed, std=0.02)
+    nn.init.normal_(self.cls_token, std=1e-6)
+    if self.register_tokens is not None:
+      nn.init.normal_(self.register_tokens, std=1e-6)
+    named_apply(init_weights_vit_timm, self)
+
+  def interpolate_pos_encoding(self, x, w, h):
+    previous_dtype = x.dtype
+    npatch = x.shape[1] - 1
+    num_patches = self.pos_embed.shape[1] - 1
+    if npatch == num_patches and w == h:
+      return self.pos_embed
+    pos_embed = self.pos_embed.float()
+    class_pos_embed = pos_embed[:, 0]
+    patch_pos_embed = pos_embed[:, 1:]
+    dim = x.shape[-1]
+    w0 = w // self.patch_size
+    h0 = h // self.patch_size
+    num_patches_dim = int(
+        math.sqrt(num_patches)
+    )  # Recover the number of patches in each dimension
+    assert num_patches == num_patches_dim * num_patches_dim
+    kwargs = {}
+    if self.interpolate_offset:
+      sx = float(w0 + self.interpolate_offset) / num_patches_dim
+      sy = float(h0 + self.interpolate_offset) / num_patches_dim
+      kwargs["scale_factor"] = (sx, sy)
+    else:
+      # Simply specify an output size instead of a scale factor
+      kwargs["size"] = (w0, h0)
+    patch_pos_embed = nn.functional.interpolate(
+        patch_pos_embed.reshape(
+            1, num_patches_dim, num_patches_dim, dim
+        ).permute(0, 3, 1, 2),
+        mode="bilinear",
+        antialias=self.interpolate_antialias,
+        **kwargs,
+    )
+    assert (w0, h0) == patch_pos_embed.shape[-2:]
+    patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+    return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(
+        previous_dtype
+    )
+
+  def prepare_tokens_with_masks(self, x, masks=None):
+    _, _, w, h = x.shape
+    x = self.patch_embed(x)
+    if masks is not None:
+      x = torch.where(
+          masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x
+      )
+
+    x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
+    x = x + self.interpolate_pos_encoding(x, w, h)
+
+    if self.register_tokens is not None:
+      x = torch.cat(
+          (
+              x[:, :1],
+              self.register_tokens.expand(x.shape[0], -1, -1),
+              x[:, 1:],
+          ),
+          dim=1,
+      )
+
+    return x
+
+  def forward_features_list(self, x_list, masks_list):
+    x = [
+        self.prepare_tokens_with_masks(x, masks)
+        for x, masks in zip(x_list, masks_list)
+    ]
+    for blk in self.blocks:
+      x = blk(x)
+
+    all_x = x
+    output = []
+    for x, masks in zip(all_x, masks_list):
+      x_norm = self.norm(x)
+      output.append({
+          "x_norm_1st_clstoken": x_norm[:, :1],
+          "x_norm_2nd_clstoken": x_norm[:, 1 : self.num_register_tokens + 1],
+          "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+          "x_prenorm": x,
+          "masks": masks,
+      })
+    return output
+
+  def forward_features(self, x, masks=None):
+    if isinstance(x, list):
+      return self.forward_features_list(x, masks)
+
+    x = self.prepare_tokens_with_masks(x, masks)
+
+    for blk in self.blocks:
+      x = blk(x)
+
+    x_norm = self.norm(x)
+    return {
+        "x_norm_1st_clstoken": x_norm[:, :1],
+        "x_norm_2nd_clstoken": x_norm[:, 1 : self.num_register_tokens + 1],
+        "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+        "x_prenorm": x,
+        "masks": masks,
+    }
+
+  def _get_intermediate_layers_not_chunked(self, x, n=1):
+    x = self.prepare_tokens_with_masks(x)
+    # If n is an int, take the n last blocks. If it's a list, take them
+    output, total_block_len = [], len(self.blocks)
+    blocks_to_take = (
+        range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+    )
+    for i, blk in enumerate(self.blocks):
+      x = blk(x)
+      if i in blocks_to_take:
+        output.append(x)
+    assert len(output) == len(
+        blocks_to_take
+    ), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+    return output
+
+  def _get_intermediate_layers_chunked(self, x, n=1):
+    x = self.prepare_tokens_with_masks(x)
+    output, i, total_block_len = [], 0, len(self.blocks[-1])
+    # If n is an int, take the n last blocks. If it's a list, take them
+    blocks_to_take = (
+        range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+    )
+    for block_chunk in self.blocks:
+      for blk in block_chunk[i:]:  # Passing the nn.Identity()
+        x = blk(x)
+        if i in blocks_to_take:
+          output.append(x)
+        i += 1
+    assert len(output) == len(
+        blocks_to_take
+    ), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+    return output
+
+  def get_intermediate_layers(
+      self,
+      x: torch.torch.Tensor,
+      n: Union[int, Sequence] = 1,  # Layers or n last layers to take  # pylint: disable=g-bare-generic
+      reshape: bool = False,
+      return_class_token: bool = False,
+      norm=True,
+  ) -> Tuple[Union[torch.torch.Tensor, Tuple[torch.torch.Tensor]]]:  # pylint: disable=g-one-element-tuple
+    if self.chunked_blocks:
+      outputs = self._get_intermediate_layers_chunked(x, n)
+    else:
+      outputs = self._get_intermediate_layers_not_chunked(x, n)
+    if norm:
+      outputs = [self.norm(out) for out in outputs]
+    class_tokens = [out[:, 0] for out in outputs]
+    outputs = [out[:, 1 + self.num_register_tokens :] for out in outputs]
+    if reshape:
+      batch_size, _, w, h = x.shape
+      outputs = [
+          out.reshape(
+              batch_size, w // self.patch_size, h // self.patch_size, -1
+          )
+          .permute(0, 3, 1, 2)
+          .contiguous()
+          for out in outputs
+      ]
+    if return_class_token:
+      return tuple(zip(outputs, class_tokens))
+    return tuple(outputs)
+
+  def forward(self, *args, is_training=False, **kwargs):
+    ret = self.forward_features(*args, **kwargs)
+    if is_training:
+      return ret
+    else:
+      return self.head(ret["x_norm_1st_clstoken"]), self.head(
+          ret["x_norm_2nd_clstoken"]
+      ), ret["x_norm_patchtokens"]
+
+
+def init_weights_vit_timm(module: nn.Module, name: str = ""):  # pylint: disable=unused-argument
+  """ViT weight initialization, original timm impl (for reproducibility)."""
+  if isinstance(module, nn.Linear):
+    nn.init.trunc_normal_(module.weight, std=0.02)
+    if module.bias is not None:
+      nn.init.zeros_(module.bias)
+
+
+def vit_small(patch_size=14, **kwargs):
+  model = VisionTransformer(
+      patch_size=patch_size,
+      embed_dim=384,
+      depth=12,
+      num_heads=6,
+      mlp_ratio=4,
+      block_fn=functools.partial(Block, attn_class=MemEffAttention),
+      num_register_tokens=1,
+      **kwargs,
+  )
+  return model
+
+
+def vit_base(patch_size=14, **kwargs):
+  model = VisionTransformer(
+      patch_size=patch_size,
+      embed_dim=768,
+      depth=12,
+      num_heads=12,
+      mlp_ratio=4,
+      block_fn=functools.partial(Block, attn_class=MemEffAttention),
+      num_register_tokens=1,
+      **kwargs,
+  )
+  return model
+
+
+def vit_large(patch_size=14, **kwargs):
+  model = VisionTransformer(
+      patch_size=patch_size,
+      embed_dim=1024,
+      depth=24,
+      num_heads=16,
+      mlp_ratio=4,
+      block_fn=functools.partial(Block, attn_class=MemEffAttention),
+      num_register_tokens=1,
+      **kwargs,
+  )
+  return model
+
+
+def vit_so400m(patch_size=14, **kwargs):
+  """SoViT 400M model (https://arxiv.org/abs/2305.13035)."""
+  model = VisionTransformer(
+      patch_size=patch_size,
+      embed_dim=1152,
+      depth=27,
+      num_heads=16,
+      mlp_ratio=4304 / 1152,
+      block_fn=functools.partial(Block, attn_class=MemEffAttention),
+      num_register_tokens=1,
+      **kwargs,
+  )
+  return model
+
+
+def vit_giant2(patch_size=14, **kwargs):
+  model = VisionTransformer(
+      patch_size=patch_size,
+      embed_dim=1536,
+      depth=40,
+      num_heads=24,
+      mlp_ratio=4,
+      block_fn=functools.partial(Block, attn_class=MemEffAttention),
+      num_register_tokens=1,
+      **kwargs,
+  )
+  return model

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:637e1095c02ef28d35a516ad5d8e56d7b6fee3570bd58fd93986fbec20f4d1c8
-size 738064900
+oid sha256:8ad9e99d69b6bc3a6c046f3b34532ddf04373adb56e71dd1ebb572f52075dc5e
+size 5281937548

modeling_dpt.py

@@ -61,6 +61,7 @@ class TIPSv2DPTModel(PreTrainedModel):
     _no_split_modules = []
     _supports_cache_class = False
     _tied_weights_keys = []
+    _keys_to_ignore_on_load_unexpected = {"backbone"}
 
     @property
     def all_tied_weights_keys(self):

preprocessor_config.json

@@ -0,0 +1,13 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": false,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_processor_type": "Tipsv2DptImageProcessor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 448,
+    "width": 448
+  }
+}