Update files for transformers integration

config.json

@@ -1,5 +1,4 @@
 {
-  "model_type": "tipsv2_dpt",
   "architectures": [
     "TIPSv2DPTModel"
   ],
@@ -7,24 +6,392 @@
     "AutoConfig": "configuration_dpt.TIPSv2DPTConfig",
     "AutoModel": "modeling_dpt.TIPSv2DPTModel"
   },
-  "channels": 256,
-  "readout_type": "project",
-  "num_depth_bins": 256,
-  "min_depth": 0.001,
+  "backbone_config": {
+    "apply_layernorm": true,
+    "attention_probs_dropout_prob": 0.0,
+    "drop_path_rate": 0.0,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1024,
+    "image_size": 448,
+    "initializer_range": 0.02,
+    "interpolate_antialias": true,
+    "interpolate_offset": 0.0,
+    "layer_norm_eps": 1e-06,
+    "layerscale_value": 1.0,
+    "mlp_ratio": 4,
+    "model_type": "tipsv2_vision_model",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "num_register_tokens": 1,
+    "out_features": [
+      "stage6",
+      "stage12",
+      "stage18",
+      "stage24"
+    ],
+    "out_indices": [
+      6,
+      12,
+      18,
+      24
+    ],
+    "patch_size": 14,
+    "qkv_bias": true,
+    "reshape_hidden_states": false,
+    "stage_names": [
+      "stem",
+      "stage1",
+      "stage2",
+      "stage3",
+      "stage4",
+      "stage5",
+      "stage6",
+      "stage7",
+      "stage8",
+      "stage9",
+      "stage10",
+      "stage11",
+      "stage12",
+      "stage13",
+      "stage14",
+      "stage15",
+      "stage16",
+      "stage17",
+      "stage18",
+      "stage19",
+      "stage20",
+      "stage21",
+      "stage22",
+      "stage23",
+      "stage24"
+    ],
+    "use_swiglu_ffn": false
+  },
+  "depth_decoder_activation": "relu",
+  "fusion_hidden_size": 256,
+  "id2label": {
+    "0": "wall",
+    "1": "building",
+    "10": "cabinet",
+    "100": "poster",
+    "101": "stage",
+    "102": "van",
+    "103": "ship",
+    "104": "fountain",
+    "105": "conveyer belt",
+    "106": "canopy",
+    "107": "washer",
+    "108": "plaything",
+    "109": "swimming pool",
+    "11": "sidewalk",
+    "110": "stool",
+    "111": "barrel",
+    "112": "basket",
+    "113": "waterfall",
+    "114": "tent",
+    "115": "bag",
+    "116": "minibike",
+    "117": "cradle",
+    "118": "oven",
+    "119": "ball",
+    "12": "person",
+    "120": "food",
+    "121": "step",
+    "122": "tank",
+    "123": "trade name",
+    "124": "microwave",
+    "125": "pot",
+    "126": "animal",
+    "127": "bicycle",
+    "128": "lake",
+    "129": "dishwasher",
+    "13": "earth",
+    "130": "screen",
+    "131": "blanket",
+    "132": "sculpture",
+    "133": "hood",
+    "134": "sconce",
+    "135": "vase",
+    "136": "traffic light",
+    "137": "tray",
+    "138": "ashcan",
+    "139": "fan",
+    "14": "door",
+    "140": "pier",
+    "141": "crt screen",
+    "142": "plate",
+    "143": "monitor",
+    "144": "bulletin board",
+    "145": "shower",
+    "146": "radiator",
+    "147": "glass",
+    "148": "clock",
+    "149": "flag",
+    "15": "table",
+    "16": "mountain",
+    "17": "plant",
+    "18": "curtain",
+    "19": "chair",
+    "2": "sky",
+    "20": "car",
+    "21": "water",
+    "22": "painting",
+    "23": "sofa",
+    "24": "shelf",
+    "25": "house",
+    "26": "sea",
+    "27": "mirror",
+    "28": "rug",
+    "29": "field",
+    "3": "floor",
+    "30": "armchair",
+    "31": "seat",
+    "32": "fence",
+    "33": "desk",
+    "34": "rock",
+    "35": "wardrobe",
+    "36": "lamp",
+    "37": "bathtub",
+    "38": "railing",
+    "39": "cushion",
+    "4": "tree",
+    "40": "base",
+    "41": "box",
+    "42": "column",
+    "43": "signboard",
+    "44": "chest of drawers",
+    "45": "counter",
+    "46": "sand",
+    "47": "sink",
+    "48": "skyscraper",
+    "49": "fireplace",
+    "5": "ceiling",
+    "50": "refrigerator",
+    "51": "grandstand",
+    "52": "path",
+    "53": "stairs",
+    "54": "runway",
+    "55": "case",
+    "56": "pool table",
+    "57": "pillow",
+    "58": "screen door",
+    "59": "stairway",
+    "6": "road",
+    "60": "river",
+    "61": "bridge",
+    "62": "bookcase",
+    "63": "blind",
+    "64": "coffee table",
+    "65": "toilet",
+    "66": "flower",
+    "67": "book",
+    "68": "hill",
+    "69": "bench",
+    "7": "bed ",
+    "70": "countertop",
+    "71": "stove",
+    "72": "palm",
+    "73": "kitchen island",
+    "74": "computer",
+    "75": "swivel chair",
+    "76": "boat",
+    "77": "bar",
+    "78": "arcade machine",
+    "79": "hovel",
+    "8": "windowpane",
+    "80": "bus",
+    "81": "towel",
+    "82": "light",
+    "83": "truck",
+    "84": "tower",
+    "85": "chandelier",
+    "86": "awning",
+    "87": "streetlight",
+    "88": "booth",
+    "89": "television receiver",
+    "9": "grass",
+    "90": "airplane",
+    "91": "dirt track",
+    "92": "apparel",
+    "93": "pole",
+    "94": "land",
+    "95": "bannister",
+    "96": "escalator",
+    "97": "ottoman",
+    "98": "bottle",
+    "99": "buffet"
+  },
+  "label2id": {
+    "airplane": 90,
+    "animal": 126,
+    "apparel": 92,
+    "arcade machine": 78,
+    "armchair": 30,
+    "ashcan": 138,
+    "awning": 86,
+    "bag": 115,
+    "ball": 119,
+    "bannister": 95,
+    "bar": 77,
+    "barrel": 111,
+    "base": 40,
+    "basket": 112,
+    "bathtub": 37,
+    "bed ": 7,
+    "bench": 69,
+    "bicycle": 127,
+    "blanket": 131,
+    "blind": 63,
+    "boat": 76,
+    "book": 67,
+    "bookcase": 62,
+    "booth": 88,
+    "bottle": 98,
+    "box": 41,
+    "bridge": 61,
+    "buffet": 99,
+    "building": 1,
+    "bulletin board": 144,
+    "bus": 80,
+    "cabinet": 10,
+    "canopy": 106,
+    "car": 20,
+    "case": 55,
+    "ceiling": 5,
+    "chair": 19,
+    "chandelier": 85,
+    "chest of drawers": 44,
+    "clock": 148,
+    "coffee table": 64,
+    "column": 42,
+    "computer": 74,
+    "conveyer belt": 105,
+    "counter": 45,
+    "countertop": 70,
+    "cradle": 117,
+    "crt screen": 141,
+    "curtain": 18,
+    "cushion": 39,
+    "desk": 33,
+    "dirt track": 91,
+    "dishwasher": 129,
+    "door": 14,
+    "earth": 13,
+    "escalator": 96,
+    "fan": 139,
+    "fence": 32,
+    "field": 29,
+    "fireplace": 49,
+    "flag": 149,
+    "floor": 3,
+    "flower": 66,
+    "food": 120,
+    "fountain": 104,
+    "glass": 147,
+    "grandstand": 51,
+    "grass": 9,
+    "hill": 68,
+    "hood": 133,
+    "house": 25,
+    "hovel": 79,
+    "kitchen island": 73,
+    "lake": 128,
+    "lamp": 36,
+    "land": 94,
+    "light": 82,
+    "microwave": 124,
+    "minibike": 116,
+    "mirror": 27,
+    "monitor": 143,
+    "mountain": 16,
+    "ottoman": 97,
+    "oven": 118,
+    "painting": 22,
+    "palm": 72,
+    "path": 52,
+    "person": 12,
+    "pier": 140,
+    "pillow": 57,
+    "plant": 17,
+    "plate": 142,
+    "plaything": 108,
+    "pole": 93,
+    "pool table": 56,
+    "poster": 100,
+    "pot": 125,
+    "radiator": 146,
+    "railing": 38,
+    "refrigerator": 50,
+    "river": 60,
+    "road": 6,
+    "rock": 34,
+    "rug": 28,
+    "runway": 54,
+    "sand": 46,
+    "sconce": 134,
+    "screen": 130,
+    "screen door": 58,
+    "sculpture": 132,
+    "sea": 26,
+    "seat": 31,
+    "shelf": 24,
+    "ship": 103,
+    "shower": 145,
+    "sidewalk": 11,
+    "signboard": 43,
+    "sink": 47,
+    "sky": 2,
+    "skyscraper": 48,
+    "sofa": 23,
+    "stage": 101,
+    "stairs": 53,
+    "stairway": 59,
+    "step": 121,
+    "stool": 110,
+    "stove": 71,
+    "streetlight": 87,
+    "swimming pool": 109,
+    "swivel chair": 75,
+    "table": 15,
+    "tank": 122,
+    "television receiver": 89,
+    "tent": 114,
+    "toilet": 65,
+    "towel": 81,
+    "tower": 84,
+    "trade name": 123,
+    "traffic light": 136,
+    "tray": 137,
+    "tree": 4,
+    "truck": 83,
+    "van": 102,
+    "vase": 135,
+    "wall": 0,
+    "wardrobe": 35,
+    "washer": 107,
+    "water": 21,
+    "waterfall": 113,
+    "windowpane": 8
+  },
   "max_depth": 10.0,
-  "num_seg_classes": 150,
-  "backbone_repo": "google/tipsv2-l14",
-  "embed_dim": 1024,
-  "block_indices": [
-    5,
-    11,
-    17,
-    23
-  ],
-  "post_process_channels": [
+  "min_depth": 0.001,
+  "model_type": "tipsv2_dpt",
+  "neck_hidden_sizes": [
     128,
     256,
     512,
     1024
-  ]
-}
+  ],
+  "num_depth_bins": 256,
+  "readout_act": "gelu_pytorch_tanh",
+  "reassemble_factors": [
+    4,
+    2,
+    1,
+    0.5
+  ],
+  "semantic_loss_ignore_index": 255,
+  "transformers_version": "5.10.0.dev0"
+}

configuration_dpt.py

@@ -3,6 +3,14 @@
 from transformers import PretrainedConfig
 
 
+_BACKBONE_REPO_BY_HIDDEN_SIZE = {
+    768: "google/tipsv2-b14",
+    1024: "google/tipsv2-l14",
+    1152: "google/tipsv2-so400m14",
+    1536: "google/tipsv2-g14",
+}
+
+
 class TIPSv2DPTConfig(PretrainedConfig):
     """Configuration for TIPSv2 DPT dense prediction heads."""
 
@@ -10,26 +18,27 @@ class TIPSv2DPTConfig(PretrainedConfig):
 
     def __init__(
         self,
-        backbone_repo="google/tipsv2-l14",
-        embed_dim=1024,
-        channels=256,
-        post_process_channels=(128, 256, 512, 1024),
-        block_indices=(5, 11, 17, 23),
+        backbone_config=None,
+        neck_hidden_sizes=(96, 192, 384, 768),
+        fusion_hidden_size=256,
+        reassemble_factors=(4, 2, 1, 0.5),
         readout_type="project",
         num_depth_bins=256,
         min_depth=1e-3,
         max_depth=10.0,
-        num_seg_classes=150,
         **kwargs,
     ):
         super().__init__(**kwargs)
-        self.backbone_repo = backbone_repo
-        self.embed_dim = embed_dim
-        self.channels = channels
-        self.post_process_channels = list(post_process_channels)
-        self.block_indices = list(block_indices)
+        backbone_config = backbone_config or {}
+        hidden_size = backbone_config.get("hidden_size", 768)
+        out_indices = backbone_config.get("out_indices", (3, 6, 9, 12))
+        self.backbone_repo = _BACKBONE_REPO_BY_HIDDEN_SIZE[hidden_size]
+        self.embed_dim = hidden_size
+        self.channels = fusion_hidden_size
+        self.post_process_channels = list(neck_hidden_sizes)
+        self.block_indices = [out_index - 1 for out_index in out_indices]
         self.readout_type = readout_type
         self.num_depth_bins = num_depth_bins
         self.min_depth = min_depth
         self.max_depth = max_depth
-        self.num_seg_classes = num_seg_classes
+        self.num_seg_classes = self.num_labels

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7610f79d02ea314818402eec5437361cf046a88d69bd170c43ae90ed36e33e6
-size 406735852
+oid sha256:38362aae91092ca0a6f2fe7f7ed29ec2e7042e3b920b7217d9317ad4c13abdfa
+size 1622839292

modeling_dpt.py

@@ -8,7 +8,7 @@ from typing import Optional
 
 import torch
 from huggingface_hub import hf_hub_download
-from transformers import AutoModel, PreTrainedModel
+from transformers import AutoConfig, AutoModel, PreTrainedModel
 
 from .configuration_dpt import TIPSv2DPTConfig
 
@@ -74,6 +74,10 @@ class TIPSv2DPTModel(PreTrainedModel):
 
         ppc = tuple(config.post_process_channels)
 
+        backbone_config = AutoConfig.from_pretrained(config.backbone_repo, trust_remote_code=True)
+        backbone = AutoModel.from_config(backbone_config, trust_remote_code=True)
+        self.vision_encoder = backbone.vision_encoder
+
         self.depth_head = dpt_mod.DPTDepthHead(
             input_embed_dim=config.embed_dim, channels=config.channels,
             post_process_channels=ppc, readout_type=config.readout_type,
@@ -89,17 +93,9 @@ class TIPSv2DPTModel(PreTrainedModel):
             post_process_channels=ppc, readout_type=config.readout_type,
             num_classes=config.num_seg_classes,
         )
-        self._backbone = None
-
-    def _get_backbone(self):
-        if self._backbone is None:
-            self._backbone = AutoModel.from_pretrained(self.config.backbone_repo, trust_remote_code=True)
-            self._backbone.to(self.device).eval()
-        return self._backbone.vision_encoder
 
     def _extract_intermediate(self, pixel_values):
-        backbone = self._get_backbone()
-        intermediate = backbone.get_intermediate_layers(
+        intermediate = self.vision_encoder.get_intermediate_layers(
             pixel_values, n=self.config.block_indices,
             reshape=True, return_class_token=True, norm=True,
         )

preprocessor_config.json

@@ -0,0 +1,13 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": false,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_processor_type": "Tipsv2DptImageProcessor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 448,
+    "width": 448
+  }
+}