Update README

README.md

@@ -90,25 +90,29 @@ We are exploring more tasks where the backbone can be leveragead are are looing
 ### Basic Feature Extraction
 
 ```python
-import torch
 from transformers import AutoImageProcessor, AutoModel
+import torch
 from PIL import Image
 
-# Load the custom processor and model
-repo_id = "theairlabcmu/AnyThermal"
-processor = AutoImageProcessor.from_pretrained(repo_id, trust_remote_code=True)
-model = AutoModel.from_pretrained(repo_id)
+# Load model and processor
+processor = AutoImageProcessor.from_pretrained("theairlabcmu/AnyThermal")
+model = AutoModel.from_pretrained("theairlabcmu/AnyThermal")
+
+# Load thermal image (grayscale)
+thermal_image = Image.open("path/to/thermal_image.png").convert("L")
+
+# Convert to 3-channel (required for ViT architecture)
+thermal_image = thermal_image.convert("RGB")
 
-# Load your image (works with RGB, Grayscale, etc.)
-image = Image.open("path_to_your_image.jpg")
+# Process and extract features
+inputs = processor(images=thermal_image, return_tensors="pt")
+outputs = model(**inputs)
 
-# Preprocess: This automatically handles RGB conversion and 
-# snaps dimensions to the nearest multiple of 14.
-inputs = processor(images=image, return_tensors="pt")
+# Get CLS token (global image representation)
+cls_features = outputs.last_hidden_state[:, 0]  # Shape: [1, 768]
 
-# Inference
-with torch.no_grad():
-    outputs = model(**inputs)
+# Get patch features (spatial feature map)
+patch_features = outputs.last_hidden_state[:, 1:]  # Shape: [1, num_patches, 768]
 ```
 
 ### Task-Specific Applications

anythermal_checkpoints.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5e32f2ab731406fa3a47d58601758fff5ed9e1d1c87fcdf1a159004c87464ac
-size 2500636617
+oid sha256:cd4574987e56f58d0c6993987ddf9c4b72dd21f3786db3facab5914762aab4ef
+size 2498757381

custom_processor.py

@@ -1,48 +0,0 @@
-from transformers import BitImageProcessor as BaseProcessor
-import numpy as np
-
-from transformers import AutoImageProcessor
-from transformers.image_utils import PILImageResampling
-
-class CustomDinov2Processor(BaseProcessor):
-    model_type = "dinov2"
-
-    def preprocess(self, images, **kwargs):
-        # 1. Handle "Already Normalized" (0-1) check
-        test_img = images[0] if isinstance(images, list) else images
-        
-        # Determine max value to see if we should rescale
-        if hasattr(test_img, "getextrema"): # PIL
-            extrema = test_img.getextrema()
-            max_val = max([b[1] for b in extrema]) if isinstance(extrema[0], tuple) else extrema[1]
-        elif hasattr(test_img, "max"): # Numpy/Tensor
-            max_val = test_img.max()
-        else:
-            max_val = 255 
-
-        # If already 0-1, disable rescaling (1/255)
-        if max_val <= 1.0:
-            kwargs["do_rescale"] = False
-        else:
-            kwargs["do_rescale"] = True
-            
-        # 2. Force RGB Conversion (handles grayscale)
-        kwargs["do_convert_rgb"] = True
-            
-        return super().preprocess(images, **kwargs)
-
-    def resize(self, image: np.ndarray, size=None, resample=PILImageResampling.BILINEAR, **kwargs) -> np.ndarray:
-        # 3. Your dynamic "nearest 14" logic
-        h, w = image.shape[:2]
-        new_h = (h // 14) * 14
-        new_w = (w // 14) * 14
-        
-        return super().resize(
-            image, 
-            size={"height": new_h, "width": new_w}, 
-            resample=resample, 
-            **kwargs
-        )
-
-# Register the class so it saves to the Hub correctly
-CustomDinov2Processor.register_for_auto_class("AutoImageProcessor")

depth/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "AnyThermalDepthModel"
+  ],
+  "model_type": "anythermal_depth",
+  "auto_map": {
+    "AutoConfig": "model.AnyThermalDepthConfig",
+    "AutoModel": "model.AnyThermalDepthModel"
+  },
+  "hidden_size": 768,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 12,
+  "mlp_ratio": 4,
+  "hidden_act": "gelu",
+  "dropout": 0.0,
+  "attention_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-06,
+  "image_size": 518,
+  "patch_size": 14,
+  "num_channels": 3,
+  "qkv_bias": true,
+  "layerscale_value": 1.0,
+  "drop_path_rate": 0.0,
+  "use_swiglu_ffn": false,
+  "features": 256
+}

depth/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d45f5b9c90e4243a85c80dda747b28efcef527195dc5eceea21bd93dd56d269
+size 425617642

preprocessor_config.json

@@ -1,30 +1,16 @@
 {
-  "crop_size": {
-    "height": 224,
-    "width": 224
-  },
   "do_center_crop": false,
   "do_convert_rgb": true,
   "do_normalize": true,
   "do_rescale": true,
   "do_resize": true,
-  "image_mean": [
-    0.48145466,
-    0.4578275,
-    0.40821073
-  ],
-  "image_processor_type": "CustomDinov2Processor",
-  "image_std": [
-    0.26862954,
-    0.26130258,
-    0.27577711
-  ],
-  "resample": 2,
+  "image_mean": [0.48145466, 0.4578275, 0.40821073],
+  "image_std": [0.26862954, 0.26130258, 0.27577711],
+  "image_processor_type": "BitImageProcessor",
+  "resample": 3,
   "rescale_factor": 0.00392156862745098,
   "size": {
-    "shortest_edge": 14
-  },
-  "auto_map": {
-    "AutoImageProcessor": "custom_processor.CustomDinov2Processor"
+    "height": 224,
+    "width": 224
   }
 }

segmentation_cart/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "AnyThermalSegmentationModel"
+  ],
+  "model_type": "anythermal",
+  "auto_map": {
+    "AutoConfig": "model.AnyThermalConfig",
+    "AutoModel": "model.AnyThermalSegmentationModel"
+  },
+  "hidden_size": 768,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 12,
+  "mlp_ratio": 4,
+  "hidden_act": "gelu",
+  "dropout": 0.0,
+  "attention_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-06,
+  "image_size": 518,
+  "patch_size": 14,
+  "num_channels": 3,
+  "qkv_bias": true,
+  "layerscale_value": 1.0,
+  "drop_path_rate": 0.0,
+  "use_swiglu_ffn": false,
+  "num_labels": 10
+}

segmentation_cart/preprocessor_config.json

@@ -0,0 +1,9 @@
+{ 
+  "do_center_crop": false, 
+  "do_convert_rgb": true, 
+  "do_normalize": false, 
+  "do_rescale": true, 
+  "do_resize": false, 
+  "image_processor_type": "BitImageProcessor", 
+  "rescale_factor": 0.00392156862745098
+}

segmentation_cart/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28f828818cdbb6dde254df59ee6b221d3f91875766c621bd5206d4f82e75bdf6
+size 348159849

segmentation_mfnet/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "AnyThermalSegmentationModel"
+  ],
+  "model_type": "anythermal",
+  "auto_map": {
+    "AutoConfig": "model.AnyThermalConfig",
+    "AutoModel": "model.AnyThermalSegmentationModel"
+  },
+  "hidden_size": 768,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 12,
+  "mlp_ratio": 4,
+  "hidden_act": "gelu",
+  "dropout": 0.0,
+  "attention_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-06,
+  "image_size": 518,
+  "patch_size": 14,
+  "num_channels": 3,
+  "qkv_bias": true,
+  "layerscale_value": 1.0,
+  "drop_path_rate": 0.0,
+  "use_swiglu_ffn": false,
+  "num_labels": 9
+}

segmentation_mfnet/preprocessor_config.json

@@ -0,0 +1,9 @@
+{ 
+  "do_center_crop": false, 
+  "do_convert_rgb": true, 
+  "do_normalize": false, 
+  "do_rescale": true, 
+  "do_resize": false, 
+  "image_processor_type": "BitImageProcessor", 
+  "rescale_factor": 0.00392156862745098
+}

segmentation_mfnet/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0841214f4ac2b7b31405037593c79960ae2b10ed4cece3c93953c3c4560b26ec
+size 348159776

vpr/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "AnyThermalVPRModel"
+  ],
+  "model_type": "anythermal_vpr",
+  "auto_map": {
+    "AutoConfig": "model.AnyThermalVPRConfig",
+    "AutoModel": "model.AnyThermalVPRModel"
+  },
+  "hidden_size": 768,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 12,
+  "mlp_ratio": 4,
+  "hidden_act": "gelu",
+  "dropout": 0.0,
+  "attention_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-06,
+  "image_size": 518,
+  "patch_size": 14,
+  "num_channels": 3,
+  "qkv_bias": true,
+  "layerscale_value": 1.0,
+  "drop_path_rate": 0.0,
+  "use_swiglu_ffn": false,
+  "num_clusters": 64,
+  "cluster_dim": 128,
+  "token_dim": 256
+}

vpr/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41f4aa7b27bae7e4ff6331b27222c703845933bf668f312a3578eeb82fcdaa25
+size 352035578