Upload folder using huggingface_hub

config.json

@@ -0,0 +1 @@
+{"num_channels": 2048, "num_freq_bands": 6, "max_freq": 10, "enc_layers": 1, "num_queries": 100, "hidden_dim": 256, "enc_nheads_cross": 8, "nheads": 1, "dropout": 0, "self_per_cross_attn": 8, "num_classes": 91}

detection_perceiver_model_weights_pytorch.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4658c298c59a2d5915ac18550a963152cdfbfb973990a3a45bc3073f8ba8f05e
+size 170587066

modeling_detection_perceiver.py

@@ -0,0 +1,64 @@
+from torch import nn
+from torchvision.models import ResNet50_Weights
+from torchvision.models._utils import IntermediateLayerGetter
+import torchvision
+from perceiver_pytorch import Perceiver
+from torchvision.ops.misc import FrozenBatchNorm2d
+
+
+class DetectionPerceiver(nn.Module):
+	def __init__(self, backbone, perceiver, class_embed, bbox_embed):
+		super().__init__()
+		self.backbone = backbone
+		self.perceiver = perceiver
+		self.class_embed = class_embed
+		self.bbox_embed = bbox_embed
+
+	def forward(self, x):
+		x = self.backbone(x)
+		x = x['0'].permute(0, 2, 3, 1)
+		x = self.perceiver(data=x, return_embeddings=True)
+		return {'pred_logits': self.class_embed(x), 'pred_boxes': self.bbox_embed(x)}
+
+
+def build_model(config):
+	# Backbone
+	backbone = IntermediateLayerGetter(
+		torchvision.models.resnet50(weights=ResNet50_Weights.IMAGENET1K_V1, norm_layer=FrozenBatchNorm2d),
+		return_layers={'layer4': "0"}
+	)
+
+	# Perceiver
+	fourier_channels = 2 * ((config["num_freq_bands"] * 2) + 1)
+	perceiver = Perceiver(
+		input_channels=config["num_channels"],
+		input_axis=2,
+		num_freq_bands=config["num_freq_bands"],
+		max_freq=config["max_freq"],
+		depth=config["enc_layers"],
+		num_latents=config["num_queries"],
+		latent_dim=config["hidden_dim"],
+		cross_heads=config["enc_nheads_cross"],
+		latent_heads=config["nheads"],
+		cross_dim_head=(config["num_channels"] + fourier_channels) // config["enc_nheads_cross"],
+		latent_dim_head=config["hidden_dim"] // config["nheads"],
+		self_per_cross_attn=config["self_per_cross_attn"],
+		fourier_encode_data=True,
+		attn_dropout=config["dropout"],
+		ff_dropout=config["dropout"],
+		final_classifier_head=False
+	)
+
+	# Embeddings
+	bbox_embed = nn.Sequential(
+		nn.Linear(config["hidden_dim"], config["hidden_dim"]),
+		nn.ReLU(),
+		nn.Linear(config["hidden_dim"], config["hidden_dim"]),
+		nn.ReLU(),
+		nn.Linear(config["hidden_dim"], 4),
+		nn.Sigmoid()
+	)
+
+	class_embed = nn.Linear(config["hidden_dim"], config["num_classes"] + 1)
+
+	return DetectionPerceiver(backbone, perceiver, class_embed, bbox_embed)