import sys import torch import torch.nn as nn from .dinov2 import hubconf class AbstractEncoder(nn.Module): def __init__(self): super().__init__() def encode(self, *args, **kwargs): raise NotImplementedError class FrozenDinoV2Encoder(AbstractEncoder): """ Uses the DINOv2 encoder for image """ def __init__(self, weight_path, device="cpu", freeze=True): super().__init__() dinov2 = hubconf.dinov2_vitl14(pretrained=False) state_dict = torch.load(weight_path) dinov2.load_state_dict(state_dict, strict=False) self.model = dinov2.to(device) # self.device = device if freeze: self.freeze() self.register_buffer('image_mean', torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)) self.register_buffer('image_std', torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)) # self.projector = nn.Linear(1536, 768) @property def dtype(self): return next(self.model.parameters()).dtype def freeze(self): self.model.eval() for param in self.model.parameters(): param.requires_grad = False # image.shape [15, 3, 224, 224] def forward(self, image, mode=None): if isinstance(image,list): image = torch.cat(image,0) image = (image - self.image_mean) / self.image_std features = self.model.forward_features(image) # dict_keys(['x_norm_clstoken', 'x_norm_regtokens', 'x_norm_patchtokens', 'x_prenorm', 'masks']) if mode is not None: return features[mode] tokens = features["x_norm_patchtokens"] # [15, 256, 1024] image_features = features["x_norm_clstoken"] # [15, 1024] image_features = image_features.unsqueeze(1) # [15, 1, 1024] hint = torch.cat([image_features,tokens],1) # [15, 257, 1024] # hint = self.projector(hint) return hint def encode(self, image): return self(image)