src/models.py

import torch
import torch.nn as nn
import torch.nn.functional as F
import timm
from transformers import AutoModel


class ImageEncoder(nn.Module):
    """
    Image encoder using a pretrained ResNet18 backbone.
    Outputs normalized embeddings.
    """

    def __init__(self, embed_dim=256):
        super().__init__()

        self.backbone = timm.create_model(
            "resnet18",
            pretrained=True,
            num_classes=0
        )

        self.proj = nn.Linear(512, embed_dim)

    def forward(self, x):
        x = self.backbone(x)
        x = self.proj(x)
        return F.normalize(x, dim=-1)


class AudioEncoder(nn.Module):
    """
    CNN-based encoder for mel spectrograms.
    """

    def __init__(self, embed_dim=256):
        super().__init__()

        self.cnn = nn.Sequential(
            nn.Conv2d(1, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2),

            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.AdaptiveAvgPool2d((1, 1))
        )

        self.proj = nn.Linear(64, embed_dim)

    def forward(self, x):
        x = self.cnn(x)
        x = x.flatten(1)
        x = self.proj(x)
        return F.normalize(x, dim=-1)


class TextEncoder(nn.Module):
    """
    DistilBERT-based text encoder.
    """

    def __init__(self, embed_dim=256):
        super().__init__()

        self.backbone = AutoModel.from_pretrained(
            "distilbert-base-uncased"
        )

        self.proj = nn.Linear(768, embed_dim)

    def forward(self, input_ids, attention_mask):
        output = self.backbone(
            input_ids=input_ids,
            attention_mask=attention_mask
        )

        cls = output.last_hidden_state[:, 0, :]
        x = self.proj(cls)

        return F.normalize(x, dim=-1)