src/models/liwtermModel.py

import torch
import torch.nn as nn
import timm


class LiwTERM(nn.Module):
    """
    Implementação fiel ao paper LiwTERM (SIBGRAPI 2024)
    """

    def __init__(
        self,
        num_classes: int,
        meta_dim: int,
        image_encoder: str = "vit_large_patch16_224",
        pretrained: bool = True,
        unfreeze_backbone: bool = False,
        dropout: float = 0.3,
    ):
        super().__init__()

        # =====================================================
        # 1) Backbone visual (ViT) — FEATURE EXTRACTOR
        # =====================================================
        self.visual = timm.create_model(
            image_encoder,
            pretrained=pretrained,
            num_classes=0 
        )

        self.visual_dim = self.visual.num_features

        if not unfreeze_backbone:
            for p in self.visual.parameters():
                p.requires_grad = False

        # Projeção ViT → 4096 (como no paper)
        self.visual_proj = nn.Sequential(
            nn.Linear(self.visual_dim, 4096),
            nn.LayerNorm(4096),
            nn.ReLU(inplace=True),
            nn.Dropout(dropout),
        )

        # =====================================================
        # 2) Projeção dos metadados (OHE)
        # =====================================================
        self.meta_fc = nn.Sequential(
            nn.LayerNorm(meta_dim),
            nn.Linear(meta_dim, 1024),
            nn.ReLU(inplace=True),
        )

        # =====================================================
        # 3) Shallow Lightweight Model (SLM)
        # =====================================================
        concat_dim = 4096 + 1024

        self.slm = nn.Sequential(
            nn.LayerNorm(concat_dim),
            nn.ReLU(inplace=True),
            nn.Dropout(dropout),

            nn.Linear(concat_dim, 2048),
            nn.LayerNorm(2048),
            nn.ReLU(inplace=True),
            nn.Dropout(dropout),

            nn.Linear(2048, 1024),
            nn.LayerNorm(1024),
            nn.ReLU(inplace=True),
            nn.Dropout(dropout),

            nn.Linear(1024, 512),
            nn.LayerNorm(512),
            nn.ReLU(inplace=True),
            nn.Dropout(dropout),

            nn.Linear(512, num_classes)
        )

    # =====================================================
    # Forward
    # =====================================================
    def forward(self, image: torch.Tensor, metadata: torch.Tensor) -> torch.Tensor:

        # --- ViT features ---
        v = self.visual.forward_features(image)
        if v.dim() == 3:
            v = v[:, 0]  # CLS token

        v = self.visual_proj(v)

        # --- Metadata features ---
        m = self.meta_fc(metadata)

        # --- Concatenação ---
        x = torch.cat([v, m], dim=1)

        # --- Classificação ---
        logits = self.slm(x)
        return logits