models/visual_model.py

import torch
import torch.nn as nn
from torch.utils.checkpoint import checkpoint_sequential
import torchvision.models as models


def build_backbone(name: str):
    if name == "efficientnet_b0":
        m = models.efficientnet_b0(
            weights=models.EfficientNet_B0_Weights.IMAGENET1K_V1
        )
        feat_dim = m.classifier[1].in_features
        m.classifier = nn.Identity()
        original_features = m.features

        def forward_with_ckpt(x):
            return checkpoint_sequential(
                original_features, segments=4, input=x, use_reentrant=False
            )

        m.forward = lambda x: m.avgpool(forward_with_ckpt(x)).flatten(1)
        return m, feat_dim

    elif name == "resnet50":
        m = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1)
        feat_dim = m.fc.in_features
        m.fc = nn.Identity()
        original_layer4 = m.layer4

        def forward_resnet(x):
            x = m.conv1(x); x = m.bn1(x); x = m.relu(x); x = m.maxpool(x)
            x = m.layer1(x); x = m.layer2(x); x = m.layer3(x)
            x = checkpoint_sequential(
                original_layer4, segments=2, input=x, use_reentrant=False
            )
            x = m.avgpool(x)
            return torch.flatten(x, 1)

        m.forward = forward_resnet
        return m, feat_dim

    else:
        return build_backbone("efficientnet_b0")


class VisualPersonalityModel(nn.Module):
    def __init__(self, cfg: dict):
        super().__init__()
        self.backbone, feat_dim = build_backbone(cfg["backbone"])
        agg_dim = feat_dim * 2
        self.projector = nn.Sequential(
            nn.Linear(agg_dim, cfg["embed_dim"] * 2),
            nn.LayerNorm(cfg["embed_dim"] * 2),
            nn.GELU(),
            nn.Dropout(cfg["dropout"]),
            nn.Linear(cfg["embed_dim"] * 2, cfg["embed_dim"]),
            nn.LayerNorm(cfg["embed_dim"]),
        )
        num_traits = len(cfg["traits"])
        self.regressor = nn.Sequential(
            nn.Dropout(cfg["dropout"] * 0.5),
            nn.Linear(cfg["embed_dim"], cfg["embed_dim"] // 2),
            nn.GELU(),
            nn.Dropout(cfg["dropout"] * 0.5),
            nn.Linear(cfg["embed_dim"] // 2, num_traits),
            nn.Sigmoid(),
        )

    def forward(self, frames, backbone_chunk: int = 128):
        B, T, C, H, W = frames.shape
        flat = frames.view(B * T, C, H, W)
        feats = []
        for i in range(0, flat.size(0), backbone_chunk):
            feats.append(self.backbone(flat[i: i + backbone_chunk]))
        x   = torch.cat(feats, dim=0).view(B, T, -1)
        mu  = x.mean(dim=1)
        std = x.std(dim=1).clamp(min=1e-6)
        agg = torch.cat([mu, std], dim=-1)
        emb  = self.projector(agg)
        pred = self.regressor(emb)
        return pred, emb

    def backbone_parameters(self):
        return list(self.backbone.parameters())

    def head_parameters(self):
        return list(self.projector.parameters()) + list(self.regressor.parameters())