src/model.py

import torch.nn as nn
from transformers import AutoModel

class MultiTaskChemBERTa(nn.Module):
    """
    Multi-task ChemBERTa model.
    Predicts BBB penetration (classification) and
    solubility (regression) simultaneously.
    
    Results:
        BBBP AUC:  0.9393
        ESOL RMSE: 0.821
    """
    def __init__(self):
        super().__init__()
        self.backbone = AutoModel.from_pretrained("seyonec/ChemBERTa-zinc-base-v1")
        hidden = self.backbone.config.hidden_size

        self.bbbp_head = nn.Sequential(
            nn.Linear(hidden, 128),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(128, 1)
        )
        self.esol_head = nn.Sequential(
            nn.Linear(hidden, 128),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(128, 1)
        )

    def forward(self, input_ids, attention_mask):
        out = self.backbone(input_ids=input_ids, attention_mask=attention_mask)
        cls = out.last_hidden_state[:, 0, :]
        return self.bbbp_head(cls), self.esol_head(cls)