modeling_tapct.py

# Copyright 2025 AI for Oncology Research Group. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Optional
from transformers import PreTrainedModel
from transformers.modeling_outputs import BaseModelOutputWithPooling
import torch

from .configuration_tapct import TAPCTConfig
from .vision_transformer import vit_small, vit_base
from .vision_transformer_3d import vit_3d_small, vit_3d_base
from .vision_transformer_base import DinoVisionTransformerBase

class TAPCTPreTrainedModel(PreTrainedModel):
    config_class = TAPCTConfig
    base_model_prefix = "tapct"


class TAPCTModel(TAPCTPreTrainedModel):
    """
    TAP-CT Vision Transformer model based on DINOv2: https://github.com/facebookresearch/dinov2.
    
    This model outputs raw hidden states and does not include any task-specific head.
    """
    
    def __init__(self, config: TAPCTConfig) -> None:
        super().__init__(config)
        self.config = config
        self.model: DinoVisionTransformerBase

        match config.model_variant:
            case "2d":
                if config.model_size == "small":
                    self.model = vit_small(
                        img_size=config.img_size,
                        patch_size=config.patch_size,
                        num_register_tokens=config.num_register_tokens,
                        in_chans=config.in_chans,
                        init_values=config.init_values,
                        block_chunks=config.block_chunks
                    )
                elif config.model_size == "base":
                    self.model = vit_base(
                        img_size=config.img_size,
                        patch_size=config.patch_size,
                        num_register_tokens=config.num_register_tokens,
                        in_chans=config.in_chans,
                        init_values=config.init_values,
                        block_chunks=config.block_chunks
                    )
                else:
                    raise ValueError(f"Model size '{config.model_size}' not supported for 2D")
                    
            case "2.5d" | "3d":
                if config.model_size == "small":
                    self.model = vit_3d_small(
                        img_size=config.img_size,
                        patch_size=config.patch_size,
                        num_register_tokens=config.num_register_tokens,
                        in_chans=config.in_chans,
                        init_values=config.init_values,
                        block_chunks=config.block_chunks
                    )
                elif config.model_size == "base":
                    self.model = vit_3d_base(
                        img_size=config.img_size,
                        patch_size=config.patch_size,
                        num_register_tokens=config.num_register_tokens,
                        in_chans=config.in_chans,
                        init_values=config.init_values,
                        block_chunks=config.block_chunks
                    )
                else:
                    raise ValueError(f"Model size '{config.model_size}' not supported for 3D")
                    
            case _:
                raise ValueError(f"Model variant '{config.model_variant}' not supported. Use '2d', '2.5d', or '3d'.")
        
        # Initialize weights
        self.post_init()

    def forward(
        self,
        pixel_values: torch.Tensor,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        reshape: bool = False
    ) -> BaseModelOutputWithPooling:
        """
        Forward pass of the TAP-CT model.
        
        Parameters
        ----------
        pixel_values : torch.Tensor
            Input images. Shape (B, C, H, W) for 2D or (B, C, D, H, W) for 3D.
        output_hidden_states : Optional[bool], optional
            Whether to return hidden states from all layers.
        return_dict : Optional[bool], optional
            Whether to return a ModelOutput instead of a plain tuple.
        reshape : bool, default=False
            Whether to reshape output features to spatial dimensions. If True,
            returns shape (B, H, W, C) for 2D or (B, D, H, W, C) for 3D instead
            of flattened (B, N, C) where N is the number of patches.
        
        Returns
        -------
        BaseModelOutputWithPooling
            Contains:
            - last_hidden_state: Patch token features from the last layer
            - pooler_output: CLS token from the last layer  
            - hidden_states: (optional) All hidden states if output_hidden_states=True
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states

        if output_hidden_states:
            outputs_tuple = self.model.get_intermediate_layers(
                pixel_values,
                n=self.model.n_blocks,
                return_class_token=True,
                reshape=reshape
            )
            outputs = tuple(o[0] for o in outputs_tuple)
            class_tokens = tuple(o[1] for o in outputs_tuple)
            
            last_hidden_state = outputs[-1]
            pooler_output = class_tokens[-1]
            hidden_states = outputs
        else:
            outputs_tuple = self.model.get_intermediate_layers(
                pixel_values,
                n=1,
                return_class_token=True,
                reshape=reshape
            )
            last_hidden_state = outputs_tuple[0][0]
            pooler_output = outputs_tuple[0][1]
            hidden_states = None

        if not return_dict:
            return tuple(
                v for v in [last_hidden_state, pooler_output, hidden_states] 
                if v is not None
            )

        return BaseModelOutputWithPooling(
            last_hidden_state=last_hidden_state,
            pooler_output=pooler_output,
            hidden_states=hidden_states
        )