import warnings import torch from transformers import ClapAudioModel, ClapAudioModelWithProjection, ClapFeatureExtractor, ClapProcessor from autrainer.models.abstract_model import AbstractModel from autrainer.models.ffnn import FFNN class CLAPBackbone(AbstractModel): def __init__( self, model_name, freeze_extractor: bool = True, time_pooling: bool = True, ) -> None: self.model_name = model_name self.freeze_extractor = freeze_extractor self.time_pooling = time_pooling model = ClapAudioModelWithProjection.from_pretrained(self.model_name) super().__init__(output_dim=model.config.hidden_size) self.model = model.audio_model.audio_encoder # self.model = model # print(self.model) if self.freeze_extractor: for param in self.model.parameters(): param.requires_grad = False def embeddings(self, x: torch.Tensor) -> torch.Tensor: inputs = x is_longer = torch.tensor([False]) x = self.model(input_features=inputs, is_longer=is_longer).last_hidden_state # Flatten and transpose for the embeddings x = x.flatten(2).transpose(1, 2) if self.time_pooling: x = x.mean(1) return x def forward(self, features: torch.Tensor) -> torch.Tensor: return self.embeddings(features) class CLAPFFNN(AbstractModel): def __init__( self, output_dim: int, model_name: str, freeze_extractor: bool, hidden_size: int, num_layers: int = 2, dropout: float = 0.5, ) -> None: """CLAP model with FFNN frontend adapted for audio classification. For more information, see: https://huggingface.co/docs/transformers/en/model_doc/clap#clap Args: output_dim: Output dimension of the FFNN. model_name: Name of the model loaded from Huggingface. freeze_extractor: Whether to freeze the feature extractor. hidden_size: Hidden size of the FFNN. num_layers: Number of layers of the FFNN. Defaults to 2. dropout: Dropout rate. Defaults to 0.5. """ super().__init__(output_dim) self.model_name = model_name self.freeze_extractor = freeze_extractor self.hidden_size = hidden_size self.num_layers = num_layers self.dropout = dropout self.backbone = CLAPBackbone( model_name=model_name, freeze_extractor=freeze_extractor, time_pooling=True, ) self.frontend = FFNN( input_size=self.backbone.output_dim, hidden_size=hidden_size, output_dim=output_dim, num_layers=num_layers, dropout=dropout, ) def embeddings(self, x: torch.Tensor) -> torch.Tensor: return self.backbone(x) def forward(self, features: torch.Tensor) -> torch.Tensor: return self.frontend(self.embeddings(features)) if __name__=='__main__': output_dim = 4 model_name = "laion/clap-htsat-fused" freeze_extractor = True time_pooling = True hidden_size = 512 model = CLAPFFNN( output_dim=output_dim, model_name = model_name, freeze_extractor = freeze_extractor, hidden_size=hidden_size ) feature_extractor = ClapFeatureExtractor.from_pretrained('laion/clap-htsat-unfused') # processor = ClapProcessor.from_pretrained('laion/clap-htsat-unfused') import librosa a, sr = librosa.load("/path/to/example.wav", sr=48000) print(a.shape, sr) audio = torch.tensor(a) # inputs = processor(audios=audio, sampling_rate=48000, return_tensors="pt") # print("Inputs:", inputs['input_features'].shape) extracted = feature_extractor(audio, sampling_rate=48000, return_tensors='pt') print("Extracted: ", extracted['input_features'].shape) extracted = extracted['input_features'] # print(type(inputs)) print(type(extracted)) # features = extracted[list(extracted.keys())[0]][0].unsqueeze(0) out = model(extracted) print(out) print(out.shape)