Upload folder using huggingface_hub

MeralionForGender.py

@@ -0,0 +1,327 @@
+# models/meralion_encoder.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Tuple, Optional
+
+from transformers import AutoModel,PretrainedConfig, PreTrainedModel, AutoConfig
+from peft import get_peft_model, LoraConfig
+from omegaconf import DictConfig
+
+# 1. Define a Config class that holds all your YAML settings
+class MeralionGenderConfig(PretrainedConfig):
+    model_type = "meralion_gender"
+    def __init__(
+        self, 
+        meralion_name="", 
+        num_classes=2, 
+        aggregator="attention", 
+        downstream_params=None,
+        **kwargs
+    ):
+        # Pass all basic types (strings, ints, dicts) to super
+        super().__init__(
+            meralion_name=meralion_name,
+            num_classes=num_classes,
+            aggregator=aggregator,
+            downstream_params=downstream_params or {},
+            **kwargs
+        )
+
+class SE1d(nn.Module):
+    """Squeeze-and-Excitation block for 1D convolutions"""
+    def __init__(self, channels: int, reduction: int = 8):
+        super().__init__()
+        hidden = max(8, channels // reduction)
+        self.avg = nn.AdaptiveAvgPool1d(1)
+        self.fc = nn.Sequential(
+            nn.Conv1d(channels, hidden, 1, bias=False),
+            nn.ReLU(inplace=True),
+            nn.Conv1d(hidden, channels, 1, bias=False),
+            nn.Sigmoid(),
+        )
+    
+    def forward(self, x: torch.Tensor):
+        w = self.fc(self.avg(x))
+        return x * w
+
+
+class Res2Block1d(nn.Module):
+    """Res2Net block adapted for 1D convolutions - BatchNorm free"""
+    def __init__(self, channels: int, scale: int = 4, kernel_size: int = 3, dilation: int = 1):
+        super().__init__()
+        assert channels % scale == 0, f"channels ({channels}) must be divisible by scale ({scale})"
+        self.scale = scale
+        self.width = channels // scale
+        pad = (kernel_size // 2) * dilation
+        
+        self.convs = nn.ModuleList([
+            nn.Conv1d(self.width, self.width, kernel_size, padding=pad, dilation=dilation, bias=True)
+            for _ in range(scale - 1)
+        ])
+        self.norm = nn.GroupNorm(num_groups=min(32, channels), num_channels=channels)
+        self.act = nn.ReLU(inplace=True)
+    
+    def forward(self, x: torch.Tensor):
+        xs = torch.split(x, self.width, dim=1)
+        out = [xs[0]]
+        for i, conv in enumerate(self.convs, start=1):
+            if i == 1:
+                s = xs[i]
+            else:
+                s = xs[i] + out[-1]  # Fixed: proper residual connection
+            out.append(conv(s))
+        y = torch.cat(out, dim=1)
+        return self.act(self.norm(y))
+
+class ECAPABlock(nn.Module):
+    """Enhanced ECAPA block with proper residual connections - BatchNorm free"""
+    def __init__(self, channels: int, scale: int = 4, kernel_size: int = 3, dilation: int = 1):
+        super().__init__()
+        self.conv1 = nn.Conv1d(channels, channels, 1, bias=True)
+        self.norm1 = nn.GroupNorm(num_groups=min(32, channels), num_channels=channels)
+        self.act1 = nn.ReLU(inplace=True)
+        
+        self.res2 = Res2Block1d(channels, scale=scale, kernel_size=kernel_size, dilation=dilation)
+        self.se = SE1d(channels)
+        
+        self.conv2 = nn.Conv1d(channels, channels, 1, bias=True)
+        self.norm2 = nn.GroupNorm(num_groups=min(32, channels), num_channels=channels)
+        self.act2 = nn.ReLU(inplace=True)
+    
+    def forward(self, x: torch.Tensor):
+        residual = x
+        
+        y = self.act1(self.norm1(self.conv1(x)))
+        y = self.res2(y)
+        y = self.se(y)
+        y = self.norm2(self.conv2(y))
+        
+        return self.act2(y + residual)
+
+
+class EmotionECAPATDNN(nn.Module):
+    """ECAPA-TDNN optimized for emotion recognition with hierarchical attention"""
+    def __init__(
+        self, 
+        input_dim: int, 
+        channels: int = 512,
+        output_dim: int = 256,
+        num_blocks: int = 3,
+        dilations: tuple = (1, 2, 3),
+        embed_dim: int = 512,
+        num_emotions: int = 8,  # Common emotion categories
+        dropout: float = 0.2,
+        pooling_type="attention"
+    ):
+        super().__init__()
+        self.output_dim = output_dim
+        # Input projection with layer norm for stability
+        self.proj_in = nn.Sequential(
+            nn.Linear(input_dim, channels),
+            nn.LayerNorm(channels),
+            nn.GELU(),
+            nn.Dropout(0.2),
+        )
+        
+        # ECAPA blocks with different dilations
+        self.blocks = nn.ModuleList([
+            ECAPABlock(channels, scale=4, kernel_size=3, dilation=d) 
+            for d in dilations
+        ])
+        
+        # Multi-scale feature aggregation
+        self.mfa = nn.Sequential(
+            nn.Conv1d(channels * (len(dilations) + 1), channels, 1, bias=True),
+            nn.GroupNorm(num_groups=min(32, channels), num_channels=channels),
+            nn.GELU()
+            #nn.ReLU(inplace=True)
+        )
+
+        self.pooling = AttentionPooling(channels)
+        
+        # Final embedding layers
+        self.embed = nn.Sequential(
+            nn.Linear(channels, channels),
+            nn.LayerNorm(channels),
+            nn.GELU(),
+            nn.Dropout(0.3),
+            nn.Linear(channels, channels // 2),
+            nn.LayerNorm(channels // 2),
+            nn.GELU(),
+            nn.Dropout(dropout)
+        )
+        
+        # Initialize weights
+        self._init_weights()
+    
+    def _init_weights(self):
+        """Initialize model weights"""
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                nn.init.trunc_normal_(m.weight, std=0.02)
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.Conv1d):
+                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+    
+    def forward(self, x: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, return_embeddings: bool = False):
+        """
+        Forward pass
+        
+        Args:
+            x: Input tensor (batch, time, features) - from Whisper encoder
+            attention_mask: Attention mask (batch, time)
+            return_embeddings: Whether to return embeddings instead of logits
+        
+        Returns:
+            If return_embeddings=False: emotion logits (batch, num_emotions)
+            If return_embeddings=True: feature embeddings (batch, embed_dim // 2)
+        """
+        # Project input and transpose for conv1d
+        x = self.proj_in(x)  # (batch, time, channels)
+        x_conv = x.transpose(1, 2)  # (batch, channels, time)
+        
+        # Apply ECAPA blocks and collect multi-scale features
+        features = [x_conv]
+        for block in self.blocks:
+            x_conv = block(x_conv)
+            features.append(x_conv)
+        
+        # Multi-scale feature aggregation
+        y = torch.cat(features, dim=1)  # (batch, channels * (n_blocks + 1), time)
+        y = self.mfa(y)  # (batch, channels, time)
+        y = y.transpose(1, 2)  # (batch, time, channels)
+        
+        # Hierarchical attention pooling
+        #pooled = self.norm_layer(self.pooling(y))  # (batch, channels)
+        pooled = self.pooling(y)  # (batch, channels)
+
+        # Generate embeddings
+        embeddings = self.embed(pooled)  # (batch, embed_dim // 2)
+        
+        return embeddings
+        
+
+class LayerAttentiveAggregation(nn.Module):
+    """
+    Smart Layer Aggregation:
+    Instead of a static weighted sum, this computes attention weights 
+    based on the hidden states themselves.
+    """
+    def __init__(self, hidden_size: int, num_layers: int):
+        super().__init__()
+        # Transformation to compute score per layer
+        self.query_proj = nn.Linear(hidden_size, 1)
+        self.num_layers = num_layers
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        # hidden_states: (L, B, T, D)
+        # We want to learn which layer 'L' is most important per timestep or globally.
+        # Let's do global (per sample) importance to save compute.
+        
+        # Mean pool over time for the scoring mechanism: (L, B, D)
+        # Using mean helps avoid noise from silence frames
+        global_repr = hidden_states.mean(dim=2) 
+        
+        # Compute scores: (L, B, 1)
+        scores = self.query_proj(global_repr)
+        
+        # Softmax over layers (dim=0) -> (L, B, 1)
+        attn_weights = F.softmax(scores, dim=0)
+        
+        # Reshape for broadcasting: (L, B, 1, 1)
+        attn_weights = attn_weights.unsqueeze(-1)
+        
+        # Weighted sum: sum((L, B, T, D) * (L, B, 1, 1)) -> (B, T, D)
+        aggregated = (hidden_states * attn_weights).sum(dim=0)
+        #print(f"DEBUG: Number of hidden states provided to aggregator: {len(hidden_states)}")
+        #exit()
+        return aggregated
+
+class MeralionForGenderClassification(PreTrainedModel):
+    config_class = MeralionGenderConfig
+
+    def __init__(self, config: MeralionGenderConfig):
+        super().__init__(config)
+        
+        
+        # 1. Load Backbone        
+        backbone_config = AutoConfig.from_pretrained(config.meralion_name, trust_remote_code=True)
+        self.backbone = AutoModel.from_config(backbone_config, trust_remote_code=True)
+        
+        hidden_size = getattr(self.backbone.config, "hidden_size", None) \
+                   or getattr(self.backbone.config, "d_model", None)
+        if hidden_size is None:
+            raise ValueError("Cannot infer hidden size from MERaLiON config.")
+        
+        num_layers = getattr(self.backbone.config, "num_hidden_layers", 0) + 1 # +1 for embeddings
+
+        # Reset all to requires_grad=True first
+        for p in self.backbone.parameters():
+            p.requires_grad = False
+
+        # 3. Layer Aggregation
+        self.backbone.config.output_hidden_states = True
+        self.layer_aggregator = LayerAttentiveAggregation(hidden_size, num_layers)
+
+        # 4. Downstream Head
+        self.downstream = EmotionECAPATDNN(
+            input_dim=hidden_size, 
+        )
+        d_out = self.downstream.output_dim
+
+        # 5. Gender Heads
+        self.gender_proj  = nn.Linear(d_out, 256)
+        self.gender_head  = nn.Sequential(
+            nn.RMSNorm(256), nn.GELU(), nn.Linear(256, config.num_classes)
+        )
+
+    def _forward_backbone(self, inputs: torch.Tensor, attention_mask: torch.Tensor):
+        # During inference, we always want hidden states for the aggregator
+        outputs = self.backbone(
+            input_values=inputs, 
+            attention_mask=attention_mask, 
+            output_hidden_states=True
+        )
+        
+        # Your Aggregation Logic
+        hs = torch.stack(outputs.hidden_states, dim=0) # (L, B, T, D)
+        return self.layer_aggregator(hs)
+
+    def forward(
+        self, 
+        input_values: torch.Tensor, 
+        attention_mask: torch.Tensor,
+        **kwargs
+    ):
+
+        inputs = input_values 
+        x = self._forward_backbone(inputs, attention_mask)   # (B, T, D)
+        feats = self.downstream(x)                   # Do Not Pass mask to downstream
+        
+        pre_final = self.gender_proj(feats)
+        logits    = self.gender_head(pre_final)
+        return pre_final, logits
+        
+   
+class AttentionPooling(nn.Module):
+    """
+    Attention-based pooling over the sequence dimension.
+    Input: (batch, seq_len, embed_dim)
+    Output: (batch, embed_dim)
+    """
+    def __init__(self, embed_dim):
+        super().__init__()
+        self.attention = nn.Linear(embed_dim, 1)
+
+    def forward(self, x, mask=None):
+        # x: (batch, seq_len, embed_dim)
+        attn_scores = self.attention(x).squeeze(-1)  # (batch, seq_len)
+        if mask is not None:
+            attn_scores = attn_scores.masked_fill(mask == 0, float('-inf'))
+        attn_weights = torch.softmax(attn_scores, dim=1)  # (batch, seq_len)
+        pooled = torch.sum(x * attn_weights.unsqueeze(-1), dim=1)  # (batch, embed_dim)
+        return pooled

config.json

@@ -0,0 +1,14 @@
+{
+  "architectures": [
+    "MeralionForGenderClassification"
+  ],
+  "auto_map": {
+    "AutoConfig": "MeralionForGender.MeralionGenderConfig",
+    "AutoModel": "MeralionForGender.MeralionForGenderClassification"
+  },
+  "dtype": "float32",
+  "meralion_name": "MERaLiON/MERaLiON-SpeechEncoder-2",
+  "model_type": "meralion_gender",
+  "num_classes": 2,
+  "transformers_version": "4.57.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e154bbf19106aeaa6a6e3481457dc5b993cb65420c63da510e0931de8cc566a
+size 2554030272

preprocessor_config.json

@@ -0,0 +1,17 @@
+{
+  "auto_map": {
+    "AutoFeatureExtractor": "processing_bestrq_conformer.ModifiedWhisperFeatureExtractor",
+    "AutoProcessor": "processing_bestrq_conformer.ModifiedWhisperFeatureExtractor"
+  },
+  "chunk_length": 120,
+  "feature_extractor_type": "ModifiedWhisperFeatureExtractor",
+  "feature_size": 80,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 1920000,
+  "nb_max_frames": 12000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

processing_bestrq_conformer.py

@@ -0,0 +1,554 @@
+# coding=utf-8
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Feature extractor class for MERaLiON-SpeechEncoder, modified from original WhisperFeatureExtractor
+"""
+
+import itertools
+import os
+from shutil import copyfile
+from typing import Dict, List, Optional, Tuple, Union
+
+import numpy as np
+
+from transformers import is_torch_available, AutoFeatureExtractor, AutoTokenizer
+from transformers.audio_utils import mel_filter_bank, spectrogram, window_function
+from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import TensorType, logging
+
+
+if is_torch_available():
+    import torch
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.model"}
+
+class ModifiedWhisperFeatureExtractor(SequenceFeatureExtractor):
+    r"""
+    Constructs a modified Whisper feature extractor.
+
+    This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
+    most of the main methods. Users should refer to this superclass for more information regarding those methods.
+
+    This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
+    Fourier Transform` which should match pytorch's `torch.stft` equivalent.
+
+    Differences from WhisperFeatureExtractor:
+    - mel_filter_bank
+        - norm: "slaney" -> None
+        - mel_scale: "slaney" -> "htk"
+    - still uses log scaling and clamp but removes additional min-max/mean normalization
+
+    Args:
+        feature_size (`int`, *optional*, defaults to 80):
+            The feature dimension of the extracted features.
+        sampling_rate (`int`, *optional*, defaults to 16000):
+            The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
+        hop_length (`int`, *optional*, defaults to 160):
+            Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
+        chunk_length (`int`, *optional*, defaults to 30):
+            The maximum number of chunks of `sampling_rate` samples used to trim and pad longer or shorter audio
+            sequences.
+        n_fft (`int`, *optional*, defaults to 400):
+            Size of the Fourier transform.
+        padding_value (`float`, *optional*, defaults to 0.0):
+            Padding value used to pad the audio. Should correspond to silences.
+    """
+
+    model_input_names = ["input_values"]
+
+    def __init__(
+        self,
+        feature_size=80,
+        sampling_rate=16000,
+        hop_length=160,
+        chunk_length=120,
+        n_fft=400,
+        padding_value=0.0,
+        return_attention_mask=True,  # pad inputs to max length with silence token (zero) and no attention mask
+        **kwargs,
+    ):
+        super().__init__(
+            feature_size=feature_size,
+            sampling_rate=sampling_rate,
+            padding_value=padding_value,
+            return_attention_mask=return_attention_mask,
+            **kwargs,
+        )
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+        self.chunk_length = chunk_length
+        self.n_samples = chunk_length * sampling_rate
+        self.nb_max_frames = self.n_samples // hop_length
+        self.sampling_rate = sampling_rate
+        self.mel_filters = mel_filter_bank(
+            num_frequency_bins=1 + n_fft // 2,
+            num_mel_filters=feature_size,
+            min_frequency=0.0,
+            max_frequency=8000.0,
+            sampling_rate=sampling_rate,
+            norm=None,
+            mel_scale="htk",
+        )
+
+    def _np_extract_fbank_features(self, waveform_batch: np.array, device: str) -> np.ndarray:
+        """
+        Compute the log-mel spectrogram of the provided audio, gives similar results to Whisper's original torch
+        implementation with 1e-5 tolerance.
+        """
+        if device != "cpu":
+            raise ValueError(
+                f"Got device `{device}` for feature extraction, but feature extraction on CUDA accelerator "
+                "devices requires torch, which is not installed. Either set `device='cpu'`, or "
+                "install torch according to the official instructions: https://pytorch.org/get-started/locally/"
+            )
+        log_spec_batch = []
+        for waveform in waveform_batch:
+            log_spec = spectrogram(
+                waveform,
+                window_function(self.n_fft, "hann"),
+                frame_length=self.n_fft,
+                hop_length=self.hop_length,
+                power=2.0,
+                mel_filters=self.mel_filters,
+                log_mel="log10",
+            )
+            log_spec = log_spec[:, :-1]
+
+            log_spec_batch.append(log_spec)
+        log_spec_batch = np.array(log_spec_batch)
+        return log_spec_batch
+
+    def _torch_extract_fbank_features(self, waveform: np.array, device: str = "cpu") -> np.ndarray:
+        """
+        Compute the log-mel spectrogram of the audio using PyTorch's GPU-accelerated STFT implementation with batching,
+        yielding results similar to cpu computing with 1e-5 tolerance.
+        """
+        waveform = torch.from_numpy(waveform).type(torch.float32)
+
+        window = torch.hann_window(self.n_fft)
+        if device != "cpu":
+            waveform = waveform.to(device)
+            window = window.to(device)
+        stft = torch.stft(waveform, self.n_fft, self.hop_length, window=window, return_complex=True)
+        magnitudes = stft[..., :-1].abs() ** 2
+
+        mel_filters = torch.from_numpy(self.mel_filters).type(torch.float32)
+        if device != "cpu":
+            mel_filters = mel_filters.to(device)
+        mel_spec = mel_filters.T @ magnitudes
+
+        log_spec = torch.clamp(mel_spec, min=1e-10).log10()
+
+        if device != "cpu":
+            log_spec = log_spec.detach().cpu()
+        return log_spec.numpy()
+
+    @staticmethod
+    # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
+    def zero_mean_unit_var_norm(
+        input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0
+    ) -> List[np.ndarray]:
+        """
+        Every array in the list is normalized to have zero mean and unit variance
+        """
+        if attention_mask is not None:
+            attention_mask = np.array(attention_mask, np.int32)
+            normed_input_values = []
+
+            for vector, length in zip(input_values, attention_mask.sum(-1)):
+                normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
+                if length < normed_slice.shape[0]:
+                    normed_slice[length:] = padding_value
+
+                normed_input_values.append(normed_slice)
+        else:
+            normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]
+
+        return normed_input_values
+
+    def __call__(
+        self,
+        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
+        truncation: bool = True,
+        pad_to_multiple_of: Optional[int] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        return_attention_mask: Optional[bool] = True,
+        padding: Optional[Union[bool, str]] = True,
+        max_length: Optional[int] = None,
+        sampling_rate: Optional[int] = None,
+        do_normalize: Optional[bool] = None,
+        device: Optional[str] = "cpu",
+        return_token_timestamps: Optional[bool] = None,
+        **kwargs,
+    ) -> BatchFeature:
+        """
+        Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for
+        the STFT computation if available, otherwise a slower NumPy based one.
+
+        Args:
+            raw_speech (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`):
+                The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
+                values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
+                stereo, i.e. single float per timestep.
+            truncation (`bool`, *optional*, default to `True`):
+                Activates truncation to cut input sequences longer than *max_length* to *max_length*.
+            pad_to_multiple_of (`int`, *optional*, defaults to None):
+                If set will pad the sequence to a multiple of the provided value.
+
+                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
+                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
+            return_attention_mask (`bool`, *optional*):
+                Whether to return the attention mask. If left to the default, will return the attention mask according
+                to the specific feature_extractor's default.
+
+                [What are attention masks?](../glossary#attention-mask)
+
+                <Tip>
+
+                For Whisper models, `attention_mask` should always be passed for batched inference, to avoid subtle
+                bugs.
+
+                </Tip>
+
+            return_tensors (`str` or [`~utils.TensorType`], *optional*):
+                If set, will return tensors instead of list of python integers. Acceptable values are:
+
+                - `'tf'`: Return TensorFlow `tf.constant` objects.
+                - `'pt'`: Return PyTorch `torch.Tensor` objects.
+                - `'np'`: Return Numpy `np.ndarray` objects.
+            sampling_rate (`int`, *optional*):
+                The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
+                `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
+                pipeline.
+            padding_value (`float`, *optional*, defaults to 0.0):
+                The value that is used to fill the padding values / vectors.
+            do_normalize (`bool`, *optional*, defaults to `False`):
+                Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
+                improve the performance of the model.
+            device (`str`, *optional*, defaults to `'cpu'`):
+                Specifies the device for computation of the log-mel spectrogram of audio signals in the
+                `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
+            return_token_timestamps (`bool`, *optional*, defaults to `None`):
+                Whether or not to return the number of frames of the input raw_speech.
+                These num_frames can be used by the model to compute word level timestamps.
+        """
+
+        if sampling_rate is not None:
+            if sampling_rate != self.sampling_rate:
+                raise ValueError(
+                    f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
+                    f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
+                    f" was sampled with {self.sampling_rate} and not {sampling_rate}."
+                )
+        else:
+            logger.warning(
+                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
+                "Failing to do so can result in silent errors that might be hard to debug."
+            )
+
+        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
+        if is_batched_numpy and len(raw_speech.shape) > 2:
+            raise ValueError(f"Only mono-channel audio is supported for input to {self}")
+        is_batched = is_batched_numpy or (
+            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
+        )
+
+        if is_batched:
+            raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
+        elif not is_batched and not isinstance(raw_speech, np.ndarray):
+            raw_speech = np.asarray(raw_speech, dtype=np.float32)
+        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
+            raw_speech = raw_speech.astype(np.float32)
+
+        # always return batch
+        if not is_batched:
+            raw_speech = [np.asarray([raw_speech]).T]
+
+        batched_speech = BatchFeature({"input_values": raw_speech})
+
+        # convert into correct format for padding
+
+        padded_inputs = self.pad(  #whisper pads first then transform, while we do the reverse
+            batched_speech,
+            padding=padding,
+            max_length=max_length if max_length else self.n_samples,
+            truncation=truncation,
+            pad_to_multiple_of=pad_to_multiple_of,
+            return_attention_mask=return_attention_mask or do_normalize,
+        )
+
+        # zero-mean and unit-variance normalization
+        if do_normalize:
+            padded_inputs["input_values"] = self.zero_mean_unit_var_norm(
+                padded_inputs["input_values"],
+                attention_mask=padded_inputs["attention_mask"],
+                padding_value=self.padding_value,
+            )
+            padded_inputs["input_values"] = np.stack(padded_inputs["input_values"], axis=0)
+
+        # make sure list is in array format
+        input_values = padded_inputs.get("input_values").transpose(2, 0, 1)
+
+        extract_fbank_features = (
+            self._torch_extract_fbank_features if is_torch_available() else self._np_extract_fbank_features
+        )
+        input_values = extract_fbank_features(input_values[0], device)
+
+        if isinstance(input_values[0], List):
+            padded_inputs["input_values"] = [np.asarray(feature, dtype=np.float32) for feature in input_values]
+
+        else:
+            padded_inputs["input_values"] = input_values
+
+        if return_attention_mask:
+            # rescale from sample (48000) to feature (3000)
+            padded_inputs["attention_mask"] = padded_inputs["attention_mask"][:, :: self.hop_length]
+
+        if return_token_timestamps is not None:
+            padded_inputs["num_frames"] = [len(raw_speech_i) // self.hop_length for raw_speech_i in raw_speech]
+
+        if return_tensors is not None:
+            padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
+
+        return padded_inputs
+
+
+class MeralionBestRqConformerTokenizer(PreTrainedTokenizer):
+    """
+    Constructs a MeralionBestRqConformer tokenizer. Based on `SentencePiece`.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        unk_token (`str`, *optional*, defaults to "<unk>"):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token.
+        pad_token (`str`, *optional*, defaults to "<pad>"):
+            The token used for padding, for example when batching sequences of different lengths.
+        bos_token (`str`, *optional*, defaults to "<s>"):
+            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
+        eos_token (`str`, *optional*, defaults to "</s>"):
+            The end of sequence token.
+        **kwargs
+            Additional keyword arguments passed along to
+            [`PreTrainedTokenizer.__init__`](https://huggingface.co/docs/transformers/main_classes/tokenizer#transformers.PreTrainedTokenizer.__init__).
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token="<unk>",
+        pad_token="<pad>",
+        bos_token="<s>",
+        eos_token="</s>",
+        blank_token="<blk>",
+        **kwargs
+    ):
+        import sentencepiece as spm
+
+        self.vocab_file = vocab_file
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(vocab_file)
+
+        super().__init__(
+            unk_token=unk_token,
+            pad_token=pad_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            blank_token=blank_token,
+            **kwargs,
+        )
+
+        self.blank_token_id = self.sp_model.piece_to_id(blank_token)
+
+    def get_special_tokens_mask(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+        already_has_special_tokens: bool = False,
+    ) -> List[int]:
+        """
+        Retrieves sequence of 0s and 1s specifying if corresponding token ID is a special token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        if token_ids_1 is None:
+            return [0] * len(token_ids_0)
+        return ([0] * len(token_ids_0)) + ([0] * len(token_ids_1))
+
+    def _tokenize(self, text: str) -> List[str]:
+        """
+        Converts a string in a sequence of tokens (string), using the `sp_model` tokenizer.
+        """
+        # SentencePiece doesn't like empty strings
+        if not text:
+            return []
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token: str) -> int:
+        """
+        Converts a token (str) in an id (integer) using the vocab.
+        """
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index: int) -> str:
+        """
+        Converts an id (integer) in a token (str) using the vocab.
+        """
+        return self.sp_model.id_to_piece(index)
+
+    def convert_tokens_to_string(self, tokens: list[str]) -> str:
+        return self.sp_model.decode(tokens)
+
+    def decode(
+        self,
+        token_ids: Union[List[int], np.ndarray, torch.Tensor],
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: bool = None,
+        group_tokens: bool = True,
+        **kwargs,
+    ) -> str:
+        """
+        Converts a sequence of ids in a string, using the tokenizer and vocabulary with CTC decoding logic.
+        """
+        if isinstance(token_ids, (np.ndarray, torch.Tensor)):
+            token_ids = token_ids.tolist()
+
+        # CTC decoding
+        if group_tokens:
+            token_ids = [token_id for token_id, _ in itertools.groupby(token_ids)]
+
+        # Remove blank tokens
+        token_ids = [token_id for token_id in token_ids if token_id != self.blank_token_id]
+
+        return super().decode(
+            token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+        )
+
+    def batch_decode(
+        self,
+        sequences: Union[List[int], List[List[int]], np.ndarray, torch.Tensor],
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: Optional[bool] = None,
+        **kwargs,
+    ) -> List[str]:
+        """
+        Convert a list of lists of token ids into a list of strings by calling decode.
+        """
+        batch_decoded = [
+            self.decode(
+                seq,
+                skip_special_tokens=skip_special_tokens,
+                clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+                **kwargs,
+            )
+            for seq in sequences
+        ]
+        return batch_decoded
+
+    def get_vocab(self) -> Dict[str, int]:
+        """
+        Returns the vocabulary as a dictionary of token to index.
+        """
+        vocab = {self.sp_model.IdToPiece(i): i for i in range(self.sp_model.GetPieceSize())}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+        """
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory)
+        
+        vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+
+        copyfile(self.vocab_file, vocab_file)
+
+        return (vocab_file,)
+
+    @property
+    def vocab_size(self) -> int:
+        return self.sp_model.get_piece_size()
+
+
+class MeralionBestRqConformerProcessor(ProcessorMixin):
+    r"""
+    Constructs a Wav2Vec2 like processor which wraps a ModifiedWhisperFeatureExtractor feature extractor and a 
+    MeralionBestRqConformerTokenizer sentencepiece tokenizer into a single
+    processor.
+
+    [`MeralionBestRqConformerProcessor`] offers all the functionalities of [`ModifiedWhisperFeatureExtractor`] and 
+    [`MeralionBestRqConformerTokenizer`].
+    See the docstring of [`~MeralionBestRqConformerProcessor.__call__`] and [`~MeralionBestRqConformerProcessor.decode`]
+    for more information.
+
+    Args:
+        feature_extractor (`ModifiedWhisperFeatureExtractor`):
+            An instance of [`ModifiedWhisperFeatureExtractor`]. The feature extractor is a required input.
+        tokenizer ([`MeralionBestRqConformerTokenizer`]):
+            An instance of [`MeralionBestRqConformerTokenizer`]. The tokenizer is a required input.
+    """
+
+    feature_extractor_class = "ModifiedWhisperFeatureExtractor"
+    tokenizer_class = "MeralionBestRqConformerTokenizer"
+
+    def __init__(self, feature_extractor, tokenizer):
+        self.feature_extractor = feature_extractor
+        self.tokenizer = tokenizer
+        self.chat_template = None
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        feature_extractor = ModifiedWhisperFeatureExtractor.from_pretrained(pretrained_model_name_or_path, **kwargs)
+        tokenizer = MeralionBestRqConformerTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs)
+        
+        return cls(feature_extractor=feature_extractor, tokenizer=tokenizer)
+
+    def __call__(
+        self,
+        audio: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
+        *args,
+        **kwds,
+    ):
+        return self.feature_extractor(audio, *args, **kwds)
+        
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to MeralionBestRqConformerTokenizer's [`~MeralionBestRqConformerTokenizer.batch_decode`].
+        Please refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to MeralionBestRqConformerTokenizer's [`~MeralionBestRqConformerTokenizer.decode`]. 
+        Please refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)