feature_extraction_msp_audio.py

from pathlib import Path

import numpy as np
import torch
from torchcodec.decoders import AudioDecoder
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
from transformers.feature_extraction_utils import BatchFeature
from transformers.utils import PaddingStrategy, TensorType, logging

logger = logging.get_logger(__name__)


class MSPAudioFeatureExtractor(SequenceFeatureExtractor):
    model_input_names = ["input_values", "padding_mask"]

    def __init__(
        self,
        feature_size: int = 1,
        sampling_rate: int = 16000,
        padding_value: float = 0.0,
        return_attention_mask: bool = True,
        do_normalize: bool = True,
        **kwargs,
    ):
        super().__init__(
            feature_size=feature_size,
            sampling_rate=sampling_rate,
            padding_value=padding_value,
            **kwargs,
        )
        self.return_attention_mask = return_attention_mask
        self.do_normalize = do_normalize

    @staticmethod
    def zero_mean_unit_var_norm(
        input_values: list[np.ndarray],
        attention_mask: list[np.ndarray] | None,
        padding_value: float = 0.0,
    ) -> list[np.ndarray]:
        """Normalize each sequence to zero mean and unit variance."""
        if attention_mask is not None:
            attention_mask = np.array(attention_mask, dtype=np.int32)
            normed = []
            for vec, length in zip(input_values, attention_mask.sum(-1)):
                normed_slice = (vec - vec[:length].mean()) / np.sqrt(
                    vec[:length].var() + 1e-7
                )
                if length < normed_slice.shape[0]:
                    normed_slice[length:] = padding_value
                normed.append(normed_slice)
        else:
            normed = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]
        return normed

    def _load_audio(
        self,
        src: str | Path | bytes | torch.Tensor,
        start_seconds: float = 0.0,
        stop_seconds: float | None = None,
    ) -> np.ndarray:
        """Load audio waveform from file path or bytes as a 1-D numpy array."""
        audio_decoder = AudioDecoder(source=src, sample_rate=self.sampling_rate)
        if stop_seconds is None:
            stop_seconds = audio_decoder.metadata.duration_seconds_from_header
        waveform = audio_decoder.get_samples_played_in_range(
            start_seconds, stop_seconds
        ).data.numpy()
        return waveform.squeeze()  # shape: (T,)

    def __call__(
        self,
        raw_speech: (
            str
            | Path
            | bytes
            | np.ndarray
            | list[str]
            | list[Path]
            | list[bytes]
            | list[float]
            | list[np.ndarray]
            | list[list[float]]
        ),
        padding: bool | str | PaddingStrategy = False,
        max_length: int | None = None,
        truncation: bool = False,
        pad_to_multiple_of: int | None = None,
        return_attention_mask: bool | None = None,
        return_tensors: str | TensorType | None = None,
        sampling_rate: int | None = None,
        **kwargs,
    ) -> BatchFeature:
        """
        Featurize and pad one or several audio sequences.
        """
        if sampling_rate is not None and sampling_rate != self.sampling_rate:
            raise ValueError(
                f"Sampling rate mismatch: expected {self.sampling_rate}, "
                f"got {sampling_rate}."
            )

        is_batched_numpy = isinstance(raw_speech, np.ndarray) and raw_speech.ndim > 1
        if is_batched_numpy and raw_speech.ndim > 2:
            raise ValueError("Only mono-channel audio is supported.")

        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple))
            and isinstance(raw_speech[0], (str, Path, bytes, np.ndarray, list, tuple))
        )

        if not is_batched:
            raw_speech = [raw_speech]

        # Load from file paths or bytes
        if isinstance(raw_speech[0], (str, Path, bytes)):
            raw_speech = [self._load_audio(src) for src in raw_speech]

        encoded = BatchFeature({"input_values": raw_speech})

        padded = self.pad(
            encoded,
            padding=padding,
            max_length=max_length,
            truncation=truncation,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
        )

        # Ensure float32
        vals = padded["input_values"]
        if not isinstance(vals[0], np.ndarray):
            padded["input_values"] = [np.asarray(a, dtype=np.float32) for a in vals]
        elif isinstance(vals[0], np.ndarray) and vals[0].dtype == np.float64:
            padded["input_values"] = [a.astype(np.float32) for a in vals]

        # Normalize
        attn = padded.get("attention_mask")
        if attn is not None:
            padded["attention_mask"] = [np.asarray(a, dtype=np.int32) for a in attn]

        if self.do_normalize:
            norm_attn = (
                attn
                if self._get_padding_strategies(padding, max_length=max_length)
                is not PaddingStrategy.DO_NOT_PAD
                else None
            )
            padded["input_values"] = self.zero_mean_unit_var_norm(
                padded["input_values"],
                attention_mask=norm_attn,
                padding_value=self.padding_value,
            )

        # Rename attention_mask -> padding_mask
        if "attention_mask" in padded:
            padded["padding_mask"] = padded.pop("attention_mask")

        if return_tensors is not None:
            padded = padded.convert_to_tensors(return_tensors)

        return padded