"""Dummy tokenizer for pipeline("feature-extraction") compatibility. The HuggingFace ``FeatureExtractionPipeline`` unconditionally requires a tokenizer, even for audio models that have no vocabulary. This thin wrapper satisfies that interface by delegating ``__call__`` to the real ``EcapaTdnnFeatureExtractor``, which computes log-mel spectrograms from raw audio. >>> pipe = pipeline("feature-extraction", model=model_id, trust_remote_code=True) >>> pipe("audio.wav") # works! """ import os import numpy as np from transformers import PreTrainedTokenizer from transformers.feature_extraction_utils import BatchFeature class EcapaTdnnDummyTokenizer(PreTrainedTokenizer): """Tokenizer shim that wraps :class:`EcapaTdnnFeatureExtractor`. This class exists *only* to make ``pipeline("feature-extraction")`` work with ECAPA-TDNN speaker encoder models. It contains no real vocabulary — all audio preprocessing is handled by the feature extractor. """ vocab_files_names: dict[str, str] = {} model_input_names = ["input_values"] def __init__(self, **kwargs): # Filter out tokenizer-specific kwargs that don't apply to us kwargs.pop("added_tokens_decoder", None) super().__init__(**kwargs) # -- abstract method stubs (unused but required) ----------------------- @property def vocab_size(self) -> int: return 0 def get_vocab(self) -> dict[str, int]: return {} def _tokenize(self, text, **kwargs): return [] def _convert_token_to_id(self, token): return 0 def _convert_id_to_token(self, index): return "" def save_vocabulary(self, save_directory, filename_prefix=None): return () # -- the only method that actually matters ------------------------------ def __call__(self, raw_speech, return_tensors="pt", **kwargs): """Preprocess audio via the feature extractor. Accepts the same inputs as :class:`EcapaTdnnFeatureExtractor`: file paths, numpy arrays, or lists thereof. """ try: from .feature_extraction_ecapa_tdnn import EcapaTdnnFeatureExtractor except ImportError: from feature_extraction_ecapa_tdnn import EcapaTdnnFeatureExtractor # Load the feature extractor config from the same directory model_dir = os.path.dirname(os.path.abspath(__file__)) try: fe = EcapaTdnnFeatureExtractor.from_pretrained(model_dir) except Exception: fe = EcapaTdnnFeatureExtractor() return fe(raw_speech, return_tensors=return_tensors, **kwargs)