"""Audio input mapper for vLLM multimodal pipeline. This module handles audio data loading and preprocessing for VibeVoice ASR inference. It converts various audio input formats (path, bytes, numpy array) into tensors that can be processed by the VibeVoice model. """ import torch import numpy as np from typing import Union, List from vllm.multimodal.inputs import MultiModalInputs from vibevoice.processor.audio_utils import load_audio_use_ffmpeg, load_audio_bytes_use_ffmpeg, AudioNormalizer def load_audio(audio_path: str, target_sr: int = 24000) -> np.ndarray: """Load and normalize audio from file path. Args: audio_path: Path to audio file target_sr: Target sample rate (default 24kHz for VibeVoice) Returns: Normalized audio waveform as numpy array """ # Load with FFmpeg (handles various formats) audio, sr = load_audio_use_ffmpeg(audio_path, resample=True, target_sr=target_sr) # Normalize audio normalizer = AudioNormalizer() audio = normalizer(audio) return audio def vibevoice_audio_input_mapper(ctx, data: Union[str, bytes, np.ndarray, List[str]]) -> MultiModalInputs: """Map audio input data to vLLM MultiModalInputs format. This function is registered as the input mapper for VibeVoice audio processing. It handles multiple input formats and converts them to normalized tensors. Args: ctx: vLLM context (unused) data: Audio data in one of these formats: - str: Path to audio file - bytes: Raw audio bytes (any format FFmpeg supports) - np.ndarray: Pre-loaded audio waveform - List[str]: List of audio paths (only first is used) Returns: MultiModalInputs containing: - audio: Audio tensor (float32) - audio_length: Length of audio in samples """ # Handle list input (take first item) if isinstance(data, list): data = data[0] audio_waveform = None if isinstance(data, str): # Load from file path audio_waveform = load_audio(data) elif isinstance(data, bytes): # Decode bytes directly via ffmpeg stdin pipe to avoid temp-file IO audio_waveform, _sr = load_audio_bytes_use_ffmpeg(data, resample=True, target_sr=24000) normalizer = AudioNormalizer() audio_waveform = normalizer(audio_waveform) elif isinstance(data, np.ndarray): # Already loaded numpy array audio_waveform = data else: raise ValueError(f"Unsupported audio data type: {type(data)}") # Convert to tensor audio_tensor = torch.from_numpy(audio_waveform).float() audio_length = audio_tensor.shape[0] return MultiModalInputs({ "audio": audio_tensor, "audio_length": audio_length })