File size: 2,881 Bytes
0a81958 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | """Audio input mapper for vLLM multimodal pipeline.
This module handles audio data loading and preprocessing for VibeVoice ASR inference.
It converts various audio input formats (path, bytes, numpy array) into tensors
that can be processed by the VibeVoice model.
"""
import torch
import numpy as np
from typing import Union, List
from vllm.multimodal.inputs import MultiModalInputs
from vibevoice.processor.audio_utils import load_audio_use_ffmpeg, load_audio_bytes_use_ffmpeg, AudioNormalizer
def load_audio(audio_path: str, target_sr: int = 24000) -> np.ndarray:
"""Load and normalize audio from file path.
Args:
audio_path: Path to audio file
target_sr: Target sample rate (default 24kHz for VibeVoice)
Returns:
Normalized audio waveform as numpy array
"""
# Load with FFmpeg (handles various formats)
audio, sr = load_audio_use_ffmpeg(audio_path, resample=True, target_sr=target_sr)
# Normalize audio
normalizer = AudioNormalizer()
audio = normalizer(audio)
return audio
def vibevoice_audio_input_mapper(ctx, data: Union[str, bytes, np.ndarray, List[str]]) -> MultiModalInputs:
"""Map audio input data to vLLM MultiModalInputs format.
This function is registered as the input mapper for VibeVoice audio processing.
It handles multiple input formats and converts them to normalized tensors.
Args:
ctx: vLLM context (unused)
data: Audio data in one of these formats:
- str: Path to audio file
- bytes: Raw audio bytes (any format FFmpeg supports)
- np.ndarray: Pre-loaded audio waveform
- List[str]: List of audio paths (only first is used)
Returns:
MultiModalInputs containing:
- audio: Audio tensor (float32)
- audio_length: Length of audio in samples
"""
# Handle list input (take first item)
if isinstance(data, list):
data = data[0]
audio_waveform = None
if isinstance(data, str):
# Load from file path
audio_waveform = load_audio(data)
elif isinstance(data, bytes):
# Decode bytes directly via ffmpeg stdin pipe to avoid temp-file IO
audio_waveform, _sr = load_audio_bytes_use_ffmpeg(data, resample=True, target_sr=24000)
normalizer = AudioNormalizer()
audio_waveform = normalizer(audio_waveform)
elif isinstance(data, np.ndarray):
# Already loaded numpy array
audio_waveform = data
else:
raise ValueError(f"Unsupported audio data type: {type(data)}")
# Convert to tensor
audio_tensor = torch.from_numpy(audio_waveform).float()
audio_length = audio_tensor.shape[0]
return MultiModalInputs({
"audio": audio_tensor,
"audio_length": audio_length
})
|