| | import math
|
| | import warnings
|
| | from typing import List, Optional, Union, Dict, Any, Tuple
|
| | import os
|
| | import re
|
| |
|
| | import numpy as np
|
| | import torch
|
| |
|
| | from transformers.tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
|
| | from transformers.utils import TensorType, logging
|
| | from .vibevoice_tokenizer_processor import AudioNormalizer
|
| |
|
| | logger = logging.get_logger(__name__)
|
| |
|
| |
|
| | class VibeVoiceProcessor:
|
| | r"""
|
| | Constructs a VibeVoice processor which wraps a VibeVoice tokenizer and audio processor into a single processor.
|
| |
|
| | [`VibeVoiceProcessor`] offers all the functionalities of [`VibeVoiceTokenizer`] and [`VibeVoiceTokenizerProcessor`].
|
| | See the [`~VibeVoiceProcessor.__call__`] and [`~VibeVoiceProcessor.decode`] for more information.
|
| |
|
| | Args:
|
| | tokenizer (`VibeVoiceTextTokenizer` or `VibeVoiceTextTokenizerFast`):
|
| | The tokenizer for text processing.
|
| | audio_processor (`VibeVoiceTokenizerProcessor`):
|
| | The audio processor for speech processing.
|
| | speech_tok_compress_ratio (`int`, *optional*, defaults to 3200):
|
| | The compression ratio for speech tokenization.
|
| | db_normalize (`bool`, *optional*, defaults to True):
|
| | Whether to apply decibel normalization to audio inputs.
|
| | """
|
| |
|
| | def __init__(self, tokenizer=None, audio_processor=None, speech_tok_compress_ratio=3200, db_normalize=True, **kwargs):
|
| | self.tokenizer = tokenizer
|
| | self.audio_processor = audio_processor
|
| | self.speech_tok_compress_ratio = speech_tok_compress_ratio
|
| | self.db_normalize = db_normalize
|
| | self.audio_normalizer = AudioNormalizer() if db_normalize else None
|
| | self.system_prompt = " Transform the text provided by various speakers into speech output, utilizing the distinct voice of each respective speaker.\n"
|
| |
|
| | @classmethod
|
| | def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
|
| | """
|
| | Instantiate a VibeVoiceProcessor from a pretrained VibeVoice processor.
|
| |
|
| | Args:
|
| | pretrained_model_name_or_path (`str` or `os.PathLike`):
|
| | This can be either:
|
| | - a string, the *model id* of a pretrained model
|
| | - a path to a *directory* containing processor config
|
| |
|
| | Returns:
|
| | [`VibeVoiceProcessor`]: The processor object instantiated from pretrained model.
|
| | """
|
| | import os
|
| | import json
|
| | from .vibevoice_tokenizer_processor import VibeVoiceTokenizerProcessor
|
| | from modular.modular_vibevoice_text_tokenizer import (
|
| | VibeVoiceTextTokenizer,
|
| | VibeVoiceTextTokenizerFast
|
| | )
|
| |
|
| |
|
| | config_path = os.path.join(pretrained_model_name_or_path, "preprocessor_config.json")
|
| | if os.path.exists(config_path):
|
| | with open(config_path, 'r') as f:
|
| | config = json.load(f)
|
| | else:
|
| | logger.warning(f"No preprocessor_config.json found at {pretrained_model_name_or_path}, using defaults")
|
| | config = {
|
| | "speech_tok_compress_ratio": 3200,
|
| | "db_normalize": True,
|
| | }
|
| |
|
| |
|
| | speech_tok_compress_ratio = config.get("speech_tok_compress_ratio", 3200)
|
| | db_normalize = config.get("db_normalize", True)
|
| |
|
| |
|
| | language_model_pretrained_name = config.get("language_model_pretrained_name", None) or kwargs.pop("language_model_pretrained_name", "Qwen/Qwen2.5-1.5B")
|
| | logger.info(f"Loading tokenizer from {language_model_pretrained_name}")
|
| | if 'qwen' in language_model_pretrained_name.lower():
|
| | tokenizer = VibeVoiceTextTokenizerFast.from_pretrained(
|
| | language_model_pretrained_name,
|
| | **kwargs
|
| | )
|
| | else:
|
| | raise ValueError(f"Unsupported tokenizer type for {language_model_pretrained_name}. Supported types: Qwen, Llama, Gemma.")
|
| |
|
| |
|
| | if "audio_processor" in config:
|
| |
|
| | audio_config = config["audio_processor"]
|
| | audio_processor = VibeVoiceTokenizerProcessor(
|
| | sampling_rate=audio_config.get("sampling_rate", 24000),
|
| | normalize_audio=audio_config.get("normalize_audio", True),
|
| | target_dB_FS=audio_config.get("target_dB_FS", -25),
|
| | eps=audio_config.get("eps", 1e-6),
|
| | )
|
| | else:
|
| |
|
| | audio_processor = VibeVoiceTokenizerProcessor()
|
| |
|
| |
|
| | return cls(
|
| | tokenizer=tokenizer,
|
| | audio_processor=audio_processor,
|
| | speech_tok_compress_ratio=speech_tok_compress_ratio,
|
| | db_normalize=db_normalize,
|
| | )
|
| |
|
| | def save_pretrained(self, save_directory: Union[str, os.PathLike], **kwargs):
|
| | """
|
| | Save a processor to a directory, so that it can be re-loaded using the
|
| | [`~VibeVoiceProcessor.from_pretrained`] class method.
|
| |
|
| | Args:
|
| | save_directory (`str` or `os.PathLike`):
|
| | Directory where the processor will be saved.
|
| | """
|
| | import os
|
| | import json
|
| |
|
| | os.makedirs(save_directory, exist_ok=True)
|
| |
|
| |
|
| | processor_config = {
|
| | "processor_class": "VibeVoiceProcessor",
|
| | "speech_tok_compress_ratio": self.speech_tok_compress_ratio,
|
| | "db_normalize": self.db_normalize,
|
| | "audio_processor": {
|
| | "feature_extractor_type": "VibeVoiceTokenizerProcessor",
|
| | "sampling_rate": getattr(self.audio_processor, 'sampling_rate', 24000),
|
| | "normalize_audio": getattr(self.audio_processor, 'normalize_audio', True),
|
| | "target_dB_FS": getattr(self.audio_processor, 'target_dB_FS', -25),
|
| | "eps": getattr(self.audio_processor, 'eps', 1e-6),
|
| | }
|
| | }
|
| |
|
| | config_path = os.path.join(save_directory, "preprocessor_config.json")
|
| | with open(config_path, 'w') as f:
|
| | json.dump(processor_config, f, indent=2)
|
| |
|
| | logger.info(f"Processor configuration saved in {config_path}")
|
| |
|
| | def __call__(
|
| | self,
|
| | text: Optional[Union[str, List[str], TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
|
| | voice_samples: Optional[Union[List[Union[str, np.ndarray]], List[List[Union[str, np.ndarray]]]]] = None,
|
| | padding: Union[bool, str, PaddingStrategy] = True,
|
| | truncation: Union[bool, str, TruncationStrategy] = False,
|
| | max_length: Optional[int] = None,
|
| | return_tensors: Optional[Union[str, TensorType]] = None,
|
| | return_attention_mask: bool = True,
|
| | **kwargs,
|
| | ) -> BatchEncoding:
|
| | """
|
| | Main method to process one or more podcast scripts with optional voice samples.
|
| |
|
| | Args:
|
| | text (`str`, `List[str]`):
|
| | The input text(s) to process. Can be:
|
| | - A single script string
|
| | - A list of script strings for batch processing
|
| | - A path to a .json or .txt file
|
| | - A list of paths
|
| | voice_samples (`List[Union[str, np.ndarray]]`, `List[List[Union[str, np.ndarray]]]`, *optional*):
|
| | Voice samples for each script. Can be:
|
| | - A list of samples for a single script
|
| | - A list of lists for batch processing
|
| | padding (`bool`, `str` or `PaddingStrategy`, defaults to `True`):
|
| | Whether to pad sequences to the same length
|
| | truncation (`bool`, `str` or `TruncationStrategy`, defaults to `False`):
|
| | Whether to truncate sequences
|
| | max_length (`int`, *optional*):
|
| | Maximum length of the returned sequences
|
| | return_tensors (`str` or `TensorType`, *optional*):
|
| | If set, will return tensors of a particular framework
|
| | return_attention_mask (`bool`, defaults to `True`):
|
| | Whether to return the attention mask
|
| |
|
| | Returns:
|
| | `BatchEncoding`: A BatchEncoding with the following fields:
|
| | - **input_ids** -- List of token id sequences or tensor
|
| | - **attention_mask** -- List of attention masks or tensor
|
| | - **speech_tensors** -- Padded speech inputs (if voice_samples provided)
|
| | - **speech_masks** -- Speech masks (if voice_samples provided)
|
| | - **speech_input_mask** -- Boolean masks indicating speech token positions
|
| | """
|
| |
|
| | if isinstance(text, str) or (isinstance(text, list) and len(text) > 0 and not isinstance(text[0], str)):
|
| |
|
| | texts = [text]
|
| | is_batched = False
|
| | else:
|
| |
|
| | texts = text
|
| | is_batched = True
|
| |
|
| |
|
| | if voice_samples is not None:
|
| | if not is_batched or (isinstance(voice_samples[0], (str, np.ndarray))):
|
| |
|
| | voice_samples_list = [voice_samples]
|
| | else:
|
| |
|
| | voice_samples_list = voice_samples
|
| | else:
|
| | voice_samples_list = [None] * len(texts)
|
| |
|
| |
|
| | all_encodings = []
|
| | for text_input, voice_input in zip(texts, voice_samples_list):
|
| | encoding = self._process_single(text_input, voice_input)
|
| | all_encodings.append(encoding)
|
| |
|
| |
|
| | batch_encoding = self._batch_encode(
|
| | all_encodings,
|
| | padding=padding,
|
| | truncation=truncation,
|
| | max_length=max_length,
|
| | return_tensors=return_tensors,
|
| | return_attention_mask=return_attention_mask,
|
| | )
|
| |
|
| | return batch_encoding
|
| |
|
| | def _process_single(
|
| | self,
|
| | text: Union[str, TextInput],
|
| | voice_samples: Optional[List[Union[str, np.ndarray]]] = None,
|
| | ) -> Dict[str, Any]:
|
| | """Process a single podcast script."""
|
| |
|
| | script = None
|
| | if isinstance(text, str):
|
| |
|
| | if text.endswith('.json') and os.path.exists(text):
|
| | script = self._convert_json_to_script(text)
|
| | elif text.endswith('.txt') and os.path.exists(text):
|
| | script = self._convert_text_to_script(text)
|
| | else:
|
| |
|
| | script = text
|
| |
|
| | if script is None:
|
| | raise ValueError(f"Could not process input text: {text}")
|
| |
|
| |
|
| | parsed_lines = self._parse_script(script)
|
| | all_speakers = list(set(speaker_id for speaker_id, _ in parsed_lines))
|
| |
|
| |
|
| |
|
| | system_tokens = self.tokenizer.encode(self.system_prompt)
|
| |
|
| |
|
| | if voice_samples:
|
| | voice_tokens, voice_speech_inputs, voice_speech_masks = self._create_voice_prompt(voice_samples[:len(all_speakers)])
|
| | else:
|
| | voice_tokens, voice_speech_inputs, voice_speech_masks = [], [], []
|
| |
|
| |
|
| | full_tokens = system_tokens + voice_tokens
|
| | speech_input_mask = [False] * len(system_tokens) + voice_speech_masks
|
| |
|
| |
|
| | full_tokens += self.tokenizer.encode(' Text input:\n', add_special_tokens=False)
|
| | speech_input_mask += [False] * len(self.tokenizer.encode(' Text input:\n', add_special_tokens=False))
|
| |
|
| | for speaker_id, speaker_text in parsed_lines:
|
| | speaker_text_tokens = self.tokenizer.encode(f" Speaker {speaker_id}:{speaker_text}\n", add_special_tokens=False)
|
| | full_tokens += speaker_text_tokens
|
| | speech_input_mask += [False] * len(speaker_text_tokens)
|
| |
|
| |
|
| | full_tokens += self.tokenizer.encode(' Speech output:\n', add_special_tokens=False) + [self.tokenizer.speech_start_id]
|
| | speech_input_mask += [False] * (len(self.tokenizer.encode(' Speech output:\n', add_special_tokens=False)) + 1)
|
| |
|
| | return {
|
| | "input_ids": full_tokens,
|
| | "speech_inputs": voice_speech_inputs if voice_speech_inputs else None,
|
| | "speech_input_mask": speech_input_mask,
|
| | "parsed_script": parsed_lines,
|
| | "all_speakers": all_speakers,
|
| | }
|
| |
|
| | def _batch_encode(
|
| | self,
|
| | encodings: List[Dict[str, Any]],
|
| | padding: Union[bool, str, PaddingStrategy] = True,
|
| | truncation: Union[bool, str, TruncationStrategy] = False,
|
| | max_length: Optional[int] = None,
|
| | return_tensors: Optional[Union[str, TensorType]] = None,
|
| | return_attention_mask: bool = True,
|
| | ) -> BatchEncoding:
|
| | """Combine multiple encodings into a batch with padding."""
|
| |
|
| | input_ids_list = [enc["input_ids"] for enc in encodings]
|
| | speech_input_masks_list = [enc["speech_input_mask"] for enc in encodings]
|
| |
|
| |
|
| | if isinstance(padding, bool):
|
| | padding_strategy = PaddingStrategy.LONGEST if padding else PaddingStrategy.DO_NOT_PAD
|
| | elif isinstance(padding, str):
|
| | padding_strategy = PaddingStrategy(padding)
|
| | else:
|
| | padding_strategy = padding
|
| |
|
| |
|
| | if padding_strategy != PaddingStrategy.DO_NOT_PAD:
|
| | if padding_strategy == PaddingStrategy.LONGEST:
|
| | max_len = max(len(ids) for ids in input_ids_list)
|
| | elif padding_strategy == PaddingStrategy.MAX_LENGTH and max_length is not None:
|
| | max_len = max_length
|
| | else:
|
| | max_len = max(len(ids) for ids in input_ids_list)
|
| |
|
| |
|
| | padded_input_ids = []
|
| | attention_masks = []
|
| | padded_speech_input_masks = []
|
| |
|
| | for input_ids, speech_mask in zip(input_ids_list, speech_input_masks_list):
|
| |
|
| | if truncation and len(input_ids) > max_len:
|
| | input_ids = input_ids[:max_len]
|
| | speech_mask = speech_mask[:max_len]
|
| |
|
| |
|
| | padding_length = max_len - len(input_ids)
|
| |
|
| | padded_ids = [self.tokenizer.pad_id] * padding_length + input_ids
|
| | attention_mask = [0] * padding_length + [1] * len(input_ids)
|
| | padded_speech_mask = [False] * padding_length + speech_mask
|
| |
|
| | padded_input_ids.append(padded_ids)
|
| | attention_masks.append(attention_mask)
|
| | padded_speech_input_masks.append(padded_speech_mask)
|
| |
|
| | input_ids_list = padded_input_ids
|
| | speech_input_masks_list = padded_speech_input_masks
|
| | else:
|
| |
|
| | attention_masks = [[1] * len(ids) for ids in input_ids_list] if return_attention_mask else None
|
| |
|
| |
|
| | all_speech_inputs = []
|
| | has_speech = False
|
| | for enc in encodings:
|
| | if enc["speech_inputs"] is not None:
|
| | all_speech_inputs.extend(enc["speech_inputs"])
|
| | has_speech = True
|
| |
|
| |
|
| | batch_encoding = BatchEncoding()
|
| |
|
| |
|
| | if return_tensors is not None:
|
| | batch_encoding["input_ids"] = torch.tensor(input_ids_list, dtype=torch.long)
|
| | if return_attention_mask and attention_masks is not None:
|
| | batch_encoding["attention_mask"] = torch.tensor(attention_masks, dtype=torch.long)
|
| | batch_encoding["speech_input_mask"] = torch.tensor(speech_input_masks_list, dtype=torch.bool)
|
| | else:
|
| | batch_encoding["input_ids"] = input_ids_list
|
| | if return_attention_mask and attention_masks is not None:
|
| | batch_encoding["attention_mask"] = attention_masks
|
| | batch_encoding["speech_input_mask"] = speech_input_masks_list
|
| |
|
| |
|
| | if has_speech:
|
| | speech_dict = self.prepare_speech_inputs(
|
| | all_speech_inputs,
|
| | return_tensors=return_tensors,
|
| | )
|
| | batch_encoding["speech_tensors"] = speech_dict["padded_speeches"]
|
| | batch_encoding["speech_masks"] = speech_dict["speech_masks"]
|
| | else:
|
| | batch_encoding["speech_tensors"] = None
|
| | batch_encoding["speech_masks"] = None
|
| |
|
| |
|
| | batch_encoding["parsed_scripts"] = [enc["parsed_script"] for enc in encodings]
|
| | batch_encoding["all_speakers_list"] = [enc["all_speakers"] for enc in encodings]
|
| |
|
| | return batch_encoding
|
| |
|
| | def _create_voice_prompt(
|
| | self,
|
| | speaker_samples: List[Union[str, np.ndarray]]
|
| | ) -> Tuple[List[int], List[np.ndarray], List[bool]]:
|
| | """
|
| | Create voice prompt tokens and process audio samples.
|
| |
|
| | Returns:
|
| | tuple: (voice_tokens, voice_speech_inputs, voice_speech_masks)
|
| | """
|
| | vae_token_id = self.tokenizer.speech_diffusion_id
|
| |
|
| | voice_full_tokens = self.tokenizer.encode(' Voice input:\n', add_special_tokens=False)
|
| | voice_speech_inputs = []
|
| | voice_speech_masks = [False] * len(voice_full_tokens)
|
| |
|
| | for speaker_id, speaker_audio in enumerate(speaker_samples):
|
| | prefix_tokens = self.tokenizer.encode(f" Speaker {speaker_id}:", add_special_tokens=False)
|
| |
|
| |
|
| | if isinstance(speaker_audio, str):
|
| |
|
| | wav = self.audio_processor._load_audio_from_path(speaker_audio)
|
| | else:
|
| | wav = np.array(speaker_audio, dtype=np.float32)
|
| |
|
| |
|
| | if self.db_normalize and self.audio_normalizer:
|
| | wav = self.audio_normalizer(wav)
|
| |
|
| |
|
| |
|
| |
|
| |
|
| | vae_tok_len = math.ceil(wav.shape[0] / self.speech_tok_compress_ratio)
|
| |
|
| |
|
| | speaker_tokens = (prefix_tokens +
|
| | [self.tokenizer.speech_start_id] +
|
| | [vae_token_id] * vae_tok_len +
|
| | [self.tokenizer.speech_end_id] +
|
| | self.tokenizer.encode('\n', add_special_tokens=False))
|
| |
|
| | vae_input_mask = ([False] * len(prefix_tokens) +
|
| | [False] +
|
| | [True] * vae_tok_len +
|
| | [False] +
|
| | [False])
|
| |
|
| | voice_full_tokens.extend(speaker_tokens)
|
| | voice_speech_masks.extend(vae_input_mask)
|
| | voice_speech_inputs.append(wav)
|
| |
|
| | return voice_full_tokens, voice_speech_inputs, voice_speech_masks
|
| |
|
| | def prepare_speech_inputs(
|
| | self,
|
| | speech_inputs: List[np.ndarray],
|
| | return_tensors: Optional[Union[str, TensorType]] = None,
|
| | device: Optional[Union[str, torch.device]] = None,
|
| | dtype: Optional[torch.dtype] = None,
|
| | ) -> Dict[str, Any]:
|
| | """
|
| | Prepare speech inputs for model consumption.
|
| |
|
| | Args:
|
| | speech_inputs: List of speech arrays
|
| | return_tensors: Output tensor type
|
| | device: Device to place tensors on
|
| | dtype: Data type for tensors
|
| |
|
| | Returns:
|
| | Dictionary with padded_speeches and speech_masks
|
| | """
|
| | if not speech_inputs:
|
| | return {"padded_speeches": None, "speech_masks": None}
|
| |
|
| |
|
| | vae_tok_seqlens = [math.ceil(s.shape[0] / self.speech_tok_compress_ratio) for s in speech_inputs]
|
| |
|
| | max_speech_length = max(s.shape[0] for s in speech_inputs)
|
| |
|
| |
|
| | if speech_inputs[0].ndim == 1:
|
| | padded_speeches = np.full((len(speech_inputs), max_speech_length), fill_value=0, dtype=np.float32)
|
| | else:
|
| | padded_speeches = np.full((len(speech_inputs), max_speech_length, speech_inputs[0].shape[-1]), fill_value=0, dtype=np.float32)
|
| | speech_masks = np.zeros((len(speech_inputs), max(vae_tok_seqlens)), dtype=np.bool_)
|
| |
|
| | for i, (speech, vae_tok_length) in enumerate(zip(speech_inputs, vae_tok_seqlens)):
|
| | padded_speeches[i, :len(speech)] = speech
|
| | speech_masks[i, :vae_tok_length] = True
|
| |
|
| | result = {
|
| | "padded_speeches": padded_speeches,
|
| | "speech_masks": speech_masks,
|
| | }
|
| |
|
| |
|
| | if return_tensors == "pt":
|
| | result["padded_speeches"] = torch.tensor(padded_speeches, device=device, dtype=dtype or torch.float32)
|
| | result["speech_masks"] = torch.tensor(speech_masks, device=device, dtype=torch.bool)
|
| |
|
| | return result
|
| |
|
| | def _convert_json_to_script(self, json_file: str) -> str:
|
| | """
|
| | Convert JSON format to script format.
|
| | Expected JSON format:
|
| | [
|
| | {"speaker": "1", "text": "Hello everyone..."},
|
| | {"speaker": "2", "text": "Great to be here..."}
|
| | ]
|
| | """
|
| | import json
|
| |
|
| | with open(json_file, 'r', encoding='utf-8') as f:
|
| | data = json.load(f)
|
| |
|
| | if not isinstance(data, list):
|
| | raise ValueError("JSON file must contain a list of speaker entries")
|
| |
|
| | script_lines = []
|
| | for item in data:
|
| | if not isinstance(item, dict):
|
| | logger.warning(f"Skipping non-dict entry: {item}")
|
| | continue
|
| |
|
| | speaker = item.get('speaker')
|
| | text = item.get('text')
|
| |
|
| | if speaker is None or text is None:
|
| | logger.warning(f"Skipping entry missing speaker or text: {item}")
|
| | continue
|
| |
|
| |
|
| | try:
|
| | speaker_id = int(speaker)
|
| | except (ValueError, TypeError):
|
| | logger.warning(f"Invalid speaker ID: {speaker}, skipping entry")
|
| | continue
|
| |
|
| |
|
| | text = text.strip()
|
| | if text:
|
| | script_lines.append(f"Speaker {speaker_id}: {text}")
|
| |
|
| | if not script_lines:
|
| | raise ValueError("No valid entries found in JSON file")
|
| |
|
| | return "\n".join(script_lines)
|
| |
|
| | def _convert_text_to_script(self, text_file: str) -> str:
|
| | """
|
| | Convert text file to script format.
|
| | Handles multiple formats:
|
| | 1. Already formatted as "Speaker X: text"
|
| | 2. Plain text (assigns to Speaker 1)
|
| |
|
| | Handles edge cases like multiple colons in a line.
|
| | """
|
| | with open(text_file, 'r', encoding='utf-8') as f:
|
| | lines = f.readlines()
|
| |
|
| | script_lines = []
|
| | current_speaker = 1
|
| |
|
| | for line in lines:
|
| | line = line.strip()
|
| | if not line:
|
| | continue
|
| |
|
| |
|
| |
|
| | speaker_match = re.match(r'^Speaker\s+(\d+)\s*:\s*(.*)$', line, re.IGNORECASE)
|
| |
|
| | if speaker_match:
|
| | speaker_id = int(speaker_match.group(1))
|
| | text = speaker_match.group(2).strip()
|
| | if text:
|
| | script_lines.append(f"Speaker {speaker_id}: {text}")
|
| | else:
|
| |
|
| | script_lines.append(f"Speaker {current_speaker}: {line}")
|
| |
|
| | if not script_lines:
|
| | raise ValueError("No valid content found in text file")
|
| |
|
| | return "\n".join(script_lines)
|
| |
|
| | def _parse_script(self, script: str) -> List[Tuple[int, str]]:
|
| | """Parse script into list of (speaker_id, text) tuples."""
|
| | lines = script.strip().split("\n")
|
| | parsed_lines = []
|
| | speaker_ids = []
|
| |
|
| |
|
| | for line in lines:
|
| | if not line.strip():
|
| | continue
|
| |
|
| |
|
| | match = re.match(r'^Speaker\s+(\d+)\s*:\s*(.*)$', line.strip(), re.IGNORECASE)
|
| |
|
| | if match:
|
| | speaker_id = int(match.group(1))
|
| | text = ' ' + match.group(2).strip()
|
| | parsed_lines.append((speaker_id, text))
|
| | speaker_ids.append(speaker_id)
|
| | else:
|
| | logger.warning(f"Could not parse line: '{line}'")
|
| |
|
| | if not parsed_lines:
|
| | raise ValueError("No valid speaker lines found in script")
|
| |
|
| |
|
| | min_speaker_id = min(speaker_ids)
|
| | if min_speaker_id > 0:
|
| |
|
| | normalized_lines = []
|
| | for speaker_id, text in parsed_lines:
|
| | normalized_lines.append((speaker_id - 1, text))
|
| | return normalized_lines
|
| | else:
|
| |
|
| | return parsed_lines
|
| |
|
| | def _merge_inputs(self, text_inputs: BatchEncoding, audio_inputs: Dict) -> BatchEncoding:
|
| | """Merge text and audio inputs into a single BatchEncoding."""
|
| |
|
| | merged = BatchEncoding(text_inputs)
|
| |
|
| |
|
| | if "audio" in audio_inputs:
|
| | merged["speech_inputs"] = audio_inputs["audio"]
|
| | if "streaming" in audio_inputs:
|
| | merged["streaming"] = audio_inputs["streaming"]
|
| |
|
| | return merged
|
| |
|
| | def batch_decode(self, *args, **kwargs):
|
| | """
|
| | This method forwards all its arguments to VibeVoiceTextTokenizer's [`~PreTrainedTokenizer.batch_decode`].
|
| | Please refer to the docstring of this method for more information.
|
| | """
|
| | return self.tokenizer.batch_decode(*args, **kwargs)
|
| |
|
| | def decode(self, *args, **kwargs):
|
| | """
|
| | This method forwards all its arguments to VibeVoiceTextTokenizer's [`~PreTrainedTokenizer.decode`].
|
| | Please refer to the docstring of this method for more information.
|
| | """
|
| | return self.tokenizer.decode(*args, **kwargs)
|
| |
|
| | @property
|
| | def model_input_names(self):
|
| | """
|
| | Return the list of inputs accepted by the model.
|
| | """
|
| | tokenizer_input_names = self.tokenizer.model_input_names
|
| | audio_processor_input_names = self.audio_processor.model_input_names
|
| | return list(dict.fromkeys(tokenizer_input_names + audio_processor_input_names + ["speech_inputs", "speech_input_mask"]))
|
| |
|
| | def save_audio(self,
|
| | audio: Union[torch.Tensor, np.ndarray, List[Union[torch.Tensor, np.ndarray]]],
|
| | output_path: str = "output.wav",
|
| | sampling_rate: Optional[int] = None,
|
| | normalize: bool = False,
|
| | batch_prefix: str = "audio_",
|
| | ) -> str:
|
| | """
|
| | Save audio data to a file.
|
| | Args:
|
| | audio (Union[torch.Tensor, np.ndarray, List[Union[torch.Tensor, np.ndarray]]]):
|
| | The audio data to save. Can be a single tensor/array or a list of them.
|
| | output_path (str, optional): Path to save the audio file. Defaults to "output.wav".
|
| | sampling_rate (int, optional): Sampling rate for the audio. If None, uses the processor's default.
|
| | normalize (bool, optional): Whether to normalize the audio before saving. Defaults to False.
|
| | batch_prefix (str, optional): Prefix for batch audio files. Defaults to "audio_".
|
| | Returns:
|
| | str: The path to the saved audio file.
|
| | """
|
| | return self.audio_processor.save_audio(audio, output_path=output_path, sampling_rate=sampling_rate, normalize=normalize, batch_prefix=batch_prefix)
|
| |
|
| | __all__ = [
|
| | "VibeVoiceProcessor",
|
| | ] |