"""Breeze TTS 2 — Gradio demo on ZeroGPU. Three modes: Voice Design, Voice Clone, Voice Direction. Model: BreezeBlue/Breeze-TTS-2 """ import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # MUST come before any torch/CUDA import import tempfile import threading import time from pathlib import Path import gradio as gr import numpy as np import soundfile as sf import torch from huggingface_hub import snapshot_download from breeze_infer.runtime import ( load_runtime, set_all_seeds, update_generation_config_for_breeze, ) from breeze_infer.templates import get_template, prepare_inputs from models.fast_streaming import FastBreezeStreamingRuntime, FastStreamingConfig MODEL_ID = "BreezeBlue/Breeze-TTS-2" ASR_MODEL_ID = "openai/whisper-small" # multilingual (en/zh), runs on CPU MAX_NEW_TOKENS = 1500 MAX_SEQ_LEN = 2048 REPETITION_PENALTY = 1.1 # Download model snapshot at module scope (ZeroGPU intercepts .to("cuda")) _ckpt_dir = snapshot_download( MODEL_ID, allow_patterns=[ "config.json", "generation_config.json", "model-*.safetensors", "model.safetensors.index.json", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "audio_tokenizer/**", ], ) _ckpt_path = Path(_ckpt_dir) print(f"Loading Breeze TTS 2 from {_ckpt_path} ...") _tokenizer, _model, _audio_tokenizer = load_runtime( _ckpt_path, device="cuda", attn_implementation="eager", ) update_generation_config_for_breeze(_model) _streaming_config = FastStreamingConfig( max_new_tokens=MAX_NEW_TOKENS, max_seq_len=MAX_SEQ_LEN, fast_all=None, fast_text_encoder=False, fast_backbone_prefill=False, fast_backbone_decode=False, fast_depth_decoder=False, fast_codec=False, repetition_penalty=REPETITION_PENALTY, ) _runtime = FastBreezeStreamingRuntime( _model, _audio_tokenizer, _streaming_config, tokenizer=_tokenizer ) print("Breeze TTS 2 loaded successfully.") print(f"Sample rate: {_runtime.sample_rate}") _asr_pipe = None _asr_lock = threading.Lock() def _get_asr(): """Lazily build the CPU Whisper pipeline used to auto-transcribe reference audio.""" global _asr_pipe with _asr_lock: if _asr_pipe is None: from transformers import pipeline _asr_pipe = pipeline( "automatic-speech-recognition", model=ASR_MODEL_ID, device="cpu", chunk_length_s=30, ) return _asr_pipe def transcribe_reference( ref_audio_path: str | None, progress=gr.Progress(track_tqdm=True), ): """Auto-transcribe uploaded/recorded reference audio with Whisper on CPU. The result populates the (still editable) reference transcript field. Args: ref_audio_path: Path to the reference audio file, or None when cleared. """ if not ref_audio_path: return gr.update() try: asr = _get_asr() except Exception as exc: # pragma: no cover - model download/load failure raise gr.Error(f"Could not load the transcription model: {exc}") from exc try: import librosa audio, _ = librosa.load(ref_audio_path, sr=16000, mono=True) asr_input = {"raw": audio, "sampling_rate": 16000} except Exception: # Fall back to letting the pipeline decode the file itself (ffmpeg). asr_input = str(ref_audio_path) try: result = asr(asr_input, generate_kwargs={"task": "transcribe"}) except Exception as exc: raise gr.Error(f"Automatic transcription failed: {exc}") from exc text = (result.get("text") or "").strip() if not text: raise gr.Error( "Automatic transcription produced no text — please type the transcript manually." ) return text @spaces.GPU(duration=120) def generate_voice_design( text: str, instruction: str, cfg_scale: float, seed: int, progress=gr.Progress(track_tqdm=True), ) -> str: """Generate speech with a designed voice from a natural-language description. Args: text: The text to speak. Supports inline vocal events like (laugh), (sigh), etc. instruction: Natural-language description of the desired voice character. cfg_scale: Classifier-free guidance scale. Higher = stronger instruction following. seed: Random seed for reproducibility. """ return _run_inference(text, instruction, None, None, cfg_scale, seed) @spaces.GPU(duration=120) def generate_voice_clone( text: str, ref_audio_path: str, ref_text: str, seed: int, progress=gr.Progress(track_tqdm=True), ) -> str: """Clone a voice from reference audio and its exact transcript. Args: text: The text to speak. Supports inline vocal events. ref_audio_path: Path to reference audio file (clean speech, minimal noise). ref_text: Exact transcript of the reference audio. seed: Random seed for reproducibility. """ return _run_inference(text, "Speak clearly and naturally.", ref_audio_path, ref_text, 1.0, seed) @spaces.GPU(duration=120) def generate_voice_direction( text: str, instruction: str, ref_audio_path: str, ref_text: str, cfg_scale: float, seed: int, progress=gr.Progress(track_tqdm=True), ) -> str: """Clone a reference voice while steering tone, emotion, and pace via instructions. Args: text: The text to speak. Supports inline vocal events. instruction: Direction for tone, emotion, pace, and delivery. ref_audio_path: Path to reference audio file. ref_text: Exact transcript of the reference audio. cfg_scale: Classifier-free guidance scale. Use 4 for stronger instruction following. seed: Random seed for reproducibility. """ return _run_inference(text, instruction, ref_audio_path, ref_text, cfg_scale, seed) def _run_inference( text: str, instruction: str, ref_audio_path: str | None, ref_text: str | None, cfg_scale: float, seed: int, ) -> str: """Run Breeze TTS 2 inference and return a path to the output WAV.""" text = text.strip() if not text: raise gr.Error("Text to speak is required.") has_ref = ref_audio_path is not None and ref_text and ref_text.strip() if ref_audio_path is not None and not has_ref: raise gr.Error("Reference audio and reference text must be provided together.") set_all_seeds(int(seed)) request = { "id": f"demo-{int(time.time())}", "text": text, "instruction": instruction, "speaker": "S0", } template_name = "tts_instruction" if has_ref: request["ref_audio_path"] = str(ref_audio_path) request["ref_text"] = ref_text.strip() template_name = "ref_edit_tata" inputs = prepare_inputs( _tokenizer, _audio_tokenizer, _model, [request], get_template(template_name), guidance_scale=cfg_scale, guidance_scale_ref=None, guidance_scale_ins=None, ) audio_chunks = [] for chunk in _runtime.iter_audio_chunks(inputs, request_id=request["id"]): audio_chunks.append(chunk.audio) if not audio_chunks: raise gr.Error("Model produced no audio output.") audio = np.concatenate(audio_chunks, axis=0) sr = _runtime.sample_rate with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: sf.write(tmp.name, audio, sr, subtype="PCM_16") return tmp.name CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo: with gr.Column(elem_id="col-container"): gr.Markdown( "# 🎙️ Breeze TTS 2\n" "Open-weight bilingual (English/Chinese) text-to-speech with voice design, " "voice cloning, and voice direction. " "Supports inline vocal events: `(laugh)`, `(sigh)`, `(cough)`, `(clears throat)`." ) with gr.Tabs(): # === Voice Design Tab === with gr.Tab("🎨 Voice Design"): gr.Markdown( "Create a distinctive voice from a natural-language description — " "no reference audio needed. Use `cfg_scale = 4` for stronger instruction following." ) with gr.Row(): with gr.Column(scale=3): design_text = gr.Textbox( label="Text to speak", placeholder="Welcome aboard. Your journey begins now.", lines=3, ) design_instruction = gr.Textbox( label="Voice description", placeholder="A warm, thoughtful young woman with a clear voice and a calm, reflective delivery.", lines=2, ) with gr.Accordion("Advanced settings", open=False): design_cfg = gr.Slider( label="CFG scale", minimum=1.0, maximum=10.0, value=4.0, step=0.5, ) design_seed = gr.Number(label="Seed", value=42, precision=0) design_btn = gr.Button("Generate", variant="primary") with gr.Column(scale=2): design_output = gr.Audio( label="Generated audio", type="filepath", ) gr.Examples( examples=[ [ "(sigh) Welcome aboard. Your journey begins now.", "A warm, thoughtful young woman with a clear voice and a calm, reflective delivery.", 4.0, 42, ], [ "The stars above are calling, and tonight we answer.", "A deep, resonant male voice with theatrical gravitas and a slow, deliberate pace.", 4.0, 42, ], [ "[笑] 欢迎来到今晚的故事时间,让我们一起开始吧。", "一位温柔自信的年轻女性,声音清晰,语气亲切,表达轻快而富有感染力。", 4.0, 42, ], ], inputs=[design_text, design_instruction, design_cfg, design_seed], outputs=design_output, fn=generate_voice_design, cache_examples=True, cache_mode="lazy", ) design_btn.click( fn=generate_voice_design, inputs=[design_text, design_instruction, design_cfg, design_seed], outputs=design_output, api_name="voice_design", ) # === Voice Clone Tab === with gr.Tab("🎙️ Voice Clone"): gr.Markdown( "Clone a speaker from clean reference audio and its exact transcript. " "Reference audio should contain clean speech with minimal background noise." ) with gr.Row(): with gr.Column(scale=3): clone_text = gr.Textbox( label="Text to speak", placeholder="It is good to hear your voice again after all this time.", lines=3, ) clone_ref_audio = gr.Audio( label="Reference audio", type="filepath", ) clone_ref_text = gr.Textbox( label="Reference transcript (exact)", info="Auto-transcribed with Whisper when you add reference audio — please review and edit if needed.", placeholder="This is the exact transcript of the reference audio.", lines=2, ) clone_ref_audio.change( fn=transcribe_reference, inputs=[clone_ref_audio], outputs=[clone_ref_text], api_name="transcribe_reference", ) with gr.Accordion("Advanced settings", open=False): clone_seed = gr.Number(label="Seed", value=42, precision=0) clone_btn = gr.Button("Generate", variant="primary") with gr.Column(scale=2): clone_output = gr.Audio( label="Generated audio", type="filepath", ) clone_btn.click( fn=generate_voice_clone, inputs=[clone_text, clone_ref_audio, clone_ref_text, clone_seed], outputs=clone_output, api_name="voice_clone", ) # === Voice Direction Tab === with gr.Tab("🎛️ Voice Direction"): gr.Markdown( "Clone a reference voice while directing tone, emotion, pace, and delivery. " "Use `cfg_scale = 4` for stronger instruction following." ) with gr.Row(): with gr.Column(scale=3): dir_text = gr.Textbox( label="Text to speak", placeholder="We need to discuss what happened last night.", lines=3, ) dir_instruction = gr.Textbox( label="Direction", placeholder="Speak slowly with a restrained, serious tone.", lines=2, ) dir_ref_audio = gr.Audio( label="Reference audio", type="filepath", ) dir_ref_text = gr.Textbox( label="Reference transcript (exact)", info="Auto-transcribed with Whisper when you add reference audio — please review and edit if needed.", placeholder="This is the exact transcript of the reference audio.", lines=2, ) dir_ref_audio.change( fn=transcribe_reference, inputs=[dir_ref_audio], outputs=[dir_ref_text], api_name="transcribe_reference_direction", ) with gr.Accordion("Advanced settings", open=False): dir_cfg = gr.Slider( label="CFG scale", minimum=1.0, maximum=10.0, value=4.0, step=0.5, ) dir_seed = gr.Number(label="Seed", value=42, precision=0) dir_btn = gr.Button("Generate", variant="primary") with gr.Column(scale=2): dir_output = gr.Audio( label="Generated audio", type="filepath", ) dir_btn.click( fn=generate_voice_direction, inputs=[ dir_text, dir_instruction, dir_ref_audio, dir_ref_text, dir_cfg, dir_seed, ], outputs=dir_output, api_name="voice_direction", ) gr.Markdown( "---\n" "Model: [BreezeBlue/Breeze-TTS-2](https://huggingface.co/BreezeBlue/Breeze-TTS-2) · " "Code: [breezeblue-ai/breeze-tts](https://github.com/breezeblue-ai/breeze-tts) · " "Weights/outputs are for research and non-commercial use only." ) try: _get_asr() print(f"Whisper ASR ({ASR_MODEL_ID}) loaded on CPU for reference transcription.") except Exception as _exc: # pragma: no cover print(f"Warning: could not preload Whisper ASR ({_exc}); will retry on first use.") demo.launch(mcp_server=True)