File size: 17,373 Bytes
878a616
 
 
 
 
 
 
 
 
 
 
8f550fc
878a616
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f550fc
878a616
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f550fc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
878a616
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f550fc
878a616
 
 
8f550fc
 
 
 
 
 
878a616
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f550fc
878a616
 
 
8f550fc
 
 
 
 
 
878a616
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f550fc
 
 
 
 
 
878a616
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
"""Breeze TTS 2 — Gradio demo on ZeroGPU.

Three modes: Voice Design, Voice Clone, Voice Direction.
Model: BreezeBlue/Breeze-TTS-2
"""

import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")

import spaces  # MUST come before any torch/CUDA import
import tempfile
import threading
import time
from pathlib import Path

import gradio as gr
import numpy as np
import soundfile as sf
import torch
from huggingface_hub import snapshot_download

from breeze_infer.runtime import (
    load_runtime,
    set_all_seeds,
    update_generation_config_for_breeze,
)
from breeze_infer.templates import get_template, prepare_inputs
from models.fast_streaming import FastBreezeStreamingRuntime, FastStreamingConfig

MODEL_ID = "BreezeBlue/Breeze-TTS-2"
ASR_MODEL_ID = "openai/whisper-small"  # multilingual (en/zh), runs on CPU

MAX_NEW_TOKENS = 1500
MAX_SEQ_LEN = 2048
REPETITION_PENALTY = 1.1

# Download model snapshot at module scope (ZeroGPU intercepts .to("cuda"))
_ckpt_dir = snapshot_download(
    MODEL_ID,
    allow_patterns=[
        "config.json",
        "generation_config.json",
        "model-*.safetensors",
        "model.safetensors.index.json",
        "special_tokens_map.json",
        "tokenizer.json",
        "tokenizer_config.json",
        "audio_tokenizer/**",
    ],
)
_ckpt_path = Path(_ckpt_dir)

print(f"Loading Breeze TTS 2 from {_ckpt_path} ...")
_tokenizer, _model, _audio_tokenizer = load_runtime(
    _ckpt_path,
    device="cuda",
    attn_implementation="eager",
)
update_generation_config_for_breeze(_model)

_streaming_config = FastStreamingConfig(
    max_new_tokens=MAX_NEW_TOKENS,
    max_seq_len=MAX_SEQ_LEN,
    fast_all=None,
    fast_text_encoder=False,
    fast_backbone_prefill=False,
    fast_backbone_decode=False,
    fast_depth_decoder=False,
    fast_codec=False,
    repetition_penalty=REPETITION_PENALTY,
)
_runtime = FastBreezeStreamingRuntime(
    _model, _audio_tokenizer, _streaming_config, tokenizer=_tokenizer
)
print("Breeze TTS 2 loaded successfully.")
print(f"Sample rate: {_runtime.sample_rate}")


_asr_pipe = None
_asr_lock = threading.Lock()


def _get_asr():
    """Lazily build the CPU Whisper pipeline used to auto-transcribe reference audio."""
    global _asr_pipe
    with _asr_lock:
        if _asr_pipe is None:
            from transformers import pipeline

            _asr_pipe = pipeline(
                "automatic-speech-recognition",
                model=ASR_MODEL_ID,
                device="cpu",
                chunk_length_s=30,
            )
    return _asr_pipe


def transcribe_reference(
    ref_audio_path: str | None,
    progress=gr.Progress(track_tqdm=True),
):
    """Auto-transcribe uploaded/recorded reference audio with Whisper on CPU.

    The result populates the (still editable) reference transcript field.

    Args:
        ref_audio_path: Path to the reference audio file, or None when cleared.
    """
    if not ref_audio_path:
        return gr.update()

    try:
        asr = _get_asr()
    except Exception as exc:  # pragma: no cover - model download/load failure
        raise gr.Error(f"Could not load the transcription model: {exc}") from exc

    try:
        import librosa

        audio, _ = librosa.load(ref_audio_path, sr=16000, mono=True)
        asr_input = {"raw": audio, "sampling_rate": 16000}
    except Exception:
        # Fall back to letting the pipeline decode the file itself (ffmpeg).
        asr_input = str(ref_audio_path)

    try:
        result = asr(asr_input, generate_kwargs={"task": "transcribe"})
    except Exception as exc:
        raise gr.Error(f"Automatic transcription failed: {exc}") from exc

    text = (result.get("text") or "").strip()
    if not text:
        raise gr.Error(
            "Automatic transcription produced no text — please type the transcript manually."
        )
    return text


@spaces.GPU(duration=120)
def generate_voice_design(
    text: str,
    instruction: str,
    cfg_scale: float,
    seed: int,
    progress=gr.Progress(track_tqdm=True),
) -> str:
    """Generate speech with a designed voice from a natural-language description.

    Args:
        text: The text to speak. Supports inline vocal events like (laugh), (sigh), etc.
        instruction: Natural-language description of the desired voice character.
        cfg_scale: Classifier-free guidance scale. Higher = stronger instruction following.
        seed: Random seed for reproducibility.
    """
    return _run_inference(text, instruction, None, None, cfg_scale, seed)


@spaces.GPU(duration=120)
def generate_voice_clone(
    text: str,
    ref_audio_path: str,
    ref_text: str,
    seed: int,
    progress=gr.Progress(track_tqdm=True),
) -> str:
    """Clone a voice from reference audio and its exact transcript.

    Args:
        text: The text to speak. Supports inline vocal events.
        ref_audio_path: Path to reference audio file (clean speech, minimal noise).
        ref_text: Exact transcript of the reference audio.
        seed: Random seed for reproducibility.
    """
    return _run_inference(text, "Speak clearly and naturally.", ref_audio_path, ref_text, 1.0, seed)


@spaces.GPU(duration=120)
def generate_voice_direction(
    text: str,
    instruction: str,
    ref_audio_path: str,
    ref_text: str,
    cfg_scale: float,
    seed: int,
    progress=gr.Progress(track_tqdm=True),
) -> str:
    """Clone a reference voice while steering tone, emotion, and pace via instructions.

    Args:
        text: The text to speak. Supports inline vocal events.
        instruction: Direction for tone, emotion, pace, and delivery.
        ref_audio_path: Path to reference audio file.
        ref_text: Exact transcript of the reference audio.
        cfg_scale: Classifier-free guidance scale. Use 4 for stronger instruction following.
        seed: Random seed for reproducibility.
    """
    return _run_inference(text, instruction, ref_audio_path, ref_text, cfg_scale, seed)


def _run_inference(
    text: str,
    instruction: str,
    ref_audio_path: str | None,
    ref_text: str | None,
    cfg_scale: float,
    seed: int,
) -> str:
    """Run Breeze TTS 2 inference and return a path to the output WAV."""
    text = text.strip()
    if not text:
        raise gr.Error("Text to speak is required.")

    has_ref = ref_audio_path is not None and ref_text and ref_text.strip()
    if ref_audio_path is not None and not has_ref:
        raise gr.Error("Reference audio and reference text must be provided together.")

    set_all_seeds(int(seed))

    request = {
        "id": f"demo-{int(time.time())}",
        "text": text,
        "instruction": instruction,
        "speaker": "S0",
    }
    template_name = "tts_instruction"
    if has_ref:
        request["ref_audio_path"] = str(ref_audio_path)
        request["ref_text"] = ref_text.strip()
        template_name = "ref_edit_tata"

    inputs = prepare_inputs(
        _tokenizer,
        _audio_tokenizer,
        _model,
        [request],
        get_template(template_name),
        guidance_scale=cfg_scale,
        guidance_scale_ref=None,
        guidance_scale_ins=None,
    )

    audio_chunks = []
    for chunk in _runtime.iter_audio_chunks(inputs, request_id=request["id"]):
        audio_chunks.append(chunk.audio)

    if not audio_chunks:
        raise gr.Error("Model produced no audio output.")

    audio = np.concatenate(audio_chunks, axis=0)
    sr = _runtime.sample_rate

    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
        sf.write(tmp.name, audio, sr, subtype="PCM_16")
        return tmp.name


CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""

with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
    with gr.Column(elem_id="col-container"):
        gr.Markdown(
            "# 🎙️ Breeze TTS 2\n"
            "Open-weight bilingual (English/Chinese) text-to-speech with voice design, "
            "voice cloning, and voice direction. "
            "Supports inline vocal events: `(laugh)`, `(sigh)`, `(cough)`, `(clears throat)`."
        )

        with gr.Tabs():
            # === Voice Design Tab ===
            with gr.Tab("🎨 Voice Design"):
                gr.Markdown(
                    "Create a distinctive voice from a natural-language description — "
                    "no reference audio needed. Use `cfg_scale = 4` for stronger instruction following."
                )
                with gr.Row():
                    with gr.Column(scale=3):
                        design_text = gr.Textbox(
                            label="Text to speak",
                            placeholder="Welcome aboard. Your journey begins now.",
                            lines=3,
                        )
                        design_instruction = gr.Textbox(
                            label="Voice description",
                            placeholder="A warm, thoughtful young woman with a clear voice and a calm, reflective delivery.",
                            lines=2,
                        )
                        with gr.Accordion("Advanced settings", open=False):
                            design_cfg = gr.Slider(
                                label="CFG scale",
                                minimum=1.0,
                                maximum=10.0,
                                value=4.0,
                                step=0.5,
                            )
                            design_seed = gr.Number(label="Seed", value=42, precision=0)
                        design_btn = gr.Button("Generate", variant="primary")
                    with gr.Column(scale=2):
                        design_output = gr.Audio(
                            label="Generated audio",
                            type="filepath",
                        )

                gr.Examples(
                    examples=[
                        [
                            "(sigh) Welcome aboard. Your journey begins now.",
                            "A warm, thoughtful young woman with a clear voice and a calm, reflective delivery.",
                            4.0,
                            42,
                        ],
                        [
                            "The stars above are calling, and tonight we answer.",
                            "A deep, resonant male voice with theatrical gravitas and a slow, deliberate pace.",
                            4.0,
                            42,
                        ],
                        [
                            "[笑] 欢迎来到今晚的故事时间,让我们一起开始吧。",
                            "一位温柔自信的年轻女性,声音清晰,语气亲切,表达轻快而富有感染力。",
                            4.0,
                            42,
                        ],
                    ],
                    inputs=[design_text, design_instruction, design_cfg, design_seed],
                    outputs=design_output,
                    fn=generate_voice_design,
                    cache_examples=True,
                    cache_mode="lazy",
                )

                design_btn.click(
                    fn=generate_voice_design,
                    inputs=[design_text, design_instruction, design_cfg, design_seed],
                    outputs=design_output,
                    api_name="voice_design",
                )

            # === Voice Clone Tab ===
            with gr.Tab("🎙️ Voice Clone"):
                gr.Markdown(
                    "Clone a speaker from clean reference audio and its exact transcript. "
                    "Reference audio should contain clean speech with minimal background noise."
                )
                with gr.Row():
                    with gr.Column(scale=3):
                        clone_text = gr.Textbox(
                            label="Text to speak",
                            placeholder="It is good to hear your voice again after all this time.",
                            lines=3,
                        )
                        clone_ref_audio = gr.Audio(
                            label="Reference audio",
                            type="filepath",
                        )
                        clone_ref_text = gr.Textbox(
                            label="Reference transcript (exact)",
                            info="Auto-transcribed with Whisper when you add reference audio — please review and edit if needed.",
                            placeholder="This is the exact transcript of the reference audio.",
                            lines=2,
                        )
                        clone_ref_audio.change(
                            fn=transcribe_reference,
                            inputs=[clone_ref_audio],
                            outputs=[clone_ref_text],
                            api_name="transcribe_reference",
                        )
                        with gr.Accordion("Advanced settings", open=False):
                            clone_seed = gr.Number(label="Seed", value=42, precision=0)
                        clone_btn = gr.Button("Generate", variant="primary")
                    with gr.Column(scale=2):
                        clone_output = gr.Audio(
                            label="Generated audio",
                            type="filepath",
                        )

                clone_btn.click(
                    fn=generate_voice_clone,
                    inputs=[clone_text, clone_ref_audio, clone_ref_text, clone_seed],
                    outputs=clone_output,
                    api_name="voice_clone",
                )

            # === Voice Direction Tab ===
            with gr.Tab("🎛️ Voice Direction"):
                gr.Markdown(
                    "Clone a reference voice while directing tone, emotion, pace, and delivery. "
                    "Use `cfg_scale = 4` for stronger instruction following."
                )
                with gr.Row():
                    with gr.Column(scale=3):
                        dir_text = gr.Textbox(
                            label="Text to speak",
                            placeholder="We need to discuss what happened last night.",
                            lines=3,
                        )
                        dir_instruction = gr.Textbox(
                            label="Direction",
                            placeholder="Speak slowly with a restrained, serious tone.",
                            lines=2,
                        )
                        dir_ref_audio = gr.Audio(
                            label="Reference audio",
                            type="filepath",
                        )
                        dir_ref_text = gr.Textbox(
                            label="Reference transcript (exact)",
                            info="Auto-transcribed with Whisper when you add reference audio — please review and edit if needed.",
                            placeholder="This is the exact transcript of the reference audio.",
                            lines=2,
                        )
                        dir_ref_audio.change(
                            fn=transcribe_reference,
                            inputs=[dir_ref_audio],
                            outputs=[dir_ref_text],
                            api_name="transcribe_reference_direction",
                        )
                        with gr.Accordion("Advanced settings", open=False):
                            dir_cfg = gr.Slider(
                                label="CFG scale",
                                minimum=1.0,
                                maximum=10.0,
                                value=4.0,
                                step=0.5,
                            )
                            dir_seed = gr.Number(label="Seed", value=42, precision=0)
                        dir_btn = gr.Button("Generate", variant="primary")
                    with gr.Column(scale=2):
                        dir_output = gr.Audio(
                            label="Generated audio",
                            type="filepath",
                        )

                dir_btn.click(
                    fn=generate_voice_direction,
                    inputs=[
                        dir_text,
                        dir_instruction,
                        dir_ref_audio,
                        dir_ref_text,
                        dir_cfg,
                        dir_seed,
                    ],
                    outputs=dir_output,
                    api_name="voice_direction",
                )

        gr.Markdown(
            "---\n"
            "Model: [BreezeBlue/Breeze-TTS-2](https://huggingface.co/BreezeBlue/Breeze-TTS-2) · "
            "Code: [breezeblue-ai/breeze-tts](https://github.com/breezeblue-ai/breeze-tts) · "
            "Weights/outputs are for research and non-commercial use only."
        )

try:
    _get_asr()
    print(f"Whisper ASR ({ASR_MODEL_ID}) loaded on CPU for reference transcription.")
except Exception as _exc:  # pragma: no cover
    print(f"Warning: could not preload Whisper ASR ({_exc}); will retry on first use.")

demo.launch(mcp_server=True)