Initial commit

.gitattributes

@@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/audio/arabic_female.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/chinese_female.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/english_female.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/english_male.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/french_female.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/german_male.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/hindi_male.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/japanese_male.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/portuguese_male.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/russian_female.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/spanish_male.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio/swedish_female.wav filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,20 @@
 ---
 title: Demo
-emoji: 🏆
-colorFrom: indigo
-colorTo: pink
+emoji: ⚡
+colorFrom: blue
+colorTo: purple
 sdk: gradio
-sdk_version: 6.15.2
-python_version: '3.12'
+python_version: 3.10
+sdk_version: 4.44.1
 app_file: app.py
-pinned: false
+pinned: true
+license: cc-by-4.0
+short_description: Zero-shot TTS with Controllable Speaking-rate
+models:
+  - voxtream2/model
+preload_from_hub:
+  - voxtream2/model model.safetensors,config.json,phoneme_to_token.json
+  - kyutai/moshiko-pytorch-bf16 tokenizer-e351c8d8-checkpoint125.safetensors
+  - sarulab-speech/sidon-v0.1 decoder_cuda.pt
+  - sarulab-speech/sidon-v0.1 feature_extractor_cuda.pt
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,718 @@
+import argparse
+import json
+import os
+import uuid
+from pathlib import Path
+
+# Disable PyTorch dynamo/inductor globally for HuggingFace ZeroGPU.
+os.environ["TORCHDYNAMO_DISABLE"] = "1"
+os.environ["TORCHINDUCTOR_DISABLE"] = "1"
+
+import torch._dynamo as dynamo
+
+dynamo.config.suppress_errors = True
+
+import gradio as gr
+import numpy as np
+import soundfile as sf
+import spaces
+import torch
+
+from voxtream.config import SpeechGeneratorConfig
+from voxtream.generator import SpeechGenerator
+from voxtream.utils.app import (
+    CUSTOM_CSS,
+    AppConfig,
+    GenerationControl,
+    SharedGenerationState,
+    SpeakingRateState,
+    VisualizationState,
+    build_low_latency_audio_head,
+    clear_outputs,
+    empty_rate_plot,
+    float32_to_int16,
+    load_app_config,
+    render_audio_stream,
+    render_text_progress,
+)
+from voxtream.utils.generator import DTYPE_MAP, existing_file, text_generator
+from voxtream.utils.generator.helpers import autocast_ctx
+from voxtream.utils.generator.text import build_text_progress_metadata
+
+
+def generation_button_updates(running: bool, paused: bool = False):
+    if not running:
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
+    if paused:
+        return (
+            gr.update(visible=False),
+            gr.update(visible=True, interactive=True),
+            gr.update(visible=True, interactive=True),
+        )
+    return (
+        gr.update(visible=True, interactive=True),
+        gr.update(visible=False),
+        gr.update(visible=False),
+    )
+
+
+def ensure_generator_on_cuda(speech_generator: SpeechGenerator) -> None:
+    if not torch.cuda.is_available():
+        return
+    if next(speech_generator.model.parameters()).device.type != "cpu":
+        return
+
+    dtype = DTYPE_MAP["cuda"]
+    speech_generator.model.to("cuda", dtype=dtype)
+    speech_generator.mimi.to("cuda", dtype=dtype)
+    speech_generator.ctx.mimi_prompt.to("cuda", dtype=dtype)
+    speech_generator.ctx.spk_enc.to("cuda", dtype=dtype)
+    speech_generator.ctx.device = "cuda"
+    speech_generator.ctx.dtype = dtype
+    speech_generator._autocast_ctx = autocast_ctx(device="cuda", dtype=dtype)
+
+    if speech_generator._mimi_streaming_started:
+        speech_generator._mimi_stream_ctx.__exit__(None, None, None)
+        speech_generator._mimi_stream_ctx = None
+        speech_generator._mimi_streaming_started = False
+
+
+def demo_app(
+    config: SpeechGeneratorConfig,
+    app_config: AppConfig,
+    demo_examples,
+    synthesize_fn,
+    speaking_rate_state: SpeakingRateState,
+    generation_control: GenerationControl,
+    shared_generation_state: SharedGenerationState,
+):
+    with gr.Blocks(
+        css=CUSTOM_CSS,
+        head=build_low_latency_audio_head(app_config),
+        title="VoXtream2",
+    ) as demo:
+        gr.Markdown("# VoXtream2 TTS demo")
+        gr.Markdown(
+            "⚠️ The initial latency can be high due to deployment on ZeroGPU. "
+            "For faster inference, please try local deployment."
+        )
+
+        with gr.Row(equal_height=True, elem_id="cols"):
+            with gr.Column(scale=1, elem_id="left-col"):
+                prompt_audio = gr.Audio(
+                    sources=["microphone", "upload"],
+                    type="filepath",
+                    label=(
+                        "Prompt audio (3-10 sec of target voice. "
+                        f"Max {config.max_prompt_sec} sec)"
+                    ),
+                )
+                with gr.Accordion("Advanced options", open=False):
+                    enable_speaking_rate = gr.Checkbox(
+                        label="Use speaking rate control", value=True
+                    )
+                    prompt_enhancement = gr.Checkbox(
+                        label="Prompt enhancement", value=False
+                    )
+                    prompt_enhancement_msg = gr.Markdown(
+                        "⚠️ First 3-5 runs may have higher latency due to model "
+                        "loading and warmup.",
+                        visible=False,
+                    )
+                    voice_activity_detection = gr.Checkbox(
+                        label="Voice activity detection", value=False
+                    )
+                    streaming_input = gr.Checkbox(label="Streaming input", value=False)
+
+            with gr.Column(scale=1, elem_id="right-col"):
+                target_text = gr.Textbox(
+                    lines=4,
+                    max_length=config.max_phone_tokens,
+                    label=(
+                        "Target text (Required, "
+                        f"max {config.max_phone_tokens} chars)"
+                    ),
+                    placeholder="What you want the model to say",
+                )
+                output_audio = gr.Audio(
+                    label="Synthesized audio",
+                    interactive=False,
+                    streaming=False,
+                    autoplay=False,
+                    show_download_button=True,
+                    show_share_button=False,
+                    visible=False,
+                )
+                stream_audio = gr.HTML(
+                    render_audio_stream(app_config), elem_id="audio-stream-container"
+                )
+
+        with gr.Row():
+            clear_btn = gr.Button("Clear", elem_id="clear", variant="secondary")
+            submit_btn = gr.Button(
+                "Submit", elem_id="submit", variant="primary", interactive=False
+            )
+            pause_btn = gr.Button(
+                "Pause", elem_id="pause", variant="secondary", visible=False
+            )
+            resume_btn = gr.Button(
+                "Resume", elem_id="resume", variant="primary", visible=False
+            )
+            stop_btn = gr.Button("Stop", elem_id="stop", variant="stop", visible=False)
+
+        validation_msg = gr.Markdown("", visible=False)
+        speaking_rate_control = gr.Slider(
+            minimum=app_config.speaking_rate_min,
+            maximum=app_config.speaking_rate_max,
+            step=app_config.speaking_rate_step,
+            value=app_config.speaking_rate_default,
+            label="Speaking rate (SPS). Change the speed of speech synthesis in real-time. ",
+        )
+        rate_plot = gr.HTML(empty_rate_plot(app_config), elem_id="rate-plot-container")
+        text_progress = gr.HTML(
+            render_text_progress(app_config, None), elem_id="text-progress-container"
+        )
+        generation_session = gr.State("")
+
+        def validate_inputs(audio, ttext):
+            if not audio:
+                return gr.update(
+                    visible=True, value="⚠️ Please provide a prompt audio."
+                ), gr.update(interactive=False)
+            if not ttext or not ttext.strip():
+                return gr.update(
+                    visible=True, value="⚠️ Please provide target text."
+                ), gr.update(interactive=False)
+            return gr.update(visible=False, value=""), gr.update(interactive=True)
+
+        enable_speaking_rate.change(
+            fn=lambda enabled: gr.update(interactive=enabled),
+            inputs=enable_speaking_rate,
+            outputs=speaking_rate_control,
+        )
+        prompt_enhancement.change(
+            fn=lambda enabled: gr.update(visible=enabled),
+            inputs=prompt_enhancement,
+            outputs=prompt_enhancement_msg,
+        )
+
+        def update_speaking_rate(value, session_id):
+            speaking_rate_state.update(value)
+            shared_generation_state.update_speaking_rate(session_id, value)
+
+        speaking_rate_control.input(
+            fn=update_speaking_rate,
+            inputs=[speaking_rate_control, generation_session],
+            queue=False,
+            show_progress="hidden",
+        )
+        speaking_rate_control.release(
+            fn=update_speaking_rate,
+            inputs=[speaking_rate_control, generation_session],
+            queue=False,
+            show_progress="hidden",
+        )
+
+        for inp in [prompt_audio, target_text]:
+            inp.change(
+                fn=validate_inputs,
+                inputs=[prompt_audio, target_text],
+                outputs=[validation_msg, submit_btn],
+            )
+
+        def prepare_generation(speaking_rate, enable_rate):
+            session_id = shared_generation_state.create(speaking_rate)
+            generation_control.start()
+            speaking_rate_state.start(speaking_rate)
+            return (
+                gr.update(value=None, visible=False),
+                gr.update(interactive=False),
+                empty_rate_plot(app_config, show_target=enable_rate),
+                render_text_progress(app_config, None),
+                render_audio_stream(app_config, session_id=session_id),
+                *generation_button_updates(running=True),
+                session_id,
+            )
+
+        submit_btn.click(
+            fn=prepare_generation,
+            inputs=[speaking_rate_control, enable_speaking_rate],
+            outputs=[
+                output_audio,
+                enable_speaking_rate,
+                rate_plot,
+                text_progress,
+                stream_audio,
+                pause_btn,
+                resume_btn,
+                stop_btn,
+                generation_session,
+            ],
+            show_progress="hidden",
+        ).then(
+            fn=synthesize_fn,
+            inputs=[
+                prompt_audio,
+                target_text,
+                prompt_enhancement,
+                voice_activity_detection,
+                streaming_input,
+                speaking_rate_control,
+                enable_speaking_rate,
+                generation_session,
+            ],
+            outputs=[
+                output_audio,
+                enable_speaking_rate,
+                rate_plot,
+                text_progress,
+                stream_audio,
+                pause_btn,
+                resume_btn,
+                stop_btn,
+                generation_session,
+            ],
+        )
+
+        def pause_generation(session_id):
+            generation_control.pause()
+            shared_generation_state.pause(session_id)
+            return generation_button_updates(running=True, paused=True)
+
+        def resume_generation(session_id):
+            generation_control.resume()
+            shared_generation_state.resume(session_id)
+            return generation_button_updates(running=True)
+
+        def stop_generation(session_id):
+            generation_control.stop()
+            speaking_rate_state.stop()
+            shared_generation_state.stop(session_id)
+            return generation_button_updates(running=False)
+
+        pause_btn.click(
+            fn=pause_generation,
+            inputs=generation_session,
+            outputs=[pause_btn, resume_btn, stop_btn],
+            js=(
+                "() => { if (window.voxtreamLowLatencyAudio) { "
+                "window.voxtreamLowLatencyAudio.pause(); } return []; }"
+            ),
+            queue=False,
+        )
+        resume_btn.click(
+            fn=resume_generation,
+            inputs=generation_session,
+            outputs=[pause_btn, resume_btn, stop_btn],
+            js=(
+                "() => { if (window.voxtreamLowLatencyAudio) { "
+                "window.voxtreamLowLatencyAudio.resume(); } return []; }"
+            ),
+            queue=False,
+        )
+        stop_btn.click(
+            fn=stop_generation,
+            inputs=generation_session,
+            outputs=[pause_btn, resume_btn, stop_btn],
+            js=(
+                "() => { if (window.voxtreamLowLatencyAudio) { "
+                "window.voxtreamLowLatencyAudio.stop(); } return []; }"
+            ),
+            queue=False,
+        )
+
+        def clear_generation(session_id):
+            generation_control.stop()
+            speaking_rate_state.stop()
+            shared_generation_state.stop(session_id)
+            return (
+                gr.update(value=None),
+                gr.update(value=""),
+                gr.update(value=None, visible=False),
+                gr.update(visible=False, value=""),
+                gr.update(interactive=False),
+                gr.update(interactive=True),
+                empty_rate_plot(app_config),
+                render_text_progress(app_config, None),
+                render_audio_stream(app_config, session_id=uuid.uuid4().hex),
+                *generation_button_updates(running=False),
+                "",
+            )
+
+        clear_btn.click(
+            fn=clear_generation,
+            inputs=generation_session,
+            outputs=[
+                prompt_audio,
+                target_text,
+                output_audio,
+                validation_msg,
+                submit_btn,
+                enable_speaking_rate,
+                rate_plot,
+                text_progress,
+                stream_audio,
+                pause_btn,
+                resume_btn,
+                stop_btn,
+                generation_session,
+            ],
+        )
+
+        gr.Markdown("### Examples")
+        ex = gr.Examples(
+            examples=demo_examples,
+            inputs=[
+                prompt_audio,
+                target_text,
+                prompt_enhancement,
+                voice_activity_detection,
+                streaming_input,
+                speaking_rate_control,
+                enable_speaking_rate,
+            ],
+            outputs=[
+                output_audio,
+                enable_speaking_rate,
+                rate_plot,
+                text_progress,
+                stream_audio,
+                pause_btn,
+                resume_btn,
+                stop_btn,
+                generation_session,
+            ],
+            fn=synthesize_fn,
+            cache_examples=False,
+        )
+
+        ex.dataset.click(
+            fn=lambda: (*clear_outputs(app_config), ""),
+            inputs=[],
+            outputs=[
+                output_audio,
+                rate_plot,
+                text_progress,
+                stream_audio,
+                generation_session,
+            ],
+            queue=False,
+        ).then(
+            fn=validate_inputs,
+            inputs=[prompt_audio, target_text],
+            outputs=[validation_msg, submit_btn],
+            queue=False,
+        )
+
+    demo.queue(default_concurrency_limit=1).launch()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=existing_file,
+        help="Path to the config file",
+        default="configs/generator.json",
+    )
+    parser.add_argument(
+        "--app-config",
+        type=existing_file,
+        help="Path to the app config file",
+        default="configs/app.json",
+    )
+    parser.add_argument(
+        "--spk-rate-config",
+        type=existing_file,
+        help="Path to the speaking rate config file",
+        default="configs/speaking_rate.json",
+    )
+    parser.add_argument(
+        "--examples-config",
+        type=existing_file,
+        help="Path to the examples config file",
+        default="assets/examples.json",
+    )
+    args = parser.parse_args()
+
+    with open(args.config) as f:
+        config = SpeechGeneratorConfig(**json.load(f))
+    config.hf_token = os.environ.get("TOKEN")
+
+    # Loading speaker encoder.
+    torch.hub.load(
+        config.spk_enc_repo,
+        config.spk_enc_model,
+        model_name=config.spk_enc_model_name,
+        train_type=config.spk_enc_train_type,
+        dataset=config.spk_enc_dataset,
+        trust_repo=True,
+        verbose=False,
+    )
+
+    with open(args.spk_rate_config) as f:
+        spk_rate_config = json.load(f)
+
+    app_config = load_app_config(args.app_config)
+
+    with open(args.examples_config) as f:
+        examples_config = json.load(f)
+    demo_examples = examples_config.get("examples", [])
+
+    speech_generator = SpeechGenerator(config, spk_rate_config)
+    speaking_rate_state = SpeakingRateState(app_config.speaking_rate_default)
+    generation_control = GenerationControl()
+    shared_generation_state = SharedGenerationState()
+    chunk_size = int(config.mimi_sr * app_config.min_chunk_sec)
+
+    @spaces.GPU
+    def synthesize_fn(
+        prompt_audio_path,
+        target_text,
+        prompt_enhancement,
+        voice_activity_detection,
+        streaming_input,
+        speaking_rate_control,
+        enable_speaking_rate=True,
+        generation_session_id="",
+    ):
+        control_session_id = generation_session_id or shared_generation_state.create(
+            speaking_rate_control
+        )
+        stream_session_id = control_session_id or uuid.uuid4().hex
+        stream_seq = 0
+
+        if not prompt_audio_path or not target_text:
+            speaking_rate_state.stop()
+            generation_control.finish()
+            shared_generation_state.finish(control_session_id)
+            yield (
+                gr.update(value=None, visible=False),
+                gr.update(interactive=True),
+                empty_rate_plot(app_config, show_target=enable_speaking_rate),
+                render_text_progress(app_config, None),
+                render_audio_stream(app_config, session_id=stream_session_id),
+                *generation_button_updates(running=False),
+                control_session_id,
+            )
+            return
+
+        if shared_generation_state.is_stopped(control_session_id):
+            speaking_rate_state.stop()
+            generation_control.finish()
+            shared_generation_state.finish(control_session_id)
+            yield (
+                gr.update(value=None, visible=False),
+                gr.update(interactive=True),
+                empty_rate_plot(app_config, show_target=enable_speaking_rate),
+                render_text_progress(app_config, None),
+                render_audio_stream(
+                    app_config,
+                    session_id=stream_session_id,
+                    active=False,
+                    final=True,
+                ),
+                *generation_button_updates(running=False),
+                control_session_id,
+            )
+            return
+
+        ensure_generator_on_cuda(speech_generator)
+        speaking_rate_state.ensure_started(speaking_rate_control)
+        speaking_rate_gen = (
+            shared_generation_state.speaking_rate_values(
+                control_session_id, speaking_rate_control
+            )
+            if enable_speaking_rate
+            else None
+        )
+        text_metadata = build_text_progress_metadata(
+            target_text,
+            config=config,
+            phone_to_token=speech_generator.ctx.phone_to_token,
+            phonemizer=speech_generator.ctx.phonemizer,
+            max_phone_tokens=config.max_phone_tokens,
+        )
+        rate_window_sec = (
+            config.spk_rate_window_sec
+            if config.spk_rate_window_sec and config.spk_rate_window_sec > 0
+            else app_config.plot_window_sec
+        )
+        frame_sec = config.mimi_frame_ms / 1000.0
+        text_progress_delay_sec = (
+            app_config.audio_stream_start_delay_sec
+            + config.audio_delay_frames * frame_sec
+        )
+        visualization = VisualizationState(
+            text_metadata=text_metadata,
+            app_config=app_config,
+            rate_window_sec=rate_window_sec,
+            frame_sec=frame_sec,
+            text_progress_delay_sec=text_progress_delay_sec,
+            show_target=enable_speaking_rate,
+        )
+
+        stream = speech_generator.generate_stream(
+            prompt_audio_path=Path(prompt_audio_path),
+            text=text_generator(target_text) if streaming_input else target_text,
+            speaking_rate=speaking_rate_gen,
+            enhance_prompt=prompt_enhancement,
+            apply_vad=voice_activity_detection,
+            return_progress=True,
+            min_streaming_rtf=app_config.min_streaming_rtf,
+        )
+
+        buffer = []
+        buffer_len = 0
+        total_buffer = []
+        stopped = False
+
+        stream_iter = iter(stream)
+        while True:
+            if not shared_generation_state.wait_if_paused(control_session_id):
+                stopped = True
+                break
+            try:
+                frame, _, progress = next(stream_iter)
+            except StopIteration:
+                break
+            if shared_generation_state.is_stopped(control_session_id):
+                stopped = True
+                break
+
+            buffer.append(frame)
+            total_buffer.append(frame)
+            buffer_len += frame.shape[0]
+            plot_update, text_update = visualization.update(progress)
+
+            if buffer_len >= chunk_size:
+                if shared_generation_state.is_stopped(control_session_id):
+                    stopped = True
+                    break
+                audio = np.concatenate(buffer)
+                stream_seq += 1
+                yield (
+                    gr.update(),
+                    gr.update(),
+                    plot_update,
+                    text_update,
+                    render_audio_stream(
+                        app_config,
+                        session_id=stream_session_id,
+                        seq=stream_seq,
+                        sample_rate=config.mimi_sr,
+                        audio=float32_to_int16(audio),
+                        active=True,
+                    ),
+                    *generation_button_updates(
+                        running=True,
+                        paused=shared_generation_state.is_paused(control_session_id),
+                    ),
+                    control_session_id,
+                )
+
+                buffer = []
+                buffer_len = 0
+
+        stopped = stopped or shared_generation_state.is_stopped(control_session_id)
+        if stopped and hasattr(stream, "close"):
+            stream.close()
+        final_text = visualization.final_text()
+
+        if buffer_len > 0 and not stopped:
+            final = np.concatenate(buffer)
+            nfade = min(int(config.mimi_sr * app_config.fade_out_sec), final.shape[0])
+            if nfade > 0:
+                fade = np.linspace(1.0, 0.0, nfade, dtype=np.float32)
+                final[-nfade:] *= fade
+            stream_seq += 1
+            yield (
+                gr.update(),
+                gr.update(),
+                visualization.latest_plot,
+                visualization.latest_text,
+                render_audio_stream(
+                    app_config,
+                    session_id=stream_session_id,
+                    seq=stream_seq,
+                    sample_rate=config.mimi_sr,
+                    audio=float32_to_int16(final),
+                    active=True,
+                ),
+                *generation_button_updates(
+                    running=True,
+                    paused=shared_generation_state.is_paused(control_session_id),
+                ),
+                control_session_id,
+            )
+
+        if len(total_buffer) > 0:
+            full_audio = np.concatenate(total_buffer)
+            nfade = min(
+                int(config.mimi_sr * app_config.fade_out_sec), full_audio.shape[0]
+            )
+            if nfade > 0:
+                fade = np.linspace(1.0, 0.0, nfade, dtype=np.float32)
+                full_audio[-nfade:] *= fade
+
+            file_path = f"/tmp/voxtream_{uuid.uuid4().hex}.wav"
+            sf.write(file_path, float32_to_int16(full_audio), config.mimi_sr)
+
+            speaking_rate_state.stop()
+            generation_control.finish()
+            shared_generation_state.finish(control_session_id)
+            yield (
+                gr.update(value=file_path, visible=True),
+                gr.update(interactive=True),
+                visualization.latest_plot,
+                final_text,
+                render_audio_stream(
+                    app_config,
+                    session_id=stream_session_id,
+                    seq=stream_seq + 1,
+                    sample_rate=config.mimi_sr,
+                    active=False,
+                    final=True,
+                ),
+                *generation_button_updates(running=False),
+                control_session_id,
+            )
+        else:
+            speaking_rate_state.stop()
+            generation_control.finish()
+            shared_generation_state.finish(control_session_id)
+            yield (
+                gr.update(value=None, visible=False),
+                gr.update(interactive=True),
+                visualization.latest_plot,
+                final_text,
+                render_audio_stream(
+                    app_config,
+                    session_id=stream_session_id,
+                    seq=stream_seq + 1,
+                    sample_rate=config.mimi_sr,
+                    active=False,
+                    final=True,
+                ),
+                *generation_button_updates(running=False),
+                control_session_id,
+            )
+
+    demo_app(
+        config,
+        app_config,
+        demo_examples,
+        synthesize_fn,
+        speaking_rate_state,
+        generation_control,
+        shared_generation_state,
+    )
+
+
+if __name__ == "__main__":
+    main()

assets/audio/arabic_female.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f9a779ccd118aa0f313571172e800aa66c3c60e442f670c9132bb9ca9145d22
+size 556844

assets/audio/chinese_female.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad17aad06257940ec078474f1a246ec3dcb996444948e28832759a5ffb8fb339
+size 912044

assets/audio/english_female.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70fae2c5d540e3ce715464731af88e3e8dbfc9c2f3da34435391f27a2cee8cca
+size 973484

assets/audio/english_male.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bc63165bf521c62a823075edcee28b207ce8b77607e160922381ed1f6d5fc4c
+size 992588

assets/audio/french_female.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4d85af9a029801bf4cacf89006245adae5c02105f057da02b3635f119f65665
+size 418136

assets/audio/german_male.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a8fb0671328979463473b3838e6d51e152397094cdd547e8743df3195d54f96
+size 432044

assets/audio/hindi_male.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae95131d80cced28c3b6e22717e41474561e7c83a091fb5b298dc668e7d66e15
+size 787244

assets/audio/japanese_male.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac181ed61dd184281353ad8a6359929fa7f014fd51e2de4f6d5bcffac6ce18c
+size 422444

assets/audio/portuguese_male.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d8b09458a9d5edd3717813507d3f31cf8ba6d880982aaedec6016cc466a4f4b
+size 345642

assets/audio/russian_female.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6aae00ae7c838c3ad9ffe410851b24427ba54d4d4c9509524d885dfc186989b
+size 710444

assets/audio/spanish_male.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:611303d7bc58797b0c9a001d0b59506a2476fe821f311a18ee6e4439879fc783
+size 494444

assets/audio/swedish_female.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:149a2781409654ffa14630dc2b94f1e07adac29a59343437fef80c7277edb3f0
+size 480044

assets/examples.json

@@ -0,0 +1,52 @@
+{
+  "examples": [
+    [
+      "assets/audio/english_male.wav",
+      "Full stream text-to-speech (TTS) for interactive systems must start speaking with minimal delay while remaining controllable as text arrives incrementally. The voice should keep a natural rhythm as each new phrase becomes available. This helps assistants respond quickly while still sounding calm and clear. Careful streaming design makes spoken interaction feel more direct and responsive. It also reduces long silent gaps during complex replies. The listener hears progress while the system continues planning the next phrase."
+    ],
+    [
+      "assets/audio/english_female.wav",
+      "We present VoXtream2, a zero shot full stream text-to-speech (TTS) model with dynamic speaking rate control that can be updated during an utterance. The system can shift its pace while speech is already being produced. This allows a speaker to slow down for difficult content or move faster through simple phrases. Flexible control supports more expressive and useful speech generation. It also lets applications adapt delivery to user attention and context. The same voice can sound measured, concise, or relaxed as needed."
+    ],
+    [
+      "assets/audio/chinese_female.wav",
+      "VoXtream2 combines distribution matching over duration states with classifier free guidance across conditioning signals to improve controllability and synthesis quality. These methods help the model follow timing instructions more reliably. They also preserve natural voice quality when several controls are active. The result is speech that can be guided without sounding rigid or unstable. Better alignment between text, timing, and voice style makes each output easier to shape."
+    ],
+    [
+      "assets/audio/hindi_male.wav",
+      "Prompt text masking enables textless audio prompting, removing the need for prompt transcription. A user can provide a short voice reference without writing down what was said. This makes voice adaptation easier when transcripts are missing or expensive to prepare. The model can still learn useful speaker cues from the audio prompt. It can focus on tone, accent, and speaking style while ignoring unavailable words. This lowers setup effort for demos, research samples, and personal voice interfaces."
+    ],
+    [
+      "assets/audio/spanish_male.wav",
+      "Across standard zero shot benchmarks and a dedicated speaking rate test set, VoXtream2 achieves competitive objective and subjective results against public baselines. The evaluation covers both measured accuracy and human listening preference. Strong results suggest that streaming control does not require a large loss in quality. This makes the approach practical for real conversational systems. Consistent performance across tests gives developers more confidence in deployment."
+    ],
+    [
+      "assets/audio/arabic_female.wav",
+      "In full stream mode, it runs 4 times faster than real time with 74 ms first packet latency on a consumer graphics processor. Low latency helps the system begin speaking before the full response is complete. Faster synthesis also leaves more room for other application work. These properties are important for smooth real time dialogue. Efficient generation can support busy products without requiring unusual hardware. It also makes testing easier because responses arrive quickly during iteration."
+    ],
+    [
+      "assets/audio/french_female.wav",
+      "It has long been argued that conversational agents must be able to generate speech incrementally. Human conversation often depends on quick turns and partial understanding. An agent that waits too long can make the exchange feel broken or unnatural. Incremental generation supports more fluid spoken interaction. It lets a system begin with a confident phrase while later content is still forming. This behavior can make spoken assistants feel attentive, present, and easier to interrupt."
+    ],
+    [
+      "assets/audio/japanese_male.wav",
+      "Japanese audio recording can be used to prepare the examples list for a future voice sample. The entry keeps the same format as the other language examples. English text makes the placeholder clear for contributors who review the file. The audio path can be updated once the final sample is created. The placeholder supports early testing of selection flows and ordering. It also makes room for future validation of playback, captions, and voice metadata. Reviewers can confirm the option appears correctly."
+    ],
+    [
+      "assets/audio/russian_female.wav",
+      "Recent progress in neural text-to-speech (TTS) synthesis has led to highly natural and intelligible speech generation. Modern models can produce voices with clear pronunciation and expressive prosody. These gains make synthetic speech useful in more demanding interactive settings. The next challenge is to keep that quality while adding fine grained control. Users expect a generated voice to remain stable when speed, style, or prompting changes. Robust models must balance realism, latency, and steering."
+    ],
+    [
+      "assets/audio/swedish_female.wav",
+      "However, most contemporary systems implicitly assume that speaking rate is static across an utterance, typically allowing only coarse, global control over speed. Real speakers often vary their pace within a single response. They may pause before important words or speed through familiar details. A useful speech system should support this kind of local timing control. This makes explanations clearer and keeps long replies from feeling flat. Local control also helps match emphasis to meaning."
+    ],
+    [
+      "assets/audio/portuguese_male.wav",
+      "Portuguese audio prompt can be used to test how the interface presents another language option. The example text stays in English while the entry reserves space for a future recording. This keeps the data structure ready for multilingual expansion. A final voice sample can later replace the placeholder path. The placeholder also helps verify menus, labels, and playback behavior before the asset exists. It gives reviewers a clear signal that Portuguese support is planned but not complete."
+    ],
+    [
+      "assets/audio/german_male.wav",
+      "German acoustic prompt can be used to check language selection and example ordering. The English text describes the expected role of the entry without adding translated content. This makes the placeholder easy to identify during development. A complete recording can be added when the voice asset is available. The entry can also reveal layout issues in lists that include many languages. It keeps the example set balanced while the final German sample is prepared. Reviewers can test the flow early."
+    ]
+  ]
+}

configs/app.json

@@ -0,0 +1,23 @@
+{
+    "min_chunk_sec": 0.01,
+    "fade_out_sec": 0.1,
+    "plot_window_sec": 10.0,
+    "visual_update_sec": 0.25,
+    "future_phone_limit": 25,
+    "plot_width": 1000,
+    "plot_height": 224,
+    "plot_left": 74,
+    "plot_right": 22,
+    "plot_top": 50,
+    "plot_bottom": 42,
+    "plot_y_max": 7,
+    "plot_y_tick": 1,
+    "plot_x_tick_sec": 1,
+    "audio_stream_start_delay_sec": 0.12,
+    "audio_stream_sample_rate": 24000,
+    "speaking_rate_min": 1.0,
+    "speaking_rate_max": 7.0,
+    "speaking_rate_step": 0.1,
+    "speaking_rate_default": 4.0,
+    "min_streaming_rtf": 0.95
+}

configs/generator.json

@@ -0,0 +1,58 @@
+{
+    "sil_token": 120,
+    "bos_token": 123,
+    "eos_token": 124,
+    "unk_token": 122,
+    "eop_token": 122,
+    "num_codebooks": 16,
+    "num_phones_per_frame": 2,
+    "audio_delay_frames": 1,
+    "temperature": 0.9,
+    "topk": 5,
+    "top_p": 0.9,
+    "max_audio_length_ms": 60000,
+    "model_repo": "voxtream2/model",
+    "model_name": "model.safetensors",
+    "model_config_name": "config.json",
+    "mimi_sr": 24000,
+    "mimi_vocab_size": 2048,
+    "mimi_frame_ms": 80,
+    "mimi_repo": "kyutai/moshiko-pytorch-bf16",
+    "mimi_name": "tokenizer-e351c8d8-checkpoint125.safetensors",
+    "spk_enc_sr": 16000,
+    "spk_enc_repo": "IDRnD/ReDimNet",
+    "spk_enc_model": "ReDimNet",
+    "spk_enc_model_name": "M",
+    "spk_enc_train_type": "ft_mix",
+    "spk_enc_dataset": "vb2+vox2+cnc",
+    "phoneme_dict_name": "phoneme_to_token.json",
+    "max_prompt_sec": 20,
+    "min_prompt_sec": 1,
+    "max_phone_tokens": 2000,
+    "cache_prompt": false,
+    "cfg_gamma": 1.5,
+    "cfg_ac_gamma": 3.0,
+    "text_context": " context",
+    "text_context_length": 18,
+    "spk_proj_weight": 1.5,
+    "audio_pad_token": 2049,
+    "enhance_prompt": false,
+    "sidon_se_reload_model": true,
+    "reset_streaming_state": true,
+    "hf_token": null,
+    "apply_vad": false,
+    "min_speech_seg_sec": 0.3,
+    "min_look_ahead_phones": 3,
+    "phonemizer": "espeak",
+    "spk_rate_window_sec": 3.0,
+    "frame_repeat_counter": 25,
+    "punct_map": {
+        ".": 117,
+        ",": 118,
+        "?": 119,
+        "!": 121
+    },
+    "phoneme_index_map":{
+        "0": [0, 1], "1": [0, 2], "2": [1, 1], "3": [1, 2], "4": [2, 1], "5": [2, 2]
+    }
+}

configs/speaking_rate.json

@@ -0,0 +1,51 @@
+{
+    "1": {
+        "duration_state": [
+            52, 15, 1, 2, 1, 3
+        ],
+        "weight": 3.0,
+        "cfg_gamma": 1.25
+    },
+    "2": {
+        "duration_state": [
+            44, 24, 2, 2, 1, 1
+        ],
+        "weight": 3.0,
+        "cfg_gamma": 1.25
+    },
+    "3": {
+        "duration_state": [
+            24, 23, 2, 3, 1, 1
+        ],
+        "weight": 5.0,
+        "cfg_gamma": 1.5
+    },
+    "4": {
+        "duration_state": [
+            40, 59, 5, 12, 1, 2
+        ],
+        "weight": 5.0,
+        "cfg_gamma": 1.5
+    },
+    "5": {
+        "duration_state": [
+            21, 43, 4, 13, 1, 2
+        ],
+        "weight": 7.0,
+        "cfg_gamma": 2.0
+    },
+    "6": {
+        "duration_state": [
+            27, 69, 6, 30, 2, 7
+        ],
+        "weight": 7.0,
+        "cfg_gamma": 2.0
+    },
+    "7": {
+        "duration_state": [
+            6, 18, 2, 9, 1, 3
+        ],
+        "weight": 10.0,
+        "cfg_gamma": 2.5
+    }
+}

packages.txt

@@ -0,0 +1 @@
+espeak-ng

requirements.txt

@@ -0,0 +1 @@
+voxtream==0.2.4