Deploy VibeVoice to Spaces (runtime model + LFS presets/images)

.gitattributes

@@ -1,35 +1,4 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,181 @@
+# Initially taken from Github's Python gitignore file
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# tests and logs
+tests/fixtures/cached_*_text.txt
+logs/
+lightning_logs/
+lang_code_data/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# vscode
+.vs
+.vscode
+
+# Pycharm
+.idea
+
+# TF code
+tensorflow_code
+
+# Models
+proc_data
+
+# examples
+runs
+/runs_old
+/wandb
+/examples/runs
+/examples/**/*.args
+/examples/rag/sweep
+
+# data
+/data
+serialization_dir
+
+# emacs
+*.*~
+debug.env
+
+# vim
+.*.swp
+
+#ctags
+tags
+
+# pre-commit
+.pre-commit*
+
+# .lock
+*.lock
+
+# DS_Store (MacOS)
+.DS_Store
+
+# ruff
+.ruff_cache
+
+# our proj
+/output/
+/outputs/
+/checkpoint/
+/checkpoints/
+exp
+.gradio/
+
+# Ignored large models
+demo/models/VibeVoice-Realtime-0.5B
+*.safetensors

Dockerfile

@@ -14,4 +14,4 @@ COPY . .
 RUN pip install -e .
 EXPOSE 7860
 WORKDIR /app/demo
-CMD ["python3", "vibevoice_realtime_demo.py", "--model_path", "../models/VibeVoice-Realtime-0.5B", "--port", "7860", "--device", "cpu"]
+CMD ["python3", "vibevoice_realtime_demo.py", "--model_path", "microsoft/VibeVoice-Realtime-0.5B", "--port", "7860", "--device", "cpu"]

README.md

@@ -1,12 +1,123 @@
----
-title: VibeVoice
-emoji: 🌍
-colorFrom: purple
-colorTo: yellow
-sdk: docker
-pinned: false
-license: mit
-short_description: microsoft VibeVoice-Realtime-0.5B
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<div align="center">
+
+## 🎙️ VibeVoice: Open-Source Frontier Voice AI
+[![Project Page](https://img.shields.io/badge/Project-Page-blue?logo=microsoft)](https://microsoft.github.io/VibeVoice)
+[![Hugging Face](https://img.shields.io/badge/HuggingFace-Collection-orange?logo=huggingface)](https://huggingface.co/collections/microsoft/vibevoice-68a2ef24a875c44be47b034f)
+[![Technical Report](https://img.shields.io/badge/Technical-Report-red?logo=adobeacrobatreader)](https://arxiv.org/pdf/2508.19205)
+
+
+</div>
+
+
+<div align="center">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="Figures/VibeVoice_logo_white.png">
+  <img src="Figures/VibeVoice_logo.png" alt="VibeVoice Logo" width="300">
+</picture>
+</div>
+
+<div align="left">
+
+<h3>📰 News</h3>
+
+<img src="https://img.shields.io/badge/Status-New-brightgreen?style=flat" alt="New" />
+<img src="https://img.shields.io/badge/Feature-Realtime_TTS-blue?style=flat&logo=soundcharts" alt="Realtime TTS" />
+
+<strong>2025-12-03: 📣 We open-sourced <a href="docs/vibevoice-realtime-0.5b.md"><strong>VibeVoice‑Realtime‑0.5B</strong></a>, a real‑time text‑to‑speech model that supports streaming text input and robust long-form speech generation. Try it on [Colab](https://colab.research.google.com/github/microsoft/VibeVoice/blob/main/demo/vibevoice_realtime_colab.ipynb).</strong>
+
+<strong>2025-12-09: 📣 We’ve added experimental speakers in nine languages (DE, FR, IT, JP, KR, NL, PL, PT, ES) for exploration—welcome to try them out and share your feedback.</strong>
+
+To mitigate deepfake risks and ensure low latency for the first speech chunk, voice prompts are provided in an embedded format. For users requiring voice customization, please reach out to our team. We will also be expanding the range of available speakers.
+<br>
+
+https://github.com/user-attachments/assets/0901d274-f6ae-46ef-a0fd-3c4fba4f76dc
+
+> (Launch your own realtime demo via the websocket example in [Usage](docs/vibevoice-realtime-0.5b.md#usage-1-launch-real-time-websocket-demo)).
+
+</div>
+
+2025-09-05: VibeVoice is an open-source research framework intended to advance collaboration in the speech synthesis community. After release, we discovered instances where the tool was used in ways inconsistent with the stated intent. Since responsible use of AI is one of Microsoft’s guiding principles, we have disabled this repo until we are confident that out-of-scope use is no longer possible.
+
+
+### Overview
+
+VibeVoice is a novel framework designed for generating **expressive**, **long-form**, **multi-speaker** conversational audio, such as podcasts, from text. It addresses significant challenges in traditional Text-to-Speech (TTS) systems, particularly in scalability, speaker consistency, and natural turn-taking.
+
+VibeVoice currently includes two model variants:
+
+- **Long-form multi-speaker model**: Synthesizes conversational/single-speaker speech up to **90 minutes** with up to **4 distinct speakers**, surpassing the typical 1–2 speaker limits of many prior models.
+- **[Realtime streaming TTS model](docs/vibevoice-realtime-0.5b.md)**: Produces initial audible speech in ~**300 ms** and supports **streaming text input** for single-speaker **real-time** speech generation; designed for low-latency generation.
+
+A core innovation of VibeVoice is its use of continuous speech tokenizers (Acoustic and Semantic) operating at an ultra-low frame rate of 7.5 Hz. These tokenizers efficiently preserve audio fidelity while significantly boosting computational efficiency for processing long sequences. VibeVoice employs a [next-token diffusion](https://arxiv.org/abs/2412.08635) framework, leveraging a Large Language Model (LLM) to understand textual context and dialogue flow, and a diffusion head to generate high-fidelity acoustic details.
+
+
+<p align="left">
+  <img src="Figures/MOS-preference.png" alt="MOS Preference Results" height="260px">
+  <img src="Figures/VibeVoice.jpg" alt="VibeVoice Overview" height="250px" style="margin-right: 10px;">
+</p>
+
+
+### 🎵 Demo Examples
+
+
+**Video Demo**
+
+We produced this video with [Wan2.2](https://github.com/Wan-Video/Wan2.2). We sincerely appreciate the Wan-Video team for their great work.
+
+**English**
+<div align="center">
+
+https://github.com/user-attachments/assets/0967027c-141e-4909-bec8-091558b1b784
+
+</div>
+
+
+**Chinese**
+<div align="center">
+
+https://github.com/user-attachments/assets/322280b7-3093-4c67-86e3-10be4746c88f
+
+</div>
+
+**Cross-Lingual**
+<div align="center">
+
+https://github.com/user-attachments/assets/838d8ad9-a201-4dde-bb45-8cd3f59ce722
+
+</div>
+
+**Spontaneous Singing**
+<div align="center">
+
+https://github.com/user-attachments/assets/6f27a8a5-0c60-4f57-87f3-7dea2e11c730
+
+</div>
+
+
+**Long Conversation with 4 people**
+<div align="center">
+
+https://github.com/user-attachments/assets/a357c4b6-9768-495c-a576-1618f6275727
+
+</div>
+
+For more examples, see the [Project Page](https://microsoft.github.io/VibeVoice).
+
+
+
+## Risks and limitations
+
+While efforts have been made to optimize it through various techniques, it may still produce outputs that are unexpected, biased, or inaccurate. VibeVoice inherits any biases, errors, or omissions produced by its base model (specifically, Qwen2.5 1.5b in this release).
+Potential for Deepfakes and Disinformation: High-quality synthetic speech can be misused to create convincing fake audio content for impersonation, fraud, or spreading disinformation. Users must ensure transcripts are reliable, check content accuracy, and avoid using generated content in misleading ways. Users are expected to use the generated content and to deploy the models in a lawful manner, in full compliance with all applicable laws and regulations in the relevant jurisdictions. It is best practice to disclose the use of AI when sharing AI-generated content.
+
+English and Chinese only: Transcripts in languages other than English or Chinese may result in unexpected audio outputs.
+
+Non-Speech Audio: The model focuses solely on speech synthesis and does not handle background noise, music, or other sound effects.
+
+Overlapping Speech: The current model does not explicitly model or generate overlapping speech segments in conversations.
+
+We do not recommend using VibeVoice in commercial or real-world applications without further testing and development. This model is intended for research and development purposes only. Please use responsibly.
+
+## Star History
+
+![Star History Chart](https://api.star-history.com/svg?repos=Microsoft/vibevoice&type=date&legend=top-left)

demo/code.sh

@@ -0,0 +1 @@
+python3 vibevoice_realtime_demo.py --model_path models/VibeVoice-Realtime-0.5B --port 8000 --device mps

demo/realtime_model_inference_from_file.py

@@ -0,0 +1,314 @@
+import argparse
+import os
+import re
+import traceback
+from typing import List, Tuple, Union, Dict, Any
+import time
+import torch
+import copy
+
+from vibevoice.modular.modeling_vibevoice_streaming_inference import VibeVoiceStreamingForConditionalGenerationInference
+from vibevoice.processor.vibevoice_streaming_processor import VibeVoiceStreamingProcessor
+from transformers.utils import logging
+
+logging.set_verbosity_info()
+logger = logging.get_logger(__name__)
+
+
+class VoiceMapper:
+    """Maps speaker names to voice file paths"""
+    
+    def __init__(self):
+        self.setup_voice_presets()
+
+        # change name according to our preset voice file
+        new_dict = {}
+        for name, path in self.voice_presets.items():
+            
+            if '_' in name:
+                name = name.split('_')[0]
+            
+            if '-' in name:
+                name = name.split('-')[-1]
+
+            new_dict[name] = path
+        self.voice_presets.update(new_dict)
+        # print(list(self.voice_presets.keys()))
+
+    def setup_voice_presets(self):
+        """Setup voice presets by scanning the voices directory."""
+        voices_dir = os.path.join(os.path.dirname(__file__), "voices/streaming_model")
+        
+        # Check if voices directory exists
+        if not os.path.exists(voices_dir):
+            print(f"Warning: Voices directory not found at {voices_dir}")
+            self.voice_presets = {}
+            self.available_voices = {}
+            return
+        
+        # Scan for all VOICE files in the voices directory
+        self.voice_presets = {}
+        
+        # Get all .pt files in the voices directory
+        pt_files = [f for f in os.listdir(voices_dir) 
+                    if f.lower().endswith('.pt') and os.path.isfile(os.path.join(voices_dir, f))]
+        
+        # Create dictionary with filename (without extension) as key
+        for pt_file in pt_files:
+            # Remove .pt extension to get the name
+            name = os.path.splitext(pt_file)[0]
+            # Create full path
+            full_path = os.path.join(voices_dir, pt_file)
+            self.voice_presets[name] = full_path
+        
+        # Sort the voice presets alphabetically by name for better UI
+        self.voice_presets = dict(sorted(self.voice_presets.items()))
+        
+        # Filter out voices that don't exist (this is now redundant but kept for safety)
+        self.available_voices = {
+            name: path for name, path in self.voice_presets.items()
+            if os.path.exists(path)
+        }
+        
+        print(f"Found {len(self.available_voices)} voice files in {voices_dir}")
+        print(f"Available voices: {', '.join(self.available_voices.keys())}")
+
+    def get_voice_path(self, speaker_name: str) -> str:
+        """Get voice file path for a given speaker name"""
+        # First try exact match
+        if speaker_name in self.voice_presets:
+            return self.voice_presets[speaker_name]
+        
+        # Try partial matching (case insensitive)
+        speaker_lower = speaker_name.lower()
+        for preset_name, path in self.voice_presets.items():
+            if preset_name.lower() in speaker_lower or speaker_lower in preset_name.lower():
+                return path
+        
+        # Default to first voice if no match found
+        default_voice = list(self.voice_presets.values())[0]
+        print(f"Warning: No voice preset found for '{speaker_name}', using default voice: {default_voice}")
+        return default_voice
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="VibeVoiceStreaming Processor TXT Input Test")
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="microsoft/VibeVoice-Realtime-0.5B",
+        help="Path to the HuggingFace model directory",
+    )
+    parser.add_argument(
+        "--txt_path",
+        type=str,
+        default="demo/text_examples/1p_vibevoice.txt",
+        help="Path to the txt file containing the script",
+    )
+    parser.add_argument(
+        "--speaker_name",
+        type=str,
+        default="Wayne",
+        help="Single speaker name (e.g., --speaker_name Wayne)",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./outputs",
+        help="Directory to save output audio files",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default=("cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")),
+        help="Device for inference: cuda | mps | cpu",
+    )
+    parser.add_argument(
+        "--cfg_scale",
+        type=float,
+        default=1.5,
+        help="CFG (Classifier-Free Guidance) scale for generation (default: 1.5)",
+    )
+    
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+
+    # Normalize potential 'mpx' typo to 'mps'
+    if args.device.lower() == "mpx":
+        print("Note: device 'mpx' detected, treating it as 'mps'.")
+        args.device = "mps"
+
+    # Validate mps availability if requested
+    if args.device == "mps" and not torch.backends.mps.is_available():
+        print("Warning: MPS not available. Falling back to CPU.")
+        args.device = "cpu"
+
+    print(f"Using device: {args.device}")
+
+    # Initialize voice mapper
+    voice_mapper = VoiceMapper()
+    
+    # Check if txt file exists
+    if not os.path.exists(args.txt_path):
+        print(f"Error: txt file not found: {args.txt_path}")
+        return
+    
+    # Read and parse txt file
+    print(f"Reading script from: {args.txt_path}")
+    with open(args.txt_path, 'r', encoding='utf-8') as f:
+        scripts = f.read().strip()
+    
+    if not scripts:
+        print("Error: No valid scripts found in the txt file")
+        return
+
+    full_script = scripts.replace("’", "'").replace('“', '"').replace('”', '"')
+
+    print(f"Loading processor & model from {args.model_path}")
+    processor = VibeVoiceStreamingProcessor.from_pretrained(args.model_path)
+
+    # Decide dtype & attention implementation
+    if args.device == "mps":
+        load_dtype = torch.float32  # MPS requires float32
+        attn_impl_primary = "sdpa"  # flash_attention_2 not supported on MPS
+    elif args.device == "cuda":
+        load_dtype = torch.bfloat16
+        attn_impl_primary = "flash_attention_2"
+    else:  # cpu
+        load_dtype = torch.float32
+        attn_impl_primary = "sdpa"
+    print(f"Using device: {args.device}, torch_dtype: {load_dtype}, attn_implementation: {attn_impl_primary}")
+    # Load model with device-specific logic
+    try:
+        if args.device == "mps":
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                attn_implementation=attn_impl_primary,
+                device_map=None,  # load then move
+            )
+            model.to("mps")
+        elif args.device == "cuda":
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map="cuda",
+                attn_implementation=attn_impl_primary,
+            )
+        else:  # cpu
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map="cpu",
+                attn_implementation=attn_impl_primary,
+            )
+    except Exception as e:
+        if attn_impl_primary == 'flash_attention_2':
+            print(f"[ERROR] : {type(e).__name__}: {e}")
+            print(traceback.format_exc())
+            print("Error loading the model. Trying to use SDPA. However, note that only flash_attention_2 has been fully tested, and using SDPA may result in lower audio quality.")
+            model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map=(args.device if args.device in ("cuda", "cpu") else None),
+                attn_implementation='sdpa'
+            )
+            if args.device == "mps":
+                model.to("mps")
+        else:
+            raise e
+
+
+    model.eval()
+    model.set_ddpm_inference_steps(num_steps=5)
+
+    if hasattr(model.model, 'language_model'):
+       print(f"Language model attention: {model.model.language_model.config._attn_implementation}")
+    
+    target_device = args.device if args.device != "cpu" else "cpu"
+    voice_sample = voice_mapper.get_voice_path(args.speaker_name)
+    all_prefilled_outputs = torch.load(voice_sample, map_location=target_device, weights_only=False)
+
+    # Prepare inputs for the model
+    inputs = processor.process_input_with_cached_prompt(
+        text=full_script,
+        cached_prompt=all_prefilled_outputs,
+        padding=True,
+        return_tensors="pt",
+        return_attention_mask=True,
+    )
+
+    # Move tensors to target device
+    for k, v in inputs.items():
+        if torch.is_tensor(v):
+            inputs[k] = v.to(target_device)
+
+    print(f"Starting generation with cfg_scale: {args.cfg_scale}")
+
+    # Generate audio
+    start_time = time.time()
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=None,
+        cfg_scale=args.cfg_scale,
+        tokenizer=processor.tokenizer,
+        generation_config={'do_sample': False},
+        verbose=True,
+        all_prefilled_outputs=copy.deepcopy(all_prefilled_outputs) if all_prefilled_outputs is not None else None,
+    )
+    generation_time = time.time() - start_time
+    print(f"Generation time: {generation_time:.2f} seconds")
+    
+    # Calculate audio duration and additional metrics
+    if outputs.speech_outputs and outputs.speech_outputs[0] is not None:
+        # Assuming 24kHz sample rate (common for speech synthesis)
+        sample_rate = 24000
+        audio_samples = outputs.speech_outputs[0].shape[-1] if len(outputs.speech_outputs[0].shape) > 0 else len(outputs.speech_outputs[0])
+        audio_duration = audio_samples / sample_rate
+        rtf = generation_time / audio_duration if audio_duration > 0 else float('inf')
+        
+        print(f"Generated audio duration: {audio_duration:.2f} seconds")
+        print(f"RTF (Real Time Factor): {rtf:.2f}x")
+    else:
+        print("No audio output generated")
+    
+    # Calculate token metrics
+    input_tokens = inputs['tts_text_ids'].shape[1]  # Number of input tokens
+    output_tokens = outputs.sequences.shape[1]  # Total tokens (input + generated)
+    generated_tokens = output_tokens - input_tokens - all_prefilled_outputs['tts_lm']['last_hidden_state'].size(1)
+    
+    print(f"Prefilling text tokens: {input_tokens}")
+    print(f"Generated speech tokens: {generated_tokens}")
+    print(f"Total tokens: {output_tokens}")
+
+    # Save output (processor handles device internally)
+    txt_filename = os.path.splitext(os.path.basename(args.txt_path))[0]
+    output_path = os.path.join(args.output_dir, f"{txt_filename}_generated.wav")
+    os.makedirs(args.output_dir, exist_ok=True)
+    
+    processor.save_audio(
+        outputs.speech_outputs[0], # First (and only) batch item
+        output_path=output_path,
+    )
+    print(f"Saved output to {output_path}")
+    
+    # Print summary
+    print("\n" + "="*50)
+    print("GENERATION SUMMARY")
+    print("="*50)
+    print(f"Input file: {args.txt_path}")
+    print(f"Output file: {output_path}")
+    print(f"Speaker names: {args.speaker_name}")
+    print(f"Prefilling text tokens: {input_tokens}")
+    print(f"Generated speech tokens: {generated_tokens}")
+    print(f"Total tokens: {output_tokens}")
+    print(f"Generation time: {generation_time:.2f} seconds")
+    print(f"Audio duration: {audio_duration:.2f} seconds")
+    print(f"RTF (Real Time Factor): {rtf:.2f}x")
+    
+    print("="*50)
+
+if __name__ == "__main__":
+    main()

demo/text_examples/1p_abs.txt

@@ -0,0 +1,2 @@
+Generating long-form, multi-speaker conversational audio like podcasts poses significant challenges for traditional Text-to-Speech (TTS) systems, particularly in scalability, speaker consistency, and natural turn-taking. This report presents VibeVoice, a novel model designed to synthesize long-form speech with multiple speakers by employing the next-token diffusion framework, a unified method for modeling continuous data by autoregressively generating latent vectors via diffusion. 
+A core component of our approach is the continuous speech tokenizers operating at an ultra-low frame rate of 7.5. This tokenizer effectively preserves audio fidelity while significantly boosting computational efficiency for processing long sequences. This enables VibeVoice to synthesize long-form speech for up to 90 minutes (in a 64K context window length) with up to 4 speakers, capturing the authentic conversational "vibe" and surpassing all known open-source and closed-source dialogue models (for example, Gemini 2.5 Pro Preview TTS). Code and checkpoint are available now. 

demo/text_examples/1p_vibevoice.txt

@@ -0,0 +1 @@
+VibeVoice is a novel framework designed for generating expressive, long-form, multi-speaker conversational audio, such as podcasts, from text. It addresses significant challenges in traditional Text-to-Speech (TTS) systems, particularly in scalability, speaker consistency, and natural turn-taking. A core innovation of VibeVoice is its use of continuous speech tokenizers operating at an ultra-low frame rate of 7.5 Hz. These tokenizers efficiently preserve audio fidelity while significantly boosting computational efficiency for processing long sequences. VibeVoice employs a next-token diffusion framework, leveraging a Large Language Model to understand textual context and dialogue flow, and a diffusion head to generate high-fidelity acoustic details. The model can synthesize speech up to 90 minutes long with up to 4 distinct speakers, surpassing the typical 1-2 speaker limits of many prior models.

demo/vibevoice_realtime_colab.ipynb

@@ -0,0 +1,198 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "id": "d1785adb",
+      "metadata": {
+        "colab_type": "text",
+        "id": "view-in-github"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/microsoft/VibeVoice/blob/main/demo/vibevoice_realtime_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "WvIaUJD2y0yU",
+      "metadata": {
+        "id": "WvIaUJD2y0yU"
+      },
+      "source": [
+        "# VibeVoice-Realtime Colab — T4 Quickstart\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "e8fTKYGx7DZk",
+      "metadata": {
+        "id": "e8fTKYGx7DZk"
+      },
+      "source": [
+        "## Step 1: Setup Environment"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "4wxJ6QHM-ZOb",
+      "metadata": {
+        "id": "4wxJ6QHM-ZOb"
+      },
+      "outputs": [],
+      "source": [
+        "# Check for T4 GPU\n",
+        "import torch\n",
+        "if torch.cuda.is_available() and \"T4\" in torch.cuda.get_device_name(0):\n",
+        "    print(\"✅ T4 GPU detected\")\n",
+        "else:\n",
+        "    print(\"\"\"\n",
+        "    ⚠️ WARNING: T4 GPU not detected\n",
+        "\n",
+        "    The recommended runtime for this Colab notebook is \"T4 GPU\".\n",
+        "\n",
+        "    To change the runtime type:\n",
+        "\n",
+        "        1. Click on \"Runtime\" in the top navigation menu\n",
+        "        2. Click on \"Change runtime type\"\n",
+        "        3. Select \"T4 GPU\"\n",
+        "        4. Click \"OK\" if a \"Disconnect and delete runtime\" window appears\n",
+        "        5. Click on \"Save\"\n",
+        "\n",
+        "    \"\"\")\n",
+        "\n",
+        "# Clone the VibeVoice repository\n",
+        "![ -d /content/VibeVoice ] || git clone --quiet --branch main --depth 1 https://github.com/microsoft/VibeVoice.git /content/VibeVoice\n",
+        "print(\"✅ Cloned VibeVoice repository\")\n",
+        "\n",
+        "# Install project dependencies\n",
+        "!uv pip --quiet install --system -e /content/VibeVoice\n",
+        "!wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -O cloudflared && chmod +x cloudflared\n",
+        "print(\"✅ Installed dependencies\")\n",
+        "\n",
+        "# Download model\n",
+        "from huggingface_hub import snapshot_download\n",
+        "snapshot_download(\"microsoft/VibeVoice-Realtime-0.5B\", local_dir=\"/content/models/VibeVoice-Realtime-0.5B\")\n",
+        "print(\"✅ Downloaded model: microsoft/VibeVoice-Realtime-0.5B\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "88c727ab",
+      "metadata": {},
+      "source": [
+        "[Optional] If the download exceeds 1 minute, it is probably stuck. You can: (1) interrupt the execution, (2) log in to Hugging Face, and (3) try download again."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "dec6b870",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from huggingface_hub import login\n",
+        "login()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "c579654b",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "snapshot_download(\"microsoft/VibeVoice-Realtime-0.5B\", local_dir=\"/content/models/VibeVoice-Realtime-0.5B\")\n",
+        "print(\"✅ Downloaded model: microsoft/VibeVoice-Realtime-0.5B\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "pgKlV7153Ifi",
+      "metadata": {
+        "id": "pgKlV7153Ifi"
+      },
+      "source": [
+        "## Step 2: Launch VibeVoice-Realtime Demo"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "Yc1N9EHswFxA",
+      "metadata": {
+        "id": "Yc1N9EHswFxA"
+      },
+      "outputs": [],
+      "source": [
+        "import subprocess, re, time, threading\n",
+        "\n",
+        "srv = subprocess.Popen(\n",
+        "    \"python /content/VibeVoice/demo/vibevoice_realtime_demo.py --model_path /content/models/VibeVoice-Realtime-0.5B --port 8000\",\n",
+        "    shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, universal_newlines=True,\n",
+        ")\n",
+        "cf = subprocess.Popen(\n",
+        "    \"./cloudflared tunnel --url http://localhost:8000 --no-autoupdate\",\n",
+        "    shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, universal_newlines=True,\n",
+        ")\n",
+        "\n",
+        "public_url = None\n",
+        "server_ready = False\n",
+        "url_pattern  = re.compile(r\"(https://[a-z0-9-]+\\.trycloudflare\\.com)\")\n",
+        "\n",
+        "def read_srv():\n",
+        "    global server_ready\n",
+        "    for ln in srv.stdout:\n",
+        "        print(ln.strip())\n",
+        "        if \"Uvicorn running on\" in ln:\n",
+        "            server_ready = True\n",
+        "\n",
+        "def read_cf():\n",
+        "    global public_url\n",
+        "    for ln in cf.stdout:\n",
+        "        m = url_pattern.search(ln)\n",
+        "        if m:\n",
+        "            public_url = m.group(1)\n",
+        "            break\n",
+        "\n",
+        "threading.Thread(target=read_srv, daemon=True).start()\n",
+        "threading.Thread(target=read_cf,  daemon=True).start()\n",
+        "\n",
+        "\n",
+        "while True:\n",
+        "    if server_ready and public_url:\n",
+        "        print(f\"✅ Public URL: {public_url}\\n\");\n",
+        "        public_url = None\n",
+        "    time.sleep(0.25)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "include_colab_link": true,
+      "machine_shape": "hm",
+      "name": "VibeVoice_Colab.ipynb",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.11"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}

demo/vibevoice_realtime_demo.py

@@ -0,0 +1,17 @@
+import argparse, os, uvicorn
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--port", type=int, default=3000)
+    p.add_argument("--model_path", type=str, default="default_model")
+    p.add_argument("--device", type=str, default="cuda", choices=["cpu", "cuda", "mpx", "mps"])
+    p.add_argument("--reload", action="store_true", help="Reload the model or not")
+    args = p.parse_args()
+    
+    os.environ["MODEL_PATH"] = args.model_path
+    os.environ["MODEL_DEVICE"] = args.device
+
+    uvicorn.run("web.app:app", host="0.0.0.0", port=args.port, reload=args.reload)
+
+if __name__ == "__main__":
+    main()

demo/voices/streaming_model/de-Spk0_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba2c3e06c52ca02a851326d8a354188c8cafe4543c717d8beb8b64fe3466913a
+size 7039666

demo/voices/streaming_model/de-Spk1_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11b4fa505f7d636af7047632793fe639b1e826d46fc6c00d64dacdcf0805ad72
+size 5290778

demo/voices/streaming_model/en-Carter_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7bfdf1cd4939c22469bcfc6f427ae9c4467b3df46c2c14303a39c294cfc6897
+size 4256002

demo/voices/streaming_model/en-Davis_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67561d63bfa2153616e4c02fd967007c182593fc53738a6ad94bf5f84e8832ac
+size 2471258

demo/voices/streaming_model/en-Emma_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75b15c481e0d848991f1789620aa9929c583ec2c5f701f8152362cf74498bbf8
+size 3343090

demo/voices/streaming_model/en-Frank_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acaa8f1a4f46a79f8f5660cfb7a3af06ef473389319df7debc07376fdc840e47
+size 3359578

demo/voices/streaming_model/en-Grace_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f0ef02a3f3cace04cf721608b65273879466bb15fe4044e46ec6842190f6bb1
+size 2772466

demo/voices/streaming_model/en-Mike_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afb64b580fbc6fab09af04572bbbd2b3906ff8ed35a28731a90b8681e47bdc89
+size 2016234

demo/voices/streaming_model/fr-Spk0_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64f12314f9df2348f0fd6cdbee9efb0f1ebfc286034560e207be5efc4108a368
+size 4386482

demo/voices/streaming_model/fr-Spk1_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1425e8b7fab2fb03d400ca23d672ca96dbbb6b7b000e52332117f2fece33077
+size 4272170

demo/voices/streaming_model/in-Samuel_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ec5e8cde4006aa3f26fde0422124f3296f83f5738b5b4506268855a305cb06
+size 3782658

demo/voices/streaming_model/it-Spk0_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a838184c0304802c696bac887d15a8143539fbb0b5371fe53de66059a010f3d
+size 2552026

demo/voices/streaming_model/it-Spk1_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:349ce0d28e93ea8b1df55800527f3a8ba8a55bb9e88990841db1bc9ee424519a
+size 2854514

demo/voices/streaming_model/jp-Spk0_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4881286aa2fe14e65a800d64a7ae7cd0a7ccda53ff6de88a8d6590980c20e16d
+size 4668234

demo/voices/streaming_model/jp-Spk1_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7a3f5ca602912152c483516bc0198ea13765affd6fbf16fc1bed5e91e1cbbda
+size 4637994

demo/voices/streaming_model/kr-Spk0_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6d15b8d66f8c271655d000b5923cb33a9b564bf84484a82b10431dc1a2fafef
+size 4154002

demo/voices/streaming_model/kr-Spk1_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65514854c7ffc7080aba964cf40811822358b19397132ddaf9b3cdb168c394f3
+size 5865138

demo/voices/streaming_model/nl-Spk0_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70e9221ed11737e2b32af3ccfebb673bd1122f9f9ea8d357efd91e9f3b8e8f72
+size 3704498

demo/voices/streaming_model/nl-Spk1_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76bfd3c13bb9d61760f6146c4eadc032e9d19ddecd4eed6832bc323a8cb5e8c5
+size 5095874

demo/voices/streaming_model/pl-Spk0_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c06a9e251ab08ade328dfe16e01eb6f5ddc4ba9c66695a73e3aae8a919396bd
+size 3750522

demo/voices/streaming_model/pl-Spk1_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a30782a49635f0e60247529b4557cd39713fd158ce2f99bea2ac506bffe7427
+size 4978626

demo/voices/streaming_model/pt-Spk0_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40f3a231e011f9c5e1eb4e852599751ce749439edceb36d0fa2bca952387f5a9
+size 2268290

demo/voices/streaming_model/pt-Spk1_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06a0052170a4ae5217931eb84318a6f86279439b536ad9780c724526d84148e5
+size 3554890

demo/voices/streaming_model/sp-Spk0_woman.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8b2c54cfe9b46a711642a96cdcc49545140374ae769cc62cf89c593c38b13c0
+size 4243906

demo/voices/streaming_model/sp-Spk1_man.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9857e3c03334b1934d34dd3aed8b36cb1e6a09367e99c9e164cd3cd8f091edcb
+size 5130522

demo/web/app.py

@@ -0,0 +1,507 @@
+import datetime
+import builtins
+import asyncio
+import json
+import os
+import threading
+import traceback
+from pathlib import Path
+from queue import Empty, Queue
+from typing import Any, Callable, Dict, Iterator, Optional, Tuple, cast
+
+import numpy as np
+import torch
+from fastapi import FastAPI, WebSocket
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from starlette.websockets import WebSocketDisconnect, WebSocketState
+
+from vibevoice.modular.modeling_vibevoice_streaming_inference import (
+    VibeVoiceStreamingForConditionalGenerationInference,
+)
+from vibevoice.processor.vibevoice_streaming_processor import (
+    VibeVoiceStreamingProcessor,
+)
+from vibevoice.modular.streamer import AudioStreamer
+
+import copy
+
+BASE = Path(__file__).parent
+SAMPLE_RATE = 24_000
+
+
+def get_timestamp():
+    timestamp = datetime.datetime.utcnow().replace(
+        tzinfo=datetime.timezone.utc
+    ).astimezone(
+        datetime.timezone(datetime.timedelta(hours=8))
+    ).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+    return timestamp
+
+class StreamingTTSService:
+    def __init__(
+        self,
+        model_path: str,
+        device: str = "cuda",
+        inference_steps: int = 5,
+    ) -> None:
+        # Keep model_path as string for HuggingFace repo IDs (Path() converts / to \ on Windows)
+        self.model_path = model_path
+        self.inference_steps = inference_steps
+        self.sample_rate = SAMPLE_RATE
+
+        self.processor: Optional[VibeVoiceStreamingProcessor] = None
+        self.model: Optional[VibeVoiceStreamingForConditionalGenerationInference] = None
+        self.voice_presets: Dict[str, Path] = {}
+        self.default_voice_key: Optional[str] = None
+        self._voice_cache: Dict[str, Tuple[object, Path, str]] = {}
+
+        if device == "mpx":
+            print("Note: device 'mpx' detected, treating it as 'mps'.")
+            device = "mps"        
+        if device == "mps" and not torch.backends.mps.is_available():
+            print("Warning: MPS not available. Falling back to CPU.")
+            device = "cpu"
+        self.device = device
+        self._torch_device = torch.device(device)
+
+    def load(self) -> None:
+        print(f"[startup] Loading processor from {self.model_path}")
+        self.processor = VibeVoiceStreamingProcessor.from_pretrained(self.model_path)
+
+        
+        # Decide dtype & attention
+        if self.device == "mps":
+            load_dtype = torch.float32
+            device_map = None
+            attn_impl_primary = "sdpa"
+        elif self.device == "cuda":
+            load_dtype = torch.bfloat16
+            device_map = 'cuda'
+            attn_impl_primary = "flash_attention_2"
+        else:
+            load_dtype = torch.float32
+            device_map = 'cpu'
+            attn_impl_primary = "sdpa"
+        print(f"Using device: {device_map}, torch_dtype: {load_dtype}, attn_implementation: {attn_impl_primary}")
+        # Load model
+        try:
+            self.model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                self.model_path,
+                torch_dtype=load_dtype,
+                device_map=device_map,
+                attn_implementation=attn_impl_primary,
+            )
+            
+            if self.device == "mps":
+                self.model.to("mps")
+        except Exception as e:
+            if attn_impl_primary == 'flash_attention_2':
+                print("Error loading the model. Trying to use SDPA. However, note that only flash_attention_2 has been fully tested, and using SDPA may result in lower audio quality.")
+                
+                self.model = VibeVoiceStreamingForConditionalGenerationInference.from_pretrained(
+                    self.model_path,
+                    torch_dtype=load_dtype,
+                    device_map=self.device,
+                    attn_implementation='sdpa',
+                )
+                print("Load model with SDPA successfully ")
+            else:
+                raise e
+
+        self.model.eval()
+
+        self.model.model.noise_scheduler = self.model.model.noise_scheduler.from_config(
+            self.model.model.noise_scheduler.config,
+            algorithm_type="sde-dpmsolver++",
+            beta_schedule="squaredcos_cap_v2",
+        )
+        self.model.set_ddpm_inference_steps(num_steps=self.inference_steps)
+
+        self.voice_presets = self._load_voice_presets()
+        preset_name = os.environ.get("VOICE_PRESET")
+        self.default_voice_key = self._determine_voice_key(preset_name)
+        self._ensure_voice_cached(self.default_voice_key)
+
+    def _load_voice_presets(self) -> Dict[str, Path]:
+        voices_dir = BASE.parent / "voices" / "streaming_model"
+        if not voices_dir.exists():
+            raise RuntimeError(f"Voices directory not found: {voices_dir}")
+
+        presets: Dict[str, Path] = {}
+        for pt_path in voices_dir.glob("*.pt"):
+            presets[pt_path.stem] = pt_path
+
+        if not presets:
+            raise RuntimeError(f"No voice preset (.pt) files found in {voices_dir}")
+
+        print(f"[startup] Found {len(presets)} voice presets")
+        return dict(sorted(presets.items()))
+
+    def _determine_voice_key(self, name: Optional[str]) -> str:
+        if name and name in self.voice_presets:
+            return name
+
+        default_key = "en-WHTest_man"
+        if default_key in self.voice_presets:
+            return default_key
+
+        first_key = next(iter(self.voice_presets))
+        print(f"[startup] Using fallback voice preset: {first_key}")
+        return first_key
+
+    def _ensure_voice_cached(self, key: str) -> Tuple[object, Path, str]:
+        if key not in self.voice_presets:
+            raise RuntimeError(f"Voice preset {key!r} not found")
+
+        if key not in self._voice_cache:
+            preset_path = self.voice_presets[key]
+            print(f"[startup] Loading voice preset {key} from {preset_path}")
+            print(f"[startup] Loading prefilled prompt from {preset_path}")
+            prefilled_outputs = torch.load(
+                preset_path,
+                map_location=self._torch_device,
+                weights_only=False,
+            )
+            self._voice_cache[key] = prefilled_outputs
+
+        return self._voice_cache[key]
+
+    def _get_voice_resources(self, requested_key: Optional[str]) -> Tuple[str, object, Path, str]:
+        key = requested_key if requested_key and requested_key in self.voice_presets else self.default_voice_key
+        if key is None:
+            key = next(iter(self.voice_presets))
+            self.default_voice_key = key
+
+        prefilled_outputs = self._ensure_voice_cached(key)
+        return key, prefilled_outputs
+
+    def _prepare_inputs(self, text: str, prefilled_outputs: object):
+        if not self.processor or not self.model:
+            raise RuntimeError("StreamingTTSService not initialized")
+
+        processor_kwargs = {
+            "text": text.strip(),
+            "cached_prompt": prefilled_outputs,
+            "padding": True,
+            "return_tensors": "pt",
+            "return_attention_mask": True,
+        }
+
+        processed = self.processor.process_input_with_cached_prompt(**processor_kwargs)
+
+        prepared = {
+            key: value.to(self._torch_device) if hasattr(value, "to") else value
+            for key, value in processed.items()
+        }
+        return prepared
+
+    def _run_generation(
+        self,
+        inputs,
+        audio_streamer: AudioStreamer,
+        errors,
+        cfg_scale: float,
+        do_sample: bool,
+        temperature: float,
+        top_p: float,
+        refresh_negative: bool,
+        prefilled_outputs,
+        stop_event: threading.Event,
+    ) -> None:
+        try:
+            self.model.generate(
+                **inputs,
+                max_new_tokens=None,
+                cfg_scale=cfg_scale,
+                tokenizer=self.processor.tokenizer,
+                generation_config={
+                    "do_sample": do_sample,
+                    "temperature": temperature if do_sample else 1.0,
+                    "top_p": top_p if do_sample else 1.0,
+                },
+                audio_streamer=audio_streamer,
+                stop_check_fn=stop_event.is_set,
+                verbose=False,
+                refresh_negative=refresh_negative,
+                all_prefilled_outputs=copy.deepcopy(prefilled_outputs),
+            )
+        except Exception as exc:  # pragma: no cover - diagnostic logging
+            errors.append(exc)
+            traceback.print_exc()
+            audio_streamer.end()
+
+    def stream(
+        self,
+        text: str,
+        cfg_scale: float = 1.5,
+        do_sample: bool = False,
+        temperature: float = 0.9,
+        top_p: float = 0.9,
+        refresh_negative: bool = True,
+        inference_steps: Optional[int] = None,
+        voice_key: Optional[str] = None,
+        log_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None,
+        stop_event: Optional[threading.Event] = None,
+    ) -> Iterator[np.ndarray]:
+        if not text.strip():
+            return
+        text = text.replace("’", "'")
+        selected_voice, prefilled_outputs = self._get_voice_resources(voice_key)
+
+        def emit(event: str, **payload: Any) -> None:
+            if log_callback:
+                try:
+                    log_callback(event, **payload)
+                except Exception as exc:
+                    print(f"[log_callback] Error while emitting {event}: {exc}")
+
+        steps_to_use = self.inference_steps
+        if inference_steps is not None:
+            try:
+                parsed_steps = int(inference_steps)
+                if parsed_steps > 0:
+                    steps_to_use = parsed_steps
+            except (TypeError, ValueError):
+                pass
+        if self.model:
+            self.model.set_ddpm_inference_steps(num_steps=steps_to_use)
+        self.inference_steps = steps_to_use
+
+        inputs = self._prepare_inputs(text, prefilled_outputs)
+        audio_streamer = AudioStreamer(batch_size=1, stop_signal=None, timeout=None)
+        errors: list = []
+        stop_signal = stop_event or threading.Event()
+
+        thread = threading.Thread(
+            target=self._run_generation,
+            kwargs={
+                "inputs": inputs,
+                "audio_streamer": audio_streamer,
+                "errors": errors,
+                "cfg_scale": cfg_scale,
+                "do_sample": do_sample,
+                "temperature": temperature,
+                "top_p": top_p,
+                "refresh_negative": refresh_negative,
+                "prefilled_outputs": prefilled_outputs,
+                "stop_event": stop_signal,
+            },
+            daemon=True,
+        )
+        thread.start()
+
+        generated_samples = 0
+
+        try:
+            stream = audio_streamer.get_stream(0)
+            for audio_chunk in stream:
+                if torch.is_tensor(audio_chunk):
+                    audio_chunk = audio_chunk.detach().cpu().to(torch.float32).numpy()
+                else:
+                    audio_chunk = np.asarray(audio_chunk, dtype=np.float32)
+
+                if audio_chunk.ndim > 1:
+                    audio_chunk = audio_chunk.reshape(-1)
+
+                peak = np.max(np.abs(audio_chunk)) if audio_chunk.size else 0.0
+                if peak > 1.0:
+                    audio_chunk = audio_chunk / peak
+
+                generated_samples += int(audio_chunk.size)
+                emit(
+                    "model_progress",
+                    generated_sec=generated_samples / self.sample_rate,
+                    chunk_sec=audio_chunk.size / self.sample_rate,
+                )
+
+                chunk_to_yield = audio_chunk.astype(np.float32, copy=False)
+
+                yield chunk_to_yield
+        finally:
+            stop_signal.set()
+            audio_streamer.end()
+            thread.join()
+            if errors:
+                emit("generation_error", message=str(errors[0]))
+                raise errors[0]
+
+    def chunk_to_pcm16(self, chunk: np.ndarray) -> bytes:
+        chunk = np.clip(chunk, -1.0, 1.0)
+        pcm = (chunk * 32767.0).astype(np.int16)
+        return pcm.tobytes()
+
+
+app = FastAPI()
+
+
+@app.on_event("startup")
+async def _startup() -> None:
+    model_path = os.environ.get("MODEL_PATH")
+    if not model_path:
+        raise RuntimeError("MODEL_PATH not set in environment")
+
+    device = os.environ.get("MODEL_DEVICE", "cuda")
+    
+    service = StreamingTTSService(
+        model_path=model_path,
+        device=device
+    )
+    service.load()
+
+    app.state.tts_service = service
+    app.state.model_path = model_path
+    app.state.device = device
+    app.state.websocket_lock = asyncio.Lock()
+    print("[startup] Model ready.")
+
+
+def streaming_tts(text: str, **kwargs) -> Iterator[np.ndarray]:
+    service: StreamingTTSService = app.state.tts_service
+    yield from service.stream(text, **kwargs)
+
+@app.websocket("/stream")
+async def websocket_stream(ws: WebSocket) -> None:
+    await ws.accept()
+    text = ws.query_params.get("text", "")
+    print(f"Client connected, text={text!r}")
+    cfg_param = ws.query_params.get("cfg")
+    steps_param = ws.query_params.get("steps")
+    voice_param = ws.query_params.get("voice")
+
+    try:
+        cfg_scale = float(cfg_param) if cfg_param is not None else 1.5
+    except ValueError:
+        cfg_scale = 1.5
+    if cfg_scale <= 0:
+        cfg_scale = 1.5
+    try:
+        inference_steps = int(steps_param) if steps_param is not None else None
+        if inference_steps is not None and inference_steps <= 0:
+            inference_steps = None
+    except ValueError:
+        inference_steps = None
+
+    service: StreamingTTSService = app.state.tts_service
+    lock: asyncio.Lock = app.state.websocket_lock
+
+    if lock.locked():
+        busy_message = {
+            "type": "log",
+            "event": "backend_busy",
+            "data": {"message": "Please wait for the other requests to complete."},
+            "timestamp": get_timestamp(),
+        }
+        print("Please wait for the other requests to complete.")
+        try:
+            await ws.send_text(json.dumps(busy_message))
+        except Exception:
+            pass
+        await ws.close(code=1013, reason="Service busy")
+        return
+
+    acquired = False
+    try:
+        await lock.acquire()
+        acquired = True
+
+        log_queue: "Queue[Dict[str, Any]]" = Queue()
+
+        def enqueue_log(event: str, **data: Any) -> None:
+            log_queue.put({"event": event, "data": data})
+
+        async def flush_logs() -> None:
+            while True:
+                try:
+                    entry = log_queue.get_nowait()
+                except Empty:
+                    break
+                message = {
+                    "type": "log",
+                    "event": entry.get("event"),
+                    "data": entry.get("data", {}),
+                    "timestamp": get_timestamp(),
+                }
+                try:
+                    await ws.send_text(json.dumps(message))
+                except Exception:
+                    break
+
+        enqueue_log(
+            "backend_request_received",
+            text_length=len(text or ""),
+            cfg_scale=cfg_scale,
+            inference_steps=inference_steps,
+            voice=voice_param,
+        )
+
+        stop_signal = threading.Event()
+
+        iterator = streaming_tts(
+            text,
+            cfg_scale=cfg_scale,
+            inference_steps=inference_steps,
+            voice_key=voice_param,
+            log_callback=enqueue_log,
+            stop_event=stop_signal,
+        )
+        sentinel = object()
+        first_ws_send_logged = False
+
+        await flush_logs()
+
+        try:
+            while ws.client_state == WebSocketState.CONNECTED:
+                await flush_logs()
+                chunk = await asyncio.to_thread(next, iterator, sentinel)
+                if chunk is sentinel:
+                    break
+                chunk = cast(np.ndarray, chunk)
+                payload = service.chunk_to_pcm16(chunk)
+                await ws.send_bytes(payload)
+                if not first_ws_send_logged:
+                    first_ws_send_logged = True
+                    enqueue_log("backend_first_chunk_sent")
+                await flush_logs()
+        except WebSocketDisconnect:
+            print("Client disconnected (WebSocketDisconnect)")
+            enqueue_log("client_disconnected")
+            stop_signal.set()
+        finally:
+            stop_signal.set()
+            enqueue_log("backend_stream_complete")
+            await flush_logs()
+            try:
+                iterator_close = getattr(iterator, "close", None)
+                if callable(iterator_close):
+                    iterator_close()
+            except Exception:
+                pass
+            # clear the log queue
+            while not log_queue.empty():
+                try:
+                    log_queue.get_nowait()
+                except Empty:
+                    break
+            if ws.client_state == WebSocketState.CONNECTED:
+                await ws.close()
+            print("WS handler exit")
+    finally:
+        if acquired:
+            lock.release()
+
+
+@app.get("/")
+def index():
+    return FileResponse(BASE / "index.html")
+
+
+@app.get("/config")
+def get_config():
+    service: StreamingTTSService = app.state.tts_service
+    voices = sorted(service.voice_presets.keys())
+    return {
+        "voices": voices,
+        "default_voice": service.default_voice_key,
+    }
+

demo/web/index.html

@@ -0,0 +1,1017 @@
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="UTF-8" />
+<title>VibeVoice-Realtime TTS Demo</title>
+<style>
+  :root {
+    --bg: #f5f7fc;
+    --surface: #ffffff;
+    --accent: #5562ff;
+    --accent-strong: #3f4dff;
+    --text-primary: #1f2742;
+    --text-muted: #5d6789;
+    --border: rgba(85, 98, 255, 0.18);
+    --shadow: 0 18px 45px rgba(31, 39, 66, 0.08);
+  }
+
+  .helper-text {
+    font-size: 12px;
+    color: #8a93b5;
+  }
+
+  * {
+    box-sizing: border-box;
+  }
+
+  body {
+    margin: 0;
+    background: var(--bg);
+    font-family: 'Inter', 'Segoe UI', Roboto, Helvetica, sans-serif;
+    color: var(--text-primary);
+    display: flex;
+    justify-content: center;
+    padding: 48px 20px;
+  }
+
+  .app-shell {
+    width: min(960px, 100%);
+    background: var(--surface);
+    border-radius: 20px;
+    padding: 36px 40px 44px;
+    box-shadow: var(--shadow);
+    display: flex;
+    flex-direction: column;
+    gap: 28px;
+  }
+
+  h1 {
+    margin: 0;
+    text-align: center;
+    font-size: 30px;
+    font-weight: 700;
+    letter-spacing: 0.01em;
+  }
+
+  .panel {
+    display: flex;
+    flex-direction: column;
+    gap: 10px;
+  }
+
+  .field {
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+  }
+
+  .field-label {
+    font-weight: 600;
+    font-size: 15px;
+    color: var(--text-primary);
+  }
+
+  .text-input {
+    width: 100%;
+    min-height: 140px;
+    max-height: 240px;
+    border: 1px solid rgba(31, 39, 66, 0.14);
+    border-radius: 12px;
+    padding: 14px 16px;
+    font-size: 15px;
+    line-height: 1.6;
+    font-family: inherit;
+    background: #f9faff;
+    transition: border-color 0.2s, box-shadow 0.2s;
+    resize: vertical;
+  }
+
+  .text-input:focus {
+    outline: none;
+    border-color: var(--accent);
+    box-shadow: 0 0 0 3px rgba(85, 98, 255, 0.18);
+    background: #fff;
+  }
+
+  #streamingPreviewContainer {
+    border-radius: 14px;
+    border: 1px solid var(--border);
+    background: linear-gradient(135deg, #eef2ff 0%, #f7f9ff 100%);
+    padding: 18px 20px;
+    box-shadow: inset 0 1px 2px rgba(85, 98, 255, 0.12);
+  }
+
+  #streamingPreviewHeader {
+    font-weight: 600;
+    color: var(--text-primary);
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    font-size: 14px;
+    margin-bottom: 8px;
+  }
+
+  #streamingPreviewNote {
+    font-weight: 400;
+    font-size: 12px;
+    color: var(--text-muted);
+  }
+
+  #streamingPreview {
+    min-height: 70px;
+    padding: 10px 12px;
+    border-radius: 10px;
+    background: rgba(255, 255, 255, 0.9);
+    border: 1px solid rgba(85, 98, 255, 0.25);
+    font-family: 'Courier New', Courier, monospace;
+    font-size: 14px;
+    line-height: 1.5;
+    color: var(--text-primary);
+    white-space: pre-wrap;
+  }
+
+  #streamingPreview.streaming-active::after {
+    content: "";
+    display: inline-block;
+    width: 2px;
+    height: 1.1em;
+    background: var(--accent);
+    margin-left: 2px;
+    animation: previewCaret 0.9s steps(1) infinite;
+    vertical-align: bottom;
+  }
+
+  @keyframes previewCaret {
+    0%, 50% {
+      opacity: 1;
+    }
+    51%, 100% {
+      opacity: 0;
+    }
+  }
+
+  .control-panel {
+    display: flex;
+    flex-direction: column;
+    gap: 18px;
+  }
+
+  .inline-field {
+    display: flex;
+    flex-direction: column;
+    gap: 6px;
+  }
+
+  .select-control {
+    width: 220px;
+    border: 1px solid rgba(31, 39, 66, 0.14);
+    border-radius: 10px;
+    padding: 8px 12px;
+    font-size: 14px;
+    font-family: inherit;
+    background: #fbfcff;
+    color: var(--text-primary);
+    transition: border-color 0.2s, box-shadow 0.2s;
+  }
+
+  .select-control:focus {
+    outline: none;
+    border-color: var(--accent);
+    box-shadow: 0 0 0 3px rgba(85, 98, 255, 0.18);
+    background: #fff;
+  }
+
+  .control-row {
+    display: flex;
+    align-items: center;
+    flex-wrap: wrap;
+    gap: 20px 28px;
+  }
+
+  .range-control {
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    font-size: 14px;
+    color: var(--text-primary);
+  }
+
+  .range-control input[type="range"] {
+    width: 200px;
+    accent-color: var(--accent);
+  }
+
+  .range-value {
+    font-weight: 600;
+    color: var(--text-primary);
+    min-width: 42px;
+    text-align: right;
+  }
+
+  #playback {
+    background: var(--accent);
+    color: #fff;
+    border: none;
+    padding: 10px 24px;
+    border-radius: 999px;
+    cursor: pointer;
+    font-weight: 600;
+    font-size: 14px;
+    box-shadow: 0 8px 16px rgba(85, 98, 255, 0.25);
+    transition: transform 0.15s, box-shadow 0.15s, background 0.15s;
+  }
+
+  #playback:hover {
+    transform: translateY(-1px);
+    box-shadow: 0 10px 20px rgba(85, 98, 255, 0.28);
+  }
+
+  #playback:active {
+    transform: translateY(0);
+  }
+
+  #playback.playing {
+    background: var(--accent-strong);
+  }
+
+  .secondary-btn {
+    border: 1px solid rgba(31, 39, 66, 0.18);
+    background: #f1f3ff;
+    color: var(--text-primary);
+    padding: 8px 18px;
+    border-radius: 999px;
+    cursor: pointer;
+    font-size: 13px;
+    font-weight: 500;
+    transition: background 0.15s, border-color 0.15s;
+  }
+
+  .secondary-btn:hover {
+    background: #e6e9ff;
+    border-color: rgba(31, 39, 66, 0.26);
+  }
+
+  .secondary-btn:disabled {
+    opacity: 0.55;
+    cursor: not-allowed;
+  }
+
+  .metrics {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 16px 32px;
+    font-size: 14px;
+    color: var(--text-muted);
+  }
+
+  .metrics span {
+    display: flex;
+    align-items: baseline;
+    gap: 6px;
+  }
+
+  .metrics span strong {
+    color: var(--text-primary);
+    font-weight: 600;
+  }
+
+  .metric-unit {
+    color: var(--text-muted);
+    font-size: 13px;
+  }
+
+  #logOutput {
+    max-height: 260px;
+    overflow-y: auto;
+    background: #f7f9ff;
+    color: var(--text-primary);
+    padding: 16px 18px;
+    border: 1px solid rgba(31, 39, 66, 0.12);
+    border-radius: 12px;
+    font-size: 13px;
+    line-height: 1.6;
+    box-shadow: inset 0 1px 2px rgba(15, 23, 42, 0.06);
+    font-family: 'Fira Code', 'Courier New', Courier, monospace;
+    margin-top: 0px;
+  }
+
+  @media (max-width: 720px) {
+    .app-shell {
+      padding: 28px 20px 36px;
+      gap: 24px;
+    }
+
+    .select-control {
+      width: 100%;
+    }
+
+    .control-row {
+      flex-direction: column;
+      align-items: flex-start;
+      gap: 16px;
+    }
+
+    #playback {
+      width: 100%;
+      text-align: center;
+    }
+  }
+</style>
+<body>
+  <div class="app-shell">
+    <h1>VibeVoice-Realtime TTS Demo</h1>
+
+    <section class="panel">
+      <label class="field">
+        <span class="field-label">Text</span>
+        <textarea
+          id="prompt"
+          class="text-input"
+          rows="4"
+        >Enter your text here and click "Start" to instantly hear the VibeVoice-Realtime TTS output audio.</textarea>
+      </label>
+
+      <div id="streamingPreviewContainer">
+        <div id="streamingPreviewHeader">
+          <span>Streaming Input Text</span>
+        </div>
+        <div id="streamingPreview" aria-live="polite">This area will display the streaming input text in real time.</div>
+      </div>
+    </section>
+          <span class="helper-text">This demo requires the full text to be provided upfront. The model then receives the text via streaming input during synthesis.<br>
+            For non-punctuation special characters, applying text normalization before processing often yields better results.</span>
+          
+    <section class="panel control-panel">
+      <div class="inline-field">
+        <span class="field-label">Speaker</span>
+        <select id="voiceSelect" class="select-control">
+          <option value="">Loading...</option>
+        </select>
+      </div>
+
+      <div class="control-row">
+        <label class="range-control">
+          <span>CFG</span>
+          <input id="cfgScale" type="range" min="1.3" max="3" step="0.05" value="1.5" />
+          <span class="range-value" id="cfgValue">1.5</span>
+        </label>
+        <label class="range-control">
+          <span>Inference Steps</span>
+          <input id="inferenceSteps" type="range" min="5" max="20" step="1" value="5" />
+          <span class="range-value" id="stepsValue">5</span>
+        </label>
+        <button id="resetControls" type="button" class="secondary-btn">Reset Controls</button>
+      </div>
+
+      <div class="control-row">
+        <button id="playback">Start</button>
+        <button id="saveAudio" type="button" class="secondary-btn" disabled>Save</button>
+      </div>
+    </section>
+
+    <section class="panel">
+      <div class="metrics">
+        <span>Model Generated Audio<strong id="modelGenerated">0.00</strong><span class="metric-unit">s</span></span>
+        <span>Audio Played<strong id="playbackElapsed">0.00</strong><span class="metric-unit">s</span></span>
+      </div>
+    </section>
+
+    <section class="panel">
+      <span class="field-label">Runtime Logs</span>
+      <pre id="logOutput"></pre>
+    </section>
+  </div>
+
+
+<script>
+(() => {
+  const SAMPLE_RATE = 24_000;
+  const BUFFER_SIZE = 2048;
+  const PREBUFFER_SEC = 0.1;
+
+  let audioCtx = null;
+  let scriptNode = null;
+  let socket = null;
+  let buffer = new Float32Array(0);
+  let isPlaying = false;
+  let hasStartedPlayback = false;
+  let silentFrameCount = 0;
+
+  const promptInput = document.getElementById('prompt');
+  const streamingPreview = document.getElementById('streamingPreview');
+  const controlBtn = document.getElementById('playback');
+  const cfgSelect = document.getElementById('cfgScale');
+  const stepsSelect = document.getElementById('inferenceSteps');
+  const voiceSelect = document.getElementById('voiceSelect');
+  const cfgValueLabel = document.getElementById('cfgValue');
+  const stepsValueLabel = document.getElementById('stepsValue');
+  const modelGeneratedLabel = document.getElementById('modelGenerated');
+  const playbackElapsedLabel = document.getElementById('playbackElapsed');
+  const logOutput = document.getElementById('logOutput');
+  const resetBtn = document.getElementById('resetControls');
+  const saveBtn = document.getElementById('saveAudio');
+
+  let playbackTimer = null;
+  let lastPlaybackElapsed = 0;
+  let playbackSamples = 0;
+  let modelGeneratedTotal = 0;
+  let firstBrowserChunkLogged = false;
+  let playbackStartedLogged = false;
+  const logEntries = [];
+  let logSequence = 0;
+  let recordedChunks = [];
+  let recordedSamples = 0;
+  let recordingComplete = false;
+  let downloadUrl = null;
+
+  const revokeDownloadUrl = () => {
+    if (downloadUrl) {
+      URL.revokeObjectURL(downloadUrl);
+      downloadUrl = null;
+    }
+  };
+
+  const updateSaveButtonState = () => {
+    if (!saveBtn) {
+      return;
+    }
+    saveBtn.disabled = recordedSamples === 0 || !recordingComplete;
+  };
+
+  const clearRecordedChunks = () => {
+    recordedChunks = [];
+    recordedSamples = 0;
+    recordingComplete = false;
+    revokeDownloadUrl();
+    updateSaveButtonState();
+  };
+
+  const createWavBlob = () => {
+    if (!recordedSamples) {
+      return null;
+    }
+    const wavBuffer = new ArrayBuffer(44 + recordedSamples * 2);
+    const view = new DataView(wavBuffer);
+    const writeString = (offset, str) => {
+      for (let i = 0; i < str.length; i += 1) {
+        view.setUint8(offset + i, str.charCodeAt(i));
+      }
+    };
+
+    writeString(0, 'RIFF');
+    view.setUint32(4, 36 + recordedSamples * 2, true);
+    writeString(8, 'WAVE');
+    writeString(12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, 1, true);
+    view.setUint32(24, SAMPLE_RATE, true);
+    view.setUint32(28, SAMPLE_RATE * 2, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    writeString(36, 'data');
+    view.setUint32(40, recordedSamples * 2, true);
+
+    const pcmData = new Int16Array(wavBuffer, 44, recordedSamples);
+    let offset = 0;
+    recordedChunks.forEach(chunk => {
+      const chunkData = new Int16Array(chunk);
+      pcmData.set(chunkData, offset);
+      offset += chunkData.length;
+    });
+    return new Blob([wavBuffer], { type: 'audio/wav' });
+  };
+
+  const updateCfgDisplay = () => {
+    cfgValueLabel.textContent = Number(cfgSelect.value).toFixed(2);
+  };
+
+  const updateStepsDisplay = () => {
+    stepsValueLabel.textContent = Number(stepsSelect.value).toString();
+  };
+
+  cfgSelect.addEventListener('input', updateCfgDisplay);
+  stepsSelect.addEventListener('input', updateStepsDisplay);
+  updateCfgDisplay();
+  updateStepsDisplay();
+
+  const pad2 = value => value.toString().padStart(2, '0');
+  const pad3 = value => value.toString().padStart(3, '0');
+
+  const formatLocalTimestamp = () => {
+    const d = new Date();
+    const year = d.getFullYear();
+    const month = pad2(d.getMonth() + 1);
+    const day = pad2(d.getDate());
+    const hours = pad2(d.getHours());
+    const minutes = pad2(d.getMinutes());
+    const seconds = pad2(d.getSeconds());
+    const millis = pad3(d.getMilliseconds());
+    return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${millis}`;
+  };
+
+  const formatSeconds = raw => {
+    const value = Number(raw);
+    return Number.isFinite(value) ? value.toFixed(2) : '0.00';
+  };
+
+  const parseTimestamp = value => {
+    if (!value) {
+      return new Date();
+    }
+    if (/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/.test(value)) {
+      return new Date(value.replace(' ', 'T'));
+    }
+    return new Date(value);
+  };
+
+  const setModelGenerated = value => {
+    const numeric = Number(value);
+    if (!Number.isFinite(numeric)) {
+      return;
+    }
+    modelGeneratedTotal = Math.max(0, numeric);
+    modelGeneratedLabel.textContent = formatSeconds(modelGeneratedTotal);
+  };
+
+  const setPlaybackElapsed = value => {
+    const capped = Math.min(modelGeneratedTotal, Math.max(0, value));
+    lastPlaybackElapsed = capped;
+    playbackElapsedLabel.textContent = formatSeconds(lastPlaybackElapsed);
+  };
+
+  const STREAMING_WPM = 180;
+  const STREAMING_INTERVAL_MS = 60000 / STREAMING_WPM;
+  let previewTimeoutId = null;
+  let previewTokens = [];
+  let previewIndex = 0;
+  let previewActive = false;
+
+  const clearPreviewTimer = () => {
+    if (previewTimeoutId) {
+      clearTimeout(previewTimeoutId);
+      previewTimeoutId = null;
+    }
+  };
+
+  const setPreviewIdle = message => {
+    if (!streamingPreview) {
+      return;
+    }
+    streamingPreview.classList.remove('streaming-active');
+    streamingPreview.textContent = message;
+  };
+
+  const schedulePreviewTick = () => {
+    if (!streamingPreview) {
+      return;
+    }
+    if (previewIndex >= previewTokens.length) {
+      streamingPreview.classList.remove('streaming-active');
+      return;
+    }
+
+    streamingPreview.classList.add('streaming-active');
+
+    streamingPreview.textContent += previewTokens[previewIndex];
+    previewIndex += 1;
+    previewTimeoutId = setTimeout(schedulePreviewTick, STREAMING_INTERVAL_MS);
+  };
+
+  const updateStreamingPreview = () => {
+    if (!streamingPreview) {
+      return;
+    }
+    clearPreviewTimer();
+    previewIndex = 0;
+    const source = (promptInput?.value || '').trimEnd();
+    streamingPreview.textContent = '';
+    previewTokens = source.match(/\S+\s*/g) || [];
+    schedulePreviewTick();
+  };
+
+  const clearLogs = () => {
+    if (logOutput) {
+      logOutput.textContent = '';
+    }
+    logEntries.length = 0;
+    modelGeneratedTotal = 0;
+    setModelGenerated(0);
+  };
+
+  const appendLog = (message, timestamp) => {
+    if (!logOutput) {
+      return;
+    }
+    const finalTimestamp = timestamp || formatLocalTimestamp();
+    const entry = {
+      timestamp: finalTimestamp,
+      date: parseTimestamp(finalTimestamp),
+      message,
+      seq: logSequence += 1,
+    };
+    logEntries.push(entry);
+    logEntries.sort((a, b) => {
+      const diff = a.date.getTime() - b.date.getTime();
+      return diff !== 0 ? diff : a.seq - b.seq;
+    });
+    if (logEntries.length > 400) {
+      logEntries.splice(0, logEntries.length - 400);
+    }
+    logOutput.textContent = logEntries
+      .map(item => `[${item.timestamp}] ${item.message}`)
+      .join('\n');
+    logOutput.scrollTop = logOutput.scrollHeight;
+  };
+
+  const handleSaveClick = () => {
+    if (!recordedSamples) {
+      appendLog('[Frontend] Save requested but no audio received yet');
+      return;
+    }
+    const wavBlob = createWavBlob();
+    if (!wavBlob) {
+      appendLog('[Error] Failed to assemble WAV data for download');
+      return;
+    }
+    revokeDownloadUrl();
+    downloadUrl = URL.createObjectURL(wavBlob);
+    const link = document.createElement('a');
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    link.href = downloadUrl;
+    link.download = `vibevoice_realtime_audio_${timestamp}.wav`;
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+    appendLog('[Frontend] Audio download triggered');
+  };
+
+  const stopPlaybackTimer = () => {
+    if (playbackTimer) {
+      clearInterval(playbackTimer);
+      playbackTimer = null;
+    }
+  };
+
+  const startPlaybackTimer = () => {
+    stopPlaybackTimer();
+    playbackTimer = setInterval(() => {
+      setPlaybackElapsed(playbackSamples / SAMPLE_RATE);
+    }, 250);
+  };
+
+  const loadVoices = async () => {
+    try {
+      voiceSelect.disabled = true;
+      const response = await fetch('/config');
+      if (!response.ok) {
+        throw new Error(`Failed to fetch config: ${response.status}`);
+      }
+      const data = await response.json();
+      const voices = Array.isArray(data.voices) ? data.voices : [];
+      voiceSelect.innerHTML = '';
+      if (voices.length === 0) {
+        const option = document.createElement('option');
+        option.value = '';
+        option.textContent = 'No voices available';
+        voiceSelect.appendChild(option);
+        voiceSelect.disabled = true;
+        appendLog('[Error] No voice presets available');
+        return;
+      }
+
+      voices.forEach(voice => {
+        const option = document.createElement('option');
+        option.value = voice;
+        option.textContent = voice;
+        voiceSelect.appendChild(option);
+      });
+
+      if (data.default_voice && voices.includes(data.default_voice)) {
+        voiceSelect.value = data.default_voice;
+      }
+      voiceSelect.disabled = false;
+      appendLog(`[Frontend] Loaded ${voices.length} voice presets`);
+    } catch (err) {
+      console.error('Failed to load voices', err);
+      voiceSelect.innerHTML = '';
+      const option = document.createElement('option');
+      option.value = '';
+      option.textContent = 'Load failed';
+      voiceSelect.appendChild(option);
+      voiceSelect.disabled = true;
+      appendLog('[Error] Failed to load voice presets');
+    }
+  };
+
+  loadVoices();
+
+  resetBtn.addEventListener('click', () => {
+    cfgSelect.value = '1.5';
+    stepsSelect.value = '5';
+    updateCfgDisplay();
+    updateStepsDisplay();
+    appendLog('[Frontend] Controls reset to defaults (CFG=1.5, Steps=5)');
+  });
+
+  if (promptInput) {
+    promptInput.addEventListener('input', () => {
+      if (previewActive) {
+        updateStreamingPreview();
+      }
+    });
+  }
+
+  const handleLogMessage = raw => {
+    let payload;
+    try {
+      payload = JSON.parse(raw);
+    } catch (err) {
+      appendLog(`[Error] Failed to parse log message: ${raw}`);
+      return;
+    }
+    if (!payload || payload.type !== 'log') {
+      appendLog(`[Log] ${raw}`);
+      return;
+    }
+
+    const { event, data = {}, timestamp } = payload;
+    switch (event) {
+      case 'backend_request_received': {
+        const cfg = typeof data.cfg_scale === 'number' ? data.cfg_scale.toFixed(3) : data.cfg_scale;
+        const steps = data.inference_steps ?? 'default';
+        const voice = data.voice || 'default';
+        const textLength = data.text_length ?? 0;
+        appendLog(`[Backend]  Received request`, timestamp);
+        break;
+      }
+      case 'backend_first_chunk_sent':
+        appendLog('[Backend]  Sent first audio chunk', timestamp);
+        break;
+      case 'model_progress':
+        if (typeof data.generated_sec !== 'undefined') {
+          const generated = Number(data.generated_sec);
+          if (Number.isFinite(generated)) {
+            setModelGenerated(generated);
+          }
+        }
+        return;
+      case 'generation_error':
+        appendLog(`[Error] Generation error: ${data.message || 'Unknown error'}`, timestamp);
+        break;
+      case 'backend_error':
+        appendLog(`[Error] Backend error: ${data.message || 'Unknown error'}`, timestamp);
+        break;
+      case 'client_disconnected':
+        appendLog('[Frontend] Client disconnected', timestamp);
+        break;
+      case 'backend_stream_complete':
+        appendLog('[Backend]  Backend finished', timestamp);
+        recordingComplete = true;
+        updateSaveButtonState();
+        break;
+      default:
+        appendLog(`[Log] Event ${event}`, timestamp);
+        break;
+    }
+  };
+
+  const updateButtonLabel = () => {
+    controlBtn.textContent = isPlaying ? 'Stop' : 'Start';
+    controlBtn.classList.toggle('playing', isPlaying);
+  };
+
+  const appendAudio = chunk => {
+    const merged = new Float32Array(buffer.length + chunk.length);
+    merged.set(buffer, 0);
+    merged.set(chunk, buffer.length);
+    buffer = merged;
+  };
+
+  const pullAudio = frameCount => {
+    const available = buffer.length;
+    if (available === 0) {
+      return new Float32Array(frameCount);
+    }
+    if (available <= frameCount) {
+      const chunk = buffer;
+      buffer = new Float32Array(0);
+      if (chunk.length < frameCount) {
+        const padded = new Float32Array(frameCount);
+        padded.set(chunk, 0);
+        return padded;
+      }
+      return chunk;
+    }
+    const chunk = buffer.subarray(0, frameCount);
+    buffer = buffer.subarray(frameCount);
+    return chunk;
+  };
+
+  const closeSocket = () => {
+    if (socket && (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING)) {
+      socket.close();
+    }
+    socket = null;
+  };
+
+  const resetPlaybackFlags = (resetSamples = true) => {
+    buffer = new Float32Array(0);
+    if (resetSamples) {
+      playbackSamples = 0;
+      setPlaybackElapsed(0);
+    }
+    hasStartedPlayback = false;
+    silentFrameCount = 0;
+    firstBrowserChunkLogged = false;
+    playbackStartedLogged = false;
+  };
+
+  const teardownAudio = () => {
+    if (scriptNode) {
+      try { scriptNode.disconnect(); } catch (err) { console.warn('disconnect error', err); }
+      scriptNode.onaudioprocess = null;
+    }
+    if (audioCtx) {
+      try { audioCtx.close(); } catch (err) { console.warn('audioCtx.close error', err); }
+    }
+    audioCtx = null;
+    scriptNode = null;
+  };
+
+  const resetState = (resetSamples = true) => {
+    closeSocket();
+    teardownAudio();
+    resetPlaybackFlags(resetSamples);
+    isPlaying = false;
+    stopPlaybackTimer();
+  };
+
+  const createAudioChain = () => {
+    teardownAudio();
+    resetPlaybackFlags();
+    audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE });
+    scriptNode = audioCtx.createScriptProcessor(BUFFER_SIZE, 0, 1);
+
+    const minBufferSamples = Math.floor(audioCtx.sampleRate * PREBUFFER_SEC);
+
+    scriptNode.onaudioprocess = event => {
+      const output = event.outputBuffer.getChannelData(0);
+      const needPrebuffer = !hasStartedPlayback;
+      const socketClosed = !socket || socket.readyState === WebSocket.CLOSED || socket.readyState === WebSocket.CLOSING;
+
+      if (needPrebuffer) {
+        if (buffer.length >= minBufferSamples || socketClosed) {
+          hasStartedPlayback = true;
+          if (!playbackStartedLogged) {
+            playbackStartedLogged = true;
+            appendLog('[Frontend] Browser started to play audio');
+            startPlaybackTimer();
+          }
+        } else {
+          output.fill(0);
+          return;
+        }
+      }
+
+      const chunk = pullAudio(output.length);
+      output.set(chunk);
+
+      if (hasStartedPlayback) {
+        playbackSamples += output.length;
+      }
+
+      if (socketClosed && buffer.length === 0 && chunk.every(sample => sample === 0)) {
+        silentFrameCount += 1;
+        if (silentFrameCount >= 4) {
+          stop();
+        }
+      } else {
+        silentFrameCount = 0;
+      }
+    };
+
+    scriptNode.connect(audioCtx.destination);
+  };
+
+  const start = () => {
+    if (isPlaying) {
+      return;
+    }
+
+    const textValue = promptInput?.value || '';
+    const cfgValue = Number(cfgSelect.value);
+    const stepsValue = Number(stepsSelect.value);
+    const voiceValue = voiceSelect.value || '';
+
+    clearLogs();
+    const cfgDisplay = Number.isFinite(cfgValue) ? cfgValue.toFixed(3) : 'default';
+    const stepsDisplay = Number.isFinite(stepsValue) ? stepsValue : 'default';
+    appendLog(`[Frontend] Start button clicked, CFG=${cfgDisplay}, Steps=${stepsDisplay}, Speaker=${voiceValue || 'default'}`);
+    setModelGenerated(0);
+    setPlaybackElapsed(0);
+
+    resetState(true);
+    clearRecordedChunks();
+    isPlaying = true;
+    previewActive = true;
+    updateStreamingPreview();
+    updateButtonLabel();
+    createAudioChain();
+
+    const params = new URLSearchParams();
+    params.set('text', textValue);
+    if (!Number.isNaN(cfgValue)) {
+      params.set('cfg', cfgValue.toFixed(3));
+    }
+    if (!Number.isNaN(stepsValue)) {
+      params.set('steps', stepsValue.toString());
+    }
+    if (voiceValue) {
+      params.set('voice', voiceValue);
+    }
+    const wsUrl = `${location.origin.replace(/^http/, 'ws')}/stream?${params.toString()}`;
+
+    socket = new WebSocket(wsUrl);
+    socket.binaryType = 'arraybuffer';
+
+    socket.onmessage = event => {
+      if (typeof event.data === 'string') {
+        handleLogMessage(event.data);
+        return;
+      }
+
+      if (!(event.data instanceof ArrayBuffer)) {
+        return;
+      }
+      const rawBuffer = event.data.slice(0);
+      const view = new DataView(rawBuffer);
+      const floatChunk = new Float32Array(view.byteLength / 2);
+      for (let i = 0; i < floatChunk.length; i += 1) {
+        floatChunk[i] = view.getInt16(i * 2, true) / 32768;
+      }
+      appendAudio(floatChunk);
+      recordedChunks.push(rawBuffer);
+      recordedSamples += floatChunk.length;
+      updateSaveButtonState();
+
+      if (!firstBrowserChunkLogged) {
+        firstBrowserChunkLogged = true;
+        appendLog('[Frontend] Received first audio chunk');
+      }
+    };
+
+    socket.onerror = err => {
+      console.error('WebSocket error', err);
+      appendLog(`[Error] WebSocket error: ${err?.message || err}`);
+      stop();
+    };
+
+    socket.onclose = () => {
+      socket = null;
+      if (recordedSamples > 0) {
+        recordingComplete = true;
+        updateSaveButtonState();
+      }
+    };
+  };
+
+  const stop = () => {
+    if (!isPlaying) {
+      resetState(false);
+      updateButtonLabel();
+      return;
+    }
+    resetState(false);
+    setPlaybackElapsed(Math.min(lastPlaybackElapsed, modelGeneratedTotal));
+    appendLog('[Frontend] Playback stopped');
+    if (recordedSamples > 0) {
+      recordingComplete = true;
+      updateSaveButtonState();
+    }
+    previewActive = false;
+    clearPreviewTimer();
+    streamingPreview?.classList.remove('streaming-active');
+    updateButtonLabel();
+  };
+
+  controlBtn.addEventListener('click', () => {
+    if (isPlaying) {
+      stop();
+    } else {
+      start();
+    }
+  });
+  if (saveBtn) {
+    saveBtn.addEventListener('click', handleSaveClick);
+  }
+  updateButtonLabel();
+  updateSaveButtonState();
+  window.addEventListener('beforeunload', () => {
+    resetState();
+    clearPreviewTimer();
+    revokeDownloadUrl();
+  });
+})();
+</script>
+</body>
+</html>

docs/vibevoice-realtime-0.5b.md

@@ -0,0 +1,139 @@
+<div align="center">
+
+## 🎙️ VibeVoice-Realtime: Real-time Long‑Form Text‑to‑Speech with Streaming Input
+[![Hugging Face](https://img.shields.io/badge/HuggingFace-Collection-orange?logo=huggingface)](https://huggingface.co/microsoft/VibeVoice-Realtime-0.5B)
+[![Colab](https://img.shields.io/badge/Run-Colab-orange?logo=googlecolab)](https://colab.research.google.com/github/microsoft/VibeVoice/blob/main/demo/vibevoice_realtime_colab.ipynb)
+</div>
+
+VibeVoice-Realtime is a **lightweight real‑time** text-to-speech model supporting **streaming text input** and **robust long-form speech generation**. It can be used to build real-time TTS services, narrate live data streams, and let different LLMs start speaking from their very first tokens (plug in your preferred model) long before a full answer is generated. It produces initial audible speech in **~300 milliseconds** (hardware dependent).
+
+<div align="center">
+
+| Model | Context Length | Generation Length |  Weight |
+|-------|----------------|----------|----------|
+| VibeVoice-Realtime-0.5B | 8K | ~10 min | [HF link](https://huggingface.co/microsoft/VibeVoice-Realtime-0.5B) |
+
+</div>
+
+> Note (multilingual exploration): Although the model is primarily built for English, we found that it still exhibits a certain level of multilingual capability—and even performs reasonably well in some languages. We provide nine additional languages (German, French, Italian, Japanese, Korean, Dutch, Polish, Portuguese, and Spanish) for users to explore. These multilingual behaviors have not been extensively tested; use with caution and share observations.
+
+The model uses an interleaved, windowed design: it incrementally encodes incoming text chunks while, in parallel, continuing diffusion-based acoustic latent generation from prior context. Unlike the full multi-speaker long-form variants, this streaming model removes the semantic tokenizer and relies solely on an efficient acoustic tokenizer operating at an ultra-low frame rate (7.5 Hz).
+
+<div align="center">
+	<picture>
+		<source media="(prefers-color-scheme: dark)" srcset="../Figures/VibeVoice_logo_white.png">
+		<img src="../Figures/VibeVoice_Realtime.png" alt="VibeVoice Realtime Overview" width="800" />
+	</picture>
+	<br>
+	<em>Overview of VibeVoice Realtime Model.</em>
+</div>
+
+Key features:
+- Parameter size: 0.5B (deployment-friendly)
+- Real-time TTS (~300 milliseconds first audible latency)
+- Streaming text input
+- Robust long-form speech generation
+
+This real-time variant supports only a single speaker. For multi‑speaker conversational speech generation, please use other VibeVoice models (long‑form multi‑speaker variants). The model is currently intended for English speech only; other languages may produce unpredictable results.
+
+To mitigate deepfake risks and ensure low latency for the first speech chunk, voice prompts are provided in an embedded format. For users requiring voice customization, please reach out to our team. We will also be expanding the range of available speakers.
+
+
+### 📋 TODO
+
+- [ ] Add more voices (expand available speakers/voice timbres)
+- [ ] Implement streaming text input function to feed new tokens while audio is still being generated
+- [ ] Merge models into official HuggingFace's `transformers` repository 
+
+
+### 🎵 Demo Examples
+
+<div align="center" id="generated-example-audio-vibevoice-realtime">
+
+https://github.com/user-attachments/assets/9aa8ab3c-681d-4a02-b9ea-3f54ffd180b2
+
+</div>
+
+
+## Results
+
+The model achieves satisfactory performance on short-sentence benchmarks, while the model is more focused on long‑form speech generation.
+
+### Zero-shot TTS performance on LibriSpeech test-clean set
+
+| Model | WER (%) ↓ | Speaker Similarity ↑ |
+|:--------------------|:---------:|:----------------:|
+| VALL-E 2            | 2.40      | 0.643            |
+| Voicebox            | 1.90      | 0.662            |
+| MELLE               | 2.10      | 0.625            |
+| **VibeVoice-Realtime-0.5B** | 2.00 | 0.695            |
+
+### Zero-shot TTS performance on SEED test-en set
+
+| Model | WER (%) ↓ | Speaker Similarity ↑ |
+|:--------------------|:---------:|:----------------:|
+| MaskGCT             | 2.62      | 0.714            |
+| Seed-TTS            | 2.25      | 0.762            |
+| FireRedTTS          | 3.82      | 0.460            |
+| SparkTTS            | 1.98      | 0.584            |
+| CosyVoice2          | 2.57      | 0.652            |
+| **VibeVoice-Realtime-0.5B** | 2.05 | 0.633            | 
+
+
+## Installation
+We recommend to use NVIDIA Deep Learning Container to manage the CUDA environment. 
+
+1. Launch docker
+```bash
+# NVIDIA PyTorch Container 24.07 / 24.10 / 24.12 verified. 
+# Later versions are also compatible.
+sudo docker run --privileged --net=host --ipc=host --ulimit memlock=-1:-1 --ulimit stack=-1:-1 --gpus all --rm -it  nvcr.io/nvidia/pytorch:24.07-py3
+
+## If flash attention is not included in your docker environment, you need to install it manually
+## Refer to https://github.com/Dao-AILab/flash-attention for installation instructions
+# pip install flash-attn --no-build-isolation
+```
+
+2. Install from github
+```bash
+git clone https://github.com/microsoft/VibeVoice.git
+cd VibeVoice/
+
+pip install -e .
+```
+
+## Usages
+
+
+### Usage 1: Launch real-time websocket demo
+Note: NVIDIA T4 / Mac M4 Pro achieve realtime in our tests; other devices with weaker inference capability may require further testing and speed optimizations. 
+
+Due to network latency, the time when audio playback is heard may exceed the ~300 ms first speech chunk generation latency.
+```bash
+python demo/vibevoice_realtime_demo.py --model_path microsoft/VibeVoice-Realtime-0.5B
+```
+
+Tip: Just try it on [Colab](https://colab.research.google.com/github/microsoft/VibeVoice/blob/main/demo/vibevoice_realtime_colab.ipynb).
+
+### Usage 2: Inference from files directly
+```bash
+# We provide some example scripts under demo/text_examples/ for demo
+python demo/realtime_model_inference_from_file.py --model_path microsoft/VibeVoice-Realtime-0.5B --txt_path demo/text_examples/1p_vibevoice.txt --speaker_name Carter
+```
+
+
+## Risks and limitations
+
+While efforts have been made to optimize it through various techniques, it may still produce outputs that are unexpected, biased, or inaccurate. VibeVoice inherits any biases, errors, or omissions produced by its base model (specifically, Qwen2.5 0.5b in this release).
+
+Potential for Deepfakes and Disinformation: High-quality synthetic speech can be misused to create convincing fake audio content for impersonation, fraud, or spreading disinformation. Users must ensure transcripts are reliable, check content accuracy, and avoid using generated content in misleading ways. Users are expected to use the generated content and to deploy the models in a lawful manner, in full compliance with all applicable laws and regulations in the relevant jurisdictions. It is best practice to disclose the use of AI when sharing AI-generated content.
+
+English only: Transcripts in languages other than English may result in unexpected audio outputs.
+
+Non-Speech Audio: The model focuses solely on speech synthesis and does not handle background noise, music, or other sound effects.
+
+Code, formulas, and special symbols: The model does not currently support reading code, mathematical formulas, or uncommon symbols. Please pre‑process input text to remove or normalize such content to avoid unpredictable results. 
+
+Very short inputs: When the input text is extremely short (three words or fewer), the model’s stability may degrade.
+
+We do not recommend using VibeVoice in commercial or real-world applications without further testing and development. This model is intended for research and development purposes only. Please use responsibly.

vibevoice/__init__.py

@@ -0,0 +1,16 @@
+# vibevoice/__init__.py
+from vibevoice.modular import (
+    VibeVoiceStreamingForConditionalGenerationInference,
+    VibeVoiceStreamingConfig,
+)
+from vibevoice.processor import (
+    VibeVoiceStreamingProcessor,
+    VibeVoiceTokenizerProcessor,
+)
+
+__all__ = [
+    "VibeVoiceStreamingForConditionalGenerationInference",
+    "VibeVoiceStreamingConfig",
+    "VibeVoiceStreamingProcessor",
+    "VibeVoiceTokenizerProcessor",
+]

vibevoice/configs/qwen2.5_1.5b_64k.json

@@ -0,0 +1,112 @@
+{
+  "_attn_implementation_autoset": true,
+  "acoustic_vae_dim": 64,
+  "acoustic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "decoder_depths": null,
+    "decoder_n_filters": 32,
+    "decoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0.5,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_acoustic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "gaussian",
+    "vae_dim": 64,
+    "weight_init_value": 0.01
+  },
+  "decoder_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "max_position_embeddings": 65536,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "diffusion_head_config": {
+    "ddpm_batch_mul": 4,
+    "ddpm_beta_schedule": "cosine",
+    "ddpm_num_inference_steps": 20,
+    "ddpm_num_steps": 1000,
+    "diffusion_type": "ddpm",
+    "head_ffn_ratio": 3.0,
+    "head_layers": 4,
+    "hidden_size": 1536,
+    "latent_size": 64,
+    "model_type": "vibepod_diffusion_head",
+    "prediction_type": "v_prediction",
+    "rms_norm_eps": 1e-05,
+    "speech_vae_dim": 64
+  },
+  "model_type": "vibepod",
+  "semantic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_semantic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "none",
+    "vae_dim": 128,
+    "weight_init_value": 0.01
+  },
+  "semantic_vae_dim": 128,
+  "torch_dtype": "bfloat16"
+}

vibevoice/configs/qwen2.5_7b_32k.json

@@ -0,0 +1,113 @@
+{
+  "_attn_implementation_autoset": true,
+  "acoustic_vae_dim": 64,
+  "acoustic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "decoder_depths": null,
+    "decoder_n_filters": 32,
+    "decoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0.5,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_acoustic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "gaussian",
+    "vae_dim": 64,
+    "weight_init_value": 0.01
+  },
+  "decoder_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 28,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.40.1",
+    "use_cache": true,
+    "use_mrope": false,
+    "use_sliding_window": false,
+    "vocab_size": 152064
+  },
+  "diffusion_head_config": {
+    "ddpm_batch_mul": 4,
+    "ddpm_beta_schedule": "cosine",
+    "ddpm_num_inference_steps": 20,
+    "ddpm_num_steps": 1000,
+    "diffusion_type": "ddpm",
+    "head_ffn_ratio": 3.0,
+    "head_layers": 4,
+    "hidden_size": 3584,
+    "latent_size": 64,
+    "model_type": "vibepod_diffusion_head",
+    "prediction_type": "v_prediction",
+    "rms_norm_eps": 1e-05,
+    "speech_vae_dim": 64
+  },
+  "model_type": "vibepod",
+  "semantic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_semantic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "none",
+    "vae_dim": 128,
+    "weight_init_value": 0.01
+  },
+  "semantic_vae_dim": 128,
+  "torch_dtype": "bfloat16"
+}

vibevoice/modular/__init__.py

@@ -0,0 +1,14 @@
+# vibevoice/modular/__init__.py
+from .modeling_vibevoice_streaming_inference import VibeVoiceStreamingForConditionalGenerationInference
+from .configuration_vibevoice_streaming import VibeVoiceStreamingConfig
+from .modeling_vibevoice_streaming import VibeVoiceStreamingModel, VibeVoiceStreamingPreTrainedModel
+from .streamer import AudioStreamer, AsyncAudioStreamer
+
+__all__ = [
+    "VibeVoiceStreamingForConditionalGenerationInference",
+    "VibeVoiceStreamingConfig",
+    "VibeVoiceStreamingModel",
+    "VibeVoiceStreamingPreTrainedModel",
+    "AudioStreamer",
+    "AsyncAudioStreamer",
+]

vibevoice/modular/configuration_vibevoice.py

@@ -0,0 +1,248 @@
+""" VibeVoice_AcousticTokenizer model configuration"""
+
+from typing import Dict, List, Optional, Tuple
+
+from transformers.configuration_utils import PretrainedConfig 
+from transformers.utils import logging
+
+from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
+
+logger = logging.get_logger(__name__)
+
+
+class VibeVoiceAcousticTokenizerConfig(PretrainedConfig):
+    model_type = "vibevoice_acoustic_tokenizer"
+
+    def __init__(
+        self,
+        channels: int = 1,
+        corpus_normalize: float = 0.0,
+        causal: bool = True,
+        vae_dim: int = 64,
+        fix_std: float = 0.5,
+        std_dist_type: str = 'gaussian',
+        # common 
+        mixer_layer: str = 'depthwise_conv',
+        conv_norm: str = 'none',
+        pad_mode: str = 'constant',
+        disable_last_norm: bool = True,
+        layernorm: str = 'RMSNorm',
+        layernorm_eps: float = 1e-5,
+        layernorm_elementwise_affine: bool = True,
+        conv_bias: bool = True,
+        layer_scale_init_value: float = 1e-6,
+        weight_init_value: float = 1e-2,
+        # encoder specific
+        encoder_n_filters: int = 32,
+        encoder_ratios: Optional[List[int]] = [8,5,5,4,2,2],
+        encoder_depths: str = "3-3-3-3-3-3-8",
+        # decoder specific
+        decoder_n_filters: int = 32,
+        decoder_ratios: Optional[List[int]] = None, # if None, same as encoder
+        decoder_depths: Optional[str] = None,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.channels = channels
+        self.corpus_normalize = corpus_normalize
+        self.causal = causal
+        self.vae_dim = vae_dim
+        self.fix_std = fix_std
+        self.std_dist_type = std_dist_type
+        
+        # common parameters
+        self.conv_norm = conv_norm
+        self.pad_mode = pad_mode
+        self.layernorm_eps = layernorm_eps
+        self.disable_last_norm = disable_last_norm
+        self.layernorm = layernorm
+        self.layernorm_elementwise_affine = layernorm_elementwise_affine
+        self.conv_bias = conv_bias
+        self.layer_scale_init_value = layer_scale_init_value
+        self.weight_init_value = weight_init_value
+        self.mixer_layer = mixer_layer
+
+        # encoder specific parameters
+        self.encoder_n_filters = encoder_n_filters
+        self.encoder_ratios = encoder_ratios
+        self.encoder_depths = encoder_depths
+        
+        # decoder specific parameters
+        self.decoder_ratios = decoder_ratios if decoder_ratios is not None else encoder_ratios
+        self.decoder_n_filters = decoder_n_filters
+        self.decoder_depths = decoder_depths
+
+
+class VibeVoiceSemanticTokenizerConfig(PretrainedConfig):
+    model_type = "vibevoice_semantic_tokenizer"
+    
+    def __init__(
+        self,
+        channels: int = 1,
+        corpus_normalize: float = 0.0,
+        causal: bool = True,
+        vae_dim: int = 64,
+        fix_std: float = 0,
+        std_dist_type: str = 'none',
+        # common 
+        mixer_layer: str = 'depthwise_conv',
+        conv_norm: str = 'none',
+        pad_mode: str = 'constant',
+        disable_last_norm: bool = True,
+        layernorm: str = 'RMSNorm',
+        layernorm_eps: float = 1e-5,
+        layernorm_elementwise_affine: bool = True,
+        conv_bias: bool = True,
+        layer_scale_init_value: float = 1e-6,
+        weight_init_value: float = 1e-2,
+        # encoder specific
+        encoder_n_filters: int = 32,
+        encoder_ratios: Optional[List[int]] = [8,5,5,4,2,2],
+        encoder_depths: str = "3-3-3-3-3-3-8",
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.channels = channels
+        self.corpus_normalize = corpus_normalize
+        self.causal = causal
+        self.vae_dim = vae_dim
+        self.fix_std = fix_std
+        self.std_dist_type = std_dist_type
+        
+        # common parameters
+        self.conv_norm = conv_norm
+        self.pad_mode = pad_mode
+        self.layernorm_eps = layernorm_eps
+        self.disable_last_norm = disable_last_norm
+        self.layernorm = layernorm
+        self.layernorm_elementwise_affine = layernorm_elementwise_affine
+        self.conv_bias = conv_bias
+        self.layer_scale_init_value = layer_scale_init_value
+        self.weight_init_value = weight_init_value
+        self.mixer_layer = mixer_layer
+
+        # encoder specific parameters
+        self.encoder_n_filters = encoder_n_filters
+        self.encoder_ratios = encoder_ratios
+        self.encoder_depths = encoder_depths
+        
+
+class VibeVoiceDiffusionHeadConfig(PretrainedConfig):
+    model_type = "vibevoice_diffusion_head"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        head_layers=4,
+        head_ffn_ratio=3.0,
+        rms_norm_eps=1e-5,
+        latent_size=64,
+        speech_vae_dim=None,
+        prediction_type="v_prediction",
+        diffusion_type="ddpm",
+        ddpm_num_steps=1000,
+        ddpm_num_inference_steps=20,
+        ddpm_beta_schedule="cosine",
+        ddpm_batch_mul=4,
+        **kwargs
+    ):
+        self.hidden_size = hidden_size
+        self.head_layers = head_layers
+        self.head_ffn_ratio = head_ffn_ratio
+        self.rms_norm_eps = rms_norm_eps
+        self.latent_size = latent_size
+        self.speech_vae_dim = speech_vae_dim
+        self.prediction_type = prediction_type
+        self.diffusion_type = diffusion_type
+        self.ddpm_num_steps = ddpm_num_steps
+        self.ddpm_num_inference_steps = ddpm_num_inference_steps
+        self.ddpm_beta_schedule = ddpm_beta_schedule
+        self.ddpm_batch_mul = ddpm_batch_mul
+        
+        super().__init__(**kwargs)
+
+class VibeVoiceConfig(PretrainedConfig):
+    model_type = "vibevoice"
+    is_composition = True
+    sub_configs = {
+        "acoustic_tokenizer_config": VibeVoiceAcousticTokenizerConfig, 
+        "semantic_tokenizer_config": VibeVoiceSemanticTokenizerConfig,
+        "decoder_config": Qwen2Config,
+        "diffusion_head_config": VibeVoiceDiffusionHeadConfig,
+    }
+    # keys_to_ignore_at_inference = ["past_key_values"]
+    # Default tensor parallel plan for base model `Qwen2`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    
+    def __init__(
+        self,
+        acoustic_tokenizer_config=None,
+        semantic_tokenizer_config=None,
+        decoder_config=None,
+        diffusion_head_config=None,
+        **kwargs
+    ):
+
+        # kwargs["_attn_implementation"] = "flash_attention_2"
+        kwargs["_attn_implementation_autoset"] = False 
+
+        if acoustic_tokenizer_config is None:
+            self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"]()
+        elif isinstance(acoustic_tokenizer_config, dict):
+            acoustic_tokenizer_config["model_type"] = "vibevoice_acoustic_tokenizer"
+            self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"](**acoustic_tokenizer_config)
+        elif isinstance(acoustic_tokenizer_config, VibeVoiceAcousticTokenizerConfig):
+            # If an instance of the config class is provided
+            self.acoustic_tokenizer_config = acoustic_tokenizer_config
+
+        if semantic_tokenizer_config is None:
+            self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"]()
+        elif isinstance(semantic_tokenizer_config, dict):
+            semantic_tokenizer_config["model_type"] = "vibevoice_semantic_tokenizer"
+            self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"](**semantic_tokenizer_config)
+        elif isinstance(semantic_tokenizer_config, VibeVoiceSemanticTokenizerConfig):
+            # If an instance of the config class is provided
+            self.semantic_tokenizer_config = semantic_tokenizer_config
+
+        if decoder_config is None:
+            self.decoder_config = self.sub_configs["decoder_config"]()
+        elif isinstance(decoder_config, dict):
+            # If a dictionary is provided, instantiate the config class with it
+            # self.decoder_config = self.sub_configs["decoder_config"](**decoder_config)
+            if decoder_config.get("model_type", '') == "qwen2":
+                self.decoder_config = Qwen2Config(**decoder_config)
+            else:
+                raise ValueError(f"Unsupported decoder model type: {decoder_config.get('model_type', '')}")
+        elif isinstance(decoder_config, (Qwen2Config,)):
+            # If an instance of the config class is provided
+            self.decoder_config = decoder_config
+
+        if diffusion_head_config is None:
+            self.diffusion_head_config = self.sub_configs["diffusion_head_config"]()
+        elif isinstance(diffusion_head_config, dict):
+            diffusion_head_config["model_type"] = "vibevoice_diffusion_head"
+            self.diffusion_head_config = self.sub_configs["diffusion_head_config"](**diffusion_head_config)
+        elif isinstance(diffusion_head_config, VibeVoiceDiffusionHeadConfig):
+            # If an instance of the config class is provided
+            self.diffusion_head_config = diffusion_head_config
+
+        # other parameters
+        self.acoustic_vae_dim = getattr(self.acoustic_tokenizer_config, 'vae_dim', 64)
+        self.semantic_vae_dim = getattr(self.semantic_tokenizer_config, 'vae_dim', 128)
+
+        super().__init__(**kwargs)
+
+__all__ = [
+    "VibeVoiceAcousticTokenizerConfig", 
+    "VibeVoiceSemanticTokenizerConfig", 
+    "VibeVoiceDiffusionHeadConfig", 
+    "VibeVoiceConfig"
+]

vibevoice/modular/configuration_vibevoice_streaming.py

@@ -0,0 +1,85 @@
+""" VibeVoice Streaming model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig 
+from transformers.utils import logging
+
+from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
+
+from .configuration_vibevoice import VibeVoiceAcousticTokenizerConfig, VibeVoiceDiffusionHeadConfig
+
+logger = logging.get_logger(__name__)
+
+
+class VibeVoiceStreamingConfig(PretrainedConfig):
+    model_type = "vibevoice_streaming"
+    is_composition = True
+    sub_configs = {
+        "acoustic_tokenizer_config": VibeVoiceAcousticTokenizerConfig,
+        "decoder_config": Qwen2Config,
+        "diffusion_head_config": VibeVoiceDiffusionHeadConfig,
+    }
+    # keys_to_ignore_at_inference = ["past_key_values"]
+    # Default tensor parallel plan for base model `Qwen2`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    
+    def __init__(
+        self,
+        acoustic_tokenizer_config=None,
+        decoder_config=None,
+        diffusion_head_config=None,
+        tts_backbone_num_hidden_layers=20,
+        **kwargs
+    ):
+
+        # kwargs["_attn_implementation"] = "flash_attention_2"
+        kwargs["_attn_implementation_autoset"] = False 
+
+        if acoustic_tokenizer_config is None:
+            self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"]()
+        elif isinstance(acoustic_tokenizer_config, dict):
+            acoustic_tokenizer_config["model_type"] = "vibevoice_acoustic_tokenizer"
+            self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"](**acoustic_tokenizer_config)
+        elif isinstance(acoustic_tokenizer_config, VibeVoiceAcousticTokenizerConfig):
+            # If an instance of the config class is provided
+            self.acoustic_tokenizer_config = acoustic_tokenizer_config
+
+        if decoder_config is None:
+            self.decoder_config = self.sub_configs["decoder_config"]()
+        elif isinstance(decoder_config, dict):
+            # If a dictionary is provided, instantiate the config class with it
+            # self.decoder_config = self.sub_configs["decoder_config"](**decoder_config)
+            if decoder_config.get("model_type", '') == "qwen2":
+                self.decoder_config = Qwen2Config(**decoder_config)
+            else:
+                raise ValueError(f"Unsupported decoder model type: {decoder_config.get('model_type', '')}")
+        elif isinstance(decoder_config, (Qwen2Config,)):
+            # If an instance of the config class is provided
+            self.decoder_config = decoder_config
+
+        if diffusion_head_config is None:
+            self.diffusion_head_config = self.sub_configs["diffusion_head_config"]()
+        elif isinstance(diffusion_head_config, dict):
+            diffusion_head_config["model_type"] = "vibevoice_diffusion_head"
+            self.diffusion_head_config = self.sub_configs["diffusion_head_config"](**diffusion_head_config)
+        elif isinstance(diffusion_head_config, VibeVoiceDiffusionHeadConfig):
+            # If an instance of the config class is provided
+            self.diffusion_head_config = diffusion_head_config
+
+        # other parameters
+        self.acoustic_vae_dim = getattr(self.acoustic_tokenizer_config, 'vae_dim', 64)
+        # The decoder of the model is divided into two components. The lower Transformer layers are only used for encoding text, while the upper Transformer layers are used for encoding text and generating speech. `tts_backbone_num_hidden_layers` indicates the number of upper layers used for TTS.
+        self.tts_backbone_num_hidden_layers = tts_backbone_num_hidden_layers
+
+        super().__init__(**kwargs)
+
+__all__ = [
+    "VibeVoiceStreamingConfig"
+]

vibevoice/modular/modeling_vibevoice_streaming.py

@@ -0,0 +1,190 @@
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple, Union, Callable
+from tqdm import tqdm
+import copy
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.distributed as dist
+
+from transformers.models.auto import AutoModel, AutoModelForCausalLM
+
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import CausalLMOutput, BaseModelOutputWithPast, ModelOutput
+from transformers.models.llama.modeling_llama import LlamaRMSNorm
+from transformers import modeling_utils
+from transformers.modeling_utils import PreTrainedModel
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.utils import logging
+
+from .modular_vibevoice_diffusion_head import VibeVoiceDiffusionHead
+from vibevoice.schedule.dpm_solver import DPMSolverMultistepScheduler
+
+from .configuration_vibevoice_streaming import VibeVoiceStreamingConfig
+
+
+logger = logging.get_logger(__name__)
+
+if not hasattr(modeling_utils, "ALL_PARALLEL_STYLES") or modeling_utils.ALL_PARALLEL_STYLES is None:
+    modeling_utils.ALL_PARALLEL_STYLES = ["tp", "none", "colwise", "rowwise"]
+
+
+class BinaryClassifier(nn.Module):
+    def __init__(self, hidden_size):
+        super(BinaryClassifier, self).__init__()
+        self.fc1 = nn.Linear(hidden_size, hidden_size)
+        self.fc2 = nn.Linear(hidden_size, 1)
+
+    def forward(self, x):
+        x = torch.relu(self.fc1(x))
+        x = self.fc2(x)
+        return x
+
+
+class SpeechConnector(nn.Module):
+    def __init__(self, input_dim, output_dim):
+        super().__init__()
+        self.fc1 = nn.Linear(input_dim, output_dim)
+        self.norm = LlamaRMSNorm(output_dim, eps=1e-6)
+        self.fc2 = nn.Linear(output_dim, output_dim)
+
+    def forward(self, features, **kwargs):    
+        x = self.fc1(features)
+        x = self.norm(x)
+        x = self.fc2(x)
+        return x
+
+
+# @auto_docstring
+class VibeVoiceStreamingPreTrainedModel(PreTrainedModel):
+    config_class = VibeVoiceStreamingConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _skip_keys_device_placement = "past_key_values"
+    _supports_cache_class = True
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_quantized_cache = True
+    _supports_static_cache = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module):
+        if isinstance(module, VibeVoiceDiffusionHead):
+            module.initialize_weights()
+            return
+
+        # Use the language model's initializer_range if available
+        if hasattr(self.config, 'language_model_config') and hasattr(self.config.language_model_config, 'initializer_range'):
+            std = self.config.language_model_config.initializer_range
+        elif hasattr(self.config, 'decoder_config') and hasattr(self.config.decoder_config, 'initializer_range'):
+            std = self.config.decoder_config.initializer_range
+        else:
+            std = 0.02  # Default value
+            
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.weight.data.fill_(1.0)
+            module.bias.data.zero_()
+
+
+# @auto_docstring
+class VibeVoiceStreamingModel(VibeVoiceStreamingPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        
+        if hasattr(config, 'torch_dtype') and config.torch_dtype is not None:
+            if isinstance(config.torch_dtype, str):
+                dtype = getattr(torch, config.torch_dtype)
+            else:
+                dtype = config.torch_dtype
+        else:
+            dtype = torch.float32
+        
+        # Initialize Qwen2 model for language modeling. 
+        # The lower Transformer layers are only used for encoding text, while the upper Transformer layers are used for encoding text and generating speech. 
+        # To keep the code clean, we constructs two language models.
+        # The final norm layer of the first language_model is set to identity and will not be used in inference.
+        lm_config = copy.deepcopy(config.decoder_config)
+        lm_backbone_num_hidden_layers = getattr(lm_config, 'num_hidden_layers', 24) - config.tts_backbone_num_hidden_layers
+        lm_config.num_hidden_layers = lm_backbone_num_hidden_layers
+        self.language_model = AutoModel.from_config(lm_config)
+        self.language_model.norm = nn.Identity()
+        
+        # We only need the Transformer layers here. Note that embed_tokens in tts_language_model is unused
+        tts_lm_config = copy.deepcopy(lm_config)
+        tts_lm_config.num_hidden_layers = config.tts_backbone_num_hidden_layers
+        self.tts_language_model = AutoModel.from_config(tts_lm_config)
+
+        # Marks the text that needs to be spoken by the TTS model.
+        self.tts_input_types = nn.Embedding(num_embeddings=2, embedding_dim=config.decoder_config.hidden_size)
+        
+        # Initialize speech components if needed
+        self.acoustic_tokenizer = AutoModel.from_config(config.acoustic_tokenizer_config).to(dtype)
+        self.acoustic_connector = SpeechConnector(config.acoustic_vae_dim, lm_config.hidden_size).to(dtype)
+        
+        # Register scaling factors as buffers - use 1D tensors for FSDP compatibility
+        self.register_buffer('speech_scaling_factor', torch.tensor(float('nan')))  
+        self.register_buffer('speech_bias_factor', torch.tensor(float('nan')))
+
+        # Initialize prediction head for speech generation
+        self.prediction_head = AutoModel.from_config(config.diffusion_head_config).to(dtype)
+
+        # Initialize noise scheduler
+        self.noise_scheduler = DPMSolverMultistepScheduler(
+            num_train_timesteps=config.diffusion_head_config.ddpm_num_steps,
+            beta_schedule=config.diffusion_head_config.ddpm_beta_schedule,
+            prediction_type=config.diffusion_head_config.prediction_type
+        )
+    
+    def get_input_embeddings(self):
+        if hasattr(self.language_model, 'embed_tokens'):
+            # If the language model has an embed_tokens attribute, return it
+            return self.language_model.embed_tokens
+        
+        for name, attr in self.language_model.fullmap.items(): # parallel by nnscaler, the name is changed
+            if attr.orig_name == 'embed_tokens.weight':
+                return getattr(self.language_model, name)
+        assert False, 'should not arrive here'
+
+    def set_input_embeddings(self, value):
+        self.language_model.embed_tokens = value
+    
+    def set_speech_tokenizers(self, acoustic_tokenizer=None):
+        """Set the speech tokenizers used for encoding and decoding speech."""
+        self.acoustic_tokenizer = acoustic_tokenizer
+        
+        # Reset the encoder to evaluation mode
+        if self.acoustic_tokenizer is not None:
+            self.acoustic_tokenizer.eval()
+    
+    def forward(self, *args, **kwargs):
+        """
+        Intentionally not implemented.
+
+        This streaming model is split into two explicit submodules:
+          - `language_model`      for plain text processing (lower layers).
+          - `tts_language_model`  for TTS-related upper layers.
+
+        We deliberately avoid a unified `forward` to prevent accidental calls
+        that mix responsibilities.
+
+        To use the model:
+          - Call `self.language_model(...)` for text embeddings / hidden states.
+          - Call `self.tts_language_model(...)` for the TTS portion.
+          - Use the dedicated inference class for combined generation logic.
+        """
+        raise RuntimeError(
+            "VibeVoiceStreamingModel.forward is intentionally disabled. "
+            "Use `model.language_model(...)` or `model.tts_language_model(...)` instead."
+        )
+
+
+AutoModel.register(VibeVoiceStreamingConfig, VibeVoiceStreamingModel)
+
+__all__ = [
+    "VibeVoiceStreamingPreTrainedModel",
+    "VibeVoiceStreamingModel",
+]