Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,34 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+VoxCPM/assets/voxcpm_model.png filter=lfs diff=lfs merge=lfs -text
+VoxCPM/ckpts/assets/voxcpm_model.png filter=lfs diff=lfs merge=lfs -text
+VoxCPM/examples/example.wav filter=lfs diff=lfs merge=lfs -text
+VoxCPM/prompt_sample.wav filter=lfs diff=lfs merge=lfs -text
+eval/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/asr_example_hotword.wav filter=lfs diff=lfs merge=lfs -text
+eval/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav filter=lfs diff=lfs merge=lfs -text
+eval/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/fig/res.png filter=lfs diff=lfs merge=lfs -text
+eval/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/fig/seaco.png filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/UniSpeech-SAT/UniSpeech_SAT_SUPERB_Results.png filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/WavLM/WavLM_ASR.PNG filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/WavLM/WavLM_SUPERB_Leaderboard.png filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/WavLM/WavLM_SUPERB_Results.png filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/David_Faustino/hn8GyCJIfLM_0000012.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/David_Faustino/xTOk1Jz-F_g_0000015.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/Josh_Gad/HXUqYaOwrxA_0000015.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/Josh_Gad/RFyw7V3SOnQ_0000001.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/Lea_Thompson/HladKGyKTLM_0000006.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/Lea_Thompson/mHTAr5dlAgc_0000004.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/Zulay_Henao/WbB8m9-wlIQ_0000001.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/downstreams/speaker_verification/vox1_data/Zulay_Henao/gFfcgOVmiO0_0000002.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/David_Faustino/hn8GyCJIfLM_0000012.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/David_Faustino/xTOk1Jz-F_g_0000015.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/Josh_Gad/HXUqYaOwrxA_0000015.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/Josh_Gad/RFyw7V3SOnQ_0000001.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/Lea_Thompson/HladKGyKTLM_0000006.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/Lea_Thompson/mHTAr5dlAgc_0000004.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/Zulay_Henao/WbB8m9-wlIQ_0000001.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/examples/speaker_verification/vox1_data/Zulay_Henao/gFfcgOVmiO0_0000002.wav filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/fairseq/data/data_utils_fast.cpython-36m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+eval/thirdparty/UniSpeech/src/fairseq/data/data_utils_fast.cpython-37m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+实验三：基于VoxCPM的音色克隆（实验指导）.pdf filter=lfs diff=lfs merge=lfs -text

VoxCPM/.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,54 @@
+name: Publish Python 🐍 distribution 📦 to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/voxcpm
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

VoxCPM/.gitignore

@@ -0,0 +1,3 @@
+launch.json
+__pycache__
+voxcpm.egg-info

VoxCPM/LICENSE

@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright OpenBMB
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

VoxCPM/README.md

@@ -0,0 +1,353 @@
+## 🎙️ VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning
+
+
+[![Project Page](https://img.shields.io/badge/Project%20Page-GitHub-blue)](https://github.com/OpenBMB/VoxCPM/) [![Technical Report](https://img.shields.io/badge/Technical%20Report-Arxiv-red)](https://arxiv.org/abs/2509.24650) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-OpenBMB-yellow)](https://huggingface.co/openbmb/VoxCPM-0.5B) [![ModelScope](https://img.shields.io/badge/ModelScope-OpenBMB-purple)](https://modelscope.cn/models/OpenBMB/VoxCPM-0.5B)  [![Live Playground](https://img.shields.io/badge/Live%20PlayGround-Demo-orange)](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) [![Samples](https://img.shields.io/badge/Audio%20Samples-Page-green)](https://openbmb.github.io/VoxCPM-demopage)
+
+
+
+<div align="center">
+  <img src="assets/voxcpm_logo.png" alt="VoxCPM Logo" width="40%">
+</div>
+
+<div align="center">
+
+👋 Contact us on [WeChat](assets/wechat.png)
+
+</div>
+
+## News 
+* [2025.09.30] 🔥 🔥 🔥  We Release VoxCPM [Technical Report](https://arxiv.org/abs/2509.24650)!
+* [2025.09.16] 🔥 🔥 🔥  We Open Source the VoxCPM-0.5B [weights](https://huggingface.co/openbmb/VoxCPM-0.5B)!
+* [2025.09.16] 🎉 🎉 🎉  We Provide the [Gradio PlayGround](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) for VoxCPM-0.5B, try it now! 
+
+## Overview
+
+VoxCPM is a novel tokenizer-free Text-to-Speech (TTS) system that redefines realism in speech synthesis. By modeling speech in a continuous space, it overcomes the limitations of discrete tokenization and enables two flagship capabilities: context-aware speech generation and true-to-life zero-shot voice cloning.
+
+Unlike mainstream approaches that convert speech to discrete tokens, VoxCPM uses an end-to-end diffusion autoregressive architecture that directly generates continuous speech representations from text. Built on [MiniCPM-4](https://huggingface.co/openbmb/MiniCPM4-0.5B) backbone, it achieves implicit semantic-acoustic decoupling through hierachical language modeling and FSQ constraints, greatly enhancing both expressiveness and generation stability.
+
+<div align="center">
+  <img src="assets/voxcpm_model.png" alt="VoxCPM Model Architecture" width="90%">
+</div>
+
+
+###  🚀 Key Features
+- **Context-Aware, Expressive Speech Generation** - VoxCPM comprehends text to infer and generate appropriate prosody, delivering speech with remarkable expressiveness and natural flow. It spontaneously adapts speaking style based on content, producing highly fitting vocal expression trained on a massive 1.8 million-hour bilingual corpus.
+- **True-to-Life Voice Cloning** - With only a short reference audio clip, VoxCPM performs accurate zero-shot voice cloning, capturing not only the speaker’s timbre but also fine-grained characteristics such as accent, emotional tone, rhythm, and pacing to create a faithful and natural replica.
+- **High-Efficiency Synthesis** - VoxCPM supports streaming synthesis with a Real-Time Factor (RTF) as low as 0.17 on a consumer-grade NVIDIA RTX 4090 GPU, making it possible for real-time applications.
+
+
+
+
+
+##  Quick Start
+
+### 🔧 Install from PyPI
+``` sh
+pip install voxcpm
+```
+### 1.  Model Download (Optional)
+By default, when you first run the script, the model will be downloaded automatically, but you can also download the model in advance.
+- Download VoxCPM-0.5B
+    ```
+    from huggingface_hub import snapshot_download
+    snapshot_download("openbmb/VoxCPM-0.5B")
+    ```
+- Download ZipEnhancer and SenseVoice-Small. We use ZipEnhancer to enhance speech prompts and SenseVoice-Small for speech prompt ASR in the web demo.
+    ```
+    from modelscope import snapshot_download
+    snapshot_download('iic/speech_zipenhancer_ans_multiloss_16k_base')
+    snapshot_download('iic/SenseVoiceSmall')
+    ```
+
+### 2. Basic Usage
+```python
+import soundfile as sf
+import numpy as np
+from voxcpm import VoxCPM
+
+model = VoxCPM.from_pretrained("openbmb/VoxCPM-0.5B")
+
+# Non-streaming
+wav = model.generate(
+    text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.",
+    prompt_wav_path=None,      # optional: path to a prompt speech for voice cloning
+    prompt_text=None,          # optional: reference text
+    cfg_value=2.0,             # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse
+    inference_timesteps=10,   # LocDiT inference timesteps, higher for better result, lower for fast speed
+    normalize=True,           # enable external TN tool
+    denoise=True,             # enable external Denoise tool
+    retry_badcase=True,        # enable retrying mode for some bad cases (unstoppable)
+    retry_badcase_max_times=3,  # maximum retrying times
+    retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech
+)
+
+sf.write("output.wav", wav, 16000)
+print("saved: output.wav")
+
+# Streaming
+chunks = []
+for chunk in model.generate_streaming(
+    text = "Streaming text to speech is easy with VoxCPM!",
+    # supports same args as above
+):
+    chunks.append(chunk)
+wav = np.concatenate(chunks)
+
+sf.write("output_streaming.wav", wav, 16000)
+print("saved: output_streaming.wav")
+```
+
+### 3. CLI Usage
+
+After installation, the entry point is `voxcpm` (or use `python -m voxcpm.cli`).
+
+```bash
+# 1) Direct synthesis (single text)
+voxcpm --text "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech." --output out.wav
+
+# 2) Voice cloning (reference audio + transcript)
+voxcpm --text "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech." \
+  --prompt-audio path/to/voice.wav \
+  --prompt-text "reference transcript" \
+  --output out.wav \
+  --denoise
+
+# (Optinal) Voice cloning (reference audio + transcript file)
+voxcpm --text "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech." \
+  --prompt-audio path/to/voice.wav \
+  --prompt-file "/path/to/text-file" \
+  --output out.wav \
+  --denoise
+
+# 3) Batch processing (one text per line)
+voxcpm --input examples/input.txt --output-dir outs
+# (optional) Batch + cloning
+voxcpm --input examples/input.txt --output-dir outs \
+  --prompt-audio path/to/voice.wav \
+  --prompt-text "reference transcript" \
+  --denoise
+
+# 4) Inference parameters (quality/speed)
+voxcpm --text "..." --output out.wav \
+  --cfg-value 2.0 --inference-timesteps 10 --normalize
+
+# 5) Model loading
+# Prefer local path
+voxcpm --text "..." --output out.wav --model-path /path/to/VoxCPM_model_dir
+# Or from Hugging Face (auto download/cache)
+voxcpm --text "..." --output out.wav \
+  --hf-model-id openbmb/VoxCPM-0.5B --cache-dir ~/.cache/huggingface --local-files-only
+
+# 6) Denoiser control
+voxcpm --text "..." --output out.wav \
+  --no-denoiser --zipenhancer-path iic/speech_zipenhancer_ans_multiloss_16k_base
+
+# 7) Help
+voxcpm --help
+python -m voxcpm.cli --help
+```
+
+### 4. Start web demo
+
+You can start the UI interface by running `python app.py`, which allows you to perform Voice Cloning and Voice Creation.
+
+## 🛠️ Fine-tune VoxCPM
+
+We provide a training pipeline mirroring the `minicpm-audio` workflow while relying purely on HuggingFace `datasets` for audio-text management.
+
+1. **Prepare a manifest (JSONL)**
+
+   ```
+   {"audio": "/path/to/audio_0001.wav", "text": "你好，世界。", "dataset_id": 0}
+   {"audio": "/path/to/audio_0002.wav", "text": "第二条语音", "dataset_id": 0}
+   ```
+   - `audio`: waveform file path (WAV/FLAC/MP3 supported)
+   - `text`: transcription
+   - `dataset_id` *(optional)*: integer identifier for multi-dataset sampling statistics
+
+2. **Copy & edit the example config**  
+   `conf/voxcpm/voxcpm_finetune_example.yaml` contains hyper-parameters (pretrained weights, tokenizer, manifests, λ-weights, etc.).
+
+3. **Launch training**
+
+   ```bash
+   CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node 2 \
+     scripts/train_voxcpm_finetune.py \
+     --config_path conf/voxcpm/voxcpm_finetune_example.yaml
+   ```
+
+   Features:
+   - Distributed + AMP training (`torchrun`).
+   - TensorBoard logging (`tensorboard --logdir logs/voxcpm_finetune`).
+   - Periodic validation & checkpointing under `checkpoints/`.
+
+4. **Key modules**
+   - `src/voxcpm/model/voxcpm.py`: unified model providing both inference and training forward。
+   - `src/voxcpm/training/`: accelerator, tracker, dataset loader & batch packer utilities。
+   - `scripts/train_voxcpm_finetune.py`: end-to-end fine-tune loop。
+
+## 👩‍🍳 A Voice Chef's Guide
+Welcome to the VoxCPM kitchen! Follow this recipe to cook up perfect generated speech. Let’s begin.
+
+---
+### 🥚 Step 1: Prepare Your Base Ingredients (Content)
+
+First, choose how you’d like to input your text:.
+1. Regular Text (Classic Mode)
+- ✅ Keep "Text Normalization" ON. Type naturally (e.g., "Hello, world! 123"). The system will automatically process numbers, abbreviations, and punctuation using WeTextProcessing library.
+2. Phoneme Input (Native Mode)
+- ❌ Turn "Text Normalization" OFF. Enter phoneme text like {HH AH0 L OW1} (EN) or {ni3}{hao3} (ZH) for precise pronunciation  control. In this mode, VoxCPM also supports native understanding of other complex non-normalized text—try it out!
+
+---
+### 🍳 Step 2: Choose Your Flavor Profile (Voice Style) 
+
+This is the secret sauce that gives your audio its unique sound.
+1. Cooking with a Prompt Speech (Following a Famous Recipe)
+  - A prompt speech provides the desired acoustic characteristics for VoxCPM. The speaker's timbre, speaking style, and even the background sounds and ambiance will be replicated.
+  - For a Clean, Studio-Quality Voice:
+    - ✅ Enable "Prompt Speech Enhancement". This acts like a noise filter, removing background hiss and rumble to give you a pure, clean voice clone.
+2. Cooking au Naturel (Letting the Model Improvise)
+  - If no reference is provided, VoxCPM becomes a creative chef! It will infer a fitting speaking style based on the text itself, thanks to the text-smartness of its foundation model, MiniCPM-4.
+  - Pro Tip: Challenge VoxCPM with any text—poetry, song lyrics, dramatic monologues—it may deliver some interesting results!
+
+---
+### 🧂 Step 3: The Final Seasoning (Fine-Tuning Your Results)
+You're ready to serve! But for master chefs who want to tweak the flavor, here are two key spices.
+- CFG Value (How Closely to Follow the Recipe)
+  - Default: A great starting point.
+  - Voice sounds strained or weird? Lower this value. It tells the model to be more relaxed and improvisational, great for expressive prompts.
+  - Need maximum clarity and adherence to the text? Raise it slightly to keep the model on a tighter leash.
+- Inference Timesteps (Simmering Time: Quality vs. Speed)
+  - Need a quick snack? Use a lower number. Perfect for fast drafts and experiments.
+  - Cooking a gourmet meal? Use a higher number. This lets the model "simmer" longer, refining the audio for superior detail and naturalness.
+
+---
+Happy creating! 🎉 Start with the default settings and tweak from there to suit your project. The kitchen is yours!
+
+
+---
+
+
+## 🌟 Community Projects
+
+We're excited to see the VoxCPM community growing! Here are some amazing projects and features built by our community:
+
+- **[ComfyUI-VoxCPM](https://github.com/wildminder/ComfyUI-VoxCPM)**
+- **[ComfyUI-VoxCPMTTS](https://github.com/1038lab/ComfyUI-VoxCPMTTS)** 
+- **[WebUI-VoxCPM](https://github.com/rsxdalv/tts_webui_extension.vox_cpm)**
+- **[PR: Streaming API Support (by AbrahamSanders)](https://github.com/OpenBMB/VoxCPM/pull/26)** 
+
+
+
+*Have you built something cool with VoxCPM? We'd love to feature it here! Please open an issue or pull request to add your project.*
+
+
+## 📊 Performance Highlights
+
+VoxCPM achieves competitive results on public zero-shot TTS benchmarks:
+
+### Seed-TTS-eval Benchmark
+
+| Model | Parameters | Open-Source | test-EN | | test-ZH | | test-Hard | |
+|------|------|------|:------------:|:--:|:------------:|:--:|:-------------:|:--:|
+| | | | WER/%⬇ | SIM/%⬆| CER/%⬇| SIM/%⬆ | CER/%⬇ | SIM/%⬆ |
+| MegaTTS3 | 0.5B | ❌ | 2.79 | 77.1 | 1.52 | 79.0 | - | - |
+| DiTAR | 0.6B | ❌ | 1.69 | 73.5 | 1.02 | 75.3 | - | - |
+| CosyVoice3 | 0.5B | ❌ | 2.02 | 71.8 | 1.16 | 78.0 | 6.08 | 75.8 |
+| CosyVoice3 | 1.5B | ❌ | 2.22 | 72.0 | 1.12 | 78.1 | 5.83 | 75.8 |
+| Seed-TTS | - | ❌ | 2.25 | 76.2 | 1.12 | 79.6 | 7.59 | 77.6 |
+| MiniMax-Speech | - | ❌ | 1.65 | 69.2 | 0.83 | 78.3 | - | - |
+| CosyVoice | 0.3B | ✅ | 4.29 | 60.9 | 3.63 | 72.3 | 11.75 | 70.9 |
+| CosyVoice2 | 0.5B | ✅ | 3.09 | 65.9 | 1.38 | 75.7 | **6.83** | 72.4 |
+| F5-TTS | 0.3B | ✅ | 2.00 | 67.0 | 1.53 | 76.0 | 8.67 | 71.3 |
+| SparkTTS | 0.5B | ✅ | 3.14 | 57.3 | 1.54 | 66.0 | - | - |
+| FireRedTTS | 0.5B | ✅ | 3.82 | 46.0 | 1.51 | 63.5 | 17.45 | 62.1 |
+| FireRedTTS-2 | 1.5B | ✅ | 1.95 | 66.5 | 1.14 | 73.6 | - | - |
+| Qwen2.5-Omni | 7B | ✅ | 2.72 | 63.2 | 1.70 | 75.2 | 7.97 | **74.7** |
+| OpenAudio-s1-mini | 0.5B | ✅ | 1.94 | 55.0 | 1.18 | 68.5 | - | - |
+| IndexTTS2 | 1.5B | ✅ | 2.23 | 70.6 | 1.03 | 76.5 | - | - |
+| VibeVoice | 1.5B | ✅ | 3.04 | 68.9 | 1.16 | 74.4 | - | - |
+| HiggsAudio-v2 | 3B | ✅ | 2.44 | 67.7 | 1.50 | 74.0 | - | - |
+| **VoxCPM** | 0.5B | ✅ | **1.85** | **72.9** | **0.93** | **77.2** | 8.87 | 73.0 |
+
+
+###  CV3-eval Benchmark
+
+| Model | zh | en | hard-zh | | | hard-en | | |
+|-------|:--:|:--:|:-------:|:--:|:--:|:-------:|:--:|:--:|
+| | CER/%⬇ | WER/%⬇ | CER/%⬇ | SIM/%⬆ | DNSMOS⬆ | WER/%⬇ | SIM/%⬆ | DNSMOS⬆ |
+| F5-TTS | 5.47 | 8.90 | - | - | - | - | - | - |
+| SparkTTS | 5.15 | 11.0 | - | - | - | - | - | - |
+| GPT-SoVits | 7.34 | 12.5 | - | - | - | - | - | - |
+| CosyVoice2 | 4.08 | 6.32 | 12.58 | 72.6 | 3.81 | 11.96 | 66.7 | 3.95 |
+| OpenAudio-s1-mini | 4.00 | 5.54 | 18.1 | 58.2 | 3.77 | 12.4 | 55.7 | 3.89 |
+| IndexTTS2 | 3.58 | 4.45 | 12.8 | 74.6 | 3.65 | - | - | - |
+| HiggsAudio-v2 | 9.54 | 7.89 | 41.0 | 60.2 | 3.39 | 10.3 | 61.8 | 3.68 |
+| CosyVoice3-0.5B | 3.89 | 5.24 | 14.15 | 78.6 | 3.75 | 9.04 | 75.9 | 3.92 |
+| CosyVoice3-1.5B | 3.91 | 4.99 | 9.77 | 78.5 | 3.79 | 10.55 | 76.1 | 3.95 |
+| **VoxCPM** | **3.40** | **4.04** | 12.9 | 66.1 | 3.59 | **7.89** | 64.3 | 3.74 |
+
+
+
+
+
+
+
+
+
+
+
+
+## ⚠️ Risks and limitations
+- General Model Behavior: While VoxCPM has been trained on a large-scale dataset, it may still produce outputs that are unexpected, biased, or contain artifacts.
+- Potential for Misuse of Voice Cloning: VoxCPM's powerful zero-shot voice cloning capability can generate highly realistic synthetic speech. This technology could be misused for creating convincing deepfakes for purposes of impersonation, fraud, or spreading disinformation. Users of this model must not use it to create content that infringes upon the rights of individuals. It is strictly forbidden to use VoxCPM for any illegal or unethical purposes. We strongly recommend that any publicly shared content generated with this model be clearly marked as AI-generated.
+- Current Technical Limitations: Although generally stable, the model may occasionally exhibit instability, especially with very long or expressive inputs. Furthermore, the current version offers limited direct control over specific speech attributes like emotion or speaking style.
+- Bilingual Model: VoxCPM is trained primarily on Chinese and English data. Performance on other languages is not guaranteed and may result in unpredictable or low-quality audio.
+- This model is released for research and development purposes only. We do not recommend its use in production or commercial applications without rigorous testing and safety evaluations. Please use VoxCPM responsibly.
+
+
+
+## 📝TO-DO List
+Please stay tuned for updates!
+- [x] Release the VoxCPM technical report.
+- [ ] Support higher sampling rate (next version).
+
+
+
+## 📄 License
+The VoxCPM model weights and code are open-sourced under the [Apache-2.0](LICENSE) license.
+
+## 🙏 Acknowledgments
+
+We extend our sincere gratitude to the following works and resources for their inspiration and contributions:
+
+- [DiTAR](https://arxiv.org/abs/2502.03930) for the diffusion autoregressive backbone used in speech generation
+- [MiniCPM-4](https://github.com/OpenBMB/MiniCPM) for serving as the language model foundation
+- [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) for the implementation of Flow Matching-based LocDiT
+- [DAC](https://github.com/descriptinc/descript-audio-codec) for providing the Audio VAE backbone
+
+## Institutions
+
+This project is developed by the following institutions:
+- <img src="assets/modelbest_logo.png" width="28px"> [ModelBest](https://modelbest.cn/)
+
+- <img src="assets/thuhcsi_logo.png" width="28px"> [THUHCSI](https://github.com/thuhcsi)
+
+
+## ⭐ Star History
+ [![Star History Chart](https://api.star-history.com/svg?repos=OpenBMB/VoxCPM&type=Date)](https://star-history.com/#OpenBMB/VoxCPM&Date)
+
+
+## 📚 Citation
+
+If you find our model helpful, please consider citing our projects 📝 and staring us ⭐️！
+
+```bib
+@article{voxcpm2025,
+  title        = {VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning},
+  author       = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and Yu, Renjie and Wang, Ziyang and Ye, Runchuan and Sun, Weiyue and Gui, Jiancheng and Li, Kehan and Wu, Zhiyong  and Liu, Zhiyuan},
+  journal      = {arXiv preprint arXiv:2509.24650},
+  year         = {2025},
+}
+```

VoxCPM/app.py

@@ -0,0 +1,274 @@
+import os
+import numpy as np
+import torch
+import gradio as gr  
+import spaces
+from typing import Optional, Tuple
+from funasr import AutoModel
+from pathlib import Path
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+if os.environ.get("HF_REPO_ID", "").strip() == "":
+    os.environ["HF_REPO_ID"] = "openbmb/VoxCPM-0.5B"
+
+import voxcpm
+
+
+class VoxCPMDemo:
+    def __init__(self) -> None:
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        print(f"🚀 Running on device: {self.device}")
+
+        # ASR model for prompt text recognition
+        self.asr_model_id = "iic/SenseVoiceSmall"
+        self.asr_model: Optional[AutoModel] = AutoModel(
+            model=self.asr_model_id,
+            disable_update=True,
+            log_level='DEBUG',
+            device="cuda:0" if self.device == "cuda" else "cpu",
+        )
+
+        # TTS model (lazy init)
+        self.voxcpm_model: Optional[voxcpm.VoxCPM] = None
+        self.default_local_model_dir = "./models/VoxCPM-0.5B"
+
+    # ---------- Model helpers ----------
+    def _resolve_model_dir(self) -> str:
+        """
+        Resolve model directory:
+        1) Use local checkpoint directory if exists
+        2) If HF_REPO_ID env is set, download into models/{repo}
+        3) Fallback to 'models'
+        """
+        if os.path.isdir(self.default_local_model_dir):
+            return self.default_local_model_dir
+
+        repo_id = os.environ.get("HF_REPO_ID", "").strip()
+        if len(repo_id) > 0:
+            target_dir = os.path.join("models", repo_id.replace("/", "__"))
+            if not os.path.isdir(target_dir):
+                try:
+                    from huggingface_hub import snapshot_download  # type: ignore
+                    os.makedirs(target_dir, exist_ok=True)
+                    print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...")
+                    snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
+                except Exception as e:
+                    print(f"Warning: HF download failed: {e}. Falling back to 'data'.")
+                    return "models"
+            return target_dir
+        return "models"
+
+    def get_or_load_voxcpm(self) -> voxcpm.VoxCPM:
+        if self.voxcpm_model is not None:
+            return self.voxcpm_model
+        print("Model not loaded, initializing...")
+        model_dir = self._resolve_model_dir()
+        print(f"Using model dir: {model_dir}")
+        self.voxcpm_model = voxcpm.VoxCPM(voxcpm_model_path=model_dir)
+        print("Model loaded successfully.")
+        return self.voxcpm_model
+
+    # ---------- Functional endpoints ----------
+    def prompt_wav_recognition(self, prompt_wav: Optional[str]) -> str:
+        if prompt_wav is None:
+            return ""
+        res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
+        text = res[0]["text"].split('|>')[-1]
+        return text
+
+    def generate_tts_audio(
+        self,
+        text_input: str,
+        prompt_wav_path_input: Optional[str] = None,
+        prompt_text_input: Optional[str] = None,
+        cfg_value_input: float = 2.0,
+        inference_timesteps_input: int = 10,
+        do_normalize: bool = True,
+        denoise: bool = True,
+    ) -> Tuple[int, np.ndarray]:
+        """
+        Generate speech from text using VoxCPM; optional reference audio for voice style guidance.
+        Returns (sample_rate, waveform_numpy)
+        """
+        current_model = self.get_or_load_voxcpm()
+
+        text = (text_input or "").strip()
+        if len(text) == 0:
+            raise ValueError("Please input text to synthesize.")
+
+        prompt_wav_path = prompt_wav_path_input if prompt_wav_path_input else None
+        prompt_text = prompt_text_input if prompt_text_input else None
+
+        print(f"Generating audio for text: '{text[:60]}...'")
+        wav = current_model.generate(
+            text=text,
+            prompt_text=prompt_text,
+            prompt_wav_path=prompt_wav_path,
+            cfg_value=float(cfg_value_input),
+            inference_timesteps=int(inference_timesteps_input),
+            normalize=do_normalize,
+            denoise=denoise,
+        )
+        return (16000, wav)
+
+
+# ---------- UI Builders ----------
+
+def create_demo_interface(demo: VoxCPMDemo):
+    """Build the Gradio UI for VoxCPM demo."""
+    # static assets (logo path)
+    gr.set_static_paths(paths=[Path.cwd().absolute()/"assets"])
+
+    with gr.Blocks(
+        theme=gr.themes.Soft(
+            primary_hue="blue",
+            secondary_hue="gray",
+            neutral_hue="slate",
+            font=[gr.themes.GoogleFont("Inter"), "Arial", "sans-serif"]
+        ),
+        css="""
+        .logo-container {
+            text-align: center;
+            margin: 0.5rem 0 1rem 0;
+        }
+        .logo-container img {
+            height: 80px;
+            width: auto;
+            max-width: 200px;
+            display: inline-block;
+        }
+        /* Bold accordion labels */
+        #acc_quick details > summary,
+        #acc_tips details > summary {
+            font-weight: 600 !important;
+            font-size: 1.1em !important;
+        }
+        /* Bold labels for specific checkboxes */
+        #chk_denoise label,
+        #chk_denoise span,
+        #chk_normalize label,
+        #chk_normalize span {
+            font-weight: 600;
+        }
+        """
+    ) as interface:
+        # Header logo
+        gr.HTML('<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>')
+
+        # Quick Start
+        with gr.Accordion("📋 Quick Start Guide ｜快速入门", open=False, elem_id="acc_quick"):
+            gr.Markdown("""
+            ### How to Use ｜使用说明
+            1. **(Optional) Provide a Voice Prompt** - Upload or record an audio clip to provide the desired voice characteristics for synthesis.  
+               **（可选）提供参考声音** - 上传或录制一段音频，为声音合成提供音色、语调和情感等个性化特征
+            2. **(Optional) Enter prompt text** - If you provided a voice prompt, enter the corresponding transcript here (auto-recognition available).  
+               **（可选项）输入参考文本** - 如果提供了参考语音，请输入其对应的文本内容（支持自动识别）。
+            3. **Enter target text** - Type the text you want the model to speak.  
+               **输入目标文本** - 输入您希望模型朗读的文字内容。
+            4. **Generate Speech** - Click the "Generate" button to create your audio.  
+               **生成语音** - 点击"生成"按钮，即可为您创造出音频。
+            """)
+
+        # Pro Tips
+        with gr.Accordion("💡 Pro Tips ｜使用建议", open=False, elem_id="acc_tips"):
+            gr.Markdown("""
+            ### Prompt Speech Enhancement｜参考语音降噪
+            - **Enable** to remove background noise for a clean, studio-like voice, with an external ZipEnhancer component.  
+              **启用**：通过 ZipEnhancer 组件消除背景噪音，获得更好的音质。
+            - **Disable** to preserve the original audio's background atmosphere.  
+              **禁用**：保留原始音频的背景环境声，如果想复刻相应声学环境。
+
+            ### Text Normalization｜文本正则化
+            - **Enable** to process general text with an external WeTextProcessing component.  
+              **启用**：使用 WeTextProcessing 组件，可处理常见文本。
+            - **Disable** to use VoxCPM's native text understanding ability. For example, it supports phonemes input ({HH AH0 L OW1}), try it!  
+              **禁用**：将使用 VoxCPM 内置的文本理解能力。如，支持音素输入（如 {da4}{jia1}好）和公式符号合成，尝试一下！
+
+            ### CFG Value｜CFG 值
+            - **Lower CFG** if the voice prompt sounds strained or expressive.  
+              **调低**：如果提示语音听起来不自然或过于夸张。
+            - **Higher CFG** for better adherence to the prompt speech style or input text.  
+              **调高**：为更好地贴合提示音频的风格或输入文本。
+
+            ### Inference Timesteps｜推理时间步
+            - **Lower** for faster synthesis speed.  
+              **调低**：合成速度更快。
+            - **Higher** for better synthesis quality.  
+              **调高**：合成质量更佳。
+            """)
+
+        # Main controls
+        with gr.Row():
+            with gr.Column():
+                prompt_wav = gr.Audio(
+                    sources=["upload", 'microphone'],
+                    type="filepath",
+                    label="Prompt Speech (Optional, or let VoxCPM improvise)",
+                    value="./examples/example.wav",
+                )
+                DoDenoisePromptAudio = gr.Checkbox(
+                    value=False,
+                    label="Prompt Speech Enhancement",
+                    elem_id="chk_denoise",
+                    info="We use ZipEnhancer model to denoise the prompt audio."
+                )
+                with gr.Row():
+                    prompt_text = gr.Textbox(
+                        value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
+                        label="Prompt Text",
+                        placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself..."
+                    )
+                run_btn = gr.Button("Generate Speech", variant="primary")
+
+            with gr.Column():
+                cfg_value = gr.Slider(
+                    minimum=1.0,
+                    maximum=3.0,
+                    value=2.0,
+                    step=0.1,
+                    label="CFG Value (Guidance Scale)",
+                    info="Higher values increase adherence to prompt, lower values allow more creativity"
+                )
+                inference_timesteps = gr.Slider(
+                    minimum=4,
+                    maximum=30,
+                    value=10,
+                    step=1,
+                    label="Inference Timesteps",
+                    info="Number of inference timesteps for generation (higher values may improve quality but slower)"
+                )
+                with gr.Row():
+                    text = gr.Textbox(
+                        value="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly realistic speech.",
+                        label="Target Text",
+                    )
+                with gr.Row():
+                    DoNormalizeText = gr.Checkbox(
+                        value=False,
+                        label="Text Normalization",
+                        elem_id="chk_normalize",
+                        info="We use wetext library to normalize the input text."
+                    )
+                audio_output = gr.Audio(label="Output Audio")
+
+        # Wiring
+        run_btn.click(
+            fn=demo.generate_tts_audio,
+            inputs=[text, prompt_wav, prompt_text, cfg_value, inference_timesteps, DoNormalizeText, DoDenoisePromptAudio],
+            outputs=[audio_output],
+            show_progress=True,
+            api_name="generate",
+        )
+        prompt_wav.change(fn=demo.prompt_wav_recognition, inputs=[prompt_wav], outputs=[prompt_text])
+
+    return interface
+
+
+def run_demo(server_name: str = "localhost", server_port: int = 7860, show_error: bool = True):
+    demo = VoxCPMDemo()
+    interface = create_demo_interface(demo)
+    # Recommended to enable queue on Spaces for better throughput
+    interface.queue(max_size=10).launch(server_name=server_name, server_port=server_port, show_error=show_error)
+
+
+if __name__ == "__main__":
+    run_demo()

VoxCPM/ckpts/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/voxcpm_model.png filter=lfs diff=lfs merge=lfs -text

VoxCPM/ckpts/README.md

@@ -0,0 +1,238 @@
+---
+license: apache-2.0
+language:
+- en
+- zh
+base_model:
+- openbmb/MiniCPM4-0.5B
+pipeline_tag: text-to-speech
+library_name: voxcpm
+tags:
+- text-to-speech
+- speech
+- speech generation
+- voice cloning
+---
+
+## 🎙️ VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning
+
+
+[![Project Page](https://img.shields.io/badge/Project%20Page-GitHub-blue)](https://github.com/OpenBMB/VoxCPM/) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-OpenBMB-yellow)](https://huggingface.co/openbmb/VoxCPM-0.5B) [![Live Playground](https://img.shields.io/badge/Live%20PlayGround-Demo-orange)](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) [![Samples](https://img.shields.io/badge/Page-Samples-red)](https://openbmb.github.io/VoxCPM-demopage/)
+
+
+<div align="center">
+  <img src="assets/voxcpm_logo.png" alt="VoxCPM Logo" width="40%">
+</div>
+
+## Overview
+
+VoxCPM is a novel tokenizer-free Text-to-Speech (TTS) system that redefines realism in speech synthesis. By modeling speech in a continuous space, it overcomes the limitations of discrete tokenization and enables two flagship capabilities: context-aware speech generation and true-to-life zero-shot voice cloning.
+
+Unlike mainstream approaches that convert speech to discrete tokens, VoxCPM uses an end-to-end diffusion autoregressive architecture that directly generates continuous speech representations from text. Built on [MiniCPM-4](https://huggingface.co/openbmb/MiniCPM4-0.5B) backbone, it achieves implicit semantic-acoustic decoupling through hierachical language modeling and FSQ constraints, greatly enhancing both expressiveness and generation stability.
+
+<div align="center">
+  <img src="assets/voxcpm_model.png" alt="VoxCPM Model Architecture" width="90%">
+</div>
+
+
+###  🚀 Key Features
+- **Context-Aware, Expressive Speech Generation** - VoxCPM comprehends text to infer and generate appropriate prosody, delivering speech with remarkable expressiveness and natural flow. It spontaneously adapts speaking style based on content, producing highly fitting vocal expression trained on a massive 1.8 million-hour bilingual corpus.
+- **True-to-Life Voice Cloning** - With only a short reference audio clip, VoxCPM performs accurate zero-shot voice cloning, capturing not only the speaker’s timbre but also fine-grained characteristics such as accent, emotional tone, rhythm, and pacing to create a faithful and natural replica.
+- **High-Efficiency Synthesis** - VoxCPM supports streaming synthesis with a Real-Time Factor (RTF) as low as 0.17 on a consumer-grade NVIDIA RTX 4090 GPU, making it possible for real-time applications.
+
+
+##  Quick Start
+
+### 🔧 Install from PyPI
+``` sh
+pip install voxcpm
+```
+### 1.  Model Download (Optional)
+By default, when you first run the script, the model will be downloaded automatically, but you can also download the model in advance.
+- Download VoxCPM-0.5B
+    ```
+    from huggingface_hub import snapshot_download
+    snapshot_download("openbmb/VoxCPM-0.5B",local_files_only=local_files_only)
+    ```
+- Download ZipEnhancer and SenseVoice-Small. We use ZipEnhancer to enhance speech prompts and SenseVoice-Small for speech prompt ASR in the web demo.
+    ```
+    from modelscope import snapshot_download
+    snapshot_download('iic/speech_zipenhancer_ans_multiloss_16k_base')
+    snapshot_download('iic/SenseVoiceSmall')
+    ```
+
+### 2. Basic Usage
+```python
+import soundfile as sf
+from voxcpm import VoxCPM
+
+model = VoxCPM.from_pretrained("openbmb/VoxCPM-0.5B")
+
+wav = model.generate(
+    text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.",
+    prompt_wav_path=None,      # optional: path to a prompt speech for voice cloning
+    prompt_text=None,          # optional: reference text
+    cfg_value=2.0,             # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse
+    inference_timesteps=10,   # LocDiT inference timesteps, higher for better result, lower for fast speed
+    normalize=True,           # enable external TN tool
+    denoise=True,             # enable external Denoise tool
+    retry_badcase=True,        # enable retrying mode for some bad cases (unstoppable)
+    retry_badcase_max_times=3,  # maximum retrying times
+    retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech
+)
+
+sf.write("output.wav", wav, 16000)
+print("saved: output.wav")
+```
+
+### 3. CLI Usage
+
+After installation, the entry point is `voxcpm` (or use `python -m voxcpm.cli`).
+
+```bash
+# 1) Direct synthesis (single text)
+voxcpm --text "Hello VoxCPM" --output out.wav
+
+# 2) Voice cloning (reference audio + transcript)
+voxcpm --text "Hello" \
+  --prompt-audio path/to/voice.wav \
+  --prompt-text "reference transcript" \
+  --output out.wav \
+  --denoise
+
+# 3) Batch processing (one text per line)
+voxcpm --input examples/input.txt --output-dir outs
+# (optional) Batch + cloning
+voxcpm --input examples/input.txt --output-dir outs \
+  --prompt-audio path/to/voice.wav \
+  --prompt-text "reference transcript" \
+  --denoise
+
+# 4) Inference parameters (quality/speed)
+voxcpm --text "..." --output out.wav \
+  --cfg-value 2.0 --inference-timesteps 10 --normalize
+
+# 5) Model loading
+# Prefer local path
+voxcpm --text "..." --output out.wav --model-path /path/to/VoxCPM_model_dir
+# Or from Hugging Face (auto download/cache)
+voxcpm --text "..." --output out.wav \
+  --hf-model-id openbmb/VoxCPM-0.5B --cache-dir ~/.cache/huggingface --local-files-only
+
+# 6) Denoiser control
+voxcpm --text "..." --output out.wav \
+  --no-denoiser --zipenhancer-path iic/speech_zipenhancer_ans_multiloss_16k_base
+
+# 7) Help
+voxcpm --help
+python -m voxcpm.cli --help
+```
+
+### 4. Start web demo
+
+You can start the UI interface by running `python app.py`, which allows you to perform Voice Cloning and Voice Creation.
+
+
+
+## 👩‍🍳 A Voice Chef's Guide
+Welcome to the VoxCPM kitchen! Follow this recipe to cook up perfect generated speech. Let’s begin.
+
+---
+### 🥚 Step 1: Prepare Your Base Ingredients (Content)
+
+First, choose how you’d like to input your text:.
+1. Regular Text (Classic Mode)
+- ✅ Keep "Text Normalization" ON. Type naturally (e.g., "Hello, world! 123"). The system will automatically process numbers, abbreviations, and punctuation using WeTextProcessing library.
+2. Phoneme Input (Native Mode)
+- ❌ Turn "Text Normalization" OFF. Enter phoneme text like {HH AH0 L OW1} (EN) or {ni3}{hao3} (ZH) for precise pronunciation  control. In this mode, VoxCPM also supports native understanding of other complex non-normalized text—try it out!
+
+---
+### 🍳 Step 2: Choose Your Flavor Profile (Voice Style) 
+
+This is the secret sauce that gives your audio its unique sound.
+1. Cooking with a Prompt Speech (Following a Famous Recipe)
+  - A prompt speech provides the desired acoustic characteristics for VoxCPM. The speaker's timbre, speaking style, and even the background sounds and ambiance will be replicated.
+  - For a Clean, Studio-Quality Voice:
+    - ✅ Enable "Prompt Speech Enhancement". This acts like a noise filter, removing background hiss and rumble to give you a pure, clean voice clone.
+2. Cooking au Naturel (Letting the Model Improvise)
+  - If no reference is provided, VoxCPM becomes a creative chef! It will infer a fitting speaking style based on the text itself, thanks to the text-smartness of its foundation model, MiniCPM-4.
+  - Pro Tip: Challenge VoxCPM with any text—poetry, song lyrics, dramatic monologues—it may deliver some interesting results!
+
+---
+### 🧂 Step 3: The Final Seasoning (Fine-Tuning Your Results)
+You're ready to serve! But for master chefs who want to tweak the flavor, here are two key spices.
+- CFG Value (How Closely to Follow the Recipe)
+  - Default: A great starting point.
+  - Voice sounds strained or weird? Lower this value. It tells the model to be more relaxed and improvisational, great for expressive prompts.
+  - Need maximum clarity and adherence to the text? Raise it slightly to keep the model on a tighter leash.
+- Inference Timesteps (Simmering Time: Quality vs. Speed)
+  - Need a quick snack? Use a lower number. Perfect for fast drafts and experiments.
+  - Cooking a gourmet meal? Use a higher number. This lets the model "simmer" longer, refining the audio for superior detail and naturalness.
+
+---
+Happy creating! 🎉 Start with the default settings and tweak from there to suit your project. The kitchen is yours!
+
+
+---
+
+
+
+## 📊 Performance Highlights
+
+VoxCPM achieves competitive results on public zero-shot TTS benchmarks:
+
+### Seed-TTS-eval Benchmark
+
+| Model | Parameters | Open-Source | test-EN | | test-ZH | | test-Hard | |
+|------|------|------|:------------:|:--:|:------------:|:--:|:-------------:|:--:|
+| | | | WER/%⬇ | SIM/%⬆| CER/%⬇| SIM/%⬆ | CER/%⬇ | SIM/%⬆ |
+| MegaTTS3 | 0.5B | ❌ | 2.79 | 77.1 | 1.52 | 79.0 | - | - |
+| DiTAR | 0.6B | ❌ | 1.69 | 73.5 | 1.02 | 75.3 | - | - |
+| CosyVoice3 | 0.5B | ❌ | 2.02 | 71.8 | 1.16 | 78.0 | 6.08 | 75.8 |
+| CosyVoice3 | 1.5B | ❌ | 2.22 | 72.0 | 1.12 | 78.1 | 5.83 | 75.8 |
+| Seed-TTS | - | ❌ | 2.25 | 76.2 | 1.12 | 79.6 | 7.59 | 77.6 |
+| MiniMax-Speech | - | ❌ | 1.65 | 69.2 | 0.83 | 78.3 | - | - |
+| CosyVoice | 0.3B | ✅ | 4.29 | 60.9 | 3.63 | 72.3 | 11.75 | 70.9 |
+| CosyVoice2 | 0.5B | ✅ | 3.09 | 65.9 | 1.38 | 75.7 | **6.83** | 72.4 |
+| F5-TTS | 0.3B | ✅ | 2.00 | 67.0 | 1.53 | 76.0 | 8.67 | 71.3 |
+| SparkTTS | 0.5B | ✅ | 3.14 | 57.3 | 1.54 | 66.0 | - | - |
+| FireRedTTS | 0.5B | ✅ | 3.82 | 46.0 | 1.51 | 63.5 | 17.45 | 62.1 |
+| FireRedTTS-2 | 1.5B | ✅ | 1.95 | 66.5 | 1.14 | 73.6 | - | - |
+| Qwen2.5-Omni | 7B | ✅ | 2.72 | 63.2 | 1.70 | 75.2 | 7.97 | **74.7** |
+| OpenAudio-s1-mini | 0.5B | ✅ | 1.94 | 55.0 | 1.18 | 68.5 | - | - |
+| IndexTTS2 | 1.5B | ✅ | 2.23 | 70.6 | 1.03 | 76.5 | - | - |
+| VibeVoice | 1.5B | ✅ | 3.04 | 68.9 | 1.16 | 74.4 | - | - |
+| HiggsAudio-v2 | 3B | ✅ | 2.44 | 67.7 | 1.50 | 74.0 | - | - |
+| **VoxCPM** | 0.5B | ✅ | **1.85** | **72.9** | **0.93** | **77.2** | 8.87 | 73.0 |
+
+
+###  CV3-eval Benchmark
+
+| Model | zh | en | hard-zh | | | hard-en | | |
+|-------|:--:|:--:|:-------:|:--:|:--:|:-------:|:--:|:--:|
+| | CER/%⬇ | WER/%⬇ | CER/%⬇ | SIM/%⬆ | DNSMOS⬆ | WER/%⬇ | SIM/%⬆ | DNSMOS⬆ |
+| F5-TTS | 5.47 | 8.90 | - | - | - | - | - | - |
+| SparkTTS | 5.15 | 11.0 | - | - | - | - | - | - |
+| GPT-SoVits | 7.34 | 12.5 | - | - | - | - | - | - |
+| CosyVoice2 | 4.08 | 6.32 | 12.58 | 72.6 | 3.81 | 11.96 | 66.7 | 3.95 |
+| OpenAudio-s1-mini | 4.00 | 5.54 | 18.1 | 58.2 | 3.77 | 12.4 | 55.7 | 3.89 |
+| IndexTTS2 | 3.58 | 4.45 | 12.8 | 74.6 | 3.65 | - | - | - |
+| HiggsAudio-v2 | 9.54 | 7.89 | 41.0 | 60.2 | 3.39 | 10.3 | 61.8 | 3.68 |
+| CosyVoice3-0.5B | 3.89 | 5.24 | 14.15 | 78.6 | 3.75 | 9.04 | 75.9 | 3.92 |
+| CosyVoice3-1.5B | 3.91 | 4.99 | 9.77 | 78.5 | 3.79 | 10.55 | 76.1 | 3.95 |
+| **VoxCPM** | **3.40** | **4.04** | 12.9 | 66.1 | 3.59 | **7.89** | 64.3 | 3.74 |
+
+
+## ⚠️ Risks and limitations
+- General Model Behavior: While VoxCPM has been trained on a large-scale dataset, it may still produce outputs that are unexpected, biased, or contain artifacts.
+- Potential for Misuse of Voice Cloning: VoxCPM's powerful zero-shot voice cloning capability can generate highly realistic synthetic speech. This technology could be misused for creating convincing deepfakes for purposes of impersonation, fraud, or spreading disinformation. Users of this model must not use it to create content that infringes upon the rights of individuals. It is strictly forbidden to use VoxCPM for any illegal or unethical purposes. We strongly recommend that any publicly shared content generated with this model be clearly marked as AI-generated.
+- Current Technical Limitations: Although generally stable, the model may occasionally exhibit instability, especially with very long or expressive inputs. Furthermore, the current version offers limited direct control over specific speech attributes like emotion or speaking style.
+- Bilingual Model: VoxCPM is trained primarily on Chinese and English data. Performance on other languages is not guaranteed and may result in unpredictable or low-quality audio.
+- This model is released for research and development purposes only. We do not recommend its use in production or commercial applications without rigorous testing and safety evaluations. Please use VoxCPM responsibly.
+
+
+
+## 📄 License
+The VoxCPM model weights and code are open-sourced under the Apache-2.0 license.
+
+

VoxCPM/ckpts/audiovae.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b93f34c771281679b0ff93be3d1c1681eb1f301c3892e701db8f10a725b20a9
+size 301494192

VoxCPM/ckpts/config.json

@@ -0,0 +1,52 @@
+{
+    "architecture": "voxcpm",
+    "lm_config": {
+        "bos_token_id": 1,
+        "eos_token_id": 2,
+        "hidden_size": 1024,
+        "intermediate_size": 4096,
+        "max_position_embeddings": 32768,
+        "num_attention_heads": 16,
+        "num_hidden_layers": 24,
+        "num_key_value_heads": 2,
+        "rms_norm_eps": 1e-05,
+        "rope_theta": 10000,
+        "rope_scaling": {
+            "type": "longrope", 
+            "long_factor": [1.0004360675811768, 1.0668443441390991, 1.1631425619125366, 1.3025742769241333, 1.5040205717086792, 1.7941505908966064, 2.2101221084594727, 2.802666664123535, 3.6389970779418945, 4.804192543029785, 6.39855432510376, 8.527148246765137, 11.277542114257812, 14.684998512268066, 18.69317054748535, 23.13019371032715, 27.72362518310547, 32.1606559753418, 36.168827056884766, 39.57627868652344, 42.32667541503906, 44.45526885986328, 46.04962921142578, 47.21482849121094, 48.05115509033203, 48.64370346069336, 49.05967712402344, 49.34980392456055, 49.551246643066406, 49.69068145751953, 49.78697967529297, 49.85338592529297],
+            "short_factor": [1.0004360675811768, 1.0668443441390991, 1.1631425619125366, 1.3025742769241333, 1.5040205717086792, 1.7941505908966064, 2.2101221084594727, 2.802666664123535, 3.6389970779418945, 4.804192543029785, 6.39855432510376, 8.527148246765137, 11.277542114257812, 14.684998512268066, 18.69317054748535, 23.13019371032715, 27.72362518310547, 32.1606559753418, 36.168827056884766, 39.57627868652344, 42.32667541503906, 44.45526885986328, 46.04962921142578, 47.21482849121094, 48.05115509033203, 48.64370346069336, 49.05967712402344, 49.34980392456055, 49.551246643066406, 49.69068145751953, 49.78697967529297, 49.85338592529297],
+            "original_max_position_embeddings": 32768
+        },    
+        "vocab_size": 73448,
+        "scale_emb": 12,
+        "dim_model_base": 256,
+        "scale_depth": 1.4,
+        "use_mup": false
+    },
+    "patch_size": 2,
+    "feat_dim": 64,
+    "scalar_quantization_latent_dim": 256,
+    "scalar_quantization_scale": 9,
+    "residual_lm_num_layers": 6,
+    "encoder_config": {
+        "hidden_dim": 1024,
+        "ffn_dim": 4096,
+        "num_heads": 16,
+        "num_layers": 4
+    },
+    "dit_config": {
+        "hidden_dim": 1024,
+        "ffn_dim": 4096,
+        "num_heads": 16,
+        "num_layers": 4,
+        "cfm_config": {
+            "sigma_min": 1e-06,
+            "solver": "euler",
+            "t_scheduler": "log-norm",
+            "inference_cfg_rate": 2.0
+        }
+    },
+    "max_length": 4096,
+    "device": "cuda",
+    "dtype": "bfloat16"
+}

VoxCPM/ckpts/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd0581582d410d35a41ec991dd78c774134ee7c18d74d6e99707ceae5f3566f
+size 1304698606

VoxCPM/ckpts/special_tokens_map.json

@@ -0,0 +1,81 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|execute_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|execute_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

VoxCPM/ckpts/tokenizer_config.json

@@ -0,0 +1,212 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "<|audio_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "<|audio_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "<|audio_prompt_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "104": {
+      "content": "<|audio_prompt_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "105": {
+      "content": "<|background|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<|/background|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<|characters|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "<|/characters|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "109": {
+      "content": "<|speaker_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },   
+    "110": {
+      "content": "<|/speaker_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "111": {
+      "content": "<|span|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "112": {
+      "content": "<|/span|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73440": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73441": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73442": {
+      "content": "<|tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73443": {
+      "content": "<|execute_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73444": {
+      "content": "<|execute_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73445": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73446": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73447": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_end|>",
+    "<|im_start|>",
+    "<|tool_call|>",
+    "<|execute_start|>",
+    "<|execute_end|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
+}

VoxCPM/conf/voxcpm/experiments/README.md

@@ -0,0 +1,60 @@
+# LoRA 实验对照组
+
+## 实验目的
+探索不同 LoRA 配置对 VoxCPM 微调效果的影响，找出最佳配置。
+
+## 已知结论
+- **仅 DiT LoRA + 小 scaling (0.5)**: ✅ Work
+- **LM + DiT LoRA + 大 scaling (2.0)**: ❌ 噪声
+
+## 实验列表
+
+| 实验 | enable_lm | enable_dit | r | alpha | scaling | target_modules | 预期 |
+|------|-----------|------------|---|-------|---------|----------------|------|
+| exp_01 | ❌ | ✅ | 32 | 16 | 0.5 | DiT: qkvo | ✅ Work (已验证) |
+| exp_02 | ❌ | ✅ | 32 | 32 | 1.0 | DiT: qkvo | 可能 Work |
+| exp_03 | ❌ | ✅ | 32 | 64 | 2.0 | DiT: qkvo | 风险较高 |
+| exp_04 | ✅ | ❌ | 32 | 16 | 0.5 | LM: qkvo | 可能不 Work |
+| exp_05 | ✅ | ❌ | 32 | 8 | 0.25 | LM: qkvo | 边界测试 |
+| exp_06 | ✅ | ✅ | 32 | 16 | 0.5 | LM+DiT: qkvo | 可能不 Work |
+| exp_07 | ✅ | ✅ | 32 | 8 | 0.25 | LM+DiT: qkvo | 边界测试 |
+| exp_08 | ❌ | ✅ | 8 | 8 | 1.0 | DiT: qkvo | 小 rank 测试 |
+| exp_09 | ❌ | ✅ | 64 | 64 | 1.0 | DiT: qkvo | 大 rank 测试 |
+| exp_10 | ❌ | ✅ | 32 | 16 | 0.5 | DiT: qkvo+mlp | 更多模块测试 |
+
+## 运行命令
+
+```bash
+# 训练
+CUDA_VISIBLE_DEVICES=0 python scripts/train_voxcpm_finetune.py \
+    --config_path conf/voxcpm/experiments/exp_XX.yaml
+
+# 推理测试
+python scripts/test_voxcpm_lora_infer.py \
+    --config_path conf/voxcpm/experiments/exp_XX.yaml \
+    --lora_ckpt /user/liuxin/checkpoints/voxcpm_exp/exp_XX/step_0005000 \
+    --text "你好，这是一个测试。" \
+    --output exp_XX_test.wav
+```
+
+## 评估指标
+1. **能否正常说话**: 生成的音频是否有人声（而非噪声）
+2. **能否正常停止**: 是否在合理的时间内停止（而非跑满 max_len）
+3. **音色相似度**: 与训练数据的音色是否相似（主观评估）
+4. **Loss 收敛**: 训练过程中 loss/diff 和 loss/stop 的收敛情况
+
+## 结果记录
+
+| 实验 | 能说话 | 能停止 | 音色相似度 | 备注 |
+|------|--------|--------|------------|------|
+| exp_01 | ✅ | ✅ | | |
+| exp_02 | ✅ | ✅ | | |
+| exp_03 | | | | |
+| exp_04 | | | | |
+| exp_05 | | | | |
+| exp_06 | | | | |
+| exp_07 | | | | |
+| exp_08 | | | | |
+| exp_09 | | | | |
+| exp_10 | | | | |
+

VoxCPM/conf/voxcpm/experiments/exp_01_dit_only_scale05.yaml

@@ -0,0 +1,33 @@
+# 实验 01: 仅 DiT LoRA, scaling=0.5 (r=32, alpha=16)
+# 预期: 应该 work（你已验证）
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_01_dit_only_scale05
+tensorboard: /user/liuxin/checkpoints/logs/exp_01
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: false
+  enable_dit: true
+  enable_proj: false
+  r: 32
+  alpha: 16  # scaling = 16/32 = 0.5
+  dropout: 0.0
+  target_modules_lm: []
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+

VoxCPM/conf/voxcpm/experiments/exp_02_dit_only_scale10.yaml

@@ -0,0 +1,33 @@
+# 实验 02: 仅 DiT LoRA, scaling=1.0 (r=32, alpha=32)
+# 预期: 可能 work，比 0.5 更强的适应能力
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_02_dit_only_scale10
+tensorboard: /user/liuxin/checkpoints/logs/exp_02
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: false
+  enable_dit: true
+  enable_proj: false
+  r: 32
+  alpha: 32  # scaling = 32/32 = 1.0
+  dropout: 0.0
+  target_modules_lm: []
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+

VoxCPM/conf/voxcpm/experiments/exp_03_dit_only_scale20.yaml

@@ -0,0 +1,33 @@
+# 实验 03: 仅 DiT LoRA, scaling=2.0 (r=32, alpha=64)
+# 预期: 可能 work，但风险较高（scaling 较大）
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_03_dit_only_scale20
+tensorboard: /user/liuxin/checkpoints/logs/exp_03
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: false
+  enable_dit: true
+  enable_proj: false
+  r: 32
+  alpha: 64  # scaling = 64/32 = 2.0
+  dropout: 0.0
+  target_modules_lm: []
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+

VoxCPM/conf/voxcpm/experiments/exp_04_lm_only_scale05.yaml

@@ -0,0 +1,33 @@
+# 实验 04: 仅 LM LoRA, scaling=0.5 (r=32, alpha=16)
+# 预期: 可能不 work（LM 改变会影响 stop predictor）
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_04_lm_only_scale05
+tensorboard: /user/liuxin/checkpoints/logs/exp_04
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: true
+  enable_dit: false
+  enable_proj: false
+  r: 32
+  alpha: 16  # scaling = 16/32 = 0.5
+  dropout: 0.0
+  target_modules_lm: ["q_proj", "v_proj", "k_proj", "o_proj"]  # 仅 attention
+  target_modules_dit: []
+

VoxCPM/conf/voxcpm/experiments/exp_05_lm_only_scale025.yaml

@@ -0,0 +1,33 @@
+# 实验 05: 仅 LM LoRA, scaling=0.25 (r=32, alpha=8)
+# 预期: 可能 work（极小的 scaling 可能不会破坏 stop predictor）
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_05_lm_only_scale025
+tensorboard: /user/liuxin/checkpoints/logs/exp_05
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: true
+  enable_dit: false
+  enable_proj: false
+  r: 32
+  alpha: 8  # scaling = 8/32 = 0.25
+  dropout: 0.0
+  target_modules_lm: ["q_proj", "v_proj", "k_proj", "o_proj"]  # 仅 attention
+  target_modules_dit: []
+

VoxCPM/conf/voxcpm/experiments/exp_06_both_scale05.yaml

@@ -0,0 +1,33 @@
+# 实验 06: LM + DiT LoRA, scaling=0.5 (r=32, alpha=16)
+# 预期: 可能不 work（LM 改变会影响 stop predictor）
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_06_both_scale05
+tensorboard: /user/liuxin/checkpoints/logs/exp_06
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: true
+  enable_dit: true
+  enable_proj: false
+  r: 32
+  alpha: 16  # scaling = 16/32 = 0.5
+  dropout: 0.0
+  target_modules_lm: ["q_proj", "v_proj", "k_proj", "o_proj"]  # 仅 attention
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+

VoxCPM/conf/voxcpm/experiments/exp_07_both_scale025.yaml

@@ -0,0 +1,33 @@
+# 实验 07: LM + DiT LoRA, scaling=0.25 (r=32, alpha=8)
+# 预期: 边界实验，极小 scaling 可能让 LM+DiT 组合 work
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_07_both_scale025
+tensorboard: /user/liuxin/checkpoints/logs/exp_07
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: true
+  enable_dit: true
+  enable_proj: false
+  r: 32
+  alpha: 8  # scaling = 8/32 = 0.25
+  dropout: 0.0
+  target_modules_lm: ["q_proj", "v_proj", "k_proj", "o_proj"]  # 仅 attention
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+

VoxCPM/conf/voxcpm/experiments/exp_08_dit_only_small_r.yaml

@@ -0,0 +1,33 @@
+# 实验 08: 仅 DiT LoRA, 小 rank (r=8, alpha=8, scaling=1.0)
+# 预期: work，测试小 rank 是否足够
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_08_dit_only_small_r
+tensorboard: /user/liuxin/checkpoints/logs/exp_08
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: false
+  enable_dit: true
+  enable_proj: false
+  r: 8  # 小 rank
+  alpha: 8  # scaling = 8/8 = 1.0
+  dropout: 0.0
+  target_modules_lm: []
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+

VoxCPM/conf/voxcpm/experiments/exp_09_dit_only_large_r.yaml

@@ -0,0 +1,33 @@
+# 实验 09: 仅 DiT LoRA, 大 rank (r=64, alpha=64, scaling=1.0)
+# 预期: work，测试大 rank 是否有更好的效果
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_09_dit_only_large_r
+tensorboard: /user/liuxin/checkpoints/logs/exp_09
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: false
+  enable_dit: true
+  enable_proj: false
+  r: 64  # 大 rank
+  alpha: 64  # scaling = 64/64 = 1.0
+  dropout: 0.0
+  target_modules_lm: []
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+

VoxCPM/conf/voxcpm/experiments/exp_10_dit_only_more_modules.yaml

@@ -0,0 +1,33 @@
+# 实验 10: 仅 DiT LoRA, 更多目标模块 (包含 MLP)
+# 预期: work，测试更多模块是否有更好的效果
+pretrained_path: /user/zhouyixuan/ckpt/VoxCPM-0.5B-20250912-decay/
+train_manifest: /user/liuxin/workspace/data/mianduan/1104-miantuan-data/metadata.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 500
+save_interval: 500
+learning_rate: 0.0002
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+save_path: /user/liuxin/checkpoints/voxcpm_exp/exp_10_dit_only_more_modules
+tensorboard: /user/liuxin/checkpoints/logs/exp_10
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: false
+  enable_dit: true
+  enable_proj: false
+  r: 32
+  alpha: 16  # scaling = 0.5
+  dropout: 0.0
+  target_modules_lm: []
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "down_proj", "up_proj"]  # 包含 MLP
+

VoxCPM/conf/voxcpm/voxcpm_finetune_example.yaml

@@ -0,0 +1,23 @@
+pretrained_path: ckpts
+train_manifest: datasets/metadatas/single_speaker_train_datas.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 8
+grad_accum_steps: 1  # 梯度累积步数，>1 时可在不增大显存的情况下提升等效 batch size
+num_workers: 2
+num_iters: 1000
+log_interval: 1
+valid_interval: 100
+save_interval: 100
+learning_rate: 0.00001
+weight_decay: 0.01
+warmup_steps: 10
+max_steps: 1000
+max_batch_tokens: 0  # 0 表示不启用长度过滤；>0 时按 batch 总 token 上限过滤超长样本
+save_path: checkpoints/voxcpm_finetune
+tensorboard: logs/voxcpm_finetune
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+
+

VoxCPM/conf/voxcpm/voxcpm_finetune_lora.yaml

@@ -0,0 +1,31 @@
+pretrained_path: ckpts
+train_manifest: datasets/metadatas/single_speaker_train_datas.jsonl
+val_manifest: null
+sample_rate: 16000
+batch_size: 8
+grad_accum_steps: 1  # 梯度累积步数，>1 时可在不增大显存的情况下提升等效 batch size
+num_workers: 2
+num_iters: 2000
+log_interval: 1
+valid_interval: 100
+save_interval: 100
+learning_rate: 0.0001  # Increased for LoRA (original 1e-5 is too small for LoRA)
+weight_decay: 0.01
+warmup_steps: 10
+max_steps: 2000
+max_batch_tokens: 8192  # 例：单个 batch 最多 16k token，按 batch_size=4 则单样本约 4096 token 上限
+save_path: checkpoints/voxcpm_finetune
+tensorboard: logs/voxcpm_finetune
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+lora:
+  enable_lm: true # 是否微调LM模型
+  enable_dit: true # 是否微调DiT模型
+  enable_proj: false # 是否微调投影层
+  r: 32               # Increased rank
+  alpha: 16           # Increased alpha (scaling = alpha/r = 2.0)
+  dropout: 0.0       # Added dropout
+  target_modules_lm: ["q_proj", "v_proj", "k_proj", "o_proj"] # LM模型要微调的层
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"] # DiT模型要微调的层
+  target_proj_modules: ["enc_to_lm_proj", "lm_to_dit_proj", "res_to_dit_proj"] # 投影层要微调的层

VoxCPM/datasets.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4f3ef25fb26393336239176dcbc46266f67804990a953ba4ebb300b09eb878f
+size 28233400938

VoxCPM/docs/finetune.md

@@ -0,0 +1,260 @@
+# VoxCPM 微调指南
+
+本文档介绍如何对 VoxCPM 模型进行微调，支持全量微调和 LoRA 微调两种方式。
+
+## 目录
+
+- [数据准备](#数据准备)
+- [全量微调](#全量微调)
+- [LoRA 微调](#lora-微调)
+- [推理测试](#推理测试)
+- [LoRA 热切换](#lora-热切换)
+- [常见问题](#常见问题)
+
+---
+
+## 数据准备
+
+训练数据需要准备为 JSONL 格式的 manifest 文件，每行包含一条训练样本：
+
+```json
+{"audio_path": "/path/to/audio1.wav", "text": "对应的文本内容"}
+{"audio_path": "/path/to/audio2.wav", "text": "另一条文本"}
+```
+
+**要求**：
+- 音频格式：WAV，采样率 16kHz
+- 文本：与音频对应的转录文本
+
+---
+
+## 全量微调
+
+全量微调会更新模型的所有参数，适合数据量较大、需要显著改变模型行为的场景。
+
+### 配置文件
+
+创建配置文件 `conf/voxcpm/voxcpm_finetune_all.yaml`：
+
+```yaml
+pretrained_path: /path/to/VoxCPM-0.5B/          # 预训练模型路径
+train_manifest: /path/to/train_manifest.jsonl   # 训练数据
+val_manifest: null                               # 验证数据（可选）
+
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1      # 梯度累积步数，显存不足时可增大
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 1000
+save_interval: 1000
+
+learning_rate: 0.00001   # 全量微调建议较小的学习率
+weight_decay: 0.01
+warmup_steps: 100
+max_steps: 2000
+max_batch_tokens: 8192
+
+save_path: /path/to/checkpoints/finetune_all
+tensorboard: /path/to/logs/finetune_all
+
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+```
+
+### 启动训练
+
+```bash
+python scripts/train_voxcpm_finetune.py --config_path conf/voxcpm/voxcpm_finetune_all.yaml
+```
+
+---
+
+## LoRA 微调
+
+LoRA（Low-Rank Adaptation）是一种参数高效的微调方法，只训练少量额外参数，显著降低显存需求。
+
+### 配置文件
+
+创建配置文件 `conf/voxcpm/voxcpm_finetune_lora.yaml`：
+
+```yaml
+pretrained_path: /path/to/VoxCPM-0.5B/
+train_manifest: /path/to/train_manifest.jsonl
+val_manifest: null
+
+sample_rate: 16000
+batch_size: 16
+grad_accum_steps: 1
+num_workers: 2
+num_iters: 2000
+log_interval: 10
+valid_interval: 1000
+save_interval: 1000
+
+learning_rate: 0.0001    # LoRA 可以使用较大的学习率
+weight_decay: 0.01
+warmup_steps: 200
+max_steps: 2000
+max_batch_tokens: 8192
+
+save_path: /path/to/checkpoints/finetune_lora
+tensorboard: /path/to/logs/finetune_lora
+
+lambdas:
+  loss/diff: 1.0
+  loss/stop: 1.0
+
+# LoRA 配置
+lora:
+  enable_lm: true        # 对 Language Model 加 LoRA
+  enable_dit: true       # 对 Diffusion Transformer 加 LoRA
+  enable_proj: false     # 对投影层加 LoRA（可选）
+  
+  r: 32                  # LoRA 秩（rank），越大容量越大
+  alpha: 16              # LoRA alpha，scaling = alpha / r
+  dropout: 0.0           # LoRA dropout
+  
+  # 目标模块
+  target_modules_lm: ["q_proj", "v_proj", "k_proj", "o_proj"]
+  target_modules_dit: ["q_proj", "v_proj", "k_proj", "o_proj"]
+```
+
+### LoRA 参数说明
+
+| 参数 | 说明 | 建议值 |
+|------|------|--------|
+| `enable_lm` | 对 LM（语言模型）加 LoRA | `true` |
+| `enable_dit` | 对 DiT（扩散模型）加 LoRA | `true`（音色克隆必须） |
+| `r` | LoRA 秩，越大容量越大 | 16-64 |
+| `alpha` | 缩放因子，`scaling = alpha / r` | 通常设为 `r/2` 或 `r` |
+| `target_modules_*` | 要添加 LoRA 的层名 | attention 层 |
+
+### 启动训练
+
+```bash
+python scripts/train_voxcpm_finetune.py --config_path conf/voxcpm/voxcpm_finetune_lora.yaml
+```
+
+### Checkpoint 结构
+
+LoRA 训练保存的 checkpoint 只包含 LoRA 参数：
+
+```
+checkpoints/finetune_lora/
+└── step_0002000/
+    └── generator.pth    # 仅包含 lora_A, lora_B 参数
+```
+
+---
+
+## 推理测试
+
+### 全量微调推理
+
+```bash
+python scripts/test_voxcpm_ft_infer.py \
+    --pretrained_path /path/to/VoxCPM-0.5B/ \
+    --ft_ckpt /path/to/checkpoints/finetune_all/step_0002000 \
+    --text "你好，这是微调后的效果。" \
+    --output output.wav
+```
+
+### LoRA 推理
+
+```bash
+python scripts/test_voxcpm_lora_infer.py \
+    --config_path conf/voxcpm/voxcpm_finetune_lora.yaml \
+    --lora_ckpt /path/to/checkpoints/finetune_lora/step_0002000 \
+    --text "你好，这是 LoRA 微调后的效果。" \
+    --output lora_output.wav
+```
+
+### 带参考音频（音色克隆）
+
+```bash
+python scripts/test_voxcpm_lora_infer.py \
+    --config_path conf/voxcpm/voxcpm_finetune_lora.yaml \
+    --lora_ckpt /path/to/checkpoints/finetune_lora/step_0002000 \
+    --text "这是带参考音色的合成效果。" \
+    --prompt_audio /path/to/reference.wav \
+    --prompt_text "参考音频对应的文本" \
+    --output cloned_output.wav
+```
+
+---
+
+## LoRA 热切换
+
+LoRA 支持在推理时动态加载、卸载和切换，无需重新加载整个模型。
+
+### API 说明
+
+```python
+from voxcpm.model import VoxCPMModel
+from voxcpm.model.voxcpm import LoRAConfig
+
+# 1. 加载模型（包含 LoRA 结构）
+lora_cfg = LoRAConfig(enable_lm=True, enable_dit=True, r=32, alpha=16, ...)
+model = VoxCPMModel.from_local(
+    pretrained_path,
+    optimize=True,       # 启用 torch.compile 加速
+    lora_config=lora_cfg
+)
+
+# 2. 加载 LoRA 权重（支持 compile 后调用）
+model.load_lora_weights("/path/to/lora_checkpoint")
+
+# 3. 禁用 LoRA（使用基础模型）
+model.set_lora_enabled(False)
+
+# 4. 重新启用 LoRA
+model.set_lora_enabled(True)
+
+# 5. 卸载 LoRA（重置权重为 0）
+model.reset_lora_weights()
+
+# 6. 热切换到另一个 LoRA
+model.load_lora_weights("/path/to/another_lora_checkpoint")
+```
+
+### 方法说明
+
+| 方法 | 功能 | 兼容 torch.compile |
+|------|------|-------------------|
+| `load_lora_weights(path)` | 从文件加载 LoRA 权重 | ✅ |
+| `set_lora_enabled(bool)` | 启用/禁用 LoRA | ✅ |
+| `reset_lora_weights()` | 重置 LoRA 权重为初始值 | ✅ |
+| `get_lora_state_dict()` | 获取当前 LoRA 权重 | ✅ |
+
+---
+
+## 常见问题
+
+### 1. 显存不足
+
+- 增大 `grad_accum_steps`（梯度累积）
+- 减小 `batch_size`
+- 使用 LoRA 微调代替全量微调
+
+### 2. LoRA 效果不好
+
+- 增大 `r`（LoRA 秩）
+- 调整 `alpha`（建议 `alpha = r/2` 或 `alpha = r`）
+- 确保 `enable_dit: true`（音色克隆必须）
+- 增加训练步数
+
+### 3. 训练不收敛
+
+- 降低 `learning_rate`
+- 增加 `warmup_steps`
+- 检查数据质量
+
+### 4. 推理时 LoRA 不生效
+
+- 确保推理配置与训练配置的 LoRA 参数一致
+- 检查 `load_lora_weights` 返回的 `skipped_keys` 是否为空
+
+

VoxCPM/examples/example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:009638e7474ac4eb2ca5b23d28d9114c33377eb5c91e8d6f7000a0c36d6eaa8e
+size 1439096

VoxCPM/inference.py

@@ -0,0 +1,67 @@
+import soundfile as sf
+import numpy as np
+from voxcpm import VoxCPM
+import argparse
+import os
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_path", type=str, required=True)
+    parser.add_argument("--text", type=str)
+    parser.add_argument("--text_file", type=str)
+    parser.add_argument("--output_dir", type=str, default="outputs")
+    parser.add_argument("--cfg_value", type=float, default=2.0)
+    parser.add_argument("--inference_timesteps", type=int, default=10)
+    parser.add_argument("--prompt_wav_path", type=str)
+    parser.add_argument("--prompt_text", type=str)
+    args = parser.parse_args()
+    assert args.text or args.text_file, "Please provide either text or text_file"
+    # validate prompt_wav_path and prompt_text 必须同时提供
+    if args.prompt_wav_path or args.prompt_text:
+        assert args.prompt_wav_path and args.prompt_text, "Please provide both prompt_wav_path and prompt_text"
+    model = VoxCPM.from_pretrained(args.model_path, load_denoiser=False)
+    if args.text:
+        wav = model.generate(
+        text=args.text,
+        prompt_wav_path=args.prompt_wav_path,      # optional: path to a prompt speech for voice cloning
+        prompt_text=args.prompt_text,          # optional: reference text
+        cfg_value=args.cfg_value,             # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse
+        inference_timesteps=args.inference_timesteps,   # LocDiT inference timesteps, higher for better result, lower for fast speed
+        normalize=True,           # enable external TN tool
+        denoise=False,             # enable external Denoise tool
+        retry_badcase=True,        # enable retrying mode for some bad cases (unstoppable)
+        retry_badcase_max_times=3,  # maximum retrying times
+        retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech
+        )
+        if not os.path.exists(args.output_dir):
+            os.makedirs(args.output_dir)
+        sf.write(f"{args.output_dir}/output.wav", wav, 16000)
+        print(f"saved: {args.output_dir}/output.wav")
+    elif args.text_file:
+        texts = []
+        with open(args.text_file, "r") as f:
+            lines = f.readlines()
+            for line in lines:
+                line = line.strip().split("||")
+                wav_id = line[0]
+                text = " ".join(line[1:])
+                texts.append((wav_id, text))
+        for wav_id, text in texts:
+            wav = model.generate(
+            text=text,
+            prompt_wav_path=args.prompt_wav_path,      # optional: path to a prompt speech for voice cloning
+            prompt_text=args.prompt_text,          # optional: reference text
+            cfg_value=args.cfg_value,             # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse
+            inference_timesteps=args.inference_timesteps,   # LocDiT inference timesteps, higher for better result, lower for fast speed
+            normalize=True,           # enable external TN tool
+            denoise=False,             # enable external Denoise tool
+            retry_badcase=True,        # enable retrying mode for some bad cases (unstoppable)
+            retry_badcase_max_times=3,  # maximum retrying times
+            retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech
+            )
+            if not os.path.exists(args.output_dir):
+                os.makedirs(args.output_dir)
+            sf.write(f"{args.output_dir}/{wav_id}.wav", wav, 16000)
+            print(f"saved: {args.output_dir}/{wav_id}.wav")
+
+if __name__ == "__main__":
+    main()

VoxCPM/inference_lora.py

@@ -0,0 +1,91 @@
+import soundfile as sf
+import numpy as np
+from pathlib import Path
+from voxcpm.model import VoxCPMModel
+from voxcpm.model.voxcpm import LoRAConfig
+from voxcpm.training.config import load_yaml_config
+import argparse
+import torch
+import os
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--lora_ckpt", type=str, required=True)
+    parser.add_argument("--lora_config_path", type=str, required=True)
+    parser.add_argument("--text", type=str)
+    parser.add_argument("--text_file", type=str)
+    parser.add_argument("--output_dir", type=str, default="outputs")
+    parser.add_argument("--cfg_value", type=float, default=2.0)
+    parser.add_argument("--inference_timesteps", type=int, default=10)
+    args = parser.parse_args()
+    assert args.text or args.text_file, "Please provide either text or text_file"
+
+    # 1. 读取 YAML 配置
+    cfg = load_yaml_config(args.lora_config_path)
+    pretrained_path = cfg["pretrained_path"]
+    lora_cfg_dict = cfg.get("lora", {}) or {}
+    lora_cfg = LoRAConfig(**lora_cfg_dict) if lora_cfg_dict else None
+
+    # 2. 加载基础模型（包含 LoRA 结构，并执行 torch.compile）
+    print(f"[1/3] 加载基础模型：{pretrained_path}")
+    model = VoxCPMModel.from_local(
+        pretrained_path,
+        optimize=True,  # 先 compile，load_lora_weights 使用 named_parameters 兼容
+        training=False,
+        lora_config=lora_cfg,
+    )
+
+    # 3. 加载 LoRA 权重（在 compile 后也能正常工作）
+    ckpt_dir = Path(args.lora_ckpt)
+    if not ckpt_dir.exists():
+        raise FileNotFoundError(f"找不到 LoRA checkpoint: {ckpt_dir}")
+    
+    print(f"[2/3] 加载 LoRA 权重：{ckpt_dir}")
+    loaded, skipped = model.load_lora_weights(str(ckpt_dir))
+    print(f"       已加载 {len(loaded)} 个参数")
+    if skipped:
+        print(f"[WARNING] 跳过 {len(skipped)} 个参数")
+        print(f"       跳过的 key (前5个): {skipped[:5]}")
+    print(f"\n[3/3] 开始推理...")
+    if args.text:
+        with torch.inference_mode():
+            wav = model.generate(
+            target_text=args.text,
+            cfg_value=args.cfg_value,             # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse
+            inference_timesteps=args.inference_timesteps,   # LocDiT inference timesteps, higher for better result, lower for fast speed
+            retry_badcase=True,        # enable retrying mode for some bad cases (unstoppable)
+            retry_badcase_max_times=3,  # maximum retrying times
+            retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech
+            )
+            audio_np = wav.squeeze(0).cpu().numpy() if wav.dim() > 1 else wav.cpu().numpy()
+        if not os.path.exists(args.output_dir):
+            os.makedirs(args.output_dir)
+        sf.write(f"{args.output_dir}/output_lora.wav", audio_np, 16000)
+        print(f"saved: {args.output_dir}/output_lora.wav")
+    elif args.text_file:
+        texts = []
+        with open(args.text_file, "r") as f:
+            lines = f.readlines()
+            for line in lines:
+                line = line.strip().split("||")
+                wav_id = line[0]
+                text = " ".join(line[1:])
+                texts.append((wav_id, text))
+        for wav_id, text in texts:
+            with torch.inference_mode():
+                wav = model.generate(
+                target_text=text,
+                cfg_value=args.cfg_value,             # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse
+                inference_timesteps=args.inference_timesteps,   # LocDiT inference timesteps, higher for better result, lower for fast speed
+                retry_badcase=True,        # enable retrying mode for some bad cases (unstoppable)
+                retry_badcase_max_times=3,  # maximum retrying times
+                retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech
+                )
+                audio_np = wav.squeeze(0).cpu().numpy() if wav.dim() > 1 else wav.cpu().numpy()
+            if not os.path.exists(args.output_dir):
+                os.makedirs(args.output_dir)
+            sf.write(f"{args.output_dir}/{wav_id}.wav", audio_np, 16000)
+            print(f"saved: {args.output_dir}/{wav_id}.wav")
+
+if __name__ == "__main__":
+    main()

VoxCPM/prompt_sample.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3efe60d997679f3d6dcd94531cd93e4c9d4536c618133a4af5fa87f7981e1d5f
+size 401196

VoxCPM/pyproject.toml

@@ -0,0 +1,95 @@
+[build-system]
+requires = ["setuptools>=64", "wheel", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "voxcpm"
+dynamic = ["version"]
+description = "VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning"
+readme = "README.md"
+license = "Apache-2.0"
+authors = [
+    {name = "OpenBMB", email = "openbmb@gmail.com"}
+]
+maintainers = [
+    {name = "OpenBMB", email = "openbmb@gmail.com"}
+]
+keywords = ["voxcpm", "text-to-speech", "tts", "speech-synthesis", "voice-cloning", "ai", "deep-learning", "pytorch"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+requires-python = ">=3.10"
+dependencies = [
+    "torch>=2.5.0",
+    "torchaudio>=2.5.0",
+    "transformers>=4.36.2",
+    "einops",
+    "gradio",
+    "inflect",
+    "addict",
+    "wetext",
+    "modelscope>=1.22.0",
+    "datasets>=3,<4",
+    "huggingface-hub",
+    "pydantic",
+    "tqdm",
+    "simplejson",
+    "sortedcontainers",
+    "soundfile",
+    "funasr",
+    "spaces",
+    "argbind",
+    "tensorboardX",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=6.0",
+    "pytest-cov>=2.0",
+    "black>=21.0",
+    "flake8>=3.8",
+    "mypy>=0.800",
+    "pre-commit>=2.0",
+]
+
+[project.scripts]
+voxcpm = "voxcpm.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/OpenBMB/VoxCPM"
+Repository = "https://github.com/OpenBMB/VoxCPM.git"
+Documentation = "https://github.com/OpenBMB/VoxCPM#readme"
+"Bug Tracker" = "https://github.com/OpenBMB/VoxCPM/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["voxcpm*"]
+
+[tool.setuptools.package-dir]
+"" = "src"
+
+[tool.setuptools_scm]
+version_scheme = "post-release"
+
+[tool.black]
+line-length = 120
+target-version = ['py310']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+  # directories
+  \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | build
+  | dist
+)/
+'''

VoxCPM/requirements.txt

@@ -0,0 +1,2 @@
+argbind
+tensorboardX

VoxCPM/scripts/test_voxcpm_ft_infer.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""
+简单的全量微调权重推理测试脚本（无 LoRA）：
+
+1. 使用训练时相同的 YAML（拿到 pretrained_path）
+2. 从 pretrained_path 加载基础 VoxCPM 模型（config.json / pytorch_model.bin / audiovae.pth）
+3. 再从全量微调 checkpoint 目录加载微调后的 generator.pth（保存的是完整 state_dict）
+4. 调用模型的 generate 接口合成语音并保存为 wav
+
+用法示例（与你的 finetune 配置保持一致）：
+
+    python scripts/test_voxcpm_ft_infer.py \
+        --config_path conf/voxcpm/voxcpm_finetune_test.yaml \
+        --ckpt_dir /user/liuxin/checkpoints/voxcpm_finetune/step_0001000 \
+        --text "你好，我是全量微调后的 VoxCPM。" \
+        --output ft_test.wav
+
+如果需要参考音色克隆：
+
+    python scripts/test_voxcpm_ft_infer.py \
+        --config_path conf/voxcpm/voxcpm_finetune_test.yaml \
+        --ckpt_dir /user/liuxin/checkpoints/voxcpm_finetune/step_0001000 \
+        --text "你好，这是带参考音色的合成效果。" \
+        --prompt_audio path/to/ref.wav \
+        --prompt_text "参考音频对应的文本" \
+        --output ft_clone.wav
+"""
+
+import argparse
+from pathlib import Path
+
+import soundfile as sf
+import torch
+
+from voxcpm.model import VoxCPMModel
+from voxcpm.training.config import load_yaml_config
+
+
+def parse_args():
+    parser = argparse.ArgumentParser("VoxCPM full-finetune inference test (no LoRA)")
+    parser.add_argument(
+        "--config_path",
+        type=str,
+        required=True,
+        help="训练时使用的 YAML 配置路径（至少包含 pretrained_path）",
+    )
+    parser.add_argument(
+        "--ckpt_dir",
+        type=str,
+        required=True,
+        help="全量微调 checkpoint 目录（内含 generator.pth，保存完整 state_dict）",
+    )
+    parser.add_argument(
+        "--text",
+        type=str,
+        required=True,
+        help="待合成的目标文本（target_text）",
+    )
+    parser.add_argument(
+        "--prompt_audio",
+        type=str,
+        default="",
+        help="可选：参考音频路径，用于 voice cloning（不填则为直接 TTS）",
+    )
+    parser.add_argument(
+        "--prompt_text",
+        type=str,
+        default="",
+        help="可选：参考音频对应的文本（与 prompt_audio 搭配使用）",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="ft_test.wav",
+        help="输出 wav 文件路径",
+    )
+    parser.add_argument(
+        "--cfg_value",
+        type=float,
+        default=2.0,
+        help="推理时的 CFG scale，与训练 / 官方示例保持一致（默认 2.0）",
+    )
+    parser.add_argument(
+        "--inference_timesteps",
+        type=int,
+        default=10,
+        help="扩散推理步数，默认为 10",
+    )
+    parser.add_argument(
+        "--max_len",
+        type=int,
+        default=600,
+        help="生成阶段的最大步数（对应 _generate 的 max_len）",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    # 1. 读取 YAML 配置，拿到 pretrained_path
+    cfg = load_yaml_config(args.config_path)
+    pretrained_path = cfg["pretrained_path"]
+
+    # 2. 加载基础模型（无 LoRA），推理模式：training=False，使用 config.dtype，并执行 optimize()
+    print(f"[FT Inference] 加载基础模型：{pretrained_path}")
+    model = VoxCPMModel.from_local(
+        pretrained_path,
+        optimize=True,
+        training=False,
+        lora_config=None,
+    )
+
+    # 3. 从全量微调 ckpt 目录加载 generator.pth（完整 state_dict）
+    ckpt_dir = Path(args.ckpt_dir)
+    ckpt_path = ckpt_dir / "generator.pth"
+    if not ckpt_path.exists():
+        raise FileNotFoundError(f"找不到全量微调 checkpoint: {ckpt_path}")
+
+    print(f"[FT Inference] 从 {ckpt_path} 加载全量微调权重")
+    ckpt = torch.load(ckpt_path, map_location="cpu")
+    state_dict = ckpt.get("state_dict", ckpt)
+
+    # 为兼容可能存在的额外键，使用 strict=False，并打印 missing / unexpected 数量
+    missing, unexpected = model.load_state_dict(state_dict, strict=False)
+    print(f"[FT Inference] 加载完成：missing={len(missing)}, unexpected={len(unexpected)}")
+
+    # 4. 调用 generate 进行推理
+    prompt_wav_path = args.prompt_audio or ""
+    prompt_text = args.prompt_text or ""
+
+    print(f"[FT Inference] 开始合成：text='{args.text}'")
+    if prompt_wav_path:
+        print(f"[FT Inference] 使用参考音频：{prompt_wav_path}")
+        print(f"[FT Inference] 参考文本：{prompt_text}")
+
+    with torch.inference_mode():
+        audio = model.generate(
+            target_text=args.text,
+            prompt_text=prompt_text,
+            prompt_wav_path=prompt_wav_path,
+            max_len=args.max_len,
+            inference_timesteps=args.inference_timesteps,
+            cfg_value=args.cfg_value,
+        )
+
+    # generate 返回的是一维 / 二维 Tensor，这里做一次收缩并落盘
+    if isinstance(audio, torch.Tensor):
+        if audio.dim() > 1:
+            audio = audio.squeeze(0)
+        audio_np = audio.cpu().numpy()
+    else:
+        raise TypeError(f"model.generate 返回类型异常：{type(audio)}")
+
+    out_path = Path(args.output)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    sf.write(str(out_path), audio_np, model.sample_rate)
+
+    print(f"[FT Inference] 已保存到：{out_path}，时长约 {len(audio_np) / model.sample_rate:.2f}s")
+
+
+if __name__ == "__main__":
+    main()
+
+

VoxCPM/scripts/test_voxcpm_lora_infer.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""
+LoRA 推理测试脚本。
+
+用法：
+
+    python scripts/test_voxcpm_lora_infer.py \
+        --config_path conf/voxcpm/voxcpm_finetune_test.yaml \
+        --lora_ckpt checkpoints/step_0002000 \
+        --text "你好，这是 LoRA 微调后的效果。" \
+        --output lora_test.wav
+
+带参考音频的音色克隆：
+
+    python scripts/test_voxcpm_lora_infer.py \
+        --config_path conf/voxcpm/voxcpm_finetune_test.yaml \
+        --lora_ckpt checkpoints/step_0002000 \
+        --text "这是带参考音色的合成效果。" \
+        --prompt_audio path/to/ref.wav \
+        --prompt_text "参考音频对应的文本" \
+        --output lora_clone.wav
+"""
+
+import argparse
+from pathlib import Path
+
+import soundfile as sf
+import torch
+
+from voxcpm.model import VoxCPMModel
+from voxcpm.model.voxcpm import LoRAConfig
+from voxcpm.training.config import load_yaml_config
+
+
+def parse_args():
+    parser = argparse.ArgumentParser("VoxCPM LoRA inference test")
+    parser.add_argument(
+        "--config_path",
+        type=str,
+        required=True,
+        help="训练时使用的 YAML 配置路径（包含 pretrained_path 和 lora 配置）",
+    )
+    parser.add_argument(
+        "--lora_ckpt",
+        type=str,
+        required=True,
+        help="LoRA checkpoint 目录（内含 generator.pth，仅包含 lora_A / lora_B）",
+    )
+    parser.add_argument(
+        "--text",
+        type=str,
+        required=True,
+        help="待合成的目标文本（target_text）",
+    )
+    parser.add_argument(
+        "--prompt_audio",
+        type=str,
+        default="",
+        help="可选：参考音频路径，用于 voice cloning（不填则为直接 TTS）",
+    )
+    parser.add_argument(
+        "--prompt_text",
+        type=str,
+        default="",
+        help="可选：参考音频对应的文本（与 prompt_audio 搭配使用）",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="lora_test.wav",
+        help="输出 wav 文件路径",
+    )
+    parser.add_argument(
+        "--cfg_value",
+        type=float,
+        default=2.0,
+        help="推理时的 CFG scale，与训练 / 官方示例保持一致（默认 2.0）",
+    )
+    parser.add_argument(
+        "--inference_timesteps",
+        type=int,
+        default=10,
+        help="扩散推理步数，默认为 10",
+    )
+    parser.add_argument(
+        "--max_len",
+        type=int,
+        default=600,
+        help="生成阶段的最大步数（对应 _generate 的 max_len）",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    # 1. 读取 YAML 配置
+    cfg = load_yaml_config(args.config_path)
+    pretrained_path = cfg["pretrained_path"]
+    lora_cfg_dict = cfg.get("lora", {}) or {}
+    lora_cfg = LoRAConfig(**lora_cfg_dict) if lora_cfg_dict else None
+
+    # 2. 加载基础模型（包含 LoRA 结构，并执行 torch.compile）
+    print(f"[1/3] 加载基础模型：{pretrained_path}")
+    model = VoxCPMModel.from_local(
+        pretrained_path,
+        optimize=True,  # 先 compile，load_lora_weights 使用 named_parameters 兼容
+        training=False,
+        lora_config=lora_cfg,
+    )
+    
+    # 调试：检查 compile 后 DiT 的参数路径
+    dit_params = [n for n, _ in model.named_parameters() if 'feat_decoder' in n and 'lora' in n]
+    print(f"[DEBUG] compile 后 DiT LoRA 参数路径 (前3个): {dit_params[:3]}")
+
+    # 3. 加载 LoRA 权重（在 compile 后也能正常工作）
+    ckpt_dir = Path(args.lora_ckpt)
+    if not ckpt_dir.exists():
+        raise FileNotFoundError(f"找不到 LoRA checkpoint: {ckpt_dir}")
+    
+    print(f"[2/3] 加载 LoRA 权重：{ckpt_dir}")
+    loaded, skipped = model.load_lora_weights(str(ckpt_dir))
+    print(f"       已加载 {len(loaded)} 个参数")
+    if skipped:
+        print(f"[WARNING] 跳过 {len(skipped)} 个参数")
+        print(f"       跳过的 key (前5个): {skipped[:5]}")
+
+    # 4. 合成语音
+    prompt_wav_path = args.prompt_audio or ""
+    prompt_text = args.prompt_text or ""
+    out_path = Path(args.output)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    print(f"\n[3/3] 开始合成测试...")
+    
+    # === 测试 1: 使用 LoRA ===
+    print(f"\n  [Test 1] 使用 LoRA 合成...")
+    with torch.inference_mode():
+        audio = model.generate(
+            target_text=args.text,
+            prompt_text=prompt_text,
+            prompt_wav_path=prompt_wav_path,
+            max_len=args.max_len,
+            inference_timesteps=args.inference_timesteps,
+            cfg_value=args.cfg_value,
+        )
+    audio_np = audio.squeeze(0).cpu().numpy() if audio.dim() > 1 else audio.cpu().numpy()
+    lora_output = out_path.with_stem(out_path.stem + "_with_lora")
+    sf.write(str(lora_output), audio_np, model.sample_rate)
+    print(f"           已保存：{lora_output}，时长 {len(audio_np) / model.sample_rate:.2f}s")
+
+    # === 测试 2: 禁用 LoRA（通过 set_lora_enabled） ===
+    print(f"\n  [Test 2] 禁用 LoRA (set_lora_enabled=False)...")
+    model.set_lora_enabled(False)
+    with torch.inference_mode():
+        audio = model.generate(
+            target_text=args.text,
+            prompt_text=prompt_text,
+            prompt_wav_path=prompt_wav_path,
+            max_len=args.max_len,
+            inference_timesteps=args.inference_timesteps,
+            cfg_value=args.cfg_value,
+        )
+    audio_np = audio.squeeze(0).cpu().numpy() if audio.dim() > 1 else audio.cpu().numpy()
+    disabled_output = out_path.with_stem(out_path.stem + "_lora_disabled")
+    sf.write(str(disabled_output), audio_np, model.sample_rate)
+    print(f"           已保存：{disabled_output}，时长 {len(audio_np) / model.sample_rate:.2f}s")
+
+    # === 测试 3: 重新启用 LoRA ===
+    print(f"\n  [Test 3] 重新启用 LoRA (set_lora_enabled=True)...")
+    model.set_lora_enabled(True)
+    with torch.inference_mode():
+        audio = model.generate(
+            target_text=args.text,
+            prompt_text=prompt_text,
+            prompt_wav_path=prompt_wav_path,
+            max_len=args.max_len,
+            inference_timesteps=args.inference_timesteps,
+            cfg_value=args.cfg_value,
+        )
+    audio_np = audio.squeeze(0).cpu().numpy() if audio.dim() > 1 else audio.cpu().numpy()
+    reenabled_output = out_path.with_stem(out_path.stem + "_lora_reenabled")
+    sf.write(str(reenabled_output), audio_np, model.sample_rate)
+    print(f"           已保存：{reenabled_output}，时长 {len(audio_np) / model.sample_rate:.2f}s")
+
+    # === 测试 4: 卸载 LoRA（reset_lora_weights） ===
+    print(f"\n  [Test 4] 卸载 LoRA (reset_lora_weights)...")
+    model.reset_lora_weights()
+    with torch.inference_mode():
+        audio = model.generate(
+            target_text=args.text,
+            prompt_text=prompt_text,
+            prompt_wav_path=prompt_wav_path,
+            max_len=args.max_len,
+            inference_timesteps=args.inference_timesteps,
+            cfg_value=args.cfg_value,
+        )
+    audio_np = audio.squeeze(0).cpu().numpy() if audio.dim() > 1 else audio.cpu().numpy()
+    reset_output = out_path.with_stem(out_path.stem + "_lora_reset")
+    sf.write(str(reset_output), audio_np, model.sample_rate)
+    print(f"           已保存：{reset_output}，时长 {len(audio_np) / model.sample_rate:.2f}s")
+
+    # === 测试 5: 热加载 LoRA（重新加载权重） ===
+    print(f"\n  [Test 5] 热加载 LoRA (load_lora_weights)...")
+    loaded, _ = model.load_lora_weights(str(ckpt_dir))
+    print(f"           重新加载了 {len(loaded)} 个参数")
+    with torch.inference_mode():
+        audio = model.generate(
+            target_text=args.text,
+            prompt_text=prompt_text,
+            prompt_wav_path=prompt_wav_path,
+            max_len=args.max_len,
+            inference_timesteps=args.inference_timesteps,
+            cfg_value=args.cfg_value,
+        )
+    audio_np = audio.squeeze(0).cpu().numpy() if audio.dim() > 1 else audio.cpu().numpy()
+    reload_output = out_path.with_stem(out_path.stem + "_lora_reloaded")
+    sf.write(str(reload_output), audio_np, model.sample_rate)
+    print(f"           已保存：{reload_output}，时长 {len(audio_np) / model.sample_rate:.2f}s")
+
+    print(f"\n[完成] 所有测试完成！")
+    print(f"  - with_lora:      {lora_output}")
+    print(f"  - lora_disabled:  {disabled_output}")
+    print(f"  - lora_reenabled: {reenabled_output}")
+    print(f"  - lora_reset:     {reset_output}")
+    print(f"  - lora_reloaded:  {reload_output}")
+
+
+if __name__ == "__main__":
+    main()
+
+

VoxCPM/scripts/train_voxcpm_finetune.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+
+import itertools
+from pathlib import Path
+from typing import Dict, Optional
+
+import argbind
+import torch
+from tensorboardX import SummaryWriter
+from torch.optim import AdamW
+from transformers import get_cosine_schedule_with_warmup
+
+from voxcpm.model import VoxCPMModel
+from voxcpm.model.voxcpm import LoRAConfig
+from voxcpm.training import (
+    Accelerator,
+    BatchProcessor,
+    TrainingTracker,
+    build_dataloader,
+    load_audio_text_datasets,
+)
+
+
+@argbind.bind(without_prefix=True)
+def train(
+    pretrained_path: str,
+    train_manifest: str,
+    val_manifest: str = "",
+    sample_rate: int = 16_000,
+    batch_size: int = 1,
+    grad_accum_steps: int = 1,
+    num_workers: int = 2,
+    num_iters: int = 100_000,
+    log_interval: int = 100,
+    valid_interval: int = 1_000,
+    save_interval: int = 10_000,
+    learning_rate: float = 1e-4,
+    weight_decay: float = 1e-2,
+    warmup_steps: int = 1_000,
+    max_steps: int = 100_000,
+    max_batch_tokens: int = 0,
+    save_path: str = "checkpoints",
+    tensorboard: str = "",
+    lambdas: Dict[str, float] = {"loss/diff": 1.0, "loss/stop": 1.0},
+    lora: dict = None,
+    config_path: str = "",
+):
+    _ = config_path
+    accelerator = Accelerator(amp=True)
+
+    save_dir = Path(save_path)
+    save_dir.mkdir(parents=True, exist_ok=True)
+    tb_dir = Path(tensorboard) if tensorboard else save_dir / "logs"
+    tb_dir.mkdir(parents=True, exist_ok=True)
+
+    writer = SummaryWriter(log_dir=str(tb_dir)) if accelerator.rank == 0 else None
+    tracker = TrainingTracker(writer=writer, log_file=str(save_dir / "train.log"), rank=accelerator.rank)
+
+    base_model = VoxCPMModel.from_local(pretrained_path, optimize=False, training=True, lora_config=LoRAConfig(**lora) if lora else None)
+    tokenizer = base_model.text_tokenizer
+
+    train_ds, val_ds = load_audio_text_datasets(
+        train_manifest=train_manifest,
+        val_manifest=val_manifest,
+        sample_rate=sample_rate,
+    )
+
+    def tokenize(batch):
+        text_list = batch["text"]
+        text_ids = [tokenizer(text) for text in text_list]
+        return {"text_ids": text_ids}
+
+    train_ds = train_ds.map(tokenize, batched=True, remove_columns=["text"])
+    if val_ds is not None:
+        val_ds = val_ds.map(tokenize, batched=True, remove_columns=["text"])
+
+    dataset_cnt = int(max(train_ds["dataset_id"])) + 1 if "dataset_id" in train_ds.column_names else 1
+    num_train_samples = len(train_ds)
+
+    # ------------------------------------------------------------------ #
+    # 可选：按预估 token 数过滤超长样本，避免单个样本撑爆显存
+    # max_batch_tokens > 0 时启用：
+    #   每个样本的最大长度 = max_batch_tokens // batch_size
+    #   超过该长度的样本将被丢弃（train_ds 过滤）
+    # ------------------------------------------------------------------ #
+    if max_batch_tokens and max_batch_tokens > 0:
+        from voxcpm.training.data import compute_sample_lengths
+
+        est_lengths = compute_sample_lengths(
+            train_ds,
+            audio_vae_fps=25,
+            patch_size=base_model.config.patch_size,
+        )
+        max_sample_len = max_batch_tokens // batch_size if batch_size > 0 else max(est_lengths)
+        keep_indices = [i for i, L in enumerate(est_lengths) if L <= max_sample_len]
+
+        if len(keep_indices) < len(train_ds) and accelerator.rank == 0:
+            tracker.print(
+                f"Filtering {len(train_ds) - len(keep_indices)} / {len(train_ds)} "
+                f"training samples longer than {max_sample_len} tokens "
+                f"(max_batch_tokens={max_batch_tokens})."
+            )
+        train_ds = train_ds.select(keep_indices)
+
+    train_loader = build_dataloader(
+        train_ds,
+        accelerator=accelerator,
+        batch_size=batch_size,
+        num_workers=num_workers,
+        drop_last=True,
+    )
+    val_loader = (
+        build_dataloader(
+            val_ds,
+            accelerator=accelerator,
+            batch_size=batch_size,
+            num_workers=num_workers,
+            drop_last=False,
+        )
+        if val_ds is not None
+        else None
+    )
+
+    model = accelerator.prepare_model(base_model)
+    unwrapped_model = accelerator.unwrap(model)
+    unwrapped_model.train()
+
+    batch_processor = BatchProcessor(
+        config=unwrapped_model.config,
+        audio_vae=unwrapped_model.audio_vae,
+        dataset_cnt=dataset_cnt,
+        device=accelerator.device,
+    )
+    
+    for name, param in model.named_parameters():
+        print(name, param.requires_grad)
+
+    optimizer = AdamW(
+        (p for p in model.parameters() if p.requires_grad),
+        lr=learning_rate,
+        weight_decay=weight_decay,
+    )
+
+    # 使用 transformers 的 cosine + warmup 调度器：
+    # - num_warmup_steps: 预热步数
+    # - num_training_steps: 计划的总训练步数（按 outer step 计数）
+    total_training_steps = max_steps if max_steps > 0 else num_iters
+    scheduler = get_cosine_schedule_with_warmup(
+        optimizer,
+        num_warmup_steps=warmup_steps,
+        num_training_steps=total_training_steps,
+    )
+
+    train_iter = iter(itertools.cycle(train_loader))
+    grad_accum_steps = max(int(grad_accum_steps), 1)
+
+    with tracker.live():
+        for step in range(num_iters):
+            tracker.step = step
+            optimizer.zero_grad(set_to_none=True)
+
+            # 梯度累积：在多个 micro-batch 上累积梯度，再进行一次优化步
+            loss_dict = {}
+            for micro_step in range(grad_accum_steps):
+                batch = next(train_iter)
+                processed = batch_processor(batch)
+
+                with accelerator.autocast(dtype=torch.bfloat16):
+                    outputs = model(
+                        processed["text_tokens"],
+                        processed["text_mask"],
+                        processed["audio_feats"],
+                        processed["audio_mask"],
+                        processed["loss_mask"],
+                        processed["position_ids"],
+                        processed["labels"],
+                        progress=step / max(1, num_iters),
+                    )
+
+                total_loss = 0.0
+                for key, value in outputs.items():
+                    if key.startswith("loss/"):
+                        weight = lambdas.get(key, 1.0)
+                        loss_value = value * weight / grad_accum_steps
+                        total_loss = total_loss + loss_value
+                        # 记录最后一个 micro-batch 的原始 loss，便于日志查看
+                        loss_dict[key] = value.detach()
+
+                # 对当前 micro-batch 累积梯度（已按 grad_accum_steps 归一化）
+                accelerator.backward(total_loss)
+
+            # 在所有 micro-batch 反向完成后，再做一次 unscale / grad_norm / step
+            scaler = getattr(accelerator, "scaler", None)
+            if scaler is not None:
+                scaler.unscale_(optimizer)
+            # 使用极大 max_norm 复用实现，仅做 grad_norm 统计而不实际裁剪
+            grad_norm = torch.nn.utils.clip_grad_norm_(unwrapped_model.parameters(), max_norm=1e9)
+
+            accelerator.step(optimizer)
+            accelerator.update()
+            scheduler.step()
+
+            if step % log_interval == 0:
+                loss_values = {k: v.item() if isinstance(v, torch.Tensor) else float(v) for k, v in loss_dict.items()}
+                loss_values["lr"] = float(optimizer.param_groups[0]["lr"])
+                # 近似当前 epoch：已见样本数 / 训练集样本数（考虑梯度累积和 batch_size）
+                epoch = (step * grad_accum_steps * batch_size) / max(1, num_train_samples)
+                loss_values["epoch"] = float(epoch)
+                loss_values["grad_norm"] = float(grad_norm)
+                tracker.log_metrics(loss_values, split="train")
+
+            if val_loader is not None and step % valid_interval == 0 and step != 0:
+                validate(model, val_loader, batch_processor, accelerator, tracker, lambdas)
+
+            if step % save_interval == 0 and accelerator.rank == 0:
+                save_checkpoint(model, optimizer, scheduler, save_dir, step)
+
+    if accelerator.rank == 0:
+        save_checkpoint(model, optimizer, scheduler, save_dir, num_iters)
+    if writer:
+        writer.close()
+
+
+def validate(model, val_loader, batch_processor, accelerator, tracker, lambdas):
+    model.eval()
+    losses = []
+    with torch.no_grad():
+        for batch in itertools.islice(val_loader, 0, 10):
+            processed = batch_processor(batch)
+            with accelerator.autocast(dtype=torch.bfloat16):
+                outputs = model(
+                    processed["text_tokens"],
+                    processed["text_mask"],
+                    processed["audio_feats"],
+                    processed["audio_mask"],
+                    processed["loss_mask"],
+                    processed["position_ids"],
+                    processed["labels"],
+                    progress=0.0,
+                    sample_generate=False,
+                )
+            total = 0.0
+            for key, value in outputs.items():
+                if key.startswith("loss/"):
+                    total += lambdas.get(key, 1.0) * value
+            losses.append(total.detach())
+    if losses:
+        mean_loss = torch.stack(losses).mean()
+        tracker.log_metrics({"loss": mean_loss.item()}, split="val")
+    model.train()
+
+
+def save_checkpoint(model, optimizer, scheduler, save_dir: Path, step: int):
+    save_dir.mkdir(parents=True, exist_ok=True)
+    tag = "latest" if step == 0 else f"step_{step:07d}"
+    folder = save_dir / tag
+    folder.mkdir(parents=True, exist_ok=True)
+    # 根据是否启用 LoRA 决定保存哪些权重：
+    # - 启用 LoRA，则仅保存 LoRA 参数（lora_A / lora_B）
+    # - 否则保存完整模型权重
+    unwrapped = model.module if hasattr(model, "module") else model
+    full_state = unwrapped.state_dict()
+    lora_cfg = unwrapped.lora_config
+    if lora_cfg is not None:
+        state_dict = {k: v for k, v in full_state.items() if ("lora_A" in k or "lora_B" in k)}
+    else:
+        state_dict = full_state
+
+    torch.save({"state_dict": state_dict}, folder / "generator.pth")
+    torch.save(optimizer.state_dict(), folder / "optimizer.pth")
+    torch.save(scheduler.state_dict(), folder / "scheduler.pth")
+
+
+if __name__ == "__main__":
+    from voxcpm.training.config import load_yaml_config
+
+    args = argbind.parse_args()
+    config_file = args.get("config_path")
+    # 如果提供了 YAML 配置文件，则直接用 YAML 的参数调用 train
+    if config_file:
+        yaml_args = load_yaml_config(config_file)
+        train(**yaml_args)
+    else:
+        # 否则使用命令行参数（argbind 解析）调用 train
+        with argbind.scope(args):
+            train()
+

VoxCPM/src/voxcpm.egg-info/PKG-INFO

@@ -0,0 +1,403 @@
+Metadata-Version: 2.4
+Name: voxcpm
+Version: 1.0.5.post0+gd1bb6aaf4.d20251128
+Summary: VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning
+Author-email: OpenBMB <openbmb@gmail.com>
+Maintainer-email: OpenBMB <openbmb@gmail.com>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/OpenBMB/VoxCPM
+Project-URL: Repository, https://github.com/OpenBMB/VoxCPM.git
+Project-URL: Documentation, https://github.com/OpenBMB/VoxCPM#readme
+Project-URL: Bug Tracker, https://github.com/OpenBMB/VoxCPM/issues
+Keywords: voxcpm,text-to-speech,tts,speech-synthesis,voice-cloning,ai,deep-learning,pytorch
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.5.0
+Requires-Dist: torchaudio>=2.5.0
+Requires-Dist: transformers>=4.36.2
+Requires-Dist: einops
+Requires-Dist: gradio
+Requires-Dist: inflect
+Requires-Dist: addict
+Requires-Dist: wetext
+Requires-Dist: modelscope>=1.22.0
+Requires-Dist: datasets<4,>=3
+Requires-Dist: huggingface-hub
+Requires-Dist: pydantic
+Requires-Dist: tqdm
+Requires-Dist: simplejson
+Requires-Dist: sortedcontainers
+Requires-Dist: soundfile
+Requires-Dist: funasr
+Requires-Dist: spaces
+Requires-Dist: argbind
+Requires-Dist: tensorboardX
+Provides-Extra: dev
+Requires-Dist: pytest>=6.0; extra == "dev"
+Requires-Dist: pytest-cov>=2.0; extra == "dev"
+Requires-Dist: black>=21.0; extra == "dev"
+Requires-Dist: flake8>=3.8; extra == "dev"
+Requires-Dist: mypy>=0.800; extra == "dev"
+Requires-Dist: pre-commit>=2.0; extra == "dev"
+Dynamic: license-file
+
+## 🎙️ VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning
+
+
+[![Project Page](https://img.shields.io/badge/Project%20Page-GitHub-blue)](https://github.com/OpenBMB/VoxCPM/) [![Technical Report](https://img.shields.io/badge/Technical%20Report-Arxiv-red)](https://arxiv.org/abs/2509.24650) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-OpenBMB-yellow)](https://huggingface.co/openbmb/VoxCPM-0.5B) [![ModelScope](https://img.shields.io/badge/ModelScope-OpenBMB-purple)](https://modelscope.cn/models/OpenBMB/VoxCPM-0.5B)  [![Live Playground](https://img.shields.io/badge/Live%20PlayGround-Demo-orange)](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) [![Samples](https://img.shields.io/badge/Audio%20Samples-Page-green)](https://openbmb.github.io/VoxCPM-demopage)
+
+
+
+<div align="center">
+  <img src="assets/voxcpm_logo.png" alt="VoxCPM Logo" width="40%">
+</div>
+
+<div align="center">
+
+👋 Contact us on [WeChat](assets/wechat.png)
+
+</div>
+
+## News 
+* [2025.09.30] 🔥 🔥 🔥  We Release VoxCPM [Technical Report](https://arxiv.org/abs/2509.24650)!
+* [2025.09.16] 🔥 🔥 🔥  We Open Source the VoxCPM-0.5B [weights](https://huggingface.co/openbmb/VoxCPM-0.5B)!
+* [2025.09.16] 🎉 🎉 🎉  We Provide the [Gradio PlayGround](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) for VoxCPM-0.5B, try it now! 
+
+## Overview
+
+VoxCPM is a novel tokenizer-free Text-to-Speech (TTS) system that redefines realism in speech synthesis. By modeling speech in a continuous space, it overcomes the limitations of discrete tokenization and enables two flagship capabilities: context-aware speech generation and true-to-life zero-shot voice cloning.
+
+Unlike mainstream approaches that convert speech to discrete tokens, VoxCPM uses an end-to-end diffusion autoregressive architecture that directly generates continuous speech representations from text. Built on [MiniCPM-4](https://huggingface.co/openbmb/MiniCPM4-0.5B) backbone, it achieves implicit semantic-acoustic decoupling through hierachical language modeling and FSQ constraints, greatly enhancing both expressiveness and generation stability.
+
+<div align="center">
+  <img src="assets/voxcpm_model.png" alt="VoxCPM Model Architecture" width="90%">
+</div>
+
+
+###  🚀 Key Features
+- **Context-Aware, Expressive Speech Generation** - VoxCPM comprehends text to infer and generate appropriate prosody, delivering speech with remarkable expressiveness and natural flow. It spontaneously adapts speaking style based on content, producing highly fitting vocal expression trained on a massive 1.8 million-hour bilingual corpus.
+- **True-to-Life Voice Cloning** - With only a short reference audio clip, VoxCPM performs accurate zero-shot voice cloning, capturing not only the speaker’s timbre but also fine-grained characteristics such as accent, emotional tone, rhythm, and pacing to create a faithful and natural replica.
+- **High-Efficiency Synthesis** - VoxCPM supports streaming synthesis with a Real-Time Factor (RTF) as low as 0.17 on a consumer-grade NVIDIA RTX 4090 GPU, making it possible for real-time applications.
+
+
+
+
+
+##  Quick Start
+
+### 🔧 Install from PyPI
+``` sh
+pip install voxcpm
+```
+### 1.  Model Download (Optional)
+By default, when you first run the script, the model will be downloaded automatically, but you can also download the model in advance.
+- Download VoxCPM-0.5B
+    ```
+    from huggingface_hub import snapshot_download
+    snapshot_download("openbmb/VoxCPM-0.5B")
+    ```
+- Download ZipEnhancer and SenseVoice-Small. We use ZipEnhancer to enhance speech prompts and SenseVoice-Small for speech prompt ASR in the web demo.
+    ```
+    from modelscope import snapshot_download
+    snapshot_download('iic/speech_zipenhancer_ans_multiloss_16k_base')
+    snapshot_download('iic/SenseVoiceSmall')
+    ```
+
+### 2. Basic Usage
+```python
+import soundfile as sf
+import numpy as np
+from voxcpm import VoxCPM
+
+model = VoxCPM.from_pretrained("openbmb/VoxCPM-0.5B")
+
+# Non-streaming
+wav = model.generate(
+    text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.",
+    prompt_wav_path=None,      # optional: path to a prompt speech for voice cloning
+    prompt_text=None,          # optional: reference text
+    cfg_value=2.0,             # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse
+    inference_timesteps=10,   # LocDiT inference timesteps, higher for better result, lower for fast speed
+    normalize=True,           # enable external TN tool
+    denoise=True,             # enable external Denoise tool
+    retry_badcase=True,        # enable retrying mode for some bad cases (unstoppable)
+    retry_badcase_max_times=3,  # maximum retrying times
+    retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech
+)
+
+sf.write("output.wav", wav, 16000)
+print("saved: output.wav")
+
+# Streaming
+chunks = []
+for chunk in model.generate_streaming(
+    text = "Streaming text to speech is easy with VoxCPM!",
+    # supports same args as above
+):
+    chunks.append(chunk)
+wav = np.concatenate(chunks)
+
+sf.write("output_streaming.wav", wav, 16000)
+print("saved: output_streaming.wav")
+```
+
+### 3. CLI Usage
+
+After installation, the entry point is `voxcpm` (or use `python -m voxcpm.cli`).
+
+```bash
+# 1) Direct synthesis (single text)
+voxcpm --text "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech." --output out.wav
+
+# 2) Voice cloning (reference audio + transcript)
+voxcpm --text "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech." \
+  --prompt-audio path/to/voice.wav \
+  --prompt-text "reference transcript" \
+  --output out.wav \
+  --denoise
+
+# (Optinal) Voice cloning (reference audio + transcript file)
+voxcpm --text "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech." \
+  --prompt-audio path/to/voice.wav \
+  --prompt-file "/path/to/text-file" \
+  --output out.wav \
+  --denoise
+
+# 3) Batch processing (one text per line)
+voxcpm --input examples/input.txt --output-dir outs
+# (optional) Batch + cloning
+voxcpm --input examples/input.txt --output-dir outs \
+  --prompt-audio path/to/voice.wav \
+  --prompt-text "reference transcript" \
+  --denoise
+
+# 4) Inference parameters (quality/speed)
+voxcpm --text "..." --output out.wav \
+  --cfg-value 2.0 --inference-timesteps 10 --normalize
+
+# 5) Model loading
+# Prefer local path
+voxcpm --text "..." --output out.wav --model-path /path/to/VoxCPM_model_dir
+# Or from Hugging Face (auto download/cache)
+voxcpm --text "..." --output out.wav \
+  --hf-model-id openbmb/VoxCPM-0.5B --cache-dir ~/.cache/huggingface --local-files-only
+
+# 6) Denoiser control
+voxcpm --text "..." --output out.wav \
+  --no-denoiser --zipenhancer-path iic/speech_zipenhancer_ans_multiloss_16k_base
+
+# 7) Help
+voxcpm --help
+python -m voxcpm.cli --help
+```
+
+### 4. Start web demo
+
+You can start the UI interface by running `python app.py`, which allows you to perform Voice Cloning and Voice Creation.
+
+## 🛠️ Fine-tune VoxCPM
+
+We provide a training pipeline mirroring the `minicpm-audio` workflow while relying purely on HuggingFace `datasets` for audio-text management.
+
+1. **Prepare a manifest (JSONL)**
+
+   ```
+   {"audio": "/path/to/audio_0001.wav", "text": "你好，世界。", "dataset_id": 0}
+   {"audio": "/path/to/audio_0002.wav", "text": "第二条语音", "dataset_id": 0}
+   ```
+   - `audio`: waveform file path (WAV/FLAC/MP3 supported)
+   - `text`: transcription
+   - `dataset_id` *(optional)*: integer identifier for multi-dataset sampling statistics
+
+2. **Copy & edit the example config**  
+   `conf/voxcpm/voxcpm_finetune_example.yaml` contains hyper-parameters (pretrained weights, tokenizer, manifests, λ-weights, etc.).
+
+3. **Launch training**
+
+   ```bash
+   CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node 2 \
+     scripts/train_voxcpm_finetune.py \
+     --config_path conf/voxcpm/voxcpm_finetune_example.yaml
+   ```
+
+   Features:
+   - Distributed + AMP training (`torchrun`).
+   - TensorBoard logging (`tensorboard --logdir logs/voxcpm_finetune`).
+   - Periodic validation & checkpointing under `checkpoints/`.
+
+4. **Key modules**
+   - `src/voxcpm/model/voxcpm.py`: unified model providing both inference and training forward。
+   - `src/voxcpm/training/`: accelerator, tracker, dataset loader & batch packer utilities。
+   - `scripts/train_voxcpm_finetune.py`: end-to-end fine-tune loop。
+
+## 👩‍🍳 A Voice Chef's Guide
+Welcome to the VoxCPM kitchen! Follow this recipe to cook up perfect generated speech. Let’s begin.
+
+---
+### 🥚 Step 1: Prepare Your Base Ingredients (Content)
+
+First, choose how you’d like to input your text:.
+1. Regular Text (Classic Mode)
+- ✅ Keep "Text Normalization" ON. Type naturally (e.g., "Hello, world! 123"). The system will automatically process numbers, abbreviations, and punctuation using WeTextProcessing library.
+2. Phoneme Input (Native Mode)
+- ❌ Turn "Text Normalization" OFF. Enter phoneme text like {HH AH0 L OW1} (EN) or {ni3}{hao3} (ZH) for precise pronunciation  control. In this mode, VoxCPM also supports native understanding of other complex non-normalized text—try it out!
+
+---
+### 🍳 Step 2: Choose Your Flavor Profile (Voice Style) 
+
+This is the secret sauce that gives your audio its unique sound.
+1. Cooking with a Prompt Speech (Following a Famous Recipe)
+  - A prompt speech provides the desired acoustic characteristics for VoxCPM. The speaker's timbre, speaking style, and even the background sounds and ambiance will be replicated.
+  - For a Clean, Studio-Quality Voice:
+    - ✅ Enable "Prompt Speech Enhancement". This acts like a noise filter, removing background hiss and rumble to give you a pure, clean voice clone.
+2. Cooking au Naturel (Letting the Model Improvise)
+  - If no reference is provided, VoxCPM becomes a creative chef! It will infer a fitting speaking style based on the text itself, thanks to the text-smartness of its foundation model, MiniCPM-4.
+  - Pro Tip: Challenge VoxCPM with any text—poetry, song lyrics, dramatic monologues—it may deliver some interesting results!
+
+---
+### 🧂 Step 3: The Final Seasoning (Fine-Tuning Your Results)
+You're ready to serve! But for master chefs who want to tweak the flavor, here are two key spices.
+- CFG Value (How Closely to Follow the Recipe)
+  - Default: A great starting point.
+  - Voice sounds strained or weird? Lower this value. It tells the model to be more relaxed and improvisational, great for expressive prompts.
+  - Need maximum clarity and adherence to the text? Raise it slightly to keep the model on a tighter leash.
+- Inference Timesteps (Simmering Time: Quality vs. Speed)
+  - Need a quick snack? Use a lower number. Perfect for fast drafts and experiments.
+  - Cooking a gourmet meal? Use a higher number. This lets the model "simmer" longer, refining the audio for superior detail and naturalness.
+
+---
+Happy creating! 🎉 Start with the default settings and tweak from there to suit your project. The kitchen is yours!
+
+
+---
+
+
+## 🌟 Community Projects
+
+We're excited to see the VoxCPM community growing! Here are some amazing projects and features built by our community:
+
+- **[ComfyUI-VoxCPM](https://github.com/wildminder/ComfyUI-VoxCPM)**
+- **[ComfyUI-VoxCPMTTS](https://github.com/1038lab/ComfyUI-VoxCPMTTS)** 
+- **[WebUI-VoxCPM](https://github.com/rsxdalv/tts_webui_extension.vox_cpm)**
+- **[PR: Streaming API Support (by AbrahamSanders)](https://github.com/OpenBMB/VoxCPM/pull/26)** 
+
+
+
+*Have you built something cool with VoxCPM? We'd love to feature it here! Please open an issue or pull request to add your project.*
+
+
+## 📊 Performance Highlights
+
+VoxCPM achieves competitive results on public zero-shot TTS benchmarks:
+
+### Seed-TTS-eval Benchmark
+
+| Model | Parameters | Open-Source | test-EN | | test-ZH | | test-Hard | |
+|------|------|------|:------------:|:--:|:------------:|:--:|:-------------:|:--:|
+| | | | WER/%⬇ | SIM/%⬆| CER/%⬇| SIM/%⬆ | CER/%⬇ | SIM/%⬆ |
+| MegaTTS3 | 0.5B | ❌ | 2.79 | 77.1 | 1.52 | 79.0 | - | - |
+| DiTAR | 0.6B | ❌ | 1.69 | 73.5 | 1.02 | 75.3 | - | - |
+| CosyVoice3 | 0.5B | ❌ | 2.02 | 71.8 | 1.16 | 78.0 | 6.08 | 75.8 |
+| CosyVoice3 | 1.5B | ❌ | 2.22 | 72.0 | 1.12 | 78.1 | 5.83 | 75.8 |
+| Seed-TTS | - | ❌ | 2.25 | 76.2 | 1.12 | 79.6 | 7.59 | 77.6 |
+| MiniMax-Speech | - | ❌ | 1.65 | 69.2 | 0.83 | 78.3 | - | - |
+| CosyVoice | 0.3B | ✅ | 4.29 | 60.9 | 3.63 | 72.3 | 11.75 | 70.9 |
+| CosyVoice2 | 0.5B | ✅ | 3.09 | 65.9 | 1.38 | 75.7 | **6.83** | 72.4 |
+| F5-TTS | 0.3B | ✅ | 2.00 | 67.0 | 1.53 | 76.0 | 8.67 | 71.3 |
+| SparkTTS | 0.5B | ✅ | 3.14 | 57.3 | 1.54 | 66.0 | - | - |
+| FireRedTTS | 0.5B | ✅ | 3.82 | 46.0 | 1.51 | 63.5 | 17.45 | 62.1 |
+| FireRedTTS-2 | 1.5B | ✅ | 1.95 | 66.5 | 1.14 | 73.6 | - | - |
+| Qwen2.5-Omni | 7B | ✅ | 2.72 | 63.2 | 1.70 | 75.2 | 7.97 | **74.7** |
+| OpenAudio-s1-mini | 0.5B | ✅ | 1.94 | 55.0 | 1.18 | 68.5 | - | - |
+| IndexTTS2 | 1.5B | ✅ | 2.23 | 70.6 | 1.03 | 76.5 | - | - |
+| VibeVoice | 1.5B | ✅ | 3.04 | 68.9 | 1.16 | 74.4 | - | - |
+| HiggsAudio-v2 | 3B | ✅ | 2.44 | 67.7 | 1.50 | 74.0 | - | - |
+| **VoxCPM** | 0.5B | ✅ | **1.85** | **72.9** | **0.93** | **77.2** | 8.87 | 73.0 |
+
+
+###  CV3-eval Benchmark
+
+| Model | zh | en | hard-zh | | | hard-en | | |
+|-------|:--:|:--:|:-------:|:--:|:--:|:-------:|:--:|:--:|
+| | CER/%⬇ | WER/%⬇ | CER/%⬇ | SIM/%⬆ | DNSMOS⬆ | WER/%⬇ | SIM/%⬆ | DNSMOS⬆ |
+| F5-TTS | 5.47 | 8.90 | - | - | - | - | - | - |
+| SparkTTS | 5.15 | 11.0 | - | - | - | - | - | - |
+| GPT-SoVits | 7.34 | 12.5 | - | - | - | - | - | - |
+| CosyVoice2 | 4.08 | 6.32 | 12.58 | 72.6 | 3.81 | 11.96 | 66.7 | 3.95 |
+| OpenAudio-s1-mini | 4.00 | 5.54 | 18.1 | 58.2 | 3.77 | 12.4 | 55.7 | 3.89 |
+| IndexTTS2 | 3.58 | 4.45 | 12.8 | 74.6 | 3.65 | - | - | - |
+| HiggsAudio-v2 | 9.54 | 7.89 | 41.0 | 60.2 | 3.39 | 10.3 | 61.8 | 3.68 |
+| CosyVoice3-0.5B | 3.89 | 5.24 | 14.15 | 78.6 | 3.75 | 9.04 | 75.9 | 3.92 |
+| CosyVoice3-1.5B | 3.91 | 4.99 | 9.77 | 78.5 | 3.79 | 10.55 | 76.1 | 3.95 |
+| **VoxCPM** | **3.40** | **4.04** | 12.9 | 66.1 | 3.59 | **7.89** | 64.3 | 3.74 |
+
+
+
+
+
+
+
+
+
+
+
+
+## ⚠️ Risks and limitations
+- General Model Behavior: While VoxCPM has been trained on a large-scale dataset, it may still produce outputs that are unexpected, biased, or contain artifacts.
+- Potential for Misuse of Voice Cloning: VoxCPM's powerful zero-shot voice cloning capability can generate highly realistic synthetic speech. This technology could be misused for creating convincing deepfakes for purposes of impersonation, fraud, or spreading disinformation. Users of this model must not use it to create content that infringes upon the rights of individuals. It is strictly forbidden to use VoxCPM for any illegal or unethical purposes. We strongly recommend that any publicly shared content generated with this model be clearly marked as AI-generated.
+- Current Technical Limitations: Although generally stable, the model may occasionally exhibit instability, especially with very long or expressive inputs. Furthermore, the current version offers limited direct control over specific speech attributes like emotion or speaking style.
+- Bilingual Model: VoxCPM is trained primarily on Chinese and English data. Performance on other languages is not guaranteed and may result in unpredictable or low-quality audio.
+- This model is released for research and development purposes only. We do not recommend its use in production or commercial applications without rigorous testing and safety evaluations. Please use VoxCPM responsibly.
+
+
+
+## 📝TO-DO List
+Please stay tuned for updates!
+- [x] Release the VoxCPM technical report.
+- [ ] Support higher sampling rate (next version).
+
+
+
+## 📄 License
+The VoxCPM model weights and code are open-sourced under the [Apache-2.0](LICENSE) license.
+
+## 🙏 Acknowledgments
+
+We extend our sincere gratitude to the following works and resources for their inspiration and contributions:
+
+- [DiTAR](https://arxiv.org/abs/2502.03930) for the diffusion autoregressive backbone used in speech generation
+- [MiniCPM-4](https://github.com/OpenBMB/MiniCPM) for serving as the language model foundation
+- [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) for the implementation of Flow Matching-based LocDiT
+- [DAC](https://github.com/descriptinc/descript-audio-codec) for providing the Audio VAE backbone
+
+## Institutions
+
+This project is developed by the following institutions:
+- <img src="assets/modelbest_logo.png" width="28px"> [ModelBest](https://modelbest.cn/)
+
+- <img src="assets/thuhcsi_logo.png" width="28px"> [THUHCSI](https://github.com/thuhcsi)
+
+
+## ⭐ Star History
+ [![Star History Chart](https://api.star-history.com/svg?repos=OpenBMB/VoxCPM&type=Date)](https://star-history.com/#OpenBMB/VoxCPM&Date)
+
+
+## 📚 Citation
+
+If you find our model helpful, please consider citing our projects 📝 and staring us ⭐️！
+
+```bib
+@article{voxcpm2025,
+  title        = {VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning},
+  author       = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and Yu, Renjie and Wang, Ziyang and Ye, Runchuan and Sun, Weiyue and Gui, Jiancheng and Li, Kehan and Wu, Zhiyong  and Liu, Zhiyuan},
+  journal      = {arXiv preprint arXiv:2509.24650},
+  year         = {2025},
+}
+```

VoxCPM/src/voxcpm.egg-info/SOURCES.txt

@@ -0,0 +1,48 @@
+.gitignore
+LICENSE
+README.md
+app.py
+pyproject.toml
+.github/workflows/publish-to-pypi.yml
+assets/modelbest_logo.png
+assets/thuhcsi_logo.png
+assets/voxcpm_logo.png
+assets/voxcpm_model.png
+assets/wechat.png
+examples/example.wav
+src/voxcpm/__init__.py
+src/voxcpm/cli.py
+src/voxcpm/core.py
+src/voxcpm/zipenhancer.py
+src/voxcpm.egg-info/PKG-INFO
+src/voxcpm.egg-info/SOURCES.txt
+src/voxcpm.egg-info/dependency_links.txt
+src/voxcpm.egg-info/entry_points.txt
+src/voxcpm.egg-info/requires.txt
+src/voxcpm.egg-info/top_level.txt
+src/voxcpm/model/__init__.py
+src/voxcpm/model/utils.py
+src/voxcpm/model/voxcpm.py
+src/voxcpm/modules/__init__.py
+src/voxcpm/modules/audiovae/__init__.py
+src/voxcpm/modules/audiovae/audio_vae.py
+src/voxcpm/modules/layers/__init__.py
+src/voxcpm/modules/layers/lora.py
+src/voxcpm/modules/layers/scalar_quantization_layer.py
+src/voxcpm/modules/locdit/__init__.py
+src/voxcpm/modules/locdit/local_dit.py
+src/voxcpm/modules/locdit/unified_cfm.py
+src/voxcpm/modules/locenc/__init__.py
+src/voxcpm/modules/locenc/local_encoder.py
+src/voxcpm/modules/minicpm4/__init__.py
+src/voxcpm/modules/minicpm4/cache.py
+src/voxcpm/modules/minicpm4/config.py
+src/voxcpm/modules/minicpm4/model.py
+src/voxcpm/training/__init__.py
+src/voxcpm/training/accelerator.py
+src/voxcpm/training/config.py
+src/voxcpm/training/data.py
+src/voxcpm/training/packers.py
+src/voxcpm/training/state.py
+src/voxcpm/training/tracker.py
+src/voxcpm/utils/text_normalize.py

VoxCPM/src/voxcpm.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+