Clarify language-specific inference modes
Browse files
README.md
CHANGED
|
@@ -52,6 +52,15 @@ This model repository hosts the public release checkpoints for **SE-Bridge-TTS**
|
|
| 52 |
|
| 53 |
The released files are CosyVoice2 LLM checkpoints. They are intended to be loaded with a CosyVoice2-compatible checkout and the standard CosyVoice2 base model assets. The base model directory should contain the normal CosyVoice2 configuration and acoustic/vocoder weights, while this repository supplies the Thai or Lao LLM checkpoint.
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
Install or prepare CosyVoice first:
|
| 56 |
|
| 57 |
```bash
|
|
@@ -61,7 +70,7 @@ pip install -r requirements.txt
|
|
| 61 |
pip install huggingface_hub torchaudio
|
| 62 |
```
|
| 63 |
|
| 64 |
-
|
| 65 |
|
| 66 |
```python
|
| 67 |
import sys
|
|
@@ -80,7 +89,7 @@ from cosyvoice.utils.file_utils import load_wav
|
|
| 80 |
HF_REPO_ID = "isabeth/SE-Bridge-TTS"
|
| 81 |
BASE_MODEL_DIR = Path("pretrained_models/CosyVoice2-0.5B")
|
| 82 |
|
| 83 |
-
language = "
|
| 84 |
checkpoint_name = {
|
| 85 |
"thai": "thai_tts.pt",
|
| 86 |
"lao": "lao_tts.pt",
|
|
@@ -100,36 +109,30 @@ state_dict = torch.load(checkpoint_path, map_location="cpu")
|
|
| 100 |
cosyvoice.model.llm.load_state_dict(state_dict, strict=False)
|
| 101 |
|
| 102 |
prompt_speech_16k = load_wav("prompt.wav", 16000)
|
| 103 |
-
prompt_text = "Transcript of prompt.wav."
|
| 104 |
tts_text = "Text to synthesize in the selected language."
|
| 105 |
|
| 106 |
-
|
| 107 |
-
|
|
|
|
| 108 |
tts_text,
|
| 109 |
prompt_text,
|
| 110 |
prompt_speech_16k,
|
| 111 |
stream=False,
|
| 112 |
)
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
output["tts_speech"],
|
| 117 |
-
cosyvoice.sample_rate,
|
| 118 |
-
)
|
| 119 |
-
```
|
| 120 |
-
|
| 121 |
-
For cross-lingual prompting, use the same loaded model and replace the generation loop with:
|
| 122 |
-
|
| 123 |
-
```python
|
| 124 |
-
for idx, output in enumerate(
|
| 125 |
-
cosyvoice.inference_cross_lingual(
|
| 126 |
tts_text,
|
| 127 |
prompt_speech_16k,
|
| 128 |
stream=False,
|
| 129 |
)
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
torchaudio.save(
|
| 132 |
-
f"
|
| 133 |
output["tts_speech"],
|
| 134 |
cosyvoice.sample_rate,
|
| 135 |
)
|
|
|
|
| 52 |
|
| 53 |
The released files are CosyVoice2 LLM checkpoints. They are intended to be loaded with a CosyVoice2-compatible checkout and the standard CosyVoice2 base model assets. The base model directory should contain the normal CosyVoice2 configuration and acoustic/vocoder weights, while this repository supplies the Thai or Lao LLM checkpoint.
|
| 54 |
|
| 55 |
+
Recommended inference mode by language:
|
| 56 |
+
|
| 57 |
+
| Checkpoint | Language | Recommended mode |
|
| 58 |
+
| --- | --- | --- |
|
| 59 |
+
| `thai_tts.pt` | Thai (`th`) | Zero-shot inference with `inference_zero_shot`. |
|
| 60 |
+
| `lao_tts.pt` | Lao (`lo`) | Cross-lingual inference only with `inference_cross_lingual`. |
|
| 61 |
+
|
| 62 |
+
For this release, Thai is the default zero-shot use case. Lao should be run through the cross-lingual path; do not treat `lao_tts.pt` as a zero-shot checkpoint.
|
| 63 |
+
|
| 64 |
Install or prepare CosyVoice first:
|
| 65 |
|
| 66 |
```bash
|
|
|
|
| 70 |
pip install huggingface_hub torchaudio
|
| 71 |
```
|
| 72 |
|
| 73 |
+
Language-aware inference example:
|
| 74 |
|
| 75 |
```python
|
| 76 |
import sys
|
|
|
|
| 89 |
HF_REPO_ID = "isabeth/SE-Bridge-TTS"
|
| 90 |
BASE_MODEL_DIR = Path("pretrained_models/CosyVoice2-0.5B")
|
| 91 |
|
| 92 |
+
language = "thai" # choose "thai" for zero-shot, or "lao" for cross-lingual only
|
| 93 |
checkpoint_name = {
|
| 94 |
"thai": "thai_tts.pt",
|
| 95 |
"lao": "lao_tts.pt",
|
|
|
|
| 109 |
cosyvoice.model.llm.load_state_dict(state_dict, strict=False)
|
| 110 |
|
| 111 |
prompt_speech_16k = load_wav("prompt.wav", 16000)
|
|
|
|
| 112 |
tts_text = "Text to synthesize in the selected language."
|
| 113 |
|
| 114 |
+
if language == "thai":
|
| 115 |
+
prompt_text = "Transcript of prompt.wav."
|
| 116 |
+
outputs = cosyvoice.inference_zero_shot(
|
| 117 |
tts_text,
|
| 118 |
prompt_text,
|
| 119 |
prompt_speech_16k,
|
| 120 |
stream=False,
|
| 121 |
)
|
| 122 |
+
output_prefix = "se_bridge_tts_thai_zero_shot"
|
| 123 |
+
elif language == "lao":
|
| 124 |
+
outputs = cosyvoice.inference_cross_lingual(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
tts_text,
|
| 126 |
prompt_speech_16k,
|
| 127 |
stream=False,
|
| 128 |
)
|
| 129 |
+
output_prefix = "se_bridge_tts_lao_cross_lingual"
|
| 130 |
+
else:
|
| 131 |
+
raise ValueError("language must be either 'thai' or 'lao'")
|
| 132 |
+
|
| 133 |
+
for idx, output in enumerate(outputs):
|
| 134 |
torchaudio.save(
|
| 135 |
+
f"{output_prefix}_{idx}.wav",
|
| 136 |
output["tts_speech"],
|
| 137 |
cosyvoice.sample_rate,
|
| 138 |
)
|