Add inference code (prosody v2), README, LICENSE, prompts, samples

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+prompts/clip0_waxal_yor_yor_1258.wav filter=lfs diff=lfs merge=lfs -text
+prompts/clip1_waxal_yor_yor_1205.wav filter=lfs diff=lfs merge=lfs -text
+prompts/clip2_waxal_yor_yor_1263.wav filter=lfs diff=lfs merge=lfs -text
+prompts/clip3_waxal_yor_yor_1218.wav filter=lfs diff=lfs merge=lfs -text
+samples/prosody_v2/para_hau_eng_pcm.mp3 filter=lfs diff=lfs merge=lfs -text
+samples/prosody_v2/para_ibo_pcm_eng.mp3 filter=lfs diff=lfs merge=lfs -text
+samples/prosody_v2/para_yor_eng.mp3 filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,184 @@
+---
+license: apache-2.0
+language:
+  - en
+  - pcm
+  - yo
+  - ig
+  - ha
+base_model: FunAudioLLM/CosyVoice2-0.5B
+pipeline_tag: text-to-speech
+tags:
+  - text-to-speech
+  - code-switching
+  - nigerian-languages
+  - cosyvoice2
+  - research-preview
+---
+
+# Dynamic Accent (Phase 2, research preview)
+
+Nigerian multilingual, **code-switching** text-to-speech in one consistent voice: English, Nigerian Pidgin, Yoruba, Igbo and Hausa, mixed freely inside one sentence.
+
+It is built on [FunAudioLLM/CosyVoice2-0.5B](https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B) and adds:
+- per-token **language embeddings**, a **speaker token** and an adversarial (GRL) speaker classifier on the LM, fine-tuned on about 120 h of Nigerian-language and code-switched speech with pause-normalised data (pause profile B);
+- in-context **anchor prompts** at the LM and the flow, so prosody and pause style follow the anchor voice (WAXAL Yoruba speaker 4);
+- a **flow-matching decoder fine-tuned on the anchor voice** (anchor SFT) for tone and pitch detail.
+
+This repo is a **private research preview** for testing. It is not a production model.
+
+Inference defaults to **prosody v2**: clause-level generation with anchor-matched pause gaps, and a 1.05 pitch-preserving speed-up. See the section below.
+
+## What is in the repo
+
+| path | what it is |
+|---|---|
+| `checkpoints/p2_lm_B/best.pt` | Dynamic Accent LM (CosyVoice2 Qwen2 LM + language/speaker embeddings), pause profile B, 4,000 steps, best validation CE |
+| `checkpoints/p2_lm_B/spk2id.json` | speaker-token map (the anchor is `waxal_yor_4`) |
+| `checkpoints/p2_flow_sft/best.pt` | anchor-SFT flow-matching decoder (replaces the CosyVoice2 flow weights) |
+| `prompts/clip{0..3}_*.wav`, `prompts/prompts.json` | the 4 anchor prompt clips (24 kHz) with their transcripts and S3 speech tokens. The final system uses clip 0 |
+| `infer.py` | self-contained inference: the same path as the Phase 2 final samples |
+| `code/` | the original training and synthesis code (`train_b.py`, `train_b2.py`, `da/`) for reference |
+| `cosyvoice_requirements.txt` | CosyVoice's own requirements file, at the commit used |
+| `samples/` | example renders |
+
+Everything else comes from the stock base model **`FunAudioLLM/CosyVoice2-0.5B`** (Qwen tokenizer `CosyVoice-BlankEN`, `llm.pt`, `hift.pt` vocoder, `campplus.onnx`, `speech_tokenizer_v2.onnx`, `cosyvoice2.yaml`). Nothing in the base needs changing; `infer.py` downloads it on first run.
+
+## Setup (single GPU, e.g. Colab T4 or A100)
+
+```bash
+# 1. CosyVoice code (the commit the model was built and tested on)
+git clone --recursive https://github.com/FunAudioLLM/CosyVoice.git
+cd CosyVoice && git checkout 074ca6dc9e80a2f424f1f74b48bdd7d3fea531cc && git submodule update --init --recursive && cd ..
+
+# 2. This repo (private: log in with a token that can read Modularcomputing)
+huggingface-cli login
+huggingface-cli download Modularcomputing/dynamic-accent-preview --local-dir dynamic-accent-preview
+
+# 3. Python deps (Python 3.10; on Colab keep the preinstalled torch/torchaudio)
+pip install -r dynamic-accent-preview/requirements.txt
+```
+
+The base model downloads to `dynamic-accent-preview/pretrained/CosyVoice2-0.5B` the first time you run `infer.py`. You can also point `--model_dir` (or `COSYVOICE2_DIR`) at an existing copy. Point `--cosyvoice_dir` (or `COSYVOICE_REPO`) at your CosyVoice clone if it is not inside the repo folder.
+
+The 0.5B model should fit comfortably on a 16 GB T4. This build was tested on an H100 only, not yet on Colab.
+
+## Usage
+
+```bash
+cd dynamic-accent-preview
+python infer.py --cosyvoice_dir ../CosyVoice \
+  --text "<eng>this is dynamic accent,</eng> <pcm>e dey sound nice abi?</pcm>" \
+  --seed 0 --out out.wav                     # defaults: --pause_mode clause --speed 1.05
+# Phase 2 final behaviour: add  --pause_mode model --speed 1.0
+```
+
+- **Language tags:** wrap each span in `<eng>`, `<pcm>`, `<yor>`, `<ibo>` or `<hau>`. Untagged text is read as English. Keep punctuation inside the span it belongs to.
+- **`--seed`:** the final system is a single sample at seed 0. Try seeds 1 and 2 (or more) and keep the take you like. Best-of-N picking lowered CER by about a third in evaluation.
+- **`--pause_mode clause`** (default, prosody v2): the text is cut at `, ; : . ? !` and at language switches. Each chunk is generated separately and its edge silence is trimmed (energy VAD, 30 ms kept). Chunks are then joined with set gaps taken from the real anchor's pause statistics: 0.40 s ± 0.05 at commas and `;:`, 0.60 s at sentence ends, and 0.12 s at an unpunctuated language switch, with 10 ms fades. A one-word chunk is merged into the next chunk of the same language.
+- **`--speed`** (default 1.05): a pitch-preserving tempo factor (WSOLA via `audiotsm`) applied to the speech only, so the gaps stay as set. The allowed range is 1.0–1.2.
+- **`--pause_mode model --speed 1.0`** reproduces the Phase 2 final system: model-generated pauses, sentence-by-sentence generation, internal silences capped at 0.6 s, and 0.6 s between sentences. Both modes use anchor prompt clip 0 at the LM and flow (`--prompt_clip 0`).
+- **`--dtype`:** the LM ran under bf16 autocast. T4 has no bf16, so `auto` uses fp16 there. fp16 has not been tested yet; if a T4 run sounds wrong, compare against an A100 (bf16) run.
+- **Determinism:** generation is not bit-reproducible across runs on GPU. Sampling under bf16 can pick a different token on near-ties, so the same seed can give a slightly different take.
+
+From Python:
+
+```python
+from infer import DynamicAccent
+tts = DynamicAccent("pretrained/CosyVoice2-0.5B", "../CosyVoice",
+                    "checkpoints/p2_lm_B/best.pt", "checkpoints/p2_flow_sft/best.pt", "prompts/prompts.json")
+wav = tts.say("<hau>Ina kwana, yaya aiki?</hau> <eng>The meeting has been moved to Friday morning.</eng>",
+              seed=0, pause_mode="clause", speed=1.05)
+import soundfile as sf; sf.write("hau_eng.wav", wav, tts.sr)
+```
+
+### Example code-switched inputs
+
+```text
+<eng>this is dynamic accent,</eng> <pcm>e dey sound nice abi?</pcm>
+<yor>Ẹ káàárọ̀ o, ṣé dáadáa ni?</yor> <eng>I will send you the report this afternoon.</eng>
+<ibo>Kedu ka ị mere?</ibo> <pcm>Abeg no vex, I go call you for evening.</pcm>
+<hau>Ina kwana, yaya aiki?</hau> <eng>The meeting has been moved to Friday morning.</eng>
+<ibo>Ndewo nụ, ụmụnna m.</ibo> <pcm>Una well done o, I know say the road no easy.</pcm> <eng>We have three things to discuss this evening.</eng>
+<hau>Barka da yamma, yaya kuke?</hau> <eng>The weather report says it will rain tomorrow morning.</eng> <pcm>Abeg no forget o, the rain fit heavy well well.</pcm>
+```
+
+The Yoruba, Igbo and Hausa example lines are hand-written and have not been checked by native speakers.
+
+## Samples
+
+- `samples/pcm_eng_dynamic_accent_seed{0,1,2}.mp3`: "this is dynamic accent, e dey sound nice abi?" (English then Pidgin), seeds 0, 1 and 2. These are Phase 2 final renders (`--pause_mode model --speed 1.0`).
+- `samples/prosody_v2/*.mp3`: the same custom sentence plus `para_ibo_pcm_eng`, `para_yor_eng`, `para_hau_eng_pcm`, `ibo_000` and `yor_000`, rendered with the defaults (`--pause_mode clause --speed 1.05`, seed 0).
+
+## Prosody v2 (inference-time pause and pace fix)
+
+**Speaking rate** is articulation rate: syllables per second of speech, with pauses of 0.15 s or more excluded. It uses the same syllable counter for every source. Reference speech is 150 clips per source. The system rate comes from the Phase 2 final renders of the eval set.
+
+| | anchor (yor) | yor | ibo | hau | pcm | eng |
+|---|---|---|---|---|---|---|
+| reference speech (syl/s) | 4.45 | 4.78 (WAXAL, other speakers) | 4.73 | 5.71 | 3.87 | 4.53 (FLEURS) |
+| Phase 2 final system (syl/s) | 4.20 (yor) | 4.20 | 4.50 | 3.98 | 3.63 | 4.77 |
+| reference ÷ system | 1.06 | 1.14 | 1.05 | 1.43 | 1.06 | 0.95 |
+
+The median ratio is 1.06, so the default `--speed` is **1.05** (rounded to 0.05 and clamped to 1.0–1.2). Hausa is the outlier. Its vowel-based syllable count is the roughest of the five.
+
+**Before and after** on the six sample items (custom sentence, 3 paragraphs, ibo_000, yor_000), seed 0. Pauses are measured at forced-aligned word boundaries with the Phase 2 report harness. The anchor row is real WAXAL recordings. With only 6 items, read these as a direction, not a result.
+
+| | comma pause (s) | sentence pause (s) | unpunctuated pause rate (>0.25 s) | articulation rate (syl/s) | para CER | para CER (no tones) | all-6 CER | SIM |
+|---|---|---|---|---|---|---|---|---|
+| real anchor | 0.417 | 0.543 | 0.137 | 4.45 | – | – | – | ceiling ≈0.84 |
+| before: Phase 2 final (`model`, 1.0) | 0.339 | 0.488 | 0.057 | 4.26 | 0.175 | 0.124 | 0.175 | 0.814 |
+| clause, speed 1.0 | 0.339 | 0.526 | 0.095 | 3.95 | 0.179 | 0.123 | 0.166 | 0.775 |
+| **after: clause, speed 1.05 (default)** | 0.275 (≈0.37 *) | 0.550 | 0.086 | 4.18 | 0.193 | 0.143 | 0.207 | 0.750 |
+| clause, speed 1.10 | 0.315 | 0.550 | 0.067 | 4.37 | 0.244 | 0.194 | 0.224 | 0.748 |
+| model pauses, speed 1.05 | 0.235 | 0.441 | 0.029 | 4.35 | 0.189 | 0.149 | 0.188 | 0.788 |
+
+\* At two commas that follow the one-letter Pidgin particle "o," the aligner moves the inserted gap to the boundary before "o". This reads as a 0.0 s comma plus one unpunctuated pause. The other comma gaps in clause mode measure 0.37–0.39 s, against the 0.40 s target.
+
+- **Pauses:** clause mode moves comma and sentence pauses and the phrase-break rate toward the anchor's numbers. Sentence pauses go from 0.49 to 0.55 s against the anchor's 0.54 s.
+- **Pace:** generating short chunks one at a time makes speech about 7% slower (4.26 to 3.95 syl/s). So 1.05 only brings the rate back to about where the final system was, and it is still below the anchor.
+- **Cost:** WSOLA speed-up and chunking each lower ECAPA similarity (0.81 to 0.75–0.79). At 1.10, paragraph CER rises clearly (0.175 to 0.244). At 1.05 the rise is small (0.175 to 0.193), and mixed across paragraphs: Hausa got better, Igbo and Yoruba got worse. A blind listening check should decide between 1.0 and 1.05.
+
+## Evaluation (Phase 2 report)
+
+CER is ASR round-trip: MMS-1b-all for ibo/hau/yor, Whisper-large-v3-turbo for eng and pcm, and min(MMS, Whisper) for code-switched (CS) items and paragraphs. "No tones" strips the diacritics. SIM is ECAPA cosine to the anchor centroid; the ceiling is about 0.84 on real held-out anchor clips. Pidgin CER is limited by the judge: real Pidgin recordings score 0.43. Full eval set: 30 items per language + 9 CS sentences + 3 paragraphs = 162 items, single sample (seed 0, anchor prompt).
+
+| system | ibo CER | hau CER | yor CER (no tones) | pcm CER | eng CER | CS CER | para CER | SIM to anchor (mean of 5 langs) | SIM spread | CS half-vs-half SIM |
+|---|---|---|---|---|---|---|---|---|---|---|
+| Phase 1 Arm B (smoke) | 0.212 | 0.244 | 0.407 (0.294) | 0.412 | 0.035 | 0.201 | 0.195 | 0.804 | 0.030 | 0.649 |
+| p2_lm (profile A) + SFT flow | 0.199 | 0.183 | 0.223 (0.140) | 0.190 | 0.038 | 0.184 | 0.176 | 0.769 | 0.069 | 0.634 |
+| **p2_lm_B (profile B) + SFT flow (this release)** | 0.188 | 0.188 | 0.224 (0.137) | 0.174 | 0.034 | 0.167 | 0.173 | 0.782 | 0.039 | 0.642 |
+
+Pause and pitch (energy VAD on forced-aligned word boundaries):
+
+| system | int. pause p90 (s) | pauses / s | spurious pause rate | comma pause (s) | pause-length W1 vs anchor | para: sentence pause (s) | para: pause p90 (s) | F0 std (st) | F0 range (st) | yor tone-direction acc (pairs) |
+|---|---|---|---|---|---|---|---|---|---|---|
+| **real anchor recordings** (WAXAL val+test, raw) | 1.071 | 0.624 | 0.137 | 0.417 | 0 | 0.543 | – | 3.152 | 10.173 | 0.750 (2990) |
+| Phase 1 Arm B (smoke) | 1.334 | 0.584 | 0.044 | 0.245 | 0.147 | 0.315 | 1.035 | 3.058 | 9.999 | 0.777 (251) |
+| p2_lm (profile A) + SFT flow | 0.414 | 0.328 | 0.008 | 0.263 | 0.330 | 0.509 | 0.547 | 3.038 | 10.069 | 0.752 (254) |
+| **p2_lm_B (profile B) + SFT flow (this release)** | 0.460 | 0.288 | 0.087 | 0.251 | 0.262 | 0.488 | 0.531 | 3.057 | 10.134 | 0.749 (255) |
+
+Zero-shot baselines on the 62-item stage subset (for context): CosyVoice2 zero-shot ibo/hau/yor CER 0.689 / 0.281 / 0.852, and CosyVoice3 zero-shot 0.629 / 0.197 / 0.717, against 0.173 / 0.220 / 0.226 for Phase 2 (p2_lm + SFT flow).
+
+## Known limitations
+
+- In `model` mode, comma pauses are short (0.25 s on the full eval set, against the anchor's 0.42 s). `clause` mode sets them directly (see Prosody v2).
+- Speaker similarity is 0.78 against Phase 1's 0.80. English and Pidgin drift the most from the anchor.
+- English after a Nigerian-language span keeps a strong local accent.
+- Pidgin has only about 6 h of training data and no reliable ASR judge.
+- Yoruba tone quality needs human listening; the automatic tone metric is saturated.
+- Naturalness has not been rated by native speakers yet (no MOS / A/B).
+
+## Training data
+
+The training data includes CC-BY-SA sources. Datasets used:
+- **WAXAL** (Yoruba, Igbo, Hausa, Nigerian Pidgin; includes the anchor voice, Yoruba speaker 4), CC-BY-SA-4.0
+- **BibleTTS** (Yoruba and Hausa), CC-BY-SA
+- **FLEURS** (English, Hausa, Igbo, Yoruba)
+- **OpenSLR SLR70** (Nigerian English)
+- **OpenSLR SLR86** (Yoruba)
+- 4,000 code-switched pairs built from the sets above
+
+## Licence
+
+Apache-2.0 (see `LICENSE`). The base model, CosyVoice2-0.5B, is Apache-2.0.

code/da/data.py

@@ -0,0 +1,95 @@
+"""Arm A data: memmapped Vocos mels + byte tokens with per-token language ids; frame-budget bucketing.
+Beta-binomial alignment priors are computed in DataLoader workers."""
+import json, os, random
+import numpy as np
+import torch
+from torch.utils.data import Dataset, Sampler
+
+from .text import encode, LANG2ID
+from .model import beta_binomial_prior
+
+N_MELS = 100
+
+
+def load_meta(root, datasets, split="train", max_frames=1400, max_tokens=420):
+    items, mm = [], {}
+    for ds in datasets:
+        p = f"{root}/{ds}/meta.jsonl"
+        if not os.path.exists(p) or os.path.getsize(p) == 0:
+            continue
+        mm[ds] = np.memmap(f"{root}/{ds}/mels.f16", dtype=np.float16, mode="r").reshape(-1, N_MELS)
+        for l in open(p, encoding="utf-8"):
+            m = json.loads(l)
+            if m["split"] != split or m["nf"] > max_frames:
+                continue
+            if len(m["text"].encode("utf-8")) + 2 > max_tokens:
+                continue
+            items.append(m)
+    return items, mm
+
+
+class TTSData(Dataset):
+    def __init__(self, items, mm, spk2id):
+        self.items, self.mm, self.spk2id = items, mm, spk2id
+
+    def __len__(self):
+        return len(self.items)
+
+    def __getitem__(self, i):
+        m = self.items[i]
+        mel = np.asarray(self.mm[m["ds"]][m["off"]: m["off"] + m["nf"]], dtype=np.float32)
+        tok, _ = encode(f"<{m['lang']}>{m['text']}</{m['lang']}>")
+        lang = [LANG2ID[m["lang"]]] * len(tok)
+        return dict(tok=tok, lang=lang, mel=mel, spk=self.spk2id[m["spk"]], id=m["id"])
+
+
+def collate(batch):
+    B = len(batch)
+    N = max(len(b["tok"]) for b in batch)
+    T = max(len(b["mel"]) for b in batch)
+    tok = torch.zeros(B, N, dtype=torch.long)
+    lang = torch.zeros(B, N, dtype=torch.long)
+    mel = torch.zeros(B, T, N_MELS)
+    prior = torch.full((B, T, N), -1e4)
+    tl, ml = torch.zeros(B, dtype=torch.long), torch.zeros(B, dtype=torch.long)
+    for i, b in enumerate(batch):
+        n, t = len(b["tok"]), len(b["mel"])
+        tok[i, :n] = torch.tensor(b["tok"]); lang[i, :n] = torch.tensor(b["lang"])
+        mel[i, :t] = torch.from_numpy(b["mel"]); tl[i], ml[i] = n, t
+        prior[i, :t, :n] = beta_binomial_prior(n, t)
+    spk = torch.tensor([b["spk"] for b in batch])
+    return dict(tok=tok, lang=lang, tlens=tl, mel=mel, mlens=ml, spk=spk, prior=prior)
+
+
+class FrameBucketSampler(Sampler):
+    """Batches of similar length under a padded-frame budget; dataset weights by repetition."""
+
+    def __init__(self, items, frame_budget=40000, weights=None, seed=0):
+        self.items, self.budget, self.seed, self.epoch = items, frame_budget, seed, 0
+        self.weights = weights or {}
+
+    def _batches(self):
+        rng = random.Random(self.seed + self.epoch)
+        idx = []
+        for i, m in enumerate(self.items):
+            w = self.weights.get(m["ds"], 1.0)
+            idx += [i] * int(w) + ([i] if rng.random() < (w - int(w)) else [])
+        idx.sort(key=lambda i: self.items[i]["nf"] + rng.random() * 30)
+        batches, cur, mx = [], [], 0
+        for i in idx:
+            nf = self.items[i]["nf"]
+            if cur and max(mx, nf) * (len(cur) + 1) > self.budget:
+                batches.append(cur); cur, mx = [], 0
+            cur.append(i); mx = max(mx, nf)
+        if cur:
+            batches.append(cur)
+        rng.shuffle(batches)
+        return batches
+
+    def __iter__(self):
+        b = self._batches()
+        self.epoch += 1
+        return iter(b)
+
+    def __len__(self):
+        return len(self._batches())

code/da/model.py

@@ -0,0 +1,368 @@
+"""Dynamic Accent - Arm A, written from scratch in PyTorch.
+
+Backbone: Parallel Tacotron 2 (Elias et al. 2021, arXiv 2103.14574): non-autoregressive, token durations,
+  differentiable learned upsampling (MLP over start/end offsets -> token->frame weights W), iterative mel loss.
+Multilingual recipe: Zhang et al. 2019 (arXiv 1907.04448): separate speaker + language embeddings,
+  domain-adversarial speaker classifier (gradient reversal) on text-encoder outputs, residual VAE.
+
+Documented deviation (alignment): PT2 trains durations end-to-end with Soft-DTW. Soft-DTW is O(T*N)-sequential and
+too slow for a 2 h smoke budget, so we use the internal unsupervised alignment-learning objective of
+Badlani et al. 2022 ("One TTS Alignment To Rule Them All", arXiv 2108.10447): a small text/mel attention trained with
+a CTC forward-sum loss + beta-binomial prior, hardened with monotonic alignment search (MAS). The hard durations
+supervise the duration predictor and drive the SAME differentiable PT2 upsampler at train time; predicted durations
+drive it at inference. No external aligner is used.
+"""
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .text import VOCAB, LANGS
+
+N_MELS = 100
+
+
+# ---------------------------------------------------------------- utils
+def seq_mask(lens, maxlen=None):
+    maxlen = maxlen or int(lens.max())
+    return torch.arange(maxlen, device=lens.device)[None] < lens[:, None]   # True = valid
+
+
+def sinusoid(T, d, device):
+    pos = torch.arange(T, device=device).float()[:, None]
+    i = torch.arange(0, d, 2, device=device).float()[None]
+    ang = pos / (10000 ** (i / d))
+    pe = torch.zeros(T, d, device=device)
+    pe[:, 0::2] = torch.sin(ang)
+    pe[:, 1::2] = torch.cos(ang)
+    return pe
+
+
+class GradReverse(torch.autograd.Function):
+    """Gradient reversal. As in Zhang et al. 2019 the reversed gradient sent into the text encoder is clipped
+    (here: its L2 norm is capped at `clip`), which keeps the min-max game from blowing up the encoder."""
+
+    @staticmethod
+    def forward(ctx, x, lam, clip):
+        ctx.lam, ctx.clip = lam, clip
+        return x.view_as(x)
+
+    @staticmethod
+    def backward(ctx, g):
+        g = -ctx.lam * g
+        if ctx.clip is not None:
+            n = g.norm()
+            if n > ctx.clip:
+                g = g * (ctx.clip / (n + 1e-12))
+        return g, None, None
+
+
+def grad_reverse(x, lam=1.0, clip=0.5):
+    return GradReverse.apply(x, lam, clip)
+
+
+class ConvNorm(nn.Module):
+    def __init__(self, cin, cout, k=5, drop=0.1):
+        super().__init__()
+        self.conv = nn.Conv1d(cin, cout, k, padding=k // 2)
+        self.ln = nn.LayerNorm(cout)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x, mask):           # x [B,T,C], mask [B,T]
+        x = x.masked_fill(~mask[..., None], 0)
+        y = self.conv(x.transpose(1, 2)).transpose(1, 2)
+        return self.drop(F.relu(self.ln(y)))
+
+
+class FFTBlock(nn.Module):
+    """Pre-LN self-attention + convolutional feed-forward (FastSpeech/PT2-style lightweight block)."""
+
+    def __init__(self, d, heads=4, ff=4, k=3, drop=0.1):
+        super().__init__()
+        self.ln1 = nn.LayerNorm(d)
+        self.att = nn.MultiheadAttention(d, heads, dropout=drop, batch_first=True)
+        self.ln2 = nn.LayerNorm(d)
+        self.c1 = nn.Conv1d(d, d * ff, k, padding=k // 2)
+        self.c2 = nn.Conv1d(d * ff, d, 1)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x, mask):
+        h = self.ln1(x)
+        a, _ = self.att(h, h, h, key_padding_mask=~mask, need_weights=False)
+        x = x + self.drop(a)
+        h = self.ln2(x).masked_fill(~mask[..., None], 0).transpose(1, 2)
+        h = self.c2(F.gelu(self.c1(h))).transpose(1, 2)
+        x = x + self.drop(h)
+        return x.masked_fill(~mask[..., None], 0)
+
+
+# ---------------------------------------------------------------- modules
+class TextEncoder(nn.Module):
+    def __init__(self, d, n_lang, layers=6):
+        super().__init__()
+        self.emb = nn.Embedding(VOCAB, d, padding_idx=0)
+        self.lang_in = nn.Embedding(n_lang, d)          # per-token language id at the input
+        self.convs = nn.ModuleList([ConvNorm(d, d, 5) for _ in range(3)])
+        self.blocks = nn.ModuleList([FFTBlock(d) for _ in range(layers)])
+
+    def forward(self, tok, lang, mask):
+        x = self.emb(tok) + self.lang_in(lang)
+        for c in self.convs:
+            x = x + c(x, mask)
+        x = x + sinusoid(x.shape[1], x.shape[2], x.device)[None]
+        for b in self.blocks:
+            x = b(x, mask)
+        return x
+
+
+class ResidualVAE(nn.Module):
+    """Global residual (prosody) encoder: mel -> q(z|x); z broadcast to every token (Zhang 2019 / Hsu 2019)."""
+
+    def __init__(self, zdim=16, h=256):
+        super().__init__()
+        self.convs = nn.Sequential(
+            nn.Conv1d(N_MELS, h, 3, padding=1), nn.ReLU(),
+            nn.Conv1d(h, h, 3, stride=2, padding=1), nn.ReLU(),
+            nn.Conv1d(h, h, 3, stride=2, padding=1), nn.ReLU())
+        self.gru = nn.GRU(h, h // 2, batch_first=True, bidirectional=True)
+        self.mu = nn.Linear(h, zdim)
+        self.logvar = nn.Linear(h, zdim)
+
+    def forward(self, mel, mel_lens):
+        h = self.convs(mel.transpose(1, 2)).transpose(1, 2)
+        lens = torch.clamp((mel_lens + 3) // 4, min=1)
+        h, _ = self.gru(h)
+        m = seq_mask(lens, h.shape[1]).float()[..., None]
+        h = (h * m).sum(1) / m.sum(1)
+        mu, lv = self.mu(h), self.logvar(h).clamp(-8, 8)
+        z = mu + torch.randn_like(mu) * torch.exp(0.5 * lv) if self.training else mu
+        kl = 0.5 * (mu.pow(2) + lv.exp() - 1 - lv).sum(-1).mean()
+        return z, kl
+
+
+class Aligner(nn.Module):
+    """Unsupervised text-mel alignment (Badlani et al. 2022)."""
+
+    def __init__(self, d, h=256, temp=0.0005):
+        super().__init__()
+        self.temp = temp
+        self.key = nn.Sequential(nn.Conv1d(d, h * 2, 3, padding=1), nn.ReLU(), nn.Conv1d(h * 2, 80, 1))
+        self.query = nn.Sequential(nn.Conv1d(N_MELS, h * 2, 3, padding=1), nn.ReLU(),
+                                   nn.Conv1d(h * 2, h, 1), nn.ReLU(), nn.Conv1d(h, 80, 1))
+
+    def forward(self, txt, mel, tmask, mmask, prior_log):
+        k = self.key(txt.transpose(1, 2))                         # B,80,N
+        q = self.query(mel.transpose(1, 2))                       # B,80,T
+        q, k = q.float(), k.float()
+        dist = (q.pow(2).sum(1)[:, :, None] + k.pow(2).sum(1)[:, None, :]
+                - 2 * torch.bmm(q.transpose(1, 2), k)).clamp_min(0)        # B,T,N  (memory-light ||q-k||^2)
+        logits = -self.temp * dist
+        logits = logits.masked_fill(~tmask[:, None, :], -1e4)
+        logp = F.log_softmax(logits, -1) + prior_log                # prior in log domain
+        logp = logp.masked_fill(~tmask[:, None, :], -1e4)
+        soft = F.softmax(logp, -1) * mmask[..., None]
+        return logits, logp, soft
+
+
+def forward_sum_loss(logits, tlens, mlens, blank_logprob=-1.0):
+    """CTC forward-sum: every token must be visited monotonically. logits [B,T,N] (pre-softmax, masked)."""
+    B, T, N = logits.shape
+    lp = F.pad(logits, (1, 0), value=blank_logprob)
+    lp = F.log_softmax(lp, -1).transpose(0, 1)                      # T,B,N+1
+    tgt = torch.arange(1, N + 1, device=logits.device)[None].expand(B, N)
+    return F.ctc_loss(lp.float(), tgt, mlens, tlens, blank=0, reduction="mean", zero_infinity=True)
+
+
+def _mas_np(logp, N, T):
+    """Monotonic alignment search on one [T,N] log-prob matrix -> hard [T,N]."""
+    Q = np.full((T, N), -np.inf, dtype=np.float32)
+    Q[0, 0] = logp[0, 0]
+    for t in range(1, T):
+        for n in range(min(N, t + 1)):
+            stay = Q[t - 1, n]
+            move = Q[t - 1, n - 1] if n > 0 else -np.inf
+            Q[t, n] = max(stay, move) + logp[t, n]
+    path = np.zeros((T, N), dtype=np.float32)
+    n = N - 1
+    for t in range(T - 1, -1, -1):
+        path[t, n] = 1
+        if t > 0 and n > 0 and (n == t or Q[t - 1, n - 1] > Q[t - 1, n]):
+            n -= 1
+    return path
+
+
+try:
+    import numba
+
+    _mas_np = numba.njit(cache=True)(_mas_np)  # noqa: F811
+except Exception:  # numba optional (CPU unit tests)
+    pass
+
+
+@torch.no_grad()
+def mas(logp, tlens, mlens):
+    out = torch.zeros_like(logp)
+    lp = logp.float().cpu().numpy()
+    for b in range(lp.shape[0]):
+        T, N = int(mlens[b]), int(tlens[b])
+        out[b, :T, :N] = torch.from_numpy(_mas_np(np.ascontiguousarray(lp[b, :T, :N]), N, T))
+    return out
+
+
+_prior_cache = {}
+
+
+def beta_binomial_prior(N, T, scale=1.0):
+    from scipy.stats import betabinom
+    key = (N, T)
+    if key not in _prior_cache:
+        x = np.arange(N)[None]
+        t = np.arange(1, T + 1)[:, None]
+        pmf = betabinom.pmf(x, N - 1, scale * t, scale * (T + 1 - t))
+        _prior_cache[key] = torch.from_numpy(np.log(pmf + 1e-8)).float()
+        if len(_prior_cache) > 20000:
+            _prior_cache.clear()
+    return _prior_cache[key]
+
+
+def batch_prior(tlens, mlens, N, T, device):
+    p = torch.full((len(tlens), T, N), -1e4)
+    for b in range(len(tlens)):
+        n, t = int(tlens[b]), int(mlens[b])
+        p[b, :t, :n] = beta_binomial_prior(n, t)
+    return p.to(device)
+
+
+class DurationPredictor(nn.Module):
+    def __init__(self, d, h=256):
+        super().__init__()
+        self.convs = nn.ModuleList([ConvNorm(d, h, 3), ConvNorm(h, h, 3), ConvNorm(h, h, 3)])
+        self.out = nn.Linear(h, 1)
+
+    def forward(self, x, mask):
+        for c in self.convs:
+            x = c(x, mask)
+        return self.out(x).squeeze(-1).masked_fill(~mask, 0)       # log(1 + d)
+
+
+class LearnedUpsampler(nn.Module):
+    """PT2 differentiable upsampling: W[t,k] = softmax_k MLP(S_tk, E_tk, h_k);  O = W H + proj(sum_k W C)."""
+
+    def __init__(self, d, hw=16, hc=8):
+        super().__init__()
+        self.se_w = nn.Linear(2, hw)
+        self.h_w = nn.Linear(d, hw)
+        self.w_out = nn.Linear(hw, 1)
+        self.se_c = nn.Linear(2, hc)
+        self.h_c = nn.Linear(d, hc)
+        self.c_out = nn.Linear(hc, d)
+
+    def forward(self, H, dur, tmask, T):
+        dur = dur.masked_fill(~tmask, 0)
+        e = torch.cumsum(dur, 1)
+        s = e - dur
+        t = torch.arange(T, device=H.device).float()[None, :, None] + 0.5
+        S = (t - s[:, None, :]) / 10.0
+        E = (e[:, None, :] - t) / 10.0
+        se = torch.stack([S, E], -1)                                   # B,T,N,2
+        lw = self.w_out(F.silu(self.se_w(se) + self.h_w(H)[:, None])).squeeze(-1)
+        lw = lw.masked_fill(~tmask[:, None, :], -1e4)
+        W = torch.softmax(lw, -1)                                      # B,T,N
+        C = F.silu(self.se_c(se) + self.h_c(H)[:, None])               # B,T,N,hc
+        out = torch.bmm(W, H) + self.c_out(torch.einsum("btn,btnc->btc", W, C))
+        return out, W
+
+
+class MelDecoder(nn.Module):
+    def __init__(self, d, layers=6, n_iter=3):
+        super().__init__()
+        self.blocks = nn.ModuleList([FFTBlock(d) for _ in range(layers)])
+        self.n_iter = n_iter
+        self.outs = nn.ModuleList([nn.Linear(d, N_MELS) for _ in range(n_iter)])
+
+    def forward(self, x, mask):
+        x = x + sinusoid(x.shape[1], x.shape[2], x.device)[None]
+        mels = []
+        L = len(self.blocks)
+        for i, b in enumerate(self.blocks):
+            x = b(x, mask)
+            j = i - (L - self.n_iter)
+            if j >= 0:
+                mels.append(self.outs[j](x))
+        return mels
+
+
+# ---------------------------------------------------------------- full model
+class DynamicAccentTTS(nn.Module):
+    def __init__(self, n_spk, n_lang=len(LANGS), d=384, enc_layers=6, dec_layers=6, spk_dim=128, lang_dim=64,
+                 zdim=16, adv_hidden=256, mel_mean=None, mel_std=None, align_temp=0.0005):
+        super().__init__()
+        self.n_spk = n_spk
+        self.encoder = TextEncoder(d, n_lang, enc_layers)
+        self.spk_emb = nn.Embedding(n_spk, spk_dim)
+        self.lang_out = nn.Embedding(n_lang, lang_dim)                 # per-token, concatenated to encoder outputs
+        self.vae = ResidualVAE(zdim)
+        self.cond = nn.Linear(d + spk_dim + lang_dim + zdim, d)
+        self.adv = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, adv_hidden), nn.ReLU(), nn.Linear(adv_hidden, n_spk))
+        self.aligner = Aligner(d, temp=align_temp)
+        self.dur = DurationPredictor(d)
+        self.up = LearnedUpsampler(d)
+        self.dec_spk = nn.Linear(spk_dim, d)
+        self.decoder = MelDecoder(d, dec_layers)
+        self.register_buffer("mel_mean", torch.tensor(mel_mean if mel_mean is not None else -2.5))
+        self.register_buffer("mel_std", torch.tensor(mel_std if mel_std is not None else 2.2))
+
+    def _cond(self, Ht, spk, lang, z):
+        B, N, _ = Ht.shape
+        s = self.spk_emb(spk)[:, None].expand(B, N, -1)
+        zz = z[:, None].expand(B, N, -1)
+        return self.cond(torch.cat([Ht, s, self.lang_out(lang), zz], -1)), s
+
+    def forward(self, tok, lang, tlens, mel, mlens, spk, step=0, adv_lambda=1.0, bin_start=4000, prior=None):
+        tmask, mmask = seq_mask(tlens, tok.shape[1]), seq_mask(mlens, mel.shape[1])
+        meln = (mel - self.mel_mean) / self.mel_std
+        Ht = self.encoder(tok, lang, tmask)
+        # adversarial speaker classifier on text encodings (Zhang 2019): GRL removes speaker info from text path
+        adv_logits = self.adv(grad_reverse(Ht, adv_lambda))
+        adv_ce = F.cross_entropy(adv_logits[tmask], spk[:, None].expand_as(tok)[tmask])
+        adv_acc = (adv_logits[tmask].argmax(-1) == spk[:, None].expand_as(tok)[tmask]).float().mean()
+        z, kl = self.vae(meln, mlens)
+        H, _ = self._cond(Ht, spk, lang, z)
+        # alignment (internal, unsupervised)
+        if prior is None:
+            prior = batch_prior(tlens, mlens, tok.shape[1], mel.shape[1], tok.device)
+        logits, logp, soft = self.aligner(self.encoder.emb(tok) + Ht.detach(), meln, tmask, mmask, prior)
+        l_fs = forward_sum_loss(logits.masked_fill(~mmask[..., None], -1e4) + prior, tlens, mlens)
+        hard = mas(logp, tlens, mlens)
+        l_bin = -(hard * torch.log(soft.clamp_min(1e-8))).sum() / hard.sum() if step >= bin_start else soft.new_zeros(())
+        dur_t = hard.sum(1)                                            # B,N frames
+        # durations
+        logd = self.dur(H.detach() if step < 1000 else H, tmask)
+        l_dur = F.mse_loss(logd[tmask], torch.log1p(dur_t[tmask]))
+        # upsample with target durations (train) -> decoder
+        up, W = self.up(H, dur_t, tmask, mel.shape[1])
+        x = up + self.dec_spk(self.spk_emb(spk))[:, None]
+        mels = self.decoder(x, mmask)
+        m = mmask[..., None].float()
+        l_mel = sum((F.l1_loss(o, meln, reduction="none") * m).sum() / (m.sum() * N_MELS) for o in mels) / len(mels)
+        return dict(mel=l_mel, dur=l_dur, fs=l_fs, bin=l_bin, kl=kl, adv=adv_ce, adv_acc=adv_acc.detach()), mels[-1]
+
+    @torch.no_grad()
+    def infer(self, tok, lang, spk, pace=1.0, z=None):
+        tlens = torch.full((tok.shape[0],), tok.shape[1], device=tok.device)
+        tmask = seq_mask(tlens, tok.shape[1])
+        Ht = self.encoder(tok, lang, tmask)
+        if z is None:
+            z = torch.zeros(tok.shape[0], self.vae.mu.out_features, device=tok.device)
+        H, _ = self._cond(Ht, spk, lang, z)
+        dur = (torch.expm1(self.dur(H, tmask)).clamp(min=0) * pace).masked_fill(~tmask, 0)
+        T = max(int(dur.sum(1).max().round().item()), 1)
+        up, W = self.up(H, dur, tmask, T)
+        x = up + self.dec_spk(self.spk_emb(spk))[:, None]
+        mel = self.decoder(x, torch.ones(tok.shape[0], T, dtype=torch.bool, device=tok.device))[-1]
+        return mel * self.mel_std + self.mel_mean, dur
+
+
+def count_params(m):
+    return sum(p.numel() for p in m.parameters())

code/da/p2common.py

@@ -0,0 +1,308 @@
+"""Phase 2 shared tools: romanisation for MMS forced alignment, chunked aligner, pause normalisation,
+pause / F0 / Yoruba-tone metrics.  Used by prep2.py (data) and eval2.py (metrics)."""
+import re, unicodedata
+import numpy as np
+
+# ------------------------------------------------------------------ text
+_MAP = {"ọ": "o", "ẹ": "e", "ṣ": "s", "ị": "i", "ụ": "u", "ṅ": "n", "ń": "n", "ǹ": "n", "ḿ": "m", "ɓ": "b", "ɗ": "d",
+        "ƙ": "k", "ƴ": "y", "ŋ": "n", "ɛ": "e", "ɔ": "o", "ß": "s", "æ": "ae", "œ": "oe", "ø": "o", "đ": "d", "ł": "l"}
+TONE_MARK = {"\u0301": 2, "\u0300": 0, "\u0304": 1}          # acute=H, grave=L, macron=M
+SENT_END = set(".?!")
+CLAUSE = set(";:")
+COMMA = set(",")
+_VOW = set("aeiou")
+
+
+def roman_char(c):
+    """one NFC char -> (romanised latin string, tone or None)."""
+    lc = c.lower()
+    if lc in _MAP:
+        return _MAP[lc], None
+    d = unicodedata.normalize("NFD", lc)
+    base = "".join(x for x in d if unicodedata.category(x) != "Mn")
+    tone = None
+    for x in d:
+        if x in TONE_MARK:
+            tone = TONE_MARK[x]
+    base = "".join(_MAP.get(b, b) for b in base)
+    base = "".join(b for b in base if ("a" <= b <= "z") or b == "'")
+    return base, tone
+
+
+def words_for_align(text):
+    """text -> list of dict(word, rom, punct_after, chars=[(rom_char, orig_index, tone)]).
+    Words that romanise to nothing (digits, symbols) are aligned with the '*' star token."""
+    out = []
+    for m in re.finditer(r"\S+", text):
+        w = m.group(0)
+        chars = []
+        for k, c in enumerate(w):
+            r, t = roman_char(c)
+            for j, rc in enumerate(r):
+                chars.append((rc, m.start() + k, t if j == len(r) - 1 else None))
+        trail = re.sub(r"[\w'’]+", "", w[-3:]) if w else ""
+        p = None
+        if any(c in SENT_END for c in trail):
+            p = "sent"
+        elif any(c in CLAUSE for c in trail):
+            p = "clause"
+        elif any(c in COMMA for c in trail):
+            p = "comma"
+        rom = "".join(c[0] for c in chars)
+        out.append(dict(word=w, rom=rom if rom else "*", punct=p, chars=chars))
+    return out
+
+
+# ------------------------------------------------------------------ aligner
+class Aligner:
+    """torchaudio MMS_FA (CC-BY-NC weights, used only as a data tool/judge) with chunked emissions for long audio."""
+
+    def __init__(self, device="cuda"):
+        import torch, torchaudio
+        self.torch = torch
+        self.F = torchaudio.functional
+        b = torchaudio.pipelines.MMS_FA
+        self.model = b.get_model().to(device).eval().half()
+        self.dict = b.get_dict()
+        self.dev = device
+        self.rate = 320                         # samples per emission frame at 16 kHz (20 ms)
+
+    def emission(self, y16, chunk_s=30, ctx_s=2):
+        torch = self.torch
+        sr = 16000
+        x = torch.from_numpy(y16).float().to(self.dev)
+        if len(y16) <= (chunk_s + 2 * ctx_s) * sr:
+            with torch.inference_mode():
+                e, _ = self.model(x[None].half())
+            return e[0].float().log_softmax(-1)
+        outs = []
+        step, ctx = chunk_s * sr, ctx_s * sr
+        nfr_total = len(y16) // self.rate
+        for s in range(0, len(y16), step):
+            a, b = max(0, s - ctx), min(len(y16), s + step + ctx)
+            with torch.inference_mode():
+                e, _ = self.model(x[a:b][None].half())
+            e = e[0].float().log_softmax(-1)
+            f0 = (s - a) // self.rate
+            nfr = min(step, len(y16) - s) // self.rate
+            outs.append(e[f0:f0 + nfr])
+        e = torch.cat(outs, 0)
+        return e[:max(nfr_total, 1)]
+
+    def align(self, y16, words):
+        """-> per word (start_s, end_s, score) and per char spans [(start_s, end_s)], or None on failure."""
+        torch = self.torch
+        em = self.emission(y16)
+        toks, wid = [], []
+        for i, w in enumerate(words):
+            for c in w["rom"]:
+                if c in self.dict:
+                    toks.append(self.dict[c]); wid.append(i)
+        if not toks or len(toks) * 1 > em.shape[0]:
+            return None
+        t = torch.tensor([toks], dtype=torch.int32, device=self.dev)
+        try:
+            ali, sc = self.F.forced_align(em[None], t, blank=0)
+        except Exception:
+            return None
+        ali, sc = ali[0].cpu().numpy(), sc[0].exp().cpu().numpy()
+        # token spans (merge repeats; blank=0)
+        spans, i, k = [], 0, 0
+        T = len(ali)
+        while i < T:
+            if ali[i] == 0:
+                i += 1; continue
+            j = i
+            while j + 1 < T and ali[j + 1] == ali[i] and not (j + 1 < T and False):
+                j += 1
+            spans.append((i, j + 1, float(sc[i:j + 1].mean())))
+            i = j + 1
+        # repeated identical consecutive tokens (e.g. "aa") collapse in the path above only if no blank between;
+        # forced_align always inserts a blank between identical tokens, so len(spans)==len(toks)
+        if len(spans) != len(toks):
+            return None
+        r = self.rate / 16000
+        W = [[None, None, []] for _ in words]
+        chars = [[] for _ in words]
+        for (a, b, s), w in zip(spans, wid):
+            if W[w][0] is None:
+                W[w][0] = a * r
+            W[w][1] = b * r; W[w][2].append(s); chars[w].append((a * r, b * r))
+        res = []
+        for i, (a, b, s) in enumerate(W):
+            if a is None:
+                return None
+            res.append(dict(start=a, end=b, score=float(np.mean(s)), chars=chars[i]))
+        return res
+
+
+# ------------------------------------------------------------------ energy / pauses
+def frame_db(y, sr, hop_s=0.01):
+    hop = int(sr * hop_s)
+    n = max(1, len(y) // hop)
+    fr = y[: n * hop].reshape(n, hop)
+    e = 10 * np.log10((fr.astype(np.float64) ** 2).mean(1) + 1e-10)
+    return e
+
+
+def silence_mask(y, sr, rel_db=-40.0, hop_s=0.01):
+    e = frame_db(y, sr, hop_s)
+    ref = np.percentile(e, 98)
+    return e < ref + rel_db
+
+
+def runs(mask):
+    """-> list of (start, end) index runs where mask is True."""
+    out, i, n = [], 0, len(mask)
+    while i < n:
+        if mask[i]:
+            j = i
+            while j < n and mask[j]:
+                j += 1
+            out.append((i, j)); i = j
+        else:
+            i += 1
+    return out
+
+
+PAUSE_TARGET = {           # (min, max) silence seconds after a word, by the punctuation that follows it
+    "sent": (0.30, 0.60),
+    "clause": (0.20, 0.50),
+    "comma": (0.00, 0.40),
+    None: (0.00, 0.20),
+}
+# profile B (Phase 2 v2): keeps natural phrase breaks where there is no punctuation (the real anchor pauses at
+# ~14% of unpunctuated word boundaries), still removes hesitations; set P2PAUSE=B
+import os as _os
+if _os.environ.get("P2PAUSE") == "B":
+    PAUSE_TARGET = {"sent": (0.35, 0.70), "clause": (0.25, 0.55), "comma": (0.00, 0.50), None: (0.00, 0.35)}
+LEAD_S, TAIL_S = 0.12, 0.18
+
+
+def normalise_pauses(y, sr, words, ali, seg_a, seg_b, sil=None, lo_s=0.0, hi_s=None):
+    """Rebuild one segment [word seg_a .. seg_b] of y: edge silence fixed to LEAD_S/TAIL_S, every inter-word
+    silence clamped to PAUSE_TARGET[punct] (only truly silent frames are removed; silence is inserted, as a
+    copy of the quietest local frames, when a sentence/clause boundary has too little).  -> (y_out, pauses)"""
+    hop = int(0.01 * sr)
+    if sil is None:
+        sil = silence_mask(y, sr)
+    hi_s = len(y) / sr if hi_s is None else hi_s
+    xf = int(0.005 * sr)
+    pieces, pauses = [], []
+    a0 = ali[seg_a]["start"]
+    start = int(max(lo_s, a0 - LEAD_S, 0.0) * sr)
+    cur = start
+    for i in range(seg_a, seg_b):
+        e_i, s_n = ali[i]["end"], ali[i + 1]["start"]
+        f_a, f_b = int(e_i / 0.01), int(s_n / 0.01) + 1
+        # longest silent run inside the gap (with 20 ms guard to the word edges)
+        best = None
+        for ra, rb in runs(sil[f_a:f_b]):
+            if best is None or rb - ra > best[1] - best[0]:
+                best = (ra, rb)
+        lo, hi = PAUSE_TARGET[words[i]["punct"]]
+        if best is None:
+            cur_sil = 0.0; mid = int((e_i + s_n) / 2 * sr)
+        else:
+            ra, rb = (f_a + best[0]) * hop, (f_a + best[1]) * hop
+            cur_sil = (rb - ra) / sr; mid = (ra + rb) // 2
+        if cur_sil > hi and best is not None:            # shrink: cut the middle of the silent run
+            cut = int((cur_sil - hi) * sr)
+            c0 = mid - cut // 2
+            pieces.append(y[cur:c0]); cur = c0 + cut
+            pauses.append(hi)
+        elif cur_sil < lo:                               # insert quiet frames at the quietest point of the gap
+            add = int((lo - cur_sil) * sr)
+            g0, g1 = int(e_i * sr), max(int(s_n * sr), int(e_i * sr) + hop)
+            seg = y[g0:g1]
+            if len(seg) >= hop:
+                k = int(np.argmin(frame_db(seg, sr))) * hop
+                ins_at = g0 + k
+                q = y[ins_at:ins_at + hop]
+            else:
+                ins_at, q = mid, np.zeros(hop, dtype=y.dtype)
+            filler = np.tile(q, add // hop + 1)[:add] * 0.5
+            pieces.append(y[cur:ins_at]); pieces.append(filler); cur = ins_at
+            pauses.append(lo)
+        else:
+            pauses.append(cur_sil)
+    end = min(len(y), int(min(hi_s, ali[seg_b]["end"] + TAIL_S) * sr))
+    pieces.append(y[cur:end])
+    out = np.concatenate([p for p in pieces if len(p)]) if pieces else y[start:end]
+    # pad edges with silence if the source had less than LEAD/TAIL available
+    lead_have = a0 - start / sr
+    if lead_have < LEAD_S - 0.02:
+        out = np.concatenate([np.zeros(int((LEAD_S - lead_have) * sr), out.dtype), out])
+    tail_have = end / sr - ali[seg_b]["end"]
+    if tail_have < TAIL_S - 0.02:
+        out = np.concatenate([out, np.zeros(int((TAIL_S - tail_have) * sr), out.dtype)])
+    # tiny fades at the edges
+    f = min(xf, len(out) // 4)
+    if f > 0:
+        out[:f] *= np.linspace(0, 1, f); out[-f:] *= np.linspace(1, 0, f)
+    return out.astype(np.float32), pauses
+
+
+# ------------------------------------------------------------------ metrics on audio
+def pause_stats(y, sr, min_pause=0.15):
+    """VAD (energy, -40 dB vs 98th pct) pauses: leading/trailing silence and internal pauses >= min_pause."""
+    sil = silence_mask(y, sr)
+    R = runs(sil)
+    n = len(sil)
+    lead = R[0][1] * 0.01 if R and R[0][0] == 0 else 0.0
+    tail = (n - R[-1][0]) * 0.01 if R and R[-1][1] == n else 0.0
+    internal = [(b - a) * 0.01 for a, b in R if a > 0 and b < n and (b - a) * 0.01 >= min_pause]
+    speech = max(1e-3, n * 0.01 - lead - tail)
+    return dict(lead=round(lead, 3), tail=round(tail, 3), pauses=[round(p, 3) for p in internal],
+                n_pauses=len(internal), pause_rate=round(len(internal) / speech, 4),
+                pause_frac=round(sum(internal) / speech, 4), speech_s=round(speech, 3))
+
+
+def f0_track(y16):
+    import pyworld as pw
+    x = y16.astype(np.float64)
+    f0, t = pw.dio(x, 16000, f0_floor=60, f0_ceil=500, frame_period=10)
+    return pw.stonemask(x, f0, t, 16000)
+
+
+def f0_stats(y16):
+    f0 = f0_track(y16)
+    v = f0[f0 > 0]
+    if len(v) < 20:
+        return dict(voiced=float(len(v)), f0_med=None, st_std=None, st_range=None)
+    st = 12 * np.log2(v / np.median(v))
+    d = np.diff(st)
+    return dict(voiced_frac=round(len(v) / len(f0), 3), f0_med=round(float(np.median(v)), 1),
+                st_std=round(float(st.std()), 3), st_range=round(float(np.percentile(st, 95) - np.percentile(st, 5)), 3),
+                st_vel=round(float(np.abs(d[np.abs(d) < 3]).mean()), 4))
+
+
+def tone_pairs(y16, words, ali):
+    """Yoruba tone-direction accuracy: for consecutive tone-bearing vowels whose tones differ, is the sign of the
+    median-F0 change (semitones) the same as the sign of the tone change?  -> (n_pairs, n_correct)"""
+    f0 = f0_track(y16)
+    vals = []
+    for w, a in zip(words, ali):
+        k = 0
+        for (rc, oi, tone), (s, e) in zip([c for c in w["chars"] if c[0] in "abcdefghijklmnopqrstuvwxyz'"], a["chars"]):
+            if rc in _VOW or (rc in "nm" and tone is not None):
+                t = 1 if tone is None else tone
+                fr = f0[int(s / 0.01): int(e / 0.01) + 1]
+                fr = fr[fr > 0]
+                vals.append((t, float(np.median(12 * np.log2(fr))) if len(fr) else None))
+    n = c = 0
+    for (t1, f1), (t2, f2) in zip(vals, vals[1:]):
+        if t1 == t2 or f1 is None or f2 is None:
+            continue
+        n += 1
+        c += int(np.sign(f2 - f1) == np.sign(t2 - t1))
+    return n, c
+
+
+def wasserstein1(a, b):
+    """1-D earth mover's distance between two samples."""
+    if not len(a) or not len(b):
+        return None
+    a, b = np.sort(a), np.sort(b)
+    q = np.linspace(0, 1, 101)
+    return float(np.mean(np.abs(np.quantile(a, q) - np.quantile(b, q))))

code/da/prep2.py

@@ -0,0 +1,259 @@
+"""Phase 2 data prep (H100 node): every source -> MMS forced alignment -> sentence segmentation of long clips ->
+punctuation-consistent pause normalisation -> 24 kHz int16 + meta.  Output ~/data2/<ds>/{pcm24k.i16, meta.jsonl, stats.json}
+
+  python -m da.prep2 waxal bible_hau bible_yor slr86 slr70 fleurs
+"""
+import os, sys, io, json, glob, csv, hashlib, time
+import numpy as np, soundfile as sf, librosa
+from multiprocessing import Pool
+sys.path.insert(0, os.path.expanduser("~/da"))
+from da.p2common import words_for_align, Aligner, normalise_pauses   # noqa: E402
+
+RAW = "/scratch/raw"
+OUT = os.path.expanduser(os.environ.get("P2OUT", "~/data2"))
+OLD = os.path.expanduser("~/data")
+SR = 24000
+BIBLE_HAU_CAP_H = 30.0
+FLOW_OK = {"waxal", "bible_yor", "bible_hau", "slr86", "slr70"}     # 48 kHz studio-ish sources (FLEURS is 16 kHz)
+
+
+def clean_text(t):
+    import unicodedata, re
+    t = unicodedata.normalize("NFC", t or "")
+    t = t.replace("\u2019", "'").replace("\u2018", "'").replace("\u201c", '"').replace("\u201d", '"')
+    return re.sub(r"\s+", " ", t).strip()
+
+
+def hsplit(i):
+    return "val" if int(hashlib.md5(i.encode()).hexdigest(), 16) % 100 == 0 else "train"
+
+
+# ------------------------------------------------------------------ enumerators (yield meta + loader args)
+def items_waxal():
+    import pyarrow.parquet as pq
+    for lang in ["yor", "ibo", "hau", "pcm"]:
+        files = sorted(glob.glob(f"{RAW}/waxal/data/TTS/{lang}/*.parquet"))
+        files = [f for f in files if not os.path.basename(f).startswith("yor_tts")]
+        for f in files:
+            bn = os.path.basename(f)
+            split = "test" if "test" in bn else ("val" if "validation" in bn else "train")
+            pf = pq.ParquetFile(f)
+            for b in pf.iter_batches(batch_size=8):
+                for r in b.to_pylist():
+                    yield dict(id=f"waxal_{lang}_{r['id']}", ds="waxal", lang=lang,
+                               spk=f"waxal_{lang}_{r.get('speaker_id') or 'spk'}", gender=r.get("gender") or "",
+                               split=split, text=r["text"]), ("bytes", r["audio"]["bytes"])
+
+
+FLEURS = {"hau": "ha_ng", "ibo": "ig_ng", "yor": "yo_ng", "eng": "en_us"}
+
+
+def items_fleurs():
+    """raw FLEURS train audio (16 kHz); speaker ids = Phase 1 ECAPA pseudo-speakers from ~/data/fleurs/meta.jsonl"""
+    for l in open(f"{OLD}/fleurs/meta.jsonl", encoding="utf-8"):
+        m = json.loads(l)
+        fn = m["id"].split("_", 2)[2] + ".wav"
+        p = f"{RAW}/fleurs/data/{FLEURS[m['lang']]}/audio/train/{fn}"
+        yield dict(id=m["id"], ds="fleurs", lang=m["lang"], spk=m["spk"], gender=m.get("gender", ""),
+                   split=hsplit(m["id"]), text=m["text"]), ("path", p)
+
+
+def items_slr(num, lang, prefix):
+    for idx in sorted(glob.glob(f"{RAW}/slr{num}/**/line_index.tsv", recursive=True)):
+        d = os.path.dirname(idx)
+        for line in open(idx, encoding="utf-8"):
+            parts = line.rstrip("\n").split("\t")
+            if len(parts) < 2:
+                continue
+            fid, text = parts[0].strip(), parts[-1]
+            p = os.path.join(d, fid + ".wav")
+            if os.path.exists(p):
+                spk = "_".join(fid.split("_")[:2])
+                yield dict(id=f"{prefix}_{fid}", ds=prefix, lang=lang, spk=f"{prefix}_{spk}",
+                           gender="female" if fid[2] == "f" else "male", split=hsplit(fid), text=text), ("path", p)
+
+
+def items_bible(lang_dir, lang, ds, cap_h=None):
+    root = f"{RAW}/bible/{lang_dir}"
+    auds = sorted(glob.glob(f"{root}/**/*.flac", recursive=True) + glob.glob(f"{root}/**/*.wav", recursive=True))
+    if cap_h:   # deterministic random subset of the train split, roughly cap_h hours (verse ~ 9 s on average)
+        auds.sort(key=lambda p: hashlib.md5(p.encode()).hexdigest())
+    tot = 0.0
+    for p in auds:
+        txt = os.path.splitext(p)[0] + ".txt"
+        if not os.path.exists(txt):
+            continue
+        split = "test" if "/test/" in p else ("val" if "/dev/" in p else "train")
+        if cap_h and split == "train":
+            if tot > cap_h * 3600:
+                continue
+            try:
+                tot += sf.info(p).duration
+            except Exception:
+                continue
+        yield dict(id=f"{ds}_{os.path.splitext(os.path.basename(p))[0]}", ds=ds, lang=lang, spk=ds, gender="",
+                   split=split, text=open(txt, encoding="utf-8").read()), ("path", p)
+
+
+def enum(ds):
+    return {"waxal": items_waxal, "fleurs": items_fleurs,
+            "slr86": lambda: items_slr(86, "yor", "slr86"), "slr70": lambda: items_slr(70, "eng", "slr70"),
+            "bible_yor": lambda: items_bible("yoruba", "yor", "bible_yor"),
+            "bible_hau": lambda: items_bible("hausa", "hau", "bible_hau", BIBLE_HAU_CAP_H)}[ds]()
+
+
+# ------------------------------------------------------------------ workers
+def load(arg):
+    meta, src = arg
+    try:
+        if src[0] == "bytes":
+            y, sr = sf.read(io.BytesIO(src[1]), dtype="float32", always_2d=True); y = y.mean(1)
+        elif src[0] == "path":
+            y, sr = sf.read(src[1], dtype="float32", always_2d=True); y = y.mean(1)
+        else:
+            pcm = np.memmap(src[1], dtype=np.int16, mode="r")
+            y, sr = pcm[src[2]:src[2] + src[3]].astype(np.float32) / 32768.0, 16000
+        meta["text"] = clean_text(meta["text"])
+        meta["sr_in"] = int(sr)
+        if not meta["text"] or len(y) < 0.5 * sr:
+            return meta, None, None, "empty"
+        y24 = librosa.resample(y, orig_sr=sr, target_sr=SR, res_type="soxr_hq") if sr != SR else y
+        y16 = librosa.resample(y, orig_sr=sr, target_sr=16000, res_type="soxr_hq") if sr != 16000 else y
+        return meta, y16, y24, None
+    except Exception as e:  # noqa
+        return meta, None, None, f"err:{type(e).__name__}"
+
+
+def plan_segments(words, ali, dur):
+    """word-index ranges [(a, b)] (inclusive).  Short clips: one segment.  Long clips: whole sentences grouped
+    into 3-14 s segments; an over-long sentence is cut at clause/comma marks."""
+    n = len(words)
+    if dur <= 20.0:
+        return [(0, n - 1)]
+    sents, a = [], 0
+    for i, w in enumerate(words):
+        if w["punct"] == "sent" or i == n - 1:
+            sents.append((a, i)); a = i + 1
+    pieces = []
+    for a, b in sents:
+        if ali[b]["end"] - ali[a]["start"] <= 16.0:
+            pieces.append((a, b)); continue
+        s = a
+        for i in range(a, b + 1):
+            if (words[i]["punct"] in ("clause", "comma") and ali[i]["end"] - ali[s]["start"] > 3.0) or i == b:
+                pieces.append((s, i)); s = i + 1
+    segs, cur = [], None
+    for a, b in pieces:
+        if cur is None:
+            cur = [a, b]; continue
+        if ali[b]["end"] - ali[cur[0]]["start"] <= 14.0 and ali[cur[1]]["end"] - ali[cur[0]]["start"] < 4.0:
+            cur[1] = b
+        else:
+            segs.append(tuple(cur)); cur = [a, b]
+    if cur:
+        segs.append(tuple(cur))
+    return segs
+
+
+def post(arg):
+    meta, y24, words, ali = arg
+    out, why = [], {}
+    segs = plan_segments(words, ali, len(y24) / SR)
+    from da.p2common import silence_mask
+    sil = silence_mask(y24, SR)
+    for k, (a, b) in enumerate(segs):
+        sc = float(np.mean([ali[i]["score"] for i in range(a, b + 1)]))
+        maxw = max(ali[i]["end"] - ali[i]["start"] for i in range(a, b + 1))
+        text = " ".join(words[i]["word"] for i in range(a, b + 1))
+        if sc < 0.30:
+            why["low_score"] = why.get("low_score", 0) + 1; continue
+        if maxw > 2.5:
+            why["long_word"] = why.get("long_word", 0) + 1; continue
+        lo = ali[a - 1]["end"] + 0.02 if a > 0 else 0.0
+        hi = ali[b + 1]["start"] - 0.02 if b + 1 < len(ali) else None
+        y, pauses = normalise_pauses(y24, SR, words, ali, a, b, sil=sil, lo_s=lo, hi_s=hi)
+        dur = len(y) / SR
+        cps = len(text) / dur
+        if not (0.8 <= dur <= 20.0):
+            why["dur"] = why.get("dur", 0) + 1; continue
+        if not (4.0 <= cps <= 25.0):
+            why["cps"] = why.get("cps", 0) + 1; continue
+        rms = np.sqrt((y ** 2).mean() + 1e-12)
+        y = y * (0.07 / rms)
+        pk = np.abs(y).max()
+        if pk > 0.99:
+            y = y * (0.99 / pk)
+        m = dict(meta)
+        if len(segs) > 1:
+            m["id"] = f"{meta['id']}_s{k:03d}"; m["src_id"] = meta["id"]; m["longform"] = True
+        m.update(text=text, dur=round(dur, 3), ascore=round(sc, 3), cps=round(cps, 2),
+                 pauses=[round(p, 2) for p in pauses],
+                 puncts=[words[i]["punct"] for i in range(a, b)], flow_ok=meta["ds"] in FLOW_OK)
+        out.append((m, np.clip(y * 32767, -32768, 32767).astype(np.int16)))
+    return out, why
+
+
+def run(ds):
+    od = f"{OUT}/{ds}"
+    os.makedirs(od, exist_ok=True)
+    al = Aligner()
+    stats = dict(src_items=0, src_hours=0.0, kept=0, hours=0.0, rej={}, longform_src=0, longform_hours_in=0.0,
+                 longform_segments=0)
+    t0 = time.time(); off = 0
+    pending = []
+    with open(f"{od}/pcm24k.i16", "wb") as pf, open(f"{od}/meta.jsonl", "w", encoding="utf-8") as jf, \
+            Pool(20) as lp, Pool(6) as pp:
+        def drain(block=False):
+            nonlocal off
+            keep = []
+            for r in pending:
+                if block or r.ready():
+                    segs, why = r.get()
+                    for k, v in why.items():
+                        stats["rej"][k] = stats["rej"].get(k, 0) + v
+                    for m, pcm in segs:
+                        m["off24"] = off; m["n24"] = int(len(pcm))
+                        pf.write(pcm.tobytes()); off += len(pcm)
+                        jf.write(json.dumps(m, ensure_ascii=False) + "\n")
+                        stats["kept"] += 1; stats["hours"] += m["dur"] / 3600
+                        if m.get("longform"):
+                            stats["longform_segments"] += 1
+                else:
+                    keep.append(r)
+            pending[:] = keep
+        import itertools
+        src = enum(ds)
+        if os.environ.get("LIMIT"):
+            src = itertools.islice(src, 0, None, int(os.environ.get("STRIDE", "1")))
+            src = itertools.islice(src, int(os.environ["LIMIT"]))
+        for meta, y16, y24, err in lp.imap(load, src, chunksize=2):
+            stats["src_items"] += 1
+            if err:
+                stats["rej"][err] = stats["rej"].get(err, 0) + 1; continue
+            d = len(y24) / SR
+            stats["src_hours"] += d / 3600
+            if d > 20:
+                stats["longform_src"] += 1; stats["longform_hours_in"] += d / 3600
+            if d > 2400:
+                stats["rej"]["too_long"] = stats["rej"].get("too_long", 0) + 1; continue
+            words = words_for_align(meta["text"])
+            ali = al.align(y16, words) if words else None
+            if ali is None:
+                stats["rej"]["align_fail"] = stats["rej"].get("align_fail", 0) + 1; continue
+            pending.append(pp.apply_async(post, ((meta, y24, words, ali),)))
+            if len(pending) > 64:
+                drain(block=False)
+                while len(pending) > 128:
+                    time.sleep(0.05); drain(block=False)
+            if stats["src_items"] % 2000 == 0:
+                print(ds, stats["src_items"], "kept", stats["kept"], "h", round(stats["hours"], 2),
+                      "rej", stats["rej"], round(time.time() - t0), "s", flush=True)
+        drain(block=True)
+    stats = {k: (round(v, 3) if isinstance(v, float) else v) for k, v in stats.items()}
+    json.dump(stats, open(f"{od}/stats.json", "w"), indent=1)
+    print(ds, "PREP2_DONE", json.dumps(stats), round(time.time() - t0), "s", flush=True)
+
+
+if __name__ == "__main__":
+    for ds in sys.argv[1:]:
+        run(ds)

code/da/s3_2.py

@@ -0,0 +1,130 @@
+"""Phase 2: S3 speech tokens (speech_tokenizer_v2.onnx, CUDA EP) + CAM++ speaker embeddings for ~/data2/<ds>,
+plus code-switched training pairs (ds "cs_pairs").
+
+  python -m da.s3_2 tok waxal fleurs ...        -> s3tok.i32, s3tok.jsonl, campplus.npy
+  python -m da.s3_2 cs 4000                     -> ~/data2/cs_pairs/{pcm24k.i16, meta.jsonl} (+ run tok on it)
+"""
+import os, sys, json, random, hashlib
+import numpy as np, librosa, torch
+from multiprocessing import Pool
+
+D2 = os.path.expanduser(os.environ.get("P2OUT", "~/data2"))
+MD = os.path.expanduser("~/models/CosyVoice2-0.5B")
+_S = {}
+_FB = torch.from_numpy(librosa.filters.mel(sr=16000, n_fft=400, n_mels=128)).float()
+_WIN = torch.hann_window(400)
+
+
+def log_mel_128(a):
+    st = torch.stft(a, 400, 160, window=_WIN, return_complex=True)
+    mag = st[..., :-1].abs() ** 2
+    lm = torch.clamp(_FB @ mag, min=1e-10).log10()
+    lm = torch.maximum(lm, lm.max() - 8.0)
+    return (lm + 4.0) / 4.0
+
+
+def sess(name, gpu):
+    if name not in _S:
+        import onnxruntime as ort
+        torch.set_num_threads(1)
+        o = ort.SessionOptions(); o.intra_op_num_threads = 2; o.inter_op_num_threads = 1
+        prov = ["CUDAExecutionProvider", "CPUExecutionProvider"] if gpu else ["CPUExecutionProvider"]
+        _S[name] = ort.InferenceSession(f"{MD}/{name}", sess_options=o, providers=prov)
+    return _S[name]
+
+
+def work(args):
+    path, off, n = args
+    import torchaudio.compliance.kaldi as kaldi
+    pcm = np.memmap(path, dtype=np.int16, mode="r")
+    y24 = pcm[off:off + n].astype(np.float32) / 32768.0
+    y16 = librosa.resample(y24, orig_sr=24000, target_sr=16000, res_type="soxr_hq")
+    a = torch.from_numpy(y16)
+    f = log_mel_128(a)[None]
+    s = sess("speech_tokenizer_v2.onnx", True)
+    ins = s.get_inputs()
+    tok = s.run(None, {ins[0].name: f.numpy(), ins[1].name: np.array([f.shape[2]], dtype=np.int32)})[0]
+    fb = kaldi.fbank(a[None], num_mel_bins=80, dither=0, sample_frequency=16000)
+    fb = fb - fb.mean(0, keepdim=True)
+    c = sess("campplus.onnx", False)
+    emb = c.run(None, {c.get_inputs()[0].name: fb[None].numpy()})[0].reshape(-1).astype(np.float32)
+    return np.asarray(tok).reshape(-1).astype(np.int32), emb
+
+
+def tok(ds):
+    od = f"{D2}/{ds}"
+    meta = [json.loads(l) for l in open(f"{od}/meta.jsonl", encoding="utf-8")]
+    jobs = [(f"{od}/pcm24k.i16", m["off24"], m["n24"]) for m in meta]
+    toff, embs = 0, []
+    with open(f"{od}/s3tok.i32", "wb") as tf, open(f"{od}/s3tok.jsonl", "w") as jf, Pool(12) as pool:
+        for i, (m, (t, e)) in enumerate(zip(meta, pool.imap(work, jobs, chunksize=4))):
+            tf.write(t.tobytes()); embs.append(e)
+            jf.write(json.dumps({"id": m["id"], "toff": toff, "ntok": int(len(t))}) + "\n")
+            toff += len(t)
+            if i % 5000 == 0:
+                print(ds, i, len(meta), "tok/s", round(len(t) / max(m["dur"], 1e-3), 1), flush=True)
+    np.save(f"{od}/campplus.npy", np.stack(embs))
+    print(ds, "S3_DONE", toff, flush=True)
+
+
+def build_cs(n_pairs=4000, seed=0):
+    """Join two train utterances in different languages (prefer same gender) into one code-switched item.
+    Junction silence follows the first clip's final punctuation (sentence 0.45 s, comma 0.25 s, else 0.20 s)."""
+    rng = random.Random(seed)
+    pools = {}
+    for ds in ["waxal", "slr70", "slr86", "fleurs"]:
+        p = f"{D2}/{ds}/meta.jsonl"
+        if not os.path.exists(p):
+            continue
+        for l in open(p, encoding="utf-8"):
+            m = json.loads(l)
+            if m["split"] != "train" or not (1.0 <= m["dur"] <= 6.5):
+                continue
+            if ds == "fleurs" and m["lang"] != "eng":
+                continue
+            pools.setdefault(m["lang"], []).append(m)
+    langs = sorted(pools)
+    print("cs pools", {l: len(v) for l, v in pools.items()})
+    od = f"{D2}/cs_pairs"; os.makedirs(od, exist_ok=True)
+    mm = {}
+    off = 0
+    with open(f"{od}/pcm24k.i16", "wb") as pf, open(f"{od}/meta.jsonl", "w", encoding="utf-8") as jf:
+        for k in range(n_pairs):
+            l1, l2 = rng.sample(langs, 2)
+            if rng.random() < 0.5 and "eng" in langs and "eng" not in (l1, l2):
+                l2 = "eng"          # English is the most common partner in real Nigerian code-switching
+            a = rng.choice(pools[l1])
+            g = (a.get("gender") or "").lower()[:1]
+            cand = [b for b in rng.sample(pools[l2], min(50, len(pools[l2]))) if (b.get("gender") or "").lower()[:1] == g]
+            b = cand[0] if cand else rng.choice(pools[l2])
+            ys = []
+            for m in (a, b):
+                if m["ds"] not in mm:
+                    mm[m["ds"]] = np.memmap(f"{D2}/{m['ds']}/pcm24k.i16", dtype=np.int16, mode="r")
+                ys.append(np.asarray(mm[m["ds"]][m["off24"]:m["off24"] + m["n24"]]))
+            last = a["text"].rstrip()[-1:]
+            junc = 0.45 if last in ".?!" else (0.25 if last in ",;:" else 0.20)
+            have = 0.18 + 0.12
+            y1 = ys[0]
+            if junc < have:
+                y1 = y1[: len(y1) - int((have - junc) * 24000)]
+                gap = np.zeros(0, np.int16)
+            else:
+                gap = np.zeros(int((junc - have) * 24000), np.int16)
+            y = np.concatenate([y1, gap, ys[1]])
+            text = f"<{l1}>{a['text']}</{l1}> <{l2}>{b['text']}</{l2}>"
+            m = dict(id=f"cs_{k:05d}", ds="cs_pairs", lang="cs", langs=[l1, l2], spk=a["spk"], gender=a.get("gender", ""),
+                     split="train", text=text, dur=round(len(y) / 24000, 3), off24=off, n24=int(len(y)),
+                     src=[a["id"], b["id"]], flow_ok=False)
+            pf.write(y.tobytes()); off += len(y)
+            jf.write(json.dumps(m, ensure_ascii=False) + "\n")
+    print("cs pairs", n_pairs, "hours", round(off / 24000 / 3600, 2), flush=True)
+
+
+if __name__ == "__main__":
+    if sys.argv[1] == "tok":
+        for ds in sys.argv[2:]:
+            tok(ds)
+    elif sys.argv[1] == "cs":
+        build_cs(int(sys.argv[2]))
+        tok("cs_pairs")

code/da/text.py

@@ -0,0 +1,60 @@
+"""Text front-end: Unicode NFC, tone diacritics preserved, UTF-8 byte tokens with per-token language ids.
+
+Code-switched input uses inline tags:  "<yor>Mo fẹ́ lọ sí ọjà</yor> <eng>to buy some bread</eng>"
+Untagged text takes the default language. Each byte inherits the language of the span it is in.
+"""
+import re
+import unicodedata
+
+LANGS = ["eng", "ibo", "hau", "yor", "pcm"]
+LANG2ID = {l: i for i, l in enumerate(LANGS)}
+PAD, BOS, EOS = 0, 1, 2          # byte b -> token b + 3
+VOCAB = 256 + 3
+_TAG = re.compile(r"<(eng|ibo|hau|yor|pcm)>(.*?)</\1>", re.S)
+
+
+def normalize(t: str) -> str:
+    t = unicodedata.normalize("NFC", t)
+    t = t.replace("\u2019", "'").replace("\u2018", "'").replace("\u201c", '"').replace("\u201d", '"')
+    return re.sub(r"\s+", " ", t).strip()
+
+
+def segments(text: str, default_lang: str = "eng"):
+    """-> list of (lang, normalized_text) spans, order preserved."""
+    out, pos = [], 0
+    for m in _TAG.finditer(text):
+        pre = text[pos:m.start()]
+        if pre.strip():
+            out.append((default_lang, pre))
+        out.append((m.group(1), m.group(2)))
+        pos = m.end()
+    if text[pos:].strip():
+        out.append((default_lang, text[pos:]))
+    # keep a single space between spans
+    res = []
+    for i, (l, s) in enumerate(out):
+        s = normalize(s)
+        if not s:
+            continue
+        if res:
+            s = " " + s
+        res.append((l, s))
+    return res
+
+
+def encode(text: str, default_lang: str = "eng"):
+    """-> (token ids, language ids) incl. BOS/EOS. BOS takes the first span's language, EOS the last."""
+    segs = segments(text, default_lang)
+    toks, langs = [BOS], [LANG2ID[segs[0][0]] if segs else LANG2ID[default_lang]]
+    for l, s in segs:
+        b = s.encode("utf-8")
+        toks += [x + 3 for x in b]
+        langs += [LANG2ID[l]] * len(b)
+    toks.append(EOS)
+    langs.append(langs[-1])
+    return toks, langs
+
+
+def plain(text: str) -> str:
+    """Tagged text -> plain text (for ASR references)."""
+    return " ".join(s.strip() for _, s in segments(text))

code/train_b.py

@@ -0,0 +1,353 @@
+"""Arm B: CosyVoice2-0.5B (Apache-2.0) fine-tune with the Zhang et al. 2019 recipe added to its text->speech-token LM.
+
+Additions (new, ours):
+  * per-token language embedding added to the Qwen text-token embeddings (zero-init)
+  * learned speaker embedding inserted as one token AFTER the text, before the task token (zero-init), so text
+    positions cannot see it; speech positions attend to it (anchor-voice conditioning)
+  * domain-adversarial speaker classifier with gradient reversal on the LM hidden states at text positions
+Flow-matching decoder + HiFT vocoder stay frozen; at inference they are prompted with an anchor-voice clip.
+
+  python train_b.py train --out ~/runs/b1 --minutes 30
+  python train_b.py synth --ckpt ~/runs/b1/last.pt --out ~/eval/sys/armB_ft
+  python train_b.py zeroshot --out ~/eval/sys/cosyvoice2_zeroshot      (pretrained model, cross-lingual, no fine-tune)
+"""
+import argparse, inspect, json, math, os, random, sys, time
+import numpy as np, torch, torch.nn as nn, torch.nn.functional as F, soundfile as sf, librosa
+
+HOME = os.path.expanduser("~")
+sys.path[:0] = [f"{HOME}/CosyVoice", f"{HOME}/CosyVoice/third_party/Matcha-TTS"]
+MODEL_DIR = f"{HOME}/models/CosyVoice2-0.5B"
+LANGS = ["eng", "ibo", "hau", "yor", "pcm"]
+L2I = {l: i for i, l in enumerate(LANGS)}
+sys.path.insert(0, f"{HOME}/da")
+from da.text import segments  # noqa: E402
+
+from da.model import grad_reverse  # noqa: E402
+
+
+def _sf_load_wav(wav, target_sr, min_sr=16000):
+    """soundfile replacement for cosyvoice load_wav (torchaudio>=2.9 load needs torchcodec)."""
+    y, sr = sf.read(wav, dtype="float32", always_2d=True)
+    y = y.mean(1)
+    if sr != target_sr:
+        y = librosa.resample(y, orig_sr=sr, target_sr=target_sr)
+    return torch.from_numpy(y)[None]
+
+
+def load_cosy():
+    import cosyvoice.utils.file_utils as fu
+    fu.load_wav = _sf_load_wav
+    import cosyvoice.cli.frontend as cf
+    cf.load_wav = _sf_load_wav
+    from cosyvoice.cli.cosyvoice import CosyVoice2
+    kw = dict(load_jit=False, load_trt=False, fp16=False)
+    sig = inspect.signature(CosyVoice2.__init__).parameters
+    kw = {k: v for k, v in kw.items() if k in sig}
+    return CosyVoice2(MODEL_DIR, **kw)
+
+
+def text_tokens(tok, text, default="eng"):
+    """tagged text -> (qwen ids, per-token lang ids)."""
+    ids, langs = [], []
+    for l, s in segments(text, default):
+        t = tok(s, add_special_tokens=False)["input_ids"]
+        ids += t; langs += [L2I[l]] * len(t)
+    return ids, langs
+
+
+class DALM(nn.Module):
+    def __init__(self, qlm, n_spk, d=896):
+        super().__init__()
+        self.q = qlm                                     # cosyvoice Qwen2LM
+        self.lang_emb = nn.Embedding(len(LANGS), d); nn.init.zeros_(self.lang_emb.weight)
+        self.spk_emb = nn.Embedding(n_spk, d); nn.init.normal_(self.spk_emb.weight, std=0.02)
+        self.adv = nn.Sequential(nn.Linear(d, 256), nn.ReLU(), nn.Linear(256, n_spk))
+        self.V = qlm.speech_token_size
+
+    def embed_text(self, ids, langs):
+        return self.q.llm.model.model.embed_tokens(ids) + self.lang_emb(langs)
+
+    def build(self, ids, langs, spk, speech=None):
+        """one sequence: [sos, text+lang, spk, task, speech...]"""
+        sos = self.q.llm_embedding.weight[self.q.sos][None]
+        task = self.q.llm_embedding.weight[self.q.task_id][None]
+        parts = [sos, self.embed_text(ids, langs), self.spk_emb(spk)[None], task]
+        if speech is not None:
+            parts.append(self.q.speech_embedding(speech))
+        return torch.cat(parts, 0)
+
+    def hidden(self, x, mask):
+        out = self.q.llm.model.model(inputs_embeds=x, attention_mask=mask, return_dict=True)
+        return out.last_hidden_state
+
+    def forward(self, batch, adv_lambda=1.0):
+        seqs, tgts, tpos = [], [], []
+        for ids, langs, spk, sp in batch:
+            x = self.build(ids, langs, spk, sp)
+            n_text = len(ids)
+            tgt = torch.full((x.shape[0],), -100, device=x.device, dtype=torch.long)
+            # position of task token = 1 + n_text + 1; it predicts speech[0]; last speech pos predicts EOS
+            start = 1 + n_text + 1
+            tgt[start: start + len(sp)] = sp
+            tgt[start + len(sp)] = self.V                         # EOS
+            seqs.append(x); tgts.append(tgt); tpos.append((1, 1 + n_text))
+        T = max(s.shape[0] for s in seqs) + 1
+        D = seqs[0].shape[1]
+        X = seqs[0].new_zeros(len(seqs), T, D)
+        Y = torch.full((len(seqs), T), -100, device=X.device, dtype=torch.long)
+        M = torch.zeros(len(seqs), T, device=X.device, dtype=torch.long)
+        for i, (s, t) in enumerate(zip(seqs, tgts)):
+            X[i, :len(s)] = s; Y[i, :len(t)] = t[:len(s)] if len(t) > len(s) else t; M[i, :len(s)] = 1
+        # targets were laid out on positions; shift is implicit (position p predicts tgt[p])
+        H = self.hidden(X, M)
+        logits = self.q.llm_decoder(H)
+        ce = F.cross_entropy(logits.float().reshape(-1, logits.shape[-1]), Y.reshape(-1), ignore_index=-100)
+        valid = Y != -100
+        acc = (logits.argmax(-1)[valid] == Y[valid]).float().mean()
+        # adversarial speaker classifier on text positions
+        hs, ys = [], []
+        for i, (a, b) in enumerate(tpos):
+            hs.append(H[i, a:b]); ys.append(torch.full((b - a,), int(batch[i][2]), device=X.device))
+        hs = torch.cat(hs); ys = torch.cat(ys)
+        al = self.adv(grad_reverse(hs.float(), adv_lambda))
+        adv = F.cross_entropy(al, ys)
+        adv_acc = (al.argmax(-1) == ys).float().mean()
+        return ce, acc, adv, adv_acc
+
+    @torch.no_grad()
+    def generate(self, ids, langs, spk, top_p=0.8, top_k=25, win=10, tau_r=0.1, seed=0):
+        g = torch.Generator(device=ids.device).manual_seed(seed)
+        x = self.build(ids, langs, spk)[None]
+        n = len(ids)
+        min_len, max_len = int(n * 2), int(n * 20)
+        past, out = None, []
+        inp = x
+        for i in range(max_len):
+            o = self.q.llm.model.model(inputs_embeds=inp, past_key_values=past, use_cache=True, return_dict=True)
+            past = o.past_key_values
+            logp = self.q.llm_decoder(o.last_hidden_state[:, -1]).float().log_softmax(-1)[0]
+            if i < min_len:
+                logp[self.V] = -float("inf")
+            logp[self.V + 1:] = -float("inf")
+            p = logp.exp()
+            sp, si = p.sort(descending=True)
+            keep = (sp.cumsum(0) - sp) < top_p
+            keep[top_k:] = False
+            cand_p, cand_i = sp[keep], si[keep]
+            tokid = int(cand_i[torch.multinomial(cand_p / cand_p.sum(), 1, generator=g)])
+            if out and sum(1 for t in out[-win:] if t == tokid) >= win * tau_r and tokid != self.V:  # RAS
+                tokid = int(torch.multinomial(p, 1, generator=g))
+            if tokid >= self.V:
+                break
+            out.append(tokid)
+            inp = self.q.speech_embedding.weight[tokid][None, None]
+        return out
+
+
+# ------------------------------------------------------------------ data
+def load_items(root, datasets, split, max_tok=500):
+    items, mm = [], {}
+    for ds in datasets:
+        if not os.path.exists(f"{root}/{ds}/s3tok.jsonl"):
+            continue
+        mm[ds] = np.memmap(f"{root}/{ds}/s3tok.i32", dtype=np.int32, mode="r")
+        tk = {}
+        for l in open(f"{root}/{ds}/s3tok.jsonl"):
+            r = json.loads(l); tk[r["id"]] = (r["toff"], r["ntok"])
+        for l in open(f"{root}/{ds}/meta.jsonl", encoding="utf-8"):
+            m = json.loads(l)
+            if m["split"] != split or m["id"] not in tk:
+                continue
+            m["toff"], m["ntok"] = tk[m["id"]]
+            if 5 < m["ntok"] <= max_tok:
+                items.append(m)
+    return items, mm
+
+
+def batches(items, budget, weights, rng):
+    idx = []
+    for i, m in enumerate(items):
+        w = weights.get(m["ds"], 1.0)
+        idx += [i] * int(w) + ([i] if rng.random() < w - int(w) else [])
+    idx.sort(key=lambda i: items[i]["ntok"] + rng.random() * 10)
+    out, cur, mx = [], [], 0
+    for i in idx:
+        L = items[i]["ntok"] + len(items[i]["text"]) // 2 + 4
+        if cur and max(mx, L) * (len(cur) + 1) > budget:
+            out.append(cur); cur, mx = [], 0
+        cur.append(i); mx = max(mx, L)
+    if cur:
+        out.append(cur)
+    rng.shuffle(out)
+    return out
+
+
+def train(a):
+    from transformers import AutoTokenizer
+    tok = AutoTokenizer.from_pretrained(f"{MODEL_DIR}/CosyVoice-BlankEN")
+    cosy = load_cosy()
+    items, mm = load_items(a.data, a.datasets.split(","), "train")
+    val, _ = load_items(a.data, a.datasets.split(","), "val")
+    spks = sorted({m["spk"] for m in items}); spk2id = {s: i for i, s in enumerate(spks)}
+    val = [m for m in val if m["spk"] in spk2id][:200]
+    os.makedirs(a.out, exist_ok=True)
+    json.dump(spk2id, open(f"{a.out}/spk2id.json", "w"))
+    print("armB items", len(items), "hours", round(sum(m["dur"] for m in items) / 3600, 1), "spk", len(spks), flush=True)
+    model = DALM(cosy.model.llm, len(spks)).cuda().train()
+    for p in cosy.model.flow.parameters():
+        p.requires_grad = False
+    for p in cosy.model.hift.parameters():
+        p.requires_grad = False
+    new = list(model.lang_emb.parameters()) + list(model.spk_emb.parameters()) + list(model.adv.parameters())
+    newids = {id(p) for p in new}
+    old = [p for p in model.parameters() if id(p) not in newids and p.requires_grad]
+    opt = torch.optim.AdamW([{"params": old, "lr": a.lr}, {"params": new, "lr": a.lr_new}], weight_decay=0.01)
+    base = [a.lr, a.lr_new]
+    cache = {}
+
+    def enc(m):
+        if m["id"] not in cache:
+            cache[m["id"]] = text_tokens(tok, f"<{m['lang']}>{m['text']}</{m['lang']}>")
+        ids, langs = cache[m["id"]]
+        sp = np.asarray(mm[m["ds"]][m["toff"]: m["toff"] + m["ntok"]], dtype=np.int64)
+        return (torch.tensor(ids).cuda(), torch.tensor(langs).cuda(), torch.tensor(spk2id[m["spk"]]).cuda(),
+                torch.from_numpy(sp).cuda())
+
+    rng = random.Random(0); step = 0; nskip = 0; t0 = time.time(); logf = open(f"{a.out}/log.jsonl", "a")
+
+    def validate():
+        model.eval(); L = []
+        with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
+            for i in range(0, len(val), 16):
+                ce, acc, _, _ = model([enc(m) for m in val[i:i + 16]]); L.append(float(ce))
+        model.train(); return float(np.mean(L))
+
+    v0 = validate(); print(json.dumps({"step": 0, "val_ce": v0}), flush=True); logf.write(json.dumps({"step": 0, "val_ce": v0}) + "\n")
+    done = False
+    while not done:
+        for bi in batches(items, a.tokens, {"waxal": a.waxal_w}, rng):
+            lrm = min(1.0, (step + 1) / 200)
+            for g, b in zip(opt.param_groups, base):
+                g["lr"] = b * lrm
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                ce, acc, adv, adv_acc = model([enc(items[i]) for i in bi])
+            loss = ce + a.adv_w * adv
+            opt.zero_grad(set_to_none=True)
+            if not torch.isfinite(loss):
+                nskip += 1
+                print(json.dumps({"skip_nonfinite_fwd": step, "ce": float(ce), "adv": float(adv), "B": len(bi),
+                                  "maxtok": max(items[i]["ntok"] for i in bi)}), flush=True)
+                continue
+            loss.backward()
+            gn = torch.nn.utils.clip_grad_norm_([p for p in model.parameters() if p.requires_grad], 1.0)
+            if not torch.isfinite(gn):
+                nskip += 1; opt.zero_grad(set_to_none=True)
+                print(json.dumps({"skip_nonfinite_grad": step}), flush=True); continue
+            opt.step(); step += 1
+            if step % 25 == 0:
+                r = dict(step=step, ce=round(float(ce), 4), acc=round(float(acc), 4), adv=round(float(adv), 3),
+                         adv_acc=round(float(adv_acc), 3), gn=round(float(gn), 2), B=len(bi), min=round((time.time() - t0) / 60, 2))
+                print(json.dumps(r), flush=True); logf.write(json.dumps(r) + "\n"); logf.flush()
+            if step % 500 == 0:
+                v = validate(); print(json.dumps({"step": step, "val_ce": v}), flush=True)
+                logf.write(json.dumps({"step": step, "val_ce": v}) + "\n"); logf.flush()
+            if (time.time() - t0) / 60 > a.minutes:
+                done = True; break
+    v = validate(); logf.write(json.dumps({"step": step, "val_ce": v, "final": True}) + "\n")
+    sd = {k: v for k, v in model.state_dict().items()}
+    torch.save(dict(model=sd, step=step, n_spk=len(spks)), f"{a.out}/last.pt")
+    print("TRAIN_B_DONE", step, v, flush=True)
+
+
+# ------------------------------------------------------------------ inference
+def prompt_feats(cosy, wav_path):
+    """flow prompt (speech tokens, 24k mel feat, campplus embedding) from an anchor clip."""
+    fe = cosy.frontend
+    tokn, tokl = fe._extract_speech_token(wav_path)
+    feat, featl = fe._extract_speech_feat(wav_path)
+    # CosyVoice2 aligns: feat frames = 2 * tokens
+    n = min(feat.shape[1] // 2, tokn.shape[1])
+    feat, tokn = feat[:, :2 * n], tokn[:, :n]
+    emb = fe._extract_spk_embedding(wav_path)
+    return tokn.cuda(), feat.cuda(), emb.cuda()
+
+
+@torch.no_grad()
+def token2wav(cosy, tokens, ptok, pfeat, emb):
+    flow = cosy.model.flow
+    t = torch.tensor([tokens], dtype=torch.int32).cuda()
+    kw = dict(token=t, token_len=torch.tensor([t.shape[1]], dtype=torch.int32).cuda(),
+              prompt_token=ptok, prompt_token_len=torch.tensor([ptok.shape[1]], dtype=torch.int32).cuda(),
+              prompt_feat=pfeat, prompt_feat_len=torch.tensor([pfeat.shape[1]], dtype=torch.int32).cuda(),
+              embedding=emb)
+    sig = inspect.signature(flow.inference).parameters
+    if "streaming" in sig:
+        kw["streaming"] = False
+    if "finalize" in sig:
+        kw["finalize"] = True
+    if "flow_cache" in sig:
+        kw["flow_cache"] = torch.zeros(1, 80, 0, 2).cuda()
+    mel = flow.inference(**kw)
+    mel = mel[0] if isinstance(mel, tuple) else mel
+    hs = inspect.signature(cosy.model.hift.inference).parameters
+    w = cosy.model.hift.inference(speech_feat=mel, **({"cache_source": torch.zeros(1, 1, 0).cuda()} if "cache_source" in hs else {}))
+    w = w[0] if isinstance(w, tuple) else w
+    return w.squeeze().float().cpu().numpy()
+
+
+def eval_rows():
+    return [json.loads(l) for l in open(f"{HOME}/eval/eval_set.jsonl", encoding="utf-8")]
+
+
+def synth(a):
+    from transformers import AutoTokenizer
+    tok = AutoTokenizer.from_pretrained(f"{MODEL_DIR}/CosyVoice-BlankEN")
+    cosy = load_cosy()
+    ck = torch.load(a.ckpt, map_location="cuda")
+    spk2id = json.load(open(os.path.join(os.path.dirname(a.ckpt), "spk2id.json")))
+    model = DALM(cosy.model.llm, ck["n_spk"]).cuda().eval()
+    model.load_state_dict(ck["model"])
+    anc = json.load(open(f"{HOME}/eval/anchor.json"))
+    ptok, pfeat, emb = prompt_feats(cosy, anc["ref_wavs"][0])
+    os.makedirs(a.out, exist_ok=True)
+    for r in eval_rows():
+        text = r["text"] if r["lang"] == "cs" else f"<{r['lang']}>{r['text']}</{r['lang']}>"
+        ids, langs = text_tokens(tok, text)
+        with torch.autocast("cuda", dtype=torch.bfloat16):
+            sp = model.generate(torch.tensor(ids).cuda(), torch.tensor(langs).cuda(), torch.tensor(spk2id[anc["spk"]]).cuda())
+        sf.write(f"{a.out}/{r['id']}.wav", token2wav(cosy, sp, ptok, pfeat, emb), cosy.sample_rate)
+    print("synth_b done", a.out, flush=True)
+
+
+def zeroshot(a):
+    """Unmodified pretrained CosyVoice2, cross-lingual mode with the anchor prompt (reference point)."""
+    cosy = load_cosy()
+    anc = json.load(open(f"{HOME}/eval/anchor.json"))
+    from cosyvoice.utils.file_utils import load_wav
+    pw = anc["ref_wavs"][0]
+    sig = inspect.signature(cosy.inference_cross_lingual).parameters
+    os.makedirs(a.out, exist_ok=True)
+    for r in eval_rows():
+        text = " ".join(s.strip() for _, s in segments(r["text"])) if r["lang"] == "cs" else r["text"]
+        prompt = pw
+        outs = [o["tts_speech"] for o in cosy.inference_cross_lingual(text, prompt, stream=False)]
+        w = torch.cat(outs, 1).squeeze().cpu().numpy()
+        sf.write(f"{a.out}/{r['id']}.wav", w, cosy.sample_rate)
+    print("zeroshot done", flush=True)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("cmd")
+    ap.add_argument("--data", default=f"{HOME}/data")
+    ap.add_argument("--datasets", default="waxal,fleurs,slr86,slr70,bible_yor")
+    ap.add_argument("--out", default=f"{HOME}/runs/b1")
+    ap.add_argument("--ckpt", default="")
+    ap.add_argument("--minutes", type=float, default=30)
+    ap.add_argument("--lr", type=float, default=1e-5)
+    ap.add_argument("--lr_new", type=float, default=1e-3)
+    ap.add_argument("--tokens", type=int, default=12000)
+    ap.add_argument("--waxal_w", type=float, default=2.0)
+    ap.add_argument("--adv_w", type=float, default=0.02)
+    a = ap.parse_args()
+    {"train": train, "synth": synth, "zeroshot": zeroshot}[a.cmd](a)

code/train_b2.py

@@ -0,0 +1,590 @@
+"""Arm B, Phase 2. CosyVoice2-0.5B (Apache-2.0) + language embeddings + speaker token + GRL speaker classifier.
+
+New in Phase 2
+  * data from ~/data2 (24 kHz, forced-aligned, pause-normalised; long WAXAL clips segmented; BibleTTS Hausa; CS pairs)
+  * in-context prompts during LM training (p=0.5): [sos, prompt_text+text, spk, task, prompt_speech, speech] with
+    loss on the target speech only -> at inference the anchor clip steers prosody and pause style
+  * keep-best on validation CE (every --val_every steps), cosine LR, early stop
+  * flow-matching decoder fine-tune on 24 kHz studio sources (tones / pitch / timing detail)
+  * synthesis: anchor prompt at LM and flow, sentence-level generation for multi-sentence text with anchor-matched
+    inter-sentence pauses, N candidates per text (seed + prompt clip vary)
+
+  python train_b2.py lm    --out ~/runs/p2_lm  --minutes 90
+  python train_b2.py flow  --out ~/runs/p2_flow --minutes 60
+  python train_b2.py synth --ckpt ~/runs/p2_lm/best.pt [--flow ~/runs/p2_flow/best.pt] --out DIR --n 4
+"""
+import argparse, inspect, json, math, os, random, re, sys, time
+import numpy as np, torch, torch.nn as nn, torch.nn.functional as F, soundfile as sf, librosa
+
+HOME = os.path.expanduser("~")
+sys.path[:0] = [f"{HOME}/CosyVoice", f"{HOME}/CosyVoice/third_party/Matcha-TTS", f"{HOME}/da"]
+from train_b import load_cosy, text_tokens, DALM, LANGS, L2I, MODEL_DIR, token2wav  # noqa: E402
+from da.model import grad_reverse  # noqa: E402
+from da.text import segments  # noqa: E402
+
+D2 = os.path.expanduser(os.environ.get("P2OUT", f"{HOME}/data2"))
+ANCHOR = "waxal_yor_4"
+
+
+# ------------------------------------------------------------------ model
+class DALM2(DALM):
+    def build2(self, ids, langs, spk, sp=None, pids=None, plangs=None, psp=None):
+        """[sos, (prompt_text) text, spk, task, (prompt_speech) speech] -> embeddings, n_text_total, n_prompt_speech"""
+        sos = self.q.llm_embedding.weight[self.q.sos][None]
+        task = self.q.llm_embedding.weight[self.q.task_id][None]
+        if pids is not None:
+            ids = torch.cat([pids, ids]); langs = torch.cat([plangs, langs])
+        parts = [sos, self.embed_text(ids, langs), self.spk_emb(spk)[None], task]
+        npsp = 0
+        if psp is not None:
+            parts.append(self.q.speech_embedding(psp)); npsp = len(psp)
+        if sp is not None:
+            parts.append(self.q.speech_embedding(sp))
+        return torch.cat(parts, 0), len(ids), npsp
+
+    def forward2(self, batch, adv_lambda=1.0):
+        seqs, tgts, tpos, spks = [], [], [], []
+        for b in batch:
+            ids, langs, spk, sp = b[:4]
+            pr = b[4] if len(b) > 4 else None
+            x, nt, npsp = self.build2(ids, langs, spk, sp, *(pr if pr else (None, None, None)))
+            tgt = torch.full((x.shape[0] + 1,), -100, device=x.device, dtype=torch.long)
+            start = 1 + nt + 2 + npsp - 1          # position whose output predicts speech[0]
+            tgt[start: start + len(sp)] = sp
+            tgt[start + len(sp)] = self.V
+            seqs.append(x); tgts.append(tgt[:x.shape[0]]); tpos.append((1, 1 + nt)); spks.append(int(spk))
+        T = max(s.shape[0] for s in seqs)
+        D = seqs[0].shape[1]
+        X = seqs[0].new_zeros(len(seqs), T, D)
+        Y = torch.full((len(seqs), T), -100, device=X.device, dtype=torch.long)
+        M = torch.zeros(len(seqs), T, device=X.device, dtype=torch.long)
+        for i, (s, t) in enumerate(zip(seqs, tgts)):
+            X[i, :len(s)] = s; Y[i, :len(t)] = t; M[i, :len(s)] = 1
+        H = self.hidden(X, M)
+        logits = self.q.llm_decoder(H)
+        ce = F.cross_entropy(logits.float().reshape(-1, logits.shape[-1]), Y.reshape(-1), ignore_index=-100)
+        valid = Y != -100
+        acc = (logits.argmax(-1)[valid] == Y[valid]).float().mean()
+        hs, ys = [], []
+        for i, (a, b) in enumerate(tpos):
+            hs.append(H[i, a:b]); ys.append(torch.full((b - a,), spks[i], device=X.device))
+        hs = torch.cat(hs); ys = torch.cat(ys)
+        al = self.adv(grad_reverse(hs.float(), adv_lambda))
+        adv = F.cross_entropy(al, ys)
+        return ce, acc, adv, (al.argmax(-1) == ys).float().mean()
+
+    @torch.no_grad()
+    def generate2(self, ids, langs, spk, prompt=None, top_p=0.8, top_k=25, win=10, tau_r=0.1, seed=0,
+                  min_ratio=2.0, max_ratio=20.0):
+        g = torch.Generator(device=ids.device).manual_seed(seed)
+        x, _, _ = self.build2(ids, langs, spk, None, *(prompt if prompt else (None, None, None)))
+        x = x[None]
+        n = len(ids)
+        min_len, max_len = int(n * min_ratio), int(n * max_ratio)
+        past, out, inp = None, [], x
+        for i in range(max_len):
+            o = self.q.llm.model.model(inputs_embeds=inp, past_key_values=past, use_cache=True, return_dict=True)
+            past = o.past_key_values
+            logp = self.q.llm_decoder(o.last_hidden_state[:, -1]).float().log_softmax(-1)[0]
+            if i < min_len:
+                logp[self.V] = -float("inf")
+            logp[self.V + 1:] = -float("inf")
+            p = logp.exp()
+            sp, si = p.sort(descending=True)
+            keep = (sp.cumsum(0) - sp) < top_p
+            keep[top_k:] = False
+            cp, ci = sp[keep], si[keep]
+            tokid = int(ci[torch.multinomial(cp / cp.sum(), 1, generator=g)])
+            if out and sum(1 for t in out[-win:] if t == tokid) >= win * tau_r and tokid != self.V:
+                tokid = int(torch.multinomial(p, 1, generator=g))
+            if tokid >= self.V:
+                break
+            out.append(tokid)
+            inp = self.q.speech_embedding.weight[tokid][None, None]
+        return out
+
+
+# ------------------------------------------------------------------ data
+def load_items(datasets, split, max_tok=500, flow=False):
+    items, mm = [], {}
+    for ds in datasets:
+        od = f"{D2}/{ds}"
+        if not os.path.exists(f"{od}/s3tok.jsonl"):
+            print("missing", ds, flush=True); continue
+        mm[ds] = np.memmap(f"{od}/s3tok.i32", dtype=np.int32, mode="r")
+        tk = [json.loads(l) for l in open(f"{od}/s3tok.jsonl")]
+        for k, l in enumerate(open(f"{od}/meta.jsonl", encoding="utf-8")):
+            m = json.loads(l)
+            t = tk[k]
+            assert t["id"] == m["id"]
+            if m["split"] != split:
+                continue
+            if flow and not m.get("flow_ok"):
+                continue
+            m["toff"], m["ntok"], m["row"] = t["toff"], t["ntok"], k
+            if 5 < m["ntok"] <= max_tok:
+                items.append(m)
+    return items, mm
+
+
+def tagged(m):
+    return m["text"] if m["lang"] == "cs" else f"<{m['lang']}>{m['text']}</{m['lang']}>"
+
+
+DS_W = {"waxal": 2.0, "bible_yor": 0.5, "bible_hau": 0.6, "fleurs": 1.0, "slr86": 1.0, "slr70": 1.0, "cs_pairs": 1.0}
+
+
+def batches(items, budget, rng, extra_w=None, cost=None):
+    idx = []
+    for i, m in enumerate(items):
+        w = DS_W.get(m["ds"], 1.0) * (extra_w(m) if extra_w else 1.0)
+        idx += [i] * int(w) + ([i] if rng.random() < w - int(w) else [])
+    cost = cost or (lambda m: m["ntok"] + len(m["text"]) // 2 + 4)
+    idx.sort(key=lambda i: cost(items[i]) + rng.random() * 10)
+    out, cur, mx = [], [], 0
+    for i in idx:
+        L = cost(items[i])
+        if cur and max(mx, L) * (len(cur) + 1) > budget:
+            out.append(cur); cur, mx = [], 0
+        cur.append(i); mx = max(mx, L)
+    if cur:
+        out.append(cur)
+    rng.shuffle(out)
+    return out
+
+
+# ------------------------------------------------------------------ LM training
+def train_lm(a):
+    from transformers import AutoTokenizer
+    tok = AutoTokenizer.from_pretrained(f"{MODEL_DIR}/CosyVoice-BlankEN")
+    cosy = load_cosy()
+    dss = a.datasets.split(",")
+    items, mm = load_items(dss, "train")
+    val_all, _ = load_items(dss, "val")
+    if a.anchor_only_w > 0:
+        print("anchor items", sum(m["spk"] == ANCHOR for m in items), flush=True)
+    spks = sorted({m["spk"] for m in items})
+    if a.init:
+        ck = torch.load(a.init, map_location="cpu")
+        spk2id = json.load(open(os.path.join(os.path.dirname(a.init), "spk2id.json")))
+        for s in spks:
+            if s not in spk2id:
+                spk2id[s] = len(spk2id)
+    else:
+        spk2id = {s: i for i, s in enumerate(spks)}
+    n_spk = max(len(spk2id), ck["n_spk"]) if a.init else len(spk2id)
+    # balanced val: up to 60 per language (+ anchor), never more than 15 per speaker
+    rng = random.Random(1); rng.shuffle(val_all)
+    val, per_l, per_s = [], {}, {}
+    for m in val_all:
+        if m["spk"] not in spk2id or m["lang"] == "cs":
+            continue
+        L = "anchor" if m["spk"] == ANCHOR else m["lang"]
+        if per_l.get(L, 0) >= 60 or per_s.get(m["spk"], 0) >= 15:
+            continue
+        per_l[L] = per_l.get(L, 0) + 1; per_s[m["spk"]] = per_s.get(m["spk"], 0) + 1; val.append(m)
+    by_spk = {}
+    for m in items:
+        if m["ntok"] <= 160 and m["lang"] != "cs":
+            by_spk.setdefault(m["spk"], []).append(m)
+    os.makedirs(a.out, exist_ok=True)
+    json.dump(spk2id, open(f"{a.out}/spk2id.json", "w"))
+    print("lm items", len(items), "hours", round(sum(m["dur"] for m in items) / 3600, 1), "spk", len(spk2id),
+          "val", len(val), per_l, flush=True)
+    model = DALM2(cosy.model.llm, n_spk).cuda().train()
+    if a.init:
+        sd = ck["model"]
+        if sd["spk_emb.weight"].shape[0] < n_spk:
+            for k in ["spk_emb.weight"]:
+                w = model.state_dict()[k].clone(); w[:sd[k].shape[0]] = sd[k]; sd[k] = w
+            for k in ["adv.2.weight", "adv.2.bias"]:
+                w = model.state_dict()[k].clone(); w[:sd[k].shape[0]] = sd[k]; sd[k] = w
+        print(model.load_state_dict(sd, strict=False), flush=True)
+    for p in cosy.model.flow.parameters():
+        p.requires_grad = False
+    for p in cosy.model.hift.parameters():
+        p.requires_grad = False
+    new = list(model.lang_emb.parameters()) + list(model.spk_emb.parameters()) + list(model.adv.parameters())
+    newids = {id(p) for p in new}
+    old = [p for p in model.parameters() if id(p) not in newids and p.requires_grad]
+    opt = torch.optim.AdamW([{"params": old, "lr": a.lr}, {"params": new, "lr": a.lr_new}], weight_decay=0.01)
+    base = [a.lr, a.lr_new]
+    cache = {}
+
+    def tt(text, key):
+        if key not in cache:
+            cache[key] = text_tokens(tok, text)
+        return cache[key]
+
+    def enc(m, prng=None, force_prompt=None):
+        ids, langs = tt(tagged(m), m["id"])
+        sp = np.asarray(mm[m["ds"]][m["toff"]: m["toff"] + m["ntok"]], dtype=np.int64)
+        r = [torch.tensor(ids).cuda(), torch.tensor(langs).cuda(), torch.tensor(spk2id[m["spk"]]).cuda(),
+             torch.from_numpy(sp).cuda()]
+        use = force_prompt if force_prompt is not None else (prng is not None and prng.random() < a.p_prompt)
+        if use:
+            c = by_spk.get(m["spk"], [])
+            c = [x for x in (prng or random).sample(c, min(6, len(c))) if x["id"] != m["id"]] if c else []
+            if c and c[0]["ntok"] + m["ntok"] <= 600:
+                p = c[0]
+                pids, plangs = tt(tagged(p), p["id"])
+                psp = np.asarray(mm[p["ds"]][p["toff"]: p["toff"] + p["ntok"]], dtype=np.int64)
+                r.append((torch.tensor(pids).cuda(), torch.tensor(plangs).cuda(), torch.from_numpy(psp).cuda()))
+        return r
+
+    def validate():
+        model.eval(); out = {}
+        with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
+            for mode in (False, True):
+                vr = random.Random(7); L, W = [], []
+                for i in range(0, len(val), 16):
+                    bt = val[i:i + 16]
+                    ce, _, _, _ = model.forward2([enc(m, vr, force_prompt=mode) for m in bt])
+                    L.append(float(ce) * len(bt)); W.append(len(bt))
+                out["p" if mode else "np"] = sum(L) / sum(W)
+            # anchor-only val
+            av = [m for m in val if m["spk"] == ANCHOR]
+            if av:
+                vr = random.Random(7)
+                ce, _, _, _ = model.forward2([enc(m, vr, force_prompt=True) for m in av[:32]])
+                out["anchor_p"] = float(ce)
+        model.train()
+        out["mean"] = (out["np"] + out["p"]) / 2
+        return out
+
+    extra = (lambda m: a.anchor_only_w if m["spk"] == ANCHOR else 1.0) if a.anchor_only_w > 0 else None
+    rng = random.Random(0); prng = random.Random(3)
+    step, nskip, t0 = 0, 0, time.time()
+    logf = open(f"{a.out}/log.jsonl", "a")
+    v = validate(); best = v["mean"] if not a.select_anchor else 0.5 * (v["mean"] + v.get("anchor_p", v["mean"])); bad = 0
+    logf.write(json.dumps({"step": 0, "val": v}) + "\n"); print(json.dumps({"step": 0, "val": v}), flush=True)
+    torch.save(dict(model=model.state_dict(), step=0, n_spk=n_spk, val=v), f"{a.out}/best.pt")
+    total = a.max_steps
+    done = False
+    while not done:
+        for bi in batches(items, a.tokens, rng, extra, cost=lambda m: m["ntok"] + len(m["text"]) // 2 + 4 + int(a.p_prompt * 200)):
+            lrm = min(1.0, (step + 1) / a.warmup) * (0.1 + 0.9 * 0.5 * (1 + math.cos(math.pi * min(1.0, step / total))))
+            for g, b in zip(opt.param_groups, base):
+                g["lr"] = b * lrm
+            batch = [enc(items[i], prng) for i in bi]
+            # keep the padded length inside the budget when prompts were added
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                ce, acc, adv, adv_acc = model.forward2(batch, adv_lambda=min(1.0, step / 1000))
+            loss = ce + a.adv_w * adv
+            opt.zero_grad(set_to_none=True)
+            if not torch.isfinite(loss):
+                nskip += 1; print(json.dumps({"skip_nonfinite_fwd": step, "ce": float(ce), "adv": float(adv)}), flush=True)
+                continue
+            loss.backward()
+            gn = torch.nn.utils.clip_grad_norm_([p for p in model.parameters() if p.requires_grad], 1.0)
+            if not torch.isfinite(gn):
+                nskip += 1; opt.zero_grad(set_to_none=True); print(json.dumps({"skip_nonfinite_grad": step}), flush=True)
+                continue
+            opt.step(); step += 1
+            if step % 25 == 0:
+                r = dict(step=step, ce=round(float(ce), 4), acc=round(float(acc), 4), adv=round(float(adv), 3),
+                         adv_acc=round(float(adv_acc), 3), gn=round(float(gn), 2), B=len(bi), lr=round(lrm * a.lr, 9),
+                         skips=nskip, min=round((time.time() - t0) / 60, 2))
+                print(json.dumps(r), flush=True); logf.write(json.dumps(r) + "\n"); logf.flush()
+            if step % a.val_every == 0:
+                v = validate()
+                score = v["mean"] if not a.select_anchor else 0.5 * (v["mean"] + v.get("anchor_p", v["mean"]))
+                imp = score < best - 1e-4
+                rec = {"step": step, "val": v, "best": imp}
+                print(json.dumps(rec), flush=True); logf.write(json.dumps(rec) + "\n"); logf.flush()
+                if imp:
+                    best, bad = score, 0
+                    torch.save(dict(model=model.state_dict(), step=step, n_spk=n_spk, val=v), f"{a.out}/best.pt")
+                else:
+                    bad += 1
+                    if bad >= a.patience:
+                        print("EARLY_STOP", step, flush=True); done = True; break
+            if (time.time() - t0) / 60 > a.minutes or step >= total:
+                done = True; break
+    print("TRAIN_LM_DONE", step, "best", best, "skips", nskip, flush=True)
+
+
+# ------------------------------------------------------------------ flow training
+def train_flow(a):
+    cosy = load_cosy()
+    flow = cosy.model.flow
+    if a.flow_init:
+        flow.load_state_dict(torch.load(a.flow_init, map_location="cuda")["flow"])
+    flow.train()
+    for p in cosy.model.llm.parameters():
+        p.requires_grad = False
+    fe = cosy.frontend.feat_extractor
+    dss = a.datasets.split(",")
+    items, mm = load_items(dss, "train", max_tok=400, flow=True)
+    val, _ = load_items(dss, "val", max_tok=400, flow=True)
+    rng0 = random.Random(1); rng0.shuffle(val); val = val[:256]
+    pcm = {ds: np.memmap(f"{D2}/{ds}/pcm24k.i16", dtype=np.int16, mode="r") for ds in dss if os.path.exists(f"{D2}/{ds}/pcm24k.i16")}
+    emb = {ds: np.load(f"{D2}/{ds}/campplus.npy", mmap_mode="r") for ds in dss if os.path.exists(f"{D2}/{ds}/campplus.npy")}
+    os.makedirs(a.out, exist_ok=True)
+    print("flow items", len(items), "hours", round(sum(m["dur"] for m in items) / 3600, 1), "val", len(val), flush=True)
+    extra = (lambda m: a.anchor_only_w if m["spk"] == ANCHOR else 1.0) if a.anchor_only_w > 0 else None
+
+    def make(bt):
+        toks, feats, es = [], [], []
+        for m in bt:
+            y = torch.from_numpy(np.asarray(pcm[m["ds"]][m["off24"]: m["off24"] + m["n24"]], dtype=np.float32) / 32768.0)
+            n = int(math.ceil(len(y) / 960)) * 960
+            y = F.pad(y, (0, n - len(y)))[None].cuda()
+            ft = fe(y).squeeze(0).transpose(0, 1)         # [T,80]
+            tk = torch.from_numpy(np.asarray(mm[m["ds"]][m["toff"]: m["toff"] + m["ntok"]], dtype=np.int64))
+            k = min(ft.shape[0] // 2, len(tk))
+            toks.append(tk[:k]); feats.append(ft[:2 * k]); es.append(torch.from_numpy(np.array(emb[m["ds"]][m["row"]])))
+        tl = torch.tensor([len(t) for t in toks], dtype=torch.int32)
+        fl = torch.tensor([len(f) for f in feats], dtype=torch.int32)
+        T = int(tl.max()); Fm = int(fl.max())
+        tok = torch.zeros(len(bt), T, dtype=torch.long); feat = torch.zeros(len(bt), Fm, 80)
+        for i, (t, f) in enumerate(zip(toks, feats)):
+            tok[i, :len(t)] = t; feat[i, :len(f)] = f.cpu()
+        return dict(speech_token=tok, speech_token_len=tl, speech_feat=feat, speech_feat_len=fl,
+                    embedding=torch.stack(es).float())
+
+    params = [p for p in flow.parameters() if p.requires_grad]
+    opt = torch.optim.AdamW(params, lr=a.lr, weight_decay=0.0)
+    logf = open(f"{a.out}/log.jsonl", "a")
+
+    def validate():
+        flow.eval(); L = []
+        st = torch.random.get_rng_state(); cst = torch.cuda.get_rng_state(); pst = random.getstate()
+        torch.manual_seed(0); random.seed(0)
+        with torch.no_grad():
+            for i in range(0, len(val), 16):
+                L.append(float(flow(make(val[i:i + 16]), torch.device("cuda"))["loss"]))
+        torch.random.set_rng_state(st); torch.cuda.set_rng_state(cst); random.setstate(pst)
+        flow.train(); return float(np.mean(L))
+
+    v = validate(); best, bad = v, 0
+    print(json.dumps({"step": 0, "val": v}), flush=True); logf.write(json.dumps({"step": 0, "val": v}) + "\n")
+    torch.save(dict(flow=flow.state_dict(), step=0, val=v), f"{a.out}/best.pt")
+    rng = random.Random(0); step, t0, done = 0, time.time(), False
+    while not done:
+        for bi in batches(items, a.tokens, rng, extra, cost=lambda m: m["ntok"]):
+            lrm = min(1.0, (step + 1) / a.warmup)
+            for g in opt.param_groups:
+                g["lr"] = a.lr * lrm
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                loss = flow(make([items[i] for i in bi]), torch.device("cuda"))["loss"]
+            opt.zero_grad(set_to_none=True)
+            if not torch.isfinite(loss):
+                continue
+            loss.backward()
+            gn = torch.nn.utils.clip_grad_norm_(params, 5.0)
+            if not torch.isfinite(gn):
+                continue
+            opt.step(); step += 1
+            if step % 50 == 0:
+                r = dict(step=step, loss=round(float(loss), 4), gn=round(float(gn), 2), B=len(bi), min=round((time.time() - t0) / 60, 2))
+                print(json.dumps(r), flush=True); logf.write(json.dumps(r) + "\n"); logf.flush()
+            if step % a.val_every == 0:
+                v = validate(); imp = v < best - 1e-4
+                print(json.dumps({"step": step, "val": v, "best": imp}), flush=True)
+                logf.write(json.dumps({"step": step, "val": v, "best": imp}) + "\n"); logf.flush()
+                if imp:
+                    best, bad = v, 0
+                    torch.save(dict(flow=flow.state_dict(), step=step, val=v), f"{a.out}/best.pt")
+                else:
+                    bad += 1
+                    if bad >= a.patience:
+                        done = True; break
+            if (time.time() - t0) / 60 > a.minutes or step >= a.max_steps:
+                done = True; break
+    print("TRAIN_FLOW_DONE", step, "best", best, flush=True)
+
+
+# ------------------------------------------------------------------ synthesis
+_SENT = re.compile(r"(?<=[.?!])\s+")
+
+
+def split_sentences(text):
+    """tagged text -> list of tagged sentences. A sentence ends at . ? ! inside a span or at the end of a span;
+    language spans are cut there and re-tagged, so a sentence keeps its in-sentence switches."""
+    out, cur = [], []
+    for l, s in segments(text):
+        parts = _SENT.split(s.strip())
+        for k, p in enumerate(parts):
+            p = p.strip()
+            if p:
+                cur.append(f"<{l}>{p}</{l}>")
+            if (k < len(parts) - 1 or p[-1:] in ".?!") and cur:
+                out.append(" ".join(cur)); cur = []
+    if cur:
+        out.append(" ".join(cur))
+    return [o for o in out if o.strip()]
+
+
+def trim_edges(w, sr, keep_lead=0.12, keep_tail=0.18):
+    from da.p2common import silence_mask
+    sil = silence_mask(w, sr)
+    sp = np.where(~sil)[0]
+    if not len(sp):
+        return w
+    a, b = sp[0] * 0.01, (sp[-1] + 1) * 0.01
+    out = w[int(max(0, a - keep_lead) * sr): int(min(len(w) / sr, b + keep_tail) * sr)]
+    lead = a - max(0, a - keep_lead)
+    if lead < keep_lead:
+        out = np.concatenate([np.zeros(int((keep_lead - lead) * sr), out.dtype), out])
+    return out
+
+
+def cap_internal(w, sr, cap=0.6):
+    """shorten every internal silence longer than cap seconds to cap (the centre of the silent run is removed)."""
+    from da.p2common import silence_mask, runs
+    sil = silence_mask(w, sr)
+    hop = int(0.01 * sr)
+    R = [(a, b) for a, b in runs(sil) if a > 0 and b < len(sil) and (b - a) * 0.01 > cap]
+    if not R:
+        return w
+    keep, cur = [], 0
+    for a, b in R:
+        cut0 = a * hop + int(cap / 2 * sr)
+        cut1 = b * hop - int(cap / 2 * sr)
+        keep.append(w[cur:cut0]); cur = cut1
+    keep.append(w[cur:])
+    return np.concatenate(keep)
+
+
+class Synth:
+    def __init__(self, ckpt, flow_ckpt=None, sent_pause=0.5):
+        from transformers import AutoTokenizer
+        self.tok = AutoTokenizer.from_pretrained(f"{MODEL_DIR}/CosyVoice-BlankEN")
+        self.cosy = load_cosy()
+        if flow_ckpt:
+            self.cosy.model.flow.load_state_dict(torch.load(flow_ckpt, map_location="cuda")["flow"])
+        ck = torch.load(ckpt, map_location="cuda")
+        self.spk2id = json.load(open(os.path.join(os.path.dirname(ckpt), "spk2id.json")))
+        self.m = DALM2(self.cosy.model.llm, ck["n_spk"]).cuda().eval()
+        self.m.load_state_dict(ck["model"])
+        self.sent_pause = sent_pause
+        self.cap = 0.6
+        # anchor prompts: normalised anchor val clips from data2 (never in train), 3-7 s, highest SNR first
+        meta = [json.loads(l) for l in open(f"{D2}/waxal/meta.jsonl", encoding="utf-8")]
+        tk = [json.loads(l) for l in open(f"{D2}/waxal/s3tok.jsonl")]
+        pcm = np.memmap(f"{D2}/waxal/pcm24k.i16", dtype=np.int16, mode="r")
+        s3 = np.memmap(f"{D2}/waxal/s3tok.i32", dtype=np.int32, mode="r")
+        cand = [(m, t) for m, t in zip(meta, tk) if m["spk"] == ANCHOR and m["split"] == "val" and 3.0 <= m["dur"] <= 7.0
+                and not m.get("longform")]
+        cand.sort(key=lambda x: -x[0]["ascore"])
+        self.prompts = []
+        os.makedirs(f"{HOME}/eval/prompts", exist_ok=True)
+        for m, t in cand[:4]:
+            y = np.asarray(pcm[m["off24"]: m["off24"] + m["n24"]], dtype=np.float32) / 32768.0
+            p = f"{HOME}/eval/prompts/{m['id']}.wav"; sf.write(p, y, 24000)
+            from train_b import prompt_feats
+            ptok, pfeat, pemb = prompt_feats(self.cosy, p)
+            ids, langs = text_tokens(self.tok, tagged(m))
+            lmp = (torch.tensor(ids).cuda(), torch.tensor(langs).cuda(),
+                   torch.from_numpy(np.asarray(s3[t["toff"]: t["toff"] + t["ntok"]], dtype=np.int64)).cuda())
+            self.prompts.append(dict(id=m["id"], text=m["text"], lm=lmp, flow=(ptok, pfeat, pemb)))
+        print("anchor prompts", [p["id"] for p in self.prompts], flush=True)
+
+    def one(self, tagged_text, seed=0, prompt_k=0, use_lm_prompt=True):
+        ids, langs = text_tokens(self.tok, tagged_text)
+        P = self.prompts[prompt_k % len(self.prompts)]
+        with torch.autocast("cuda", dtype=torch.bfloat16):
+            sp = self.m.generate2(torch.tensor(ids).cuda(), torch.tensor(langs).cuda(),
+                                  torch.tensor(self.spk2id[ANCHOR]).cuda(), prompt=P["lm"] if use_lm_prompt else None,
+                                  seed=seed)
+        if len(sp) < 3:
+            sp = sp + [0] * 3
+        self.last_tokens.append(sp)
+        return token2wav(self.cosy, sp, *P["flow"])
+
+    def join(self, ws):
+        sr = self.cosy.sample_rate
+        if len(ws) == 1:
+            return ws[0]
+        out = []
+        for i, w in enumerate(ws):
+            if i:
+                out.append(np.zeros(max(0, int((self.sent_pause - 0.30) * sr)), np.float32))
+            out.append(w)
+        return np.concatenate(out)
+
+    def redecode(self, toks, prompt_k=0):
+        """same LM tokens, (possibly different) flow -> identical post-processing"""
+        sr = self.cosy.sample_rate
+        P = self.prompts[prompt_k % len(self.prompts)]
+        return self.join([cap_internal(trim_edges(token2wav(self.cosy, t, *P["flow"]), sr), sr, self.cap) for t in toks])
+
+    def say(self, tagged_text, seed=0, prompt_k=0, stitch=True, use_lm_prompt=True):
+        sr = self.cosy.sample_rate
+        self.last_tokens = []
+        sents = split_sentences(tagged_text) if stitch else [tagged_text]
+        ws = [cap_internal(trim_edges(self.one(s, seed=seed + 1000 * i, prompt_k=prompt_k, use_lm_prompt=use_lm_prompt), sr), sr, self.cap)
+              for i, s in enumerate(sents)]
+        # trailing 0.18 + leading 0.12 already present; join() tops up to the anchor's sentence pause
+        return self.join(ws)
+
+
+def redecode(a):
+    """decode the LM tokens saved by synth (--src DIR) with another flow checkpoint -> --out"""
+    S = Synth(a.ckpt, a.flow or None, a.sent_pause)
+    os.makedirs(a.out, exist_ok=True)
+    import glob
+    for f in sorted(glob.glob(f"{a.src}/*.tok.json")):
+        d = json.load(open(f))
+        i = os.path.basename(f)[:-9]
+        sf.write(f"{a.out}/{i}.wav", S.redecode(d["tokens"], d["prompt_k"]), S.cosy.sample_rate)
+    print("REDECODE_DONE", a.out, flush=True)
+
+
+def synth(a):
+    S = Synth(a.ckpt, a.flow or None, a.sent_pause)
+    rows = [json.loads(l) for l in open(a.eval_set, encoding="utf-8")]
+    if a.only:
+        keep = set(a.only.split(",")); rows = [r for r in rows if r["id"] in keep or r["lang"] in keep]
+    if a.nper:
+        cnt, sel = {}, []
+        for r in rows:
+            if r["lang"] in ("cs", "para") or cnt.get(r["lang"], 0) < a.nper:
+                sel.append(r); cnt[r["lang"]] = cnt.get(r["lang"], 0) + 1
+        rows = sel
+    ks = [a.k] if a.k >= 0 else list(range(a.n))
+    for k in ks:
+        od = a.out if len(ks) == 1 else f"{a.out}/cand{k}"
+        os.makedirs(od, exist_ok=True)
+        for r in rows:
+            text = r["text"] if r["lang"] in ("cs", "para") else f"<{r['lang']}>{r['text']}</{r['lang']}>"
+            w = S.say(text, seed=k, prompt_k=k, stitch=not a.no_stitch, use_lm_prompt=not a.no_lm_prompt)
+            sf.write(f"{od}/{r['id']}.wav", w, S.cosy.sample_rate)
+            json.dump(dict(tokens=S.last_tokens, prompt_k=k), open(f"{od}/{r['id']}.tok.json", "w"))
+        print("synth cand", k, "done", od, flush=True)
+    print("SYNTH2_DONE", a.out, flush=True)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("cmd")
+    ap.add_argument("--datasets", default="waxal,fleurs,slr86,slr70,bible_yor,bible_hau,cs_pairs")
+    ap.add_argument("--out", default=f"{HOME}/runs/p2_lm")
+    ap.add_argument("--init", default="")
+    ap.add_argument("--flow_init", default="")
+    ap.add_argument("--ckpt", default="")
+    ap.add_argument("--flow", default="")
+    ap.add_argument("--minutes", type=float, default=90)
+    ap.add_argument("--max_steps", type=int, default=4000)
+    ap.add_argument("--lr", type=float, default=1e-5)
+    ap.add_argument("--lr_new", type=float, default=5e-4)
+    ap.add_argument("--warmup", type=int, default=300)
+    ap.add_argument("--tokens", type=int, default=16000)
+    ap.add_argument("--adv_w", type=float, default=0.02)
+    ap.add_argument("--p_prompt", type=float, default=0.5)
+    ap.add_argument("--val_every", type=int, default=200)
+    ap.add_argument("--patience", type=int, default=5)
+    ap.add_argument("--anchor_only_w", type=float, default=0.0)
+    ap.add_argument("--select_anchor", action="store_true")
+    ap.add_argument("--eval_set", default=f"{HOME}/eval/eval_set2.jsonl")
+    ap.add_argument("--n", type=int, default=1)
+    ap.add_argument("--nper", type=int, default=0)
+    ap.add_argument("--k", type=int, default=-1)
+    ap.add_argument("--src", default="")
+    ap.add_argument("--only", default="")
+    ap.add_argument("--sent_pause", type=float, default=0.6)
+    ap.add_argument("--no_stitch", action="store_true")
+    ap.add_argument("--no_lm_prompt", action="store_true")
+    a = ap.parse_args()
+    {"lm": train_lm, "flow": train_flow, "synth": synth, "redecode": redecode}[a.cmd](a)

cosyvoice_requirements.txt

@@ -0,0 +1,42 @@
+--extra-index-url https://download.pytorch.org/whl/cu121
+--extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/ # https://github.com/microsoft/onnxruntime/issues/21684
+conformer==0.3.2
+deepspeed==0.15.1; sys_platform == 'linux'
+diffusers==0.29.0
+fastapi==0.115.6
+fastapi-cli==0.0.4
+gdown==5.1.0
+gradio==5.4.0
+grpcio==1.57.0
+grpcio-tools==1.57.0
+hydra-core==1.3.2
+HyperPyYAML==1.2.3
+inflect==7.3.1
+librosa==0.10.2
+lightning==2.2.4
+matplotlib==3.7.5
+modelscope==1.20.0
+networkx==3.1
+numpy==1.26.4
+omegaconf==2.3.0
+onnx==1.16.0
+onnxruntime-gpu==1.18.0; sys_platform == 'linux'
+onnxruntime==1.18.0; sys_platform == 'darwin' or sys_platform == 'win32'
+openai-whisper==20231117
+protobuf==4.25
+pyarrow==18.1.0
+pydantic==2.7.0
+pyworld==0.3.4
+rich==13.7.1
+soundfile==0.12.1
+tensorboard==2.14.0
+tensorrt-cu12==10.13.3.9; sys_platform == 'linux'
+tensorrt-cu12-bindings==10.13.3.9; sys_platform == 'linux'
+tensorrt-cu12-libs==10.13.3.9; sys_platform == 'linux'
+torch==2.3.1
+torchaudio==2.3.1
+transformers==4.51.3
+x-transformers==2.11.24
+uvicorn==0.30.0
+wetext==0.0.4
+wget==3.2

infer.py

@@ -0,0 +1,418 @@
+#!/usr/bin/env python
+"""Dynamic Accent (Phase 2 preview): Nigerian multilingual, code-switching TTS on CosyVoice2-0.5B.
+
+Same inference path as the Phase 2 final samples: LM `p2_lm_B` (pause profile B) + anchor-SFT flow,
+anchor prompt clip 0 at the LM and the flow, single sample, sentence-level stitching, edge trim and
+internal-silence cap (0.6 s), 0.6 s between sentences.
+
+  python infer.py --text "<eng>this is dynamic accent,</eng> <pcm>e dey sound nice abi?</pcm>" --seed 0 --out out.wav
+
+Prosody v2 (default, `--pause_mode clause`): the text is cut at , ; : . ? ! and at language-switch points; each chunk
+is generated with the same LM/flow/prompt, its edge silence is trimmed (energy VAD, 30 ms kept), its speech is sped
+up pitch-preserving by `--speed` (WSOLA, default 1.05), and chunks are joined with set gaps from the real anchor's pause statistics:
+comma/;/: 0.40 s (+-0.05 jitter), sentence end 0.60 s, unpunctuated language switch 0.12 s, with 10 ms fades.
+`--pause_mode model --speed 1.0` reproduces the Phase 2 final samples (model-generated pauses, sentence stitching).
+
+Text: wrap each language span in <eng>, <pcm>, <yor>, <ibo> or <hau> tags. Untagged text is treated as English.
+"""
+import argparse, inspect, json, os, re, sys, unicodedata
+
+import numpy as np
+import soundfile as sf
+import librosa
+import torch
+import torch.nn as nn
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+LANGS = ["eng", "ibo", "hau", "yor", "pcm"]
+L2I = {l: i for i, l in enumerate(LANGS)}
+ANCHOR = "waxal_yor_4"
+DEFAULT_SPEED = 1.05   # from the speaking-rate measurement (README, "Prosody v2"): median ref/system ratio 1.06
+
+
+# ------------------------------------------------------------------ text front-end (tags -> spans)
+_TAG = re.compile(r"<(eng|ibo|hau|yor|pcm)>(.*?)</\1>", re.S)
+_SENT = re.compile(r"(?<=[.?!])\s+")
+
+
+def normalize(t):
+    t = unicodedata.normalize("NFC", t)
+    t = t.replace("\u2019", "'").replace("\u2018", "'").replace("\u201c", '"').replace("\u201d", '"')
+    return re.sub(r"\s+", " ", t).strip()
+
+
+def segments(text, default_lang="eng"):
+    out, pos = [], 0
+    for m in _TAG.finditer(text):
+        pre = text[pos:m.start()]
+        if pre.strip():
+            out.append((default_lang, pre))
+        out.append((m.group(1), m.group(2)))
+        pos = m.end()
+    if text[pos:].strip():
+        out.append((default_lang, text[pos:]))
+    res = []
+    for l, s in out:
+        s = normalize(s)
+        if not s:
+            continue
+        if res:
+            s = " " + s
+        res.append((l, s))
+    return res
+
+
+def text_tokens(tok, text, default="eng"):
+    ids, langs = [], []
+    for l, s in segments(text, default):
+        t = tok(s, add_special_tokens=False)["input_ids"]
+        ids += t; langs += [L2I[l]] * len(t)
+    return ids, langs
+
+
+def split_sentences(text):
+    out, cur = [], []
+    for l, s in segments(text):
+        parts = _SENT.split(s.strip())
+        for k, p in enumerate(parts):
+            p = p.strip()
+            if p:
+                cur.append(f"<{l}>{p}</{l}>")
+            if (k < len(parts) - 1 or p[-1:] in ".?!") and cur:
+                out.append(" ".join(cur)); cur = []
+    if cur:
+        out.append(" ".join(cur))
+    return [o for o in out if o.strip()]
+
+
+# ------------------------------------------------------------------ silence post-processing
+def frame_db(y, sr, hop_s=0.01):
+    hop = int(sr * hop_s)
+    n = max(1, len(y) // hop)
+    fr = y[: n * hop].reshape(n, hop)
+    return 10 * np.log10((fr.astype(np.float64) ** 2).mean(1) + 1e-10)
+
+
+def silence_mask(y, sr, rel_db=-40.0, hop_s=0.01):
+    e = frame_db(y, sr, hop_s)
+    return e < np.percentile(e, 98) + rel_db
+
+
+def runs(mask):
+    out, i, n = [], 0, len(mask)
+    while i < n:
+        if mask[i]:
+            j = i
+            while j < n and mask[j]:
+                j += 1
+            out.append((i, j)); i = j
+        else:
+            i += 1
+    return out
+
+
+def trim_edges(w, sr, keep_lead=0.12, keep_tail=0.18):
+    sil = silence_mask(w, sr)
+    sp = np.where(~sil)[0]
+    if not len(sp):
+        return w
+    a, b = sp[0] * 0.01, (sp[-1] + 1) * 0.01
+    out = w[int(max(0, a - keep_lead) * sr): int(min(len(w) / sr, b + keep_tail) * sr)]
+    lead = a - max(0, a - keep_lead)
+    if lead < keep_lead:
+        out = np.concatenate([np.zeros(int((keep_lead - lead) * sr), out.dtype), out])
+    return out
+
+
+def cap_internal(w, sr, cap=0.6):
+    sil = silence_mask(w, sr)
+    hop = int(0.01 * sr)
+    R = [(a, b) for a, b in runs(sil) if a > 0 and b < len(sil) and (b - a) * 0.01 > cap]
+    if not R:
+        return w
+    keep, cur = [], 0
+    for a, b in R:
+        keep.append(w[cur: a * hop + int(cap / 2 * sr)]); cur = b * hop - int(cap / 2 * sr)
+    keep.append(w[cur:])
+    return np.concatenate(keep)
+
+
+
+# ------------------------------------------------------------------ prosody v2: clause chunks, set gaps, pitch-preserving speed
+_PUNCT = re.compile(r"(?<=[,;:.?!])\s+")
+GAPS = {"comma": 0.40, "sent": 0.60, "switch": 0.12}
+JITTER = {"comma": 0.05, "sent": 0.03, "switch": 0.02}
+
+
+def clause_chunks(text):
+    """tagged text -> [(lang, chunk_text, boundary_after)] with boundary in comma|sent|switch|None (None = end).
+    Chunks are cut after , ; : . ? ! and at language switches; a one-word chunk is merged into the next chunk of the
+    same language (one-word LM calls are unstable), keeping its comma inside."""
+    raw = []
+    for l, s in segments(text):
+        for p in _PUNCT.split(s.strip()):
+            if p.strip():
+                raw.append([l, p.strip()])
+    merged = []
+    for c in raw:
+        if merged and merged[-1][0] == c[0] and len(merged[-1][1].split()) < 2:
+            merged[-1][1] = merged[-1][1] + " " + c[1]
+        else:
+            merged.append(c)
+    out = []
+    for i, (l, t) in enumerate(merged):
+        if i == len(merged) - 1:
+            b = None
+        elif t[-1] in ".?!":
+            b = "sent"
+        elif t[-1] in ",;:":
+            b = "comma"
+        else:
+            b = "switch"
+        out.append((l, t, b))
+    return out
+
+
+def trim_tight(w, sr, keep=0.03):
+    sil = silence_mask(w, sr)
+    sp = np.where(~sil)[0]
+    if not len(sp):
+        return w
+    a, b = sp[0] * 0.01, (sp[-1] + 1) * 0.01
+    return w[int(max(0.0, a - keep) * sr): int(min(len(w) / sr, b + keep) * sr)]
+
+
+def time_stretch(w, sr, speed):
+    """pitch-preserving tempo change (speed > 1 = faster). WSOLA via audiotsm, else librosa's phase vocoder."""
+    if abs(speed - 1.0) < 1e-3:
+        return w
+    try:
+        from audiotsm import wsola
+        from audiotsm.io.array import ArrayReader, ArrayWriter
+        r, wr = ArrayReader(w[None].astype(np.float32)), ArrayWriter(1)
+        wsola(1, speed=speed, frame_length=int(0.040 * sr), synthesis_hop=int(0.010 * sr)).run(r, wr)
+        return wr.data[0].astype(np.float32)
+    except ImportError:
+        return librosa.effects.time_stretch(w.astype(np.float32), rate=speed)
+
+
+def fade(w, sr, ms=10):
+    n = min(len(w) // 2, int(sr * ms / 1000))
+    if n > 0:
+        w = w.copy(); r = np.linspace(0, 1, n, dtype=np.float32)
+        w[:n] *= r; w[-n:] *= r[::-1]
+    return w
+
+# ------------------------------------------------------------------ CosyVoice2 base
+def _sf_load_wav(wav, target_sr, min_sr=16000):
+    y, sr = sf.read(wav, dtype="float32", always_2d=True)
+    y = y.mean(1)
+    if sr != target_sr:
+        y = librosa.resample(y, orig_sr=sr, target_sr=target_sr)
+    return torch.from_numpy(y)[None]
+
+
+def load_cosy(model_dir, cosyvoice_dir):
+    sys.path[:0] = [cosyvoice_dir, os.path.join(cosyvoice_dir, "third_party", "Matcha-TTS")]
+    import cosyvoice.utils.file_utils as fu
+    fu.load_wav = _sf_load_wav
+    import cosyvoice.cli.frontend as cf
+    cf.load_wav = _sf_load_wav
+    from cosyvoice.cli.cosyvoice import CosyVoice2
+    kw = dict(load_jit=False, load_trt=False, fp16=False)
+    sig = inspect.signature(CosyVoice2.__init__).parameters
+    return CosyVoice2(model_dir, **{k: v for k, v in kw.items() if k in sig})
+
+
+def prompt_feats(cosy, wav_path):
+    fe = cosy.frontend
+    tokn, _ = fe._extract_speech_token(wav_path)
+    feat, _ = fe._extract_speech_feat(wav_path)
+    n = min(feat.shape[1] // 2, tokn.shape[1])
+    feat, tokn = feat[:, :2 * n], tokn[:, :n]
+    emb = fe._extract_spk_embedding(wav_path)
+    return tokn.cuda(), feat.cuda(), emb.cuda()
+
+
+@torch.no_grad()
+def token2wav(cosy, tokens, ptok, pfeat, emb):
+    flow = cosy.model.flow
+    t = torch.tensor([tokens], dtype=torch.int32).cuda()
+    kw = dict(token=t, token_len=torch.tensor([t.shape[1]], dtype=torch.int32).cuda(),
+              prompt_token=ptok, prompt_token_len=torch.tensor([ptok.shape[1]], dtype=torch.int32).cuda(),
+              prompt_feat=pfeat, prompt_feat_len=torch.tensor([pfeat.shape[1]], dtype=torch.int32).cuda(),
+              embedding=emb)
+    sig = inspect.signature(flow.inference).parameters
+    if "streaming" in sig:
+        kw["streaming"] = False
+    if "finalize" in sig:
+        kw["finalize"] = True
+    if "flow_cache" in sig:
+        kw["flow_cache"] = torch.zeros(1, 80, 0, 2).cuda()
+    mel = flow.inference(**kw)
+    mel = mel[0] if isinstance(mel, tuple) else mel
+    hs = inspect.signature(cosy.model.hift.inference).parameters
+    w = cosy.model.hift.inference(speech_feat=mel, **({"cache_source": torch.zeros(1, 1, 0).cuda()} if "cache_source" in hs else {}))
+    w = w[0] if isinstance(w, tuple) else w
+    return w.squeeze().float().cpu().numpy()
+
+
+# ------------------------------------------------------------------ Dynamic Accent LM (Qwen2LM + language + speaker embeddings)
+class DALM2(nn.Module):
+    def __init__(self, qlm, n_spk, d=896):
+        super().__init__()
+        self.q = qlm
+        self.lang_emb = nn.Embedding(len(LANGS), d)
+        self.spk_emb = nn.Embedding(n_spk, d)
+        self.adv = nn.Sequential(nn.Linear(d, 256), nn.ReLU(), nn.Linear(256, n_spk))  # training-only head
+        self.V = qlm.speech_token_size
+
+    def embed_text(self, ids, langs):
+        return self.q.llm.model.model.embed_tokens(ids) + self.lang_emb(langs)
+
+    def build2(self, ids, langs, spk, pids=None, plangs=None, psp=None):
+        sos = self.q.llm_embedding.weight[self.q.sos][None]
+        task = self.q.llm_embedding.weight[self.q.task_id][None]
+        if pids is not None:
+            ids = torch.cat([pids, ids]); langs = torch.cat([plangs, langs])
+        parts = [sos, self.embed_text(ids, langs), self.spk_emb(spk)[None], task]
+        if psp is not None:
+            parts.append(self.q.speech_embedding(psp))
+        return torch.cat(parts, 0)
+
+    @torch.no_grad()
+    def generate2(self, ids, langs, spk, prompt=None, top_p=0.8, top_k=25, win=10, tau_r=0.1, seed=0,
+                  min_ratio=2.0, max_ratio=20.0):
+        g = torch.Generator(device=ids.device).manual_seed(seed)
+        x = self.build2(ids, langs, spk, *(prompt if prompt else (None, None, None)))[None]
+        n = len(ids)
+        min_len, max_len = int(n * min_ratio), int(n * max_ratio)
+        past, out, inp = None, [], x
+        for i in range(max_len):
+            o = self.q.llm.model.model(inputs_embeds=inp, past_key_values=past, use_cache=True, return_dict=True)
+            past = o.past_key_values
+            logp = self.q.llm_decoder(o.last_hidden_state[:, -1]).float().log_softmax(-1)[0]
+            if i < min_len:
+                logp[self.V] = -float("inf")
+            logp[self.V + 1:] = -float("inf")
+            p = logp.exp()
+            sp, si = p.sort(descending=True)
+            keep = (sp.cumsum(0) - sp) < top_p
+            keep[top_k:] = False
+            cp, ci = sp[keep], si[keep]
+            tokid = int(ci[torch.multinomial(cp / cp.sum(), 1, generator=g)])
+            if out and sum(1 for t in out[-win:] if t == tokid) >= win * tau_r and tokid != self.V:  # RAS
+                tokid = int(torch.multinomial(p, 1, generator=g))
+            if tokid >= self.V:
+                break
+            out.append(tokid)
+            inp = self.q.speech_embedding.weight[tokid][None, None]
+        return out
+
+
+class DynamicAccent:
+    def __init__(self, model_dir, cosyvoice_dir, lm_ckpt, flow_ckpt, prompts_json, dtype="auto", sent_pause=0.6, cap=0.6):
+        from transformers import AutoTokenizer
+        self.tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, "CosyVoice-BlankEN"))
+        self.cosy = load_cosy(model_dir, cosyvoice_dir)
+        self.sr = self.cosy.sample_rate
+        self.cosy.model.flow.load_state_dict(torch.load(flow_ckpt, map_location="cuda")["flow"])
+        ck = torch.load(lm_ckpt, map_location="cuda")
+        self.spk2id = json.load(open(os.path.join(os.path.dirname(lm_ckpt), "spk2id.json")))
+        self.m = DALM2(self.cosy.model.llm, ck["n_spk"]).cuda().eval()
+        self.m.load_state_dict(ck["model"])
+        if dtype == "auto":
+            dtype = "bf16" if torch.cuda.is_bf16_supported() else "fp16"
+        self.dtype = torch.bfloat16 if dtype == "bf16" else torch.float16
+        self.sent_pause, self.cap = sent_pause, cap
+        meta = json.load(open(prompts_json, encoding="utf-8"))
+        root = os.path.dirname(os.path.dirname(os.path.abspath(prompts_json)))
+        self.prompts = []
+        for p in meta["prompts"]:
+            ids, langs = text_tokens(self.tok, p["tagged"])
+            lmp = (torch.tensor(ids).cuda(), torch.tensor(langs).cuda(), torch.tensor(p["s3_tokens"], dtype=torch.int64).cuda())
+            self.prompts.append(dict(id=p["id"], lm=lmp, flow=prompt_feats(self.cosy, os.path.join(root, p["wav"]))))
+
+    def one(self, tagged_text, seed, prompt_k, use_lm_prompt=True):
+        ids, langs = text_tokens(self.tok, tagged_text)
+        P = self.prompts[prompt_k % len(self.prompts)]
+        with torch.autocast("cuda", dtype=self.dtype):
+            sp = self.m.generate2(torch.tensor(ids).cuda(), torch.tensor(langs).cuda(),
+                                  torch.tensor(self.spk2id[ANCHOR]).cuda(), prompt=P["lm"] if use_lm_prompt else None, seed=seed)
+        if len(sp) < 3:
+            sp = sp + [0] * 3
+        return token2wav(self.cosy, sp, *P["flow"])
+
+    def say(self, tagged_text, seed=0, prompt_k=0, stitch=True, pause_mode="clause", speed=1.10, return_info=False):
+        if pause_mode == "clause":
+            return self.say_clause(tagged_text, seed, prompt_k, speed, return_info)
+        w = self.say_model(tagged_text, seed, prompt_k, stitch)
+        w = time_stretch(w, self.sr, speed)
+        return (w, None) if return_info else w
+
+    def say_clause(self, tagged_text, seed=0, prompt_k=0, speed=1.10, return_info=False):
+        rng = np.random.default_rng(seed)
+        chunks = clause_chunks(tagged_text)
+        out, info, lead = [], [], 0.03
+        for i, (l, t, b) in enumerate(chunks):
+            w = self.one(f"<{l}>{t}</{l}>", seed + 1000 * i, prompt_k)
+            w = cap_internal(trim_tight(w, self.sr), self.sr, self.cap)
+            w = fade(time_stretch(w, self.sr, speed), self.sr)
+            if i == 0:
+                out.append(np.zeros(int(0.12 * self.sr), np.float32))       # same 0.12 s lead as model mode
+            out.append(w)
+            if b is not None:
+                g = GAPS[b] + rng.uniform(-JITTER[b], JITTER[b])
+                out.append(np.zeros(max(0, int((g - 2 * lead) * self.sr)), np.float32))  # 30 ms kept each side
+            else:
+                out.append(np.zeros(int(0.15 * self.sr), np.float32))
+            info.append(dict(lang=l, text=t, boundary=b))
+        w = np.concatenate(out)
+        return (w, info) if return_info else w
+
+    def say_model(self, tagged_text, seed=0, prompt_k=0, stitch=True):
+        sents = split_sentences(tagged_text) if stitch else [tagged_text]
+        ws = [cap_internal(trim_edges(self.one(s, seed + 1000 * i, prompt_k), self.sr), self.sr, self.cap)
+              for i, s in enumerate(sents)]
+        if len(ws) == 1:
+            return ws[0]
+        out = []
+        for i, w in enumerate(ws):
+            if i:
+                out.append(np.zeros(max(0, int((self.sent_pause - 0.30) * self.sr)), np.float32))
+            out.append(w)
+        return np.concatenate(out)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--text", required=True, help="tagged text, e.g. '<eng>Good morning,</eng> <pcm>how body?</pcm>'")
+    ap.add_argument("--seed", type=int, default=0)
+    ap.add_argument("--out", default="out.wav")
+    ap.add_argument("--prompt_clip", type=int, default=0, help="anchor prompt clip 0-3 (final system: 0)")
+    ap.add_argument("--model_dir", default=os.environ.get("COSYVOICE2_DIR", os.path.join(HERE, "pretrained", "CosyVoice2-0.5B")),
+                    help="stock FunAudioLLM/CosyVoice2-0.5B snapshot")
+    ap.add_argument("--cosyvoice_dir", default=os.environ.get("COSYVOICE_REPO", os.path.join(HERE, "CosyVoice")),
+                    help="clone of github.com/FunAudioLLM/CosyVoice (with submodules)")
+    ap.add_argument("--lm", default=os.path.join(HERE, "checkpoints", "p2_lm_B", "best.pt"))
+    ap.add_argument("--flow", default=os.path.join(HERE, "checkpoints", "p2_flow_sft", "best.pt"))
+    ap.add_argument("--prompts", default=os.path.join(HERE, "prompts", "prompts.json"))
+    ap.add_argument("--dtype", default="auto", choices=["auto", "bf16", "fp16"],
+                    help="LM autocast dtype. Samples were made with bf16; T4 has no bf16, so auto picks fp16 there")
+    ap.add_argument("--no_stitch", action="store_true", help="generate multi-sentence text in one pass")
+    ap.add_argument("--pause_mode", default="clause", choices=["clause", "model"],
+                    help="clause: clause-level chunks joined with anchor-matched gaps (default); model: Phase 2 final behaviour")
+    ap.add_argument("--speed", type=float, default=DEFAULT_SPEED,
+                    help="pitch-preserving tempo factor applied to speech (gaps are kept as set). 1.0 = Phase 2 final pace")
+    a = ap.parse_args()
+    if not os.path.isdir(a.model_dir):
+        from huggingface_hub import snapshot_download
+        snapshot_download("FunAudioLLM/CosyVoice2-0.5B", local_dir=a.model_dir)
+    S = DynamicAccent(a.model_dir, a.cosyvoice_dir, a.lm, a.flow, a.prompts, a.dtype)
+    w = S.say(a.text, seed=a.seed, prompt_k=a.prompt_clip, stitch=not a.no_stitch, pause_mode=a.pause_mode, speed=a.speed)
+    sf.write(a.out, w, S.sr)
+    print(f"wrote {a.out} ({len(w) / S.sr:.2f} s, {S.sr} Hz)")
+
+
+if __name__ == "__main__":
+    main()

prompts/clip0_waxal_yor_yor_1258.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7e368595eb4c267f124164c7d8a83b03462d549c5efd235dd8ec6eb557dc418
+size 285164

prompts/clip1_waxal_yor_yor_1205.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8731a9032a1e84a88a2494b65a576794f732cf5c8555c1b94af1a5d370ad7c9
+size 314924

prompts/clip2_waxal_yor_yor_1263.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cf10737379673b123cee008f78006f4fa2fdabc779563d466f85f0772b9cec5
+size 194924

prompts/clip3_waxal_yor_yor_1218.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0e00a437713a35429fe9423c3acf4ec668038e418fc7b85391bec53cb2368a
+size 262124

prompts/prompts.json

@@ -0,0 +1,602 @@
+{
+ "anchor_speaker": "waxal_yor_4",
+ "source": "WAXAL (CC-BY-SA-4.0), anchor val clips, pause profile B (~/data3)",
+ "prompts": [
+  {
+   "index": 0,
+   "id": "waxal_yor_yor_1258",
+   "wav": "prompts/clip0_waxal_yor_yor_1258.wav",
+   "lang": "yor",
+   "text": "Púpọ̀ jùlọ àwọn olókìkí ní ìlú Èkó ń gbé ní erékùṣù náà.",
+   "tagged": "<yor>Púpọ̀ jùlọ àwọn olókìkí ní ìlú Èkó ń gbé ní erékùṣù náà.</yor>",
+   "dur": 5.94,
+   "s3_tokens": [
+    3677,
+    4137,
+    4218,
+    2166,
+    1452,
+    5830,
+    2175,
+    2112,
+    696,
+    5027,
+    4834,
+    2344,
+    1860,
+    5973,
+    1939,
+    1862,
+    1558,
+    3880,
+    316,
+    2621,
+    2864,
+    2864,
+    2891,
+    5051,
+    3412,
+    3888,
+    3888,
+    1458,
+    1701,
+    1950,
+    1973,
+    1946,
+    3890,
+    4133,
+    692,
+    3617,
+    4346,
+    3617,
+    5030,
+    4955,
+    2768,
+    4955,
+    2789,
+    2006,
+    2903,
+    3395,
+    3771,
+    6013,
+    3853,
+    5096,
+    5095,
+    1350,
+    978,
+    2298,
+    5586,
+    1122,
+    1788,
+    2598,
+    295,
+    566,
+    51,
+    3669,
+    3748,
+    2938,
+    4751,
+    4750,
+    4741,
+    2006,
+    1349,
+    1835,
+    1510,
+    1588,
+    2180,
+    5581,
+    4606,
+    2330,
+    1127,
+    573,
+    660,
+    1364,
+    617,
+    2290,
+    5209,
+    2077,
+    2076,
+    2184,
+    967,
+    4616,
+    4607,
+    4526,
+    4769,
+    2591,
+    4526,
+    4579,
+    2446,
+    4137,
+    4137,
+    4137,
+    1959,
+    3670,
+    3671,
+    3751,
+    1479,
+    4769,
+    5021,
+    4994,
+    4751,
+    5561,
+    5561,
+    5236,
+    3255,
+    4218,
+    4218,
+    4218,
+    4137,
+    5352,
+    6081,
+    5379,
+    1006,
+    1034,
+    4679,
+    2960,
+    2411,
+    1207,
+    860,
+    4760,
+    233,
+    1407,
+    654,
+    3567,
+    5826,
+    6310,
+    5481,
+    5563,
+    3348,
+    6556,
+    2102,
+    886,
+    1563,
+    1479,
+    2300,
+    4547,
+    4546,
+    2522,
+    263,
+    263,
+    1225,
+    1953,
+    1947
+   ]
+  },
+  {
+   "index": 1,
+   "id": "waxal_yor_yor_1205",
+   "wav": "prompts/clip1_waxal_yor_yor_1205.wav",
+   "lang": "yor",
+   "text": "Mejìlá ní ìsọdipúpọ̀ nípasẹ̀ mẹ́rin dọ́gba mejì-mejìlá.",
+   "tagged": "<yor>Mejìlá ní ìsọdipúpọ̀ nípasẹ̀ mẹ́rin dọ́gba mejì-mejìlá.</yor>",
+   "dur": 6.56,
+   "s3_tokens": [
+    3677,
+    1968,
+    1804,
+    1884,
+    1883,
+    2312,
+    4579,
+    3021,
+    1041,
+    5250,
+    4858,
+    80,
+    3799,
+    1532,
+    731,
+    6158,
+    4486,
+    3669,
+    5938,
+    3748,
+    4742,
+    4751,
+    4751,
+    4751,
+    4714,
+    5605,
+    1951,
+    2031,
+    1978,
+    3481,
+    1268,
+    3721,
+    4672,
+    4999,
+    4752,
+    586,
+    2894,
+    2413,
+    1608,
+    2347,
+    1940,
+    1410,
+    2139,
+    6558,
+    5829,
+    2148,
+    2112,
+    1410,
+    5099,
+    5081,
+    2894,
+    707,
+    518,
+    1308,
+    1950,
+    4131,
+    3645,
+    1707,
+    1959,
+    1987,
+    1753,
+    1484,
+    5857,
+    1482,
+    4571,
+    3637,
+    4299,
+    1383,
+    3574,
+    4214,
+    3647,
+    56,
+    5401,
+    6374,
+    1035,
+    32,
+    2303,
+    2303,
+    227,
+    2127,
+    6502,
+    1882,
+    4742,
+    4490,
+    4579,
+    4550,
+    3016,
+    1724,
+    377,
+    377,
+    2456,
+    2456,
+    25,
+    3903,
+    3975,
+    4137,
+    4218,
+    4137,
+    4137,
+    6324,
+    6324,
+    6324,
+    1968,
+    3913,
+    1727,
+    1807,
+    1110,
+    2666,
+    5078,
+    5078,
+    1125,
+    2157,
+    1452,
+    389,
+    5186,
+    2999,
+    2513,
+    2270,
+    3242,
+    5429,
+    5105,
+    5103,
+    2922,
+    1464,
+    1950,
+    4218,
+    1987,
+    1987,
+    1987,
+    1804,
+    6256,
+    1639,
+    2573,
+    4849,
+    4696,
+    1509,
+    1779,
+    4440,
+    5911,
+    4857,
+    4777,
+    323,
+    323,
+    2445,
+    978,
+    1950,
+    1968,
+    1996,
+    1885,
+    1882,
+    4760,
+    4696,
+    3966,
+    5658,
+    4431,
+    5506,
+    78,
+    5986,
+    74,
+    1460,
+    1460,
+    982,
+    1701,
+    1947
+   ]
+  },
+  {
+   "index": 2,
+   "id": "waxal_yor_yor_1263",
+   "wav": "prompts/clip2_waxal_yor_yor_1263.wav",
+   "lang": "yor",
+   "text": "Ọmọ ọdún méji ni Nóla ní oṣù kẹwàá.",
+   "tagged": "<yor>Ọmọ ọdún méji ni Nóla ní oṣù kẹwàá.</yor>",
+   "dur": 4.06,
+   "s3_tokens": [
+    3677,
+    4218,
+    2057,
+    4217,
+    2762,
+    2127,
+    4314,
+    2852,
+    2771,
+    572,
+    4835,
+    963,
+    1914,
+    954,
+    6554,
+    5041,
+    6502,
+    4068,
+    2420,
+    4858,
+    2265,
+    1995,
+    2022,
+    4440,
+    5262,
+    5506,
+    323,
+    753,
+    1563,
+    750,
+    323,
+    2267,
+    2591,
+    2321,
+    2240,
+    4670,
+    2455,
+    4389,
+    2922,
+    3648,
+    4137,
+    1950,
+    1987,
+    1753,
+    1481,
+    1480,
+    3745,
+    6012,
+    1073,
+    4949,
+    5054,
+    2297,
+    5905,
+    5986,
+    2216,
+    5915,
+    5186,
+    5186,
+    4469,
+    2916,
+    3645,
+    1458,
+    1464,
+    4218,
+    4137,
+    1725,
+    3670,
+    3831,
+    2941,
+    5102,
+    5561,
+    6074,
+    5234,
+    2012,
+    2006,
+    2096,
+    2912,
+    5828,
+    2990,
+    6051,
+    6054,
+    1588,
+    2072,
+    2139,
+    1734,
+    2355,
+    2528,
+    4571,
+    2384,
+    881,
+    644,
+    1456,
+    2185,
+    707,
+    1946,
+    812,
+    3242,
+    5914,
+    4545,
+    1711,
+    2031,
+    6405
+   ]
+  },
+  {
+   "index": 3,
+   "id": "waxal_yor_yor_1218",
+   "wav": "prompts/clip3_waxal_yor_yor_1218.wav",
+   "lang": "yor",
+   "text": "Ìdajì àti ọ̀kan kẹ́fà dọ́gba méji kẹ́ta.",
+   "tagged": "<yor>Ìdajì àti ọ̀kan kẹ́fà dọ́gba méji kẹ́ta.</yor>",
+   "dur": 5.46,
+   "s3_tokens": [
+    3707,
+    2031,
+    566,
+    566,
+    4778,
+    861,
+    1770,
+    68,
+    3083,
+    5996,
+    4570,
+    777,
+    3939,
+    4683,
+    5169,
+    5911,
+    3239,
+    1052,
+    286,
+    259,
+    5109,
+    2922,
+    2922,
+    3894,
+    3651,
+    2031,
+    1976,
+    3161,
+    3890,
+    3890,
+    2219,
+    1527,
+    2085,
+    6380,
+    6131,
+    5401,
+    1038,
+    2267,
+    4697,
+    4697,
+    4778,
+    4697,
+    4778,
+    5507,
+    5507,
+    5507,
+    3050,
+    3255,
+    3894,
+    4218,
+    4137,
+    1979,
+    1304,
+    1385,
+    2897,
+    601,
+    4299,
+    2031,
+    2754,
+    4869,
+    5030,
+    5030,
+    665,
+    497,
+    1723,
+    2112,
+    762,
+    2271,
+    2213,
+    4523,
+    4853,
+    6372,
+    6534,
+    6534,
+    688,
+    650,
+    1055,
+    1784,
+    3161,
+    2917,
+    2916,
+    1458,
+    1458,
+    1461,
+    2031,
+    1807,
+    1726,
+    1806,
+    1806,
+    873,
+    2657,
+    5081,
+    4852,
+    1941,
+    1888,
+    2181,
+    218,
+    3242,
+    2999,
+    2999,
+    3242,
+    3170,
+    2926,
+    3645,
+    3645,
+    1701,
+    2031,
+    1959,
+    1727,
+    1727,
+    3994,
+    6256,
+    1883,
+    4841,
+    4848,
+    2346,
+    1779,
+    1284,
+    4509,
+    5181,
+    1943,
+    1923,
+    1707,
+    168,
+    4751,
+    2330,
+    62,
+    67,
+    1707,
+    2058,
+    54,
+    2189,
+    3242,
+    2927,
+    2925,
+    1944,
+    4218
+   ]
+  }
+ ]
+}

requirements.txt

@@ -0,0 +1,33 @@
+# Python 3.10. Install torch/torchaudio for your CUDA first (Colab already has them), then:
+#   pip install -r requirements.txt
+# and clone CosyVoice (see README). Versions below are the ones the samples were rendered with
+# (pins loosened where Colab's preinstalled versions also work).
+torch>=2.3
+torchaudio>=2.3
+transformers==4.51.3
+huggingface_hub>=0.23
+numpy==1.26.4
+soundfile>=0.12
+librosa>=0.10
+onnxruntime-gpu>=1.18
+HyperPyYAML>=1.2.2
+modelscope>=1.20
+conformer==0.3.2
+diffusers==0.29.0
+lightning>=2.2.4
+omegaconf>=2.3.0
+hydra-core==1.3.2
+inflect>=7.3
+x-transformers>=2.11
+openai-whisper>=20231117
+einops>=0.8
+pyworld>=0.3.4
+pyarrow>=18
+wget==3.2
+gdown>=5.1
+rich>=13.7
+matplotlib>=3.7
+protobuf>=4.25
+pydantic>=2.7
+wetext==0.0.4
+audiotsm==0.1.2   # WSOLA time-stretch for --speed (falls back to librosa if missing)

samples/prosody_v2/para_hau_eng_pcm.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce6ab62b8647c8325ffb8ba499ea2b9ec3db623aa3e68aaafe5667225751abcb
+size 289964

samples/prosody_v2/para_ibo_pcm_eng.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa85fb1f58a0778a7cc0027b4d9919ed36537f518054b7f9f8b3f9a9a5cff232
+size 291116

samples/prosody_v2/para_yor_eng.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2a0f0280ac149d9691975ba6f1e8c763c9d91a365632202d5b1a42b675e36c3
+size 241196