FireRedTTS3 demo: cloning, voice design, speech editing

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/en_prompt.wav filter=lfs diff=lfs merge=lfs -text
+examples/zh_prompt.wav filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,13 +1,43 @@
 ---
-title: Firered Tts3
+title: FireRedTTS3
 emoji: 🔥
-colorFrom: purple
-colorTo: yellow
+colorFrom: red
+colorTo: pink
 sdk: gradio
 sdk_version: 6.24.0
-python_version: '3.12'
 app_file: app.py
+python_version: "3.12"
+startup_duration_timeout: 1h
 pinned: false
+license: apache-2.0
+short_description: Voice cloning, voice design and speech editing
+models:
+  - FireRedTeam/FireRedTTS3
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🔥 FireRedTTS3
+
+Interactive demo of [FireRedTeam/FireRedTTS3](https://huggingface.co/FireRedTeam/FireRedTTS3),
+a unified speech generation and editing model (Qwen3-1.7B backbone + flow-matching
+head over the continuous RedAE audio autoencoder, 24 kHz).
+
+Three tabs:
+
+| Tab | Checkpoint | What it does |
+| --- | --- | --- |
+| 🎙️ Voice Cloning | `fireredtts3_base` | Zero-shot cloning from a short reference clip — 24 languages and 21 Chinese dialects |
+| 🎨 Voice Design | `fireredtts3_instruct` | Creates a voice from a natural-language description, no reference audio |
+| ✂️ Speech Editing | `fireredtts3_instruct` | Semantic edits (insert / delete / substitute words) and acoustic edits (speed, pitch, volume) |
+
+Text normalization runs locally via `wetext` (Chinese / English); automatic language
+routing uses fastText `lid.176`. The optional LLM-based normalizer from the upstream
+repo is disabled here (it requires external API credentials).
+
+## Credits
+
+- Model and inference code: [FireRedTeam/FireRedTTS3](https://github.com/FireRedTeam/FireRedTTS3), Apache-2.0.
+- `examples/en_prompt.wav`: from [OpenBMB/VoxCPM](https://github.com/OpenBMB/VoxCPM) (`examples/example.wav`), Apache-2.0.
+- `examples/zh_prompt.wav`: from [FireRedTeam/FireRedTTS2](https://github.com/FireRedTeam/FireRedTTS2) (`examples/chat_prompt/zh/S2.flac`), Apache-2.0.
+
+Voice cloning is provided for academic research purposes only — do not use it for
+impersonation or any illegal activity.

app.py

@@ -0,0 +1,576 @@
+"""FireRedTTS3 — unified speech generation & editing demo (ZeroGPU).
+
+Three capabilities of https://huggingface.co/FireRedTeam/FireRedTTS3 :
+  * Zero-shot voice cloning  (FireRedTTS3-Base,     24 languages + 21 ZH dialects)
+  * Voice design             (FireRedTTS3-Instruct, natural-language timbre prompt)
+  * Speech editing           (FireRedTTS3-Instruct, semantic + acoustic)
+"""
+
+import functools
+import os
+import urllib.request
+
+import spaces  # noqa: F401  (must precede torch / CUDA imports)
+
+import numpy as np
+import soundfile as sf
+import torch
+import gradio as gr
+from huggingface_hub import snapshot_download
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+# --------------------------------------------------------------------------- #
+# Text front-end assets
+# --------------------------------------------------------------------------- #
+# fastText lid.176 powers automatic language routing (see upstream README).
+_LID_URL = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz"
+_LID_PATH = os.path.join(HERE, "fireredtts3", "utils", "llm_tn", "models", "lid.176.ftz")
+os.makedirs(os.path.dirname(_LID_PATH), exist_ok=True)
+if not os.path.exists(_LID_PATH):
+    try:
+        urllib.request.urlretrieve(_LID_URL, _LID_PATH)
+        print(f"[INFO] fastText lid.176 downloaded to {_LID_PATH}", flush=True)
+    except Exception as exc:  # pragma: no cover
+        print(f"[WARN] Could not fetch fastText lid.176: {exc}", flush=True)
+
+# The upstream llm_tn TextNormalizer refuses to construct without API creds, and
+# the fastText language detector lives on that object. We only use it for
+# language *identification* (use_llm_tn=False -> local wetext TN), so give it
+# placeholder creds and disable its LLM fallback further below.
+os.environ.setdefault("LLM_TN_API_URL", "http://127.0.0.1:1/unused")
+os.environ.setdefault("LLM_TN_API_KEY", "unused")
+
+# --------------------------------------------------------------------------- #
+# Weights
+# --------------------------------------------------------------------------- #
+MODEL_REPO = "FireRedTeam/FireRedTTS3"
+MODEL_DIR = snapshot_download(MODEL_REPO)
+print(f"[INFO] weights at {MODEL_DIR}", flush=True)
+
+from fireredtts3.core import FireRedTTS3, FireRedTTS3Instruct  # noqa: E402
+from fireredtts3.redae.redae import RedAE  # noqa: E402
+from fireredtts3.utils.llm_tn.text_normalizer import TextNormalizer  # noqa: E402
+from fireredtts3.utils.text_tokenizer import (  # noqa: E402
+    MULTI_DIALECT_TAGS,
+    MULTI_LANG_TAGS,
+)
+
+# Base and Instruct each instantiate their own RedAE from the very same
+# checkpoint; share one instance instead (~3.8 GB saved, identical numerics).
+_redae_real_from_pretrained = RedAE.from_pretrained
+_redae_singleton = None
+
+
+def _shared_redae(*args, **kwargs):
+    global _redae_singleton
+    if _redae_singleton is None:
+        _redae_singleton = _redae_real_from_pretrained(*args, **kwargs)
+    return _redae_singleton
+
+
+RedAE.from_pretrained = _shared_redae
+
+tts = FireRedTTS3(MODEL_DIR, use_fasttext=True, use_llm_tn=False, use_wetext=True)
+instruct = FireRedTTS3Instruct(MODEL_DIR, use_fasttext=True, use_llm_tn=False, use_wetext=True)
+
+for _pipe in (tts, instruct):
+    _norm = getattr(_pipe, "_llm_tn", None)
+    if _norm is not None:
+        # No API creds here -> never let language ID fall back to an LLM call.
+        _norm.detect_locale = functools.partial(
+            TextNormalizer.detect_locale, _norm, use_llm_fallback=False
+        )
+print("[INFO] FireRedTTS3 Base + Instruct ready", flush=True)
+
+SAMPLE_RATE = tts.redae.sample_rate
+
+# --------------------------------------------------------------------------- #
+# Language choices
+# --------------------------------------------------------------------------- #
+AUTO = "Auto-detect"
+LANGUAGES = [t.strip("<|>") for t in MULTI_LANG_TAGS]
+DIALECTS = [t.strip("<|>") for t in MULTI_DIALECT_TAGS]
+LANG_CHOICES = (
+    [AUTO]
+    + LANGUAGES
+    + [f"{d} (Chinese dialect)" for d in DIALECTS]
+)
+
+
+def _resolve_language(choice: str):
+    if not choice or choice == AUTO:
+        return None
+    return choice.split(" (")[0]
+
+
+# --------------------------------------------------------------------------- #
+# Audio helpers
+# --------------------------------------------------------------------------- #
+MAX_PROMPT_SECONDS = 20.0
+MAX_EDIT_SECONDS = 20.0
+MAX_TEXT_CHARS = 400
+
+
+def _load_audio(path: str, max_seconds: float):
+    if not path:
+        raise gr.Error("Please provide an audio file first.")
+    wav, sr = sf.read(path, always_2d=True, dtype="float32")
+    wav = wav[:, 0]
+    if wav.shape[0] > int(max_seconds * sr):
+        wav = wav[: int(max_seconds * sr)]
+        gr.Info(f"Audio truncated to the first {max_seconds:.0f}s.")
+    peak = float(np.abs(wav).max()) if wav.size else 0.0
+    if peak > 0:
+        wav = wav / peak * 0.95
+    return torch.from_numpy(np.ascontiguousarray(wav)[None, :]), sr
+
+
+def _to_gradio_audio(audio: torch.Tensor, sr: int):
+    x = audio.detach().float().cpu().numpy()
+    if x.ndim > 1:
+        x = x[0]
+    x = np.clip(x, -1.0, 1.0)
+    return sr, (x * 32767.0).astype(np.int16)
+
+
+def _check_text(text: str, what: str = "Text"):
+    text = (text or "").strip()
+    if not text:
+        raise gr.Error(f"{what} must not be empty.")
+    if len(text) > MAX_TEXT_CHARS:
+        gr.Info(f"{what} truncated to {MAX_TEXT_CHARS} characters.")
+        text = text[:MAX_TEXT_CHARS]
+    return text
+
+
+# --------------------------------------------------------------------------- #
+# Inference
+# --------------------------------------------------------------------------- #
+@spaces.GPU(duration=150)
+def voice_clone(
+    prompt_audio,
+    prompt_text,
+    text,
+    language=AUTO,
+    inference_cfg=2.0,
+    n_timesteps=10,
+    seed=1234,
+    do_tn=True,
+):
+    """FireRedTTS3-Base zero-shot voice cloning."""
+    text = _check_text(text, "Text to synthesize")
+    prompt_text = (prompt_text or "").strip()
+    if not prompt_text:
+        raise gr.Error("Please provide the transcript of the reference audio.")
+    wav, sr = _load_audio(prompt_audio, MAX_PROMPT_SECONDS)
+
+    gen_audio, gen_sr = tts.generate(
+        text=text,
+        language=_resolve_language(language),
+        prompt_text=prompt_text,
+        prompt_audio=wav,
+        prompt_audio_sr=sr,
+        n_timesteps=int(n_timesteps),
+        inference_cfg=float(inference_cfg),
+        seed=int(seed),
+        do_tn=bool(do_tn),
+    )
+    return _to_gradio_audio(gen_audio, gen_sr)
+
+
+@spaces.GPU(duration=150)
+def voice_design(
+    instruction,
+    text,
+    inference_cfg=1.2,
+    n_timesteps=10,
+    seed=2,
+    do_tn=True,
+):
+    """FireRedTTS3-Instruct voice design (no reference audio)."""
+    instruction = _check_text(instruction, "Voice description")
+    text = _check_text(text, "Text to synthesize")
+
+    gen_audio, gen_sr, gen_text = instruct.generate_voice_design(
+        instruction=instruction,
+        text=text,
+        n_timesteps=int(n_timesteps),
+        inference_cfg=float(inference_cfg),
+        seed=int(seed),
+        do_tn=bool(do_tn),
+    )
+    return _to_gradio_audio(gen_audio, gen_sr), (gen_text or "").strip()
+
+
+@spaces.GPU(duration=150)
+def semantic_edit(
+    audio_in,
+    instruction,
+    inference_cfg=1.2,
+    n_timesteps=10,
+    seed=1234,
+):
+    """FireRedTTS3-Instruct content editing: insert / delete / substitute."""
+    instruction = _check_text(instruction, "Edit instruction")
+    wav, sr = _load_audio(audio_in, MAX_EDIT_SECONDS)
+
+    gen_audio, gen_sr, gen_text = instruct.generate_semantic_edit(
+        instruction=instruction,
+        audio_in=wav,
+        audio_in_sr=sr,
+        n_timesteps=int(n_timesteps),
+        inference_cfg=float(inference_cfg),
+        seed=int(seed),
+    )
+    return _to_gradio_audio(gen_audio, gen_sr), (gen_text or "").strip()
+
+
+def compose_acoustic_instruction(attribute: str, value: float) -> str:
+    """Acoustic edits only accept the templates the model was trained on."""
+    if attribute == "Speed":
+        return f"adjust the speed to {value:.1f}x"
+    if attribute == "Volume":
+        return f"adjust the volume to {value:.1f}"
+    steps = int(round(value))
+    return f"shift the pitch by {steps} step{'' if abs(steps) == 1 else 's'}"
+
+
+@spaces.GPU(duration=150)
+def acoustic_edit(
+    audio_in,
+    attribute="Speed",
+    value=0.8,
+    inference_cfg=1.2,
+    n_timesteps=10,
+    seed=1234,
+):
+    """FireRedTTS3-Instruct acoustic editing: speed / pitch / volume."""
+    wav, sr = _load_audio(audio_in, MAX_EDIT_SECONDS)
+    instruction = compose_acoustic_instruction(attribute, float(value))
+
+    gen_audio, gen_sr = instruct.generate_acoustic_edit(
+        instruction=instruction,
+        audio_in=wav,
+        audio_in_sr=sr,
+        n_timesteps=int(n_timesteps),
+        inference_cfg=float(inference_cfg),
+        seed=int(seed),
+    )
+    return _to_gradio_audio(gen_audio, gen_sr), instruction
+
+
+# --------------------------------------------------------------------------- #
+# UI
+# --------------------------------------------------------------------------- #
+EN_PROMPT = os.path.join(HERE, "examples", "en_prompt.wav")
+ZH_PROMPT = os.path.join(HERE, "examples", "zh_prompt.wav")
+EN_PROMPT_TEXT = (
+    "Just by listening a few minutes a day, you'll be able to eliminate negative "
+    "thoughts by conditioning your mind to be more positive."
+)
+ZH_PROMPT_TEXT = "比如具体一点的，他觉得最大的一个跟他预想的不一样的是在什么地方。"
+
+CSS = """
+.gradio-container {max-width: 1200px !important; margin: auto !important;}
+.dark .gradio-container {color: var(--body-text-color);}
+"""
+
+with gr.Blocks(theme=gr.themes.Citrus(), css=CSS, title="FireRedTTS3") as demo:
+    gr.Markdown(
+        """
+        # 🔥 FireRedTTS3 — Unified Speech Generation & Editing
+        Zero-shot voice cloning in **24 languages + 21 Chinese dialects**, natural-language
+        **voice design**, and instruction-driven **speech editing** — all from
+        [FireRedTeam/FireRedTTS3](https://huggingface.co/FireRedTeam/FireRedTTS3).
+        """
+    )
+
+    with gr.Tabs():
+        # ------------------------------------------------------------------ #
+        with gr.Tab("🎙️ Voice Cloning"):
+            gr.Markdown(
+                "Clone any voice from a short reference clip. For best quality the "
+                "reference should be spoken in the **same language / dialect** as the "
+                "text you synthesize."
+            )
+            with gr.Row():
+                with gr.Column():
+                    clone_prompt_audio = gr.Audio(
+                        label="Reference audio (5–20 s)",
+                        sources=["upload", "microphone"],
+                        type="filepath",
+                    )
+                    clone_prompt_text = gr.Textbox(
+                        label="Reference transcript",
+                        placeholder="Exactly what is said in the reference audio…",
+                        lines=2,
+                    )
+                    clone_text = gr.Textbox(
+                        label="Text to synthesize",
+                        placeholder="Type the text you want spoken in that voice…",
+                        lines=4,
+                    )
+                    clone_language = gr.Dropdown(
+                        LANG_CHOICES, value=AUTO, label="Language / dialect"
+                    )
+                    clone_btn = gr.Button("Generate speech", variant="primary")
+                with gr.Column():
+                    clone_out = gr.Audio(label="Generated speech", type="numpy")
+                    with gr.Accordion("Advanced options", open=False):
+                        clone_cfg = gr.Slider(
+                            0.0, 4.0, value=2.0, step=0.1,
+                            label="CFG strength",
+                            info="Higher sticks closer to the reference timbre.",
+                        )
+                        clone_steps = gr.Slider(
+                            4, 30, value=10, step=1, label="Flow-matching timesteps"
+                        )
+                        clone_seed = gr.Number(value=1234, precision=0, label="Seed")
+                        clone_tn = gr.Checkbox(
+                            value=True,
+                            label="Text normalization (numbers, dates, units → words)",
+                        )
+            gr.Examples(
+                examples=[
+                    [
+                        EN_PROMPT,
+                        EN_PROMPT_TEXT,
+                        "FireRedTTS3 turns a handful of seconds of speech into a voice "
+                        "that can read anything you write.",
+                        "English",
+                    ],
+                    [
+                        ZH_PROMPT,
+                        ZH_PROMPT_TEXT,
+                        "法院与不动产登记部门加强沟通，并督促银行提前办理抵押预约登记。",
+                        "Chinese",
+                    ],
+                    [
+                        EN_PROMPT,
+                        EN_PROMPT_TEXT,
+                        "Le modèle peut aussi parler français avec la même voix de référence.",
+                        "French",
+                    ],
+                ],
+                inputs=[clone_prompt_audio, clone_prompt_text, clone_text, clone_language],
+                outputs=[clone_out],
+                fn=voice_clone,
+                cache_examples=True,
+                cache_mode="lazy",
+            )
+
+        # ------------------------------------------------------------------ #
+        with gr.Tab("🎨 Voice Design"):
+            gr.Markdown(
+                "Describe a voice in plain language — no reference audio needed. The "
+                "model first writes a voice-attribute plan, then renders the audio."
+            )
+            with gr.Row():
+                with gr.Column():
+                    design_instruction = gr.Textbox(
+                        label="Voice description",
+                        placeholder="e.g. A young woman with a gentle voice, speaking slowly…",
+                        lines=3,
+                    )
+                    design_text = gr.Textbox(
+                        label="Text to synthesize", lines=4,
+                        placeholder="Type the text you want spoken…",
+                    )
+                    design_btn = gr.Button("Design voice", variant="primary")
+                with gr.Column():
+                    design_out = gr.Audio(label="Generated speech", type="numpy")
+                    design_plan = gr.Textbox(
+                        label="Voice-attribute plan (model chain-of-thought)", lines=4
+                    )
+                    with gr.Accordion("Advanced options", open=False):
+                        design_cfg = gr.Slider(
+                            0.0, 4.0, value=1.2, step=0.1, label="CFG strength"
+                        )
+                        design_steps = gr.Slider(
+                            4, 30, value=10, step=1, label="Flow-matching timesteps"
+                        )
+                        design_seed = gr.Number(value=2, precision=0, label="Seed")
+                        design_tn = gr.Checkbox(value=True, label="Text normalization")
+            gr.Examples(
+                examples=[
+                    [
+                        "一个年轻女性的温柔嗓音，语速稍慢，带一点俏皮。",
+                        "今天天气很好，我们一起去公园散步吧。",
+                    ],
+                    [
+                        "An old sailor with a deep, gravelly voice, speaking slowly and "
+                        "warmly, as if telling a story by the fire.",
+                        "The sea was calm that morning, and every rope on deck was "
+                        "still wet with salt.",
+                    ],
+                    [
+                        "A bright, energetic young man hosting a sports broadcast, fast "
+                        "paced and excited.",
+                        "And with ten seconds left on the clock, he takes the shot — "
+                        "and it is in!",
+                    ],
+                ],
+                inputs=[design_instruction, design_text],
+                outputs=[design_out, design_plan],
+                fn=voice_design,
+                cache_examples=True,
+                cache_mode="lazy",
+            )
+
+        # ------------------------------------------------------------------ #
+        with gr.Tab("✂️ Speech Editing"):
+            with gr.Tabs():
+                with gr.Tab("Semantic (content)"):
+                    gr.Markdown(
+                        "Insert, delete or substitute words in an existing recording "
+                        "while keeping the original voice. The model transcribes the "
+                        "audio itself — just say what to change."
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            sem_audio = gr.Audio(
+                                label="Input speech (≤ 20 s)",
+                                sources=["upload", "microphone"],
+                                type="filepath",
+                            )
+                            sem_instruction = gr.Textbox(
+                                label="Edit instruction",
+                                placeholder="e.g. Replace 'negative' with 'anxious'.",
+                                lines=2,
+                            )
+                            sem_btn = gr.Button("Apply edit", variant="primary")
+                        with gr.Column():
+                            sem_out = gr.Audio(label="Edited speech", type="numpy")
+                            sem_text = gr.Textbox(label="Edited transcript", lines=3)
+                            with gr.Accordion("Advanced options", open=False):
+                                sem_cfg = gr.Slider(
+                                    0.0, 4.0, value=1.2, step=0.1, label="CFG strength"
+                                )
+                                sem_steps = gr.Slider(
+                                    4, 30, value=10, step=1,
+                                    label="Flow-matching timesteps",
+                                )
+                                sem_seed = gr.Number(
+                                    value=1234, precision=0, label="Seed"
+                                )
+                    gr.Examples(
+                        examples=[
+                            [EN_PROMPT, "Replace 'negative' with 'anxious'."],
+                            [EN_PROMPT, "Delete the words 'a few minutes a day'."],
+                            [ZH_PROMPT, "把“最大的”替换成“最有意思的”。"],
+                        ],
+                        inputs=[sem_audio, sem_instruction],
+                        outputs=[sem_out, sem_text],
+                        fn=semantic_edit,
+                        cache_examples=True,
+                        cache_mode="lazy",
+                    )
+
+                with gr.Tab("Acoustic (speed / pitch / volume)"):
+                    gr.Markdown(
+                        "Re-render the same utterance with a different speaking rate, "
+                        "pitch or loudness. These edits follow fixed instruction "
+                        "templates the model was trained on."
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            aco_audio = gr.Audio(
+                                label="Input speech (≤ 20 s)",
+                                sources=["upload", "microphone"],
+                                type="filepath",
+                            )
+                            aco_attr = gr.Radio(
+                                ["Speed", "Pitch", "Volume"],
+                                value="Speed",
+                                label="Attribute",
+                            )
+                            aco_value = gr.Slider(
+                                0.5, 2.0, value=0.8, step=0.1,
+                                label="Speed (×)",
+                            )
+                            aco_btn = gr.Button("Apply edit", variant="primary")
+                        with gr.Column():
+                            aco_out = gr.Audio(label="Edited speech", type="numpy")
+                            aco_instruction = gr.Textbox(
+                                label="Instruction sent to the model", lines=1
+                            )
+                            with gr.Accordion("Advanced options", open=False):
+                                aco_cfg = gr.Slider(
+                                    0.0, 4.0, value=1.2, step=0.1, label="CFG strength"
+                                )
+                                aco_steps = gr.Slider(
+                                    4, 30, value=10, step=1,
+                                    label="Flow-matching timesteps",
+                                )
+                                aco_seed = gr.Number(
+                                    value=1234, precision=0, label="Seed"
+                                )
+                    gr.Examples(
+                        examples=[
+                            [EN_PROMPT, "Speed", 0.7],
+                            [ZH_PROMPT, "Pitch", 2],
+                            [EN_PROMPT, "Volume", 1.6],
+                        ],
+                        inputs=[aco_audio, aco_attr, aco_value],
+                        outputs=[aco_out, aco_instruction],
+                        fn=acoustic_edit,
+                        cache_examples=True,
+                        cache_mode="lazy",
+                    )
+
+    gr.Markdown(
+        """
+        ---
+        **Model:** [FireRedTeam/FireRedTTS3](https://huggingface.co/FireRedTeam/FireRedTTS3)
+        (Apache-2.0) · Base = cloning, Instruct = design + editing. Text normalization
+        runs locally through *wetext* (Chinese / English); other languages get basic
+        cleaning only. Voice cloning is provided **for academic research purposes only** —
+        do not use it for impersonation or any illegal activity.
+        """
+    )
+
+    def _attr_changed(attribute, current):
+        lo, hi, step, label = {
+            "Speed": (0.5, 2.0, 0.1, "Speed (×)"),
+            "Volume": (0.3, 2.0, 0.1, "Volume (×)"),
+        }.get(attribute, (-6, 6, 1, "Pitch shift (semitone steps)"))
+        try:
+            value = min(max(float(current), lo), hi)
+        except (TypeError, ValueError):
+            value = lo
+        if attribute == "Pitch":
+            value = int(round(value)) or 1
+        return gr.update(minimum=lo, maximum=hi, step=step, value=value, label=label)
+
+    aco_attr.change(_attr_changed, inputs=[aco_attr, aco_value], outputs=[aco_value])
+
+    clone_btn.click(
+        voice_clone,
+        inputs=[clone_prompt_audio, clone_prompt_text, clone_text, clone_language,
+                clone_cfg, clone_steps, clone_seed, clone_tn],
+        outputs=[clone_out],
+        api_name="voice_clone",
+    )
+    design_btn.click(
+        voice_design,
+        inputs=[design_instruction, design_text, design_cfg, design_steps,
+                design_seed, design_tn],
+        outputs=[design_out, design_plan],
+        api_name="voice_design",
+    )
+    sem_btn.click(
+        semantic_edit,
+        inputs=[sem_audio, sem_instruction, sem_cfg, sem_steps, sem_seed],
+        outputs=[sem_out, sem_text],
+        api_name="semantic_edit",
+    )
+    aco_btn.click(
+        acoustic_edit,
+        inputs=[aco_audio, aco_attr, aco_value, aco_cfg, aco_steps, aco_seed],
+        outputs=[aco_out, aco_instruction],
+        api_name="acoustic_edit",
+    )
+
+if __name__ == "__main__":
+    demo.queue().launch(mcp_server=True)

examples/en_prompt.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3eecb1e289b16e6038933b20d6d88a286f91b3f19e8fd1a734224a35e91747f
+size 391624

examples/zh_prompt.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18a82b883c6a18569cdb183a2b7763cbe7a7ee24c9046ed7edf4d28675118224
+size 252644

fireredtts3/campp/DTDNN.py

@@ -0,0 +1,123 @@
+# Copyright 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker). All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+
+from collections import OrderedDict
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+from fireredtts3.campp.layers import (
+    DenseLayer, 
+    StatsPool, 
+    TDNNLayer, 
+    CAMDenseTDNNBlock, 
+    TransitLayer, 
+    BasicResBlock, 
+    get_nonlinear,
+)
+
+
+class FCM(nn.Module):
+    def __init__(self,
+                block=BasicResBlock,
+                num_blocks=[2, 2],
+                m_channels=32,
+                feat_dim=80):
+        super(FCM, self).__init__()
+        self.in_planes = m_channels
+        self.conv1 = nn.Conv2d(1, m_channels, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(m_channels)
+
+        self.layer1 = self._make_layer(block, m_channels, num_blocks[0], stride=2)
+        self.layer2 = self._make_layer(block, m_channels, num_blocks[1], stride=2)
+
+        self.conv2 = nn.Conv2d(m_channels, m_channels, kernel_size=3, stride=(2, 1), padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(m_channels)
+        self.out_channels =  m_channels * (feat_dim // 8)
+
+    def _make_layer(self, block, planes, num_blocks, stride):
+        strides = [stride] + [1] * (num_blocks - 1)
+        layers = []
+        for stride in strides:
+            layers.append(block(self.in_planes, planes, stride))
+            self.in_planes = planes * block.expansion
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = x.unsqueeze(1)
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = self.layer1(out)
+        out = self.layer2(out)
+        out = F.relu(self.bn2(self.conv2(out)))
+
+        shape = out.shape
+        out = out.reshape(shape[0], shape[1]*shape[2], shape[3])
+        return out
+
+class CAMPPlus(nn.Module):
+    def __init__(self,
+                 feat_dim=80,
+                 embedding_size=512,
+                 growth_rate=32,
+                 bn_size=4,
+                 init_channels=128,
+                 config_str='batchnorm-relu',
+                 memory_efficient=True):
+        super(CAMPPlus, self).__init__()
+
+        self.head = FCM(feat_dim=feat_dim)
+        channels = self.head.out_channels
+
+        self.xvector = nn.Sequential(
+            OrderedDict([
+
+                ('tdnn',
+                 TDNNLayer(channels,
+                           init_channels,
+                           5,
+                           stride=2,
+                           dilation=1,
+                           padding=-1,
+                           config_str=config_str)),
+            ]))
+        channels = init_channels
+        for i, (num_layers, kernel_size,
+                dilation) in enumerate(zip((12, 24, 16), (3, 3, 3), (1, 2, 2))):
+            block = CAMDenseTDNNBlock(num_layers=num_layers,
+                                   in_channels=channels,
+                                   out_channels=growth_rate,
+                                   bn_channels=bn_size * growth_rate,
+                                   kernel_size=kernel_size,
+                                   dilation=dilation,
+                                   config_str=config_str,
+                                   memory_efficient=memory_efficient)
+            self.xvector.add_module('block%d' % (i + 1), block)
+            channels = channels + num_layers * growth_rate
+            self.xvector.add_module(
+                'transit%d' % (i + 1),
+                TransitLayer(channels,
+                             channels // 2,
+                             bias=False,
+                             config_str=config_str))
+            channels //= 2
+
+        self.xvector.add_module(
+            'out_nonlinear', get_nonlinear(config_str, channels))
+
+        self.xvector.add_module('stats', StatsPool())
+        self.xvector.add_module(
+            'dense',
+            DenseLayer(channels * 2, embedding_size, config_str='batchnorm_'))
+
+        for m in self.modules():
+            if isinstance(m, (nn.Conv1d, nn.Linear)):
+                nn.init.kaiming_normal_(m.weight.data)
+                if m.bias is not None:
+                    nn.init.zeros_(m.bias)
+
+    def forward(self, x):
+        x = x.permute(0, 2, 1)  # (B,T,F) => (B,F,T)
+        x = self.head(x)
+        x = self.xvector(x)
+        return x

fireredtts3/campp/campp.py

@@ -0,0 +1,58 @@
+import torch
+import torchaudio
+import torchaudio.compliance.kaldi as kaldi
+from fireredtts3.campp.DTDNN import CAMPPlus
+
+
+def extract_kaldi_mel(audio: torch.Tensor, audio_sr: int):
+    audio = audio[:1]
+    if audio_sr != 16000:
+        audio = torchaudio.functional.resample(audio, audio_sr, 16000)
+        audio_sr = 16000
+    feat = kaldi.fbank(
+        audio,
+        num_mel_bins=80,
+        dither=0,
+        sample_frequency=16000
+    )
+    feat = feat - feat.mean(dim=0, keepdim=True)    # (t, c=80)
+    return feat
+
+
+class CamppEmbedding(torch.nn.Module):
+    def __init__(self, model_path:str):
+        super().__init__()
+        self.model_path = model_path
+        model = CAMPPlus(feat_dim=80, embedding_size=512)
+        sd = torch.load(model_path, weights_only=True, map_location='cpu')
+        model.load_state_dict(sd)
+        self.model = model
+        self.model.eval()
+    
+    @property
+    def device(self):
+        return next(self.parameters()).device
+
+    def forward(self, audio:torch.Tensor, audio_sr:int):
+        """
+        Returns:
+            embedding(torch.Tensor): shape (1, 512)
+        """
+        feat = extract_kaldi_mel(audio, audio_sr)
+        feat = feat.unsqueeze(0)
+        with torch.no_grad():
+            embedding = self.model.forward(feat.to(self.device))    # (b=1, c=512)
+            embedding = embedding.cpu()
+        return embedding
+
+
+
+if __name__ == '__main__':
+    campp = CamppEmbedding(
+        model_path='pretrained_models/speech_campplus_sv_en_voxceleb_16k/campplus_voxceleb.bin',
+    )
+
+    audio, audio_sr = torchaudio.load('tests/input/siteng_clean.wav')
+
+    campp.forward(audio, audio_sr)
+

fireredtts3/campp/layers.py

@@ -0,0 +1,253 @@
+# Copyright 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker). All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint as cp
+from torch import nn
+
+
+def get_nonlinear(config_str, channels):
+    nonlinear = nn.Sequential()
+    for name in config_str.split('-'):
+        if name == 'relu':
+            nonlinear.add_module('relu', nn.ReLU(inplace=True))
+        elif name == 'prelu':
+            nonlinear.add_module('prelu', nn.PReLU(channels))
+        elif name == 'batchnorm':
+            nonlinear.add_module('batchnorm', nn.BatchNorm1d(channels))
+        elif name == 'batchnorm_':
+            nonlinear.add_module('batchnorm',
+                                 nn.BatchNorm1d(channels, affine=False))
+        else:
+            raise ValueError('Unexpected module ({}).'.format(name))
+    return nonlinear
+
+def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2):
+    mean = x.mean(dim=dim)
+    std = x.std(dim=dim, unbiased=unbiased)
+    stats = torch.cat([mean, std], dim=-1)
+    if keepdim:
+        stats = stats.unsqueeze(dim=dim)
+    return stats
+
+
+class StatsPool(nn.Module):
+    def forward(self, x):
+        return statistics_pooling(x)
+
+
+class TDNNLayer(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size,
+                 stride=1,
+                 padding=0,
+                 dilation=1,
+                 bias=False,
+                 config_str='batchnorm-relu'):
+        super(TDNNLayer, self).__init__()
+        if padding < 0:
+            assert kernel_size % 2 == 1, 'Expect equal paddings, but got even kernel size ({})'.format(
+                kernel_size)
+            padding = (kernel_size - 1) // 2 * dilation
+        self.linear = nn.Conv1d(in_channels,
+                                out_channels,
+                                kernel_size,
+                                stride=stride,
+                                padding=padding,
+                                dilation=dilation,
+                                bias=bias)
+        self.nonlinear = get_nonlinear(config_str, out_channels)
+
+    def forward(self, x):
+        x = self.linear(x)
+        x = self.nonlinear(x)
+        return x
+
+
+class CAMLayer(nn.Module):
+    def __init__(self,
+                bn_channels,
+                out_channels,
+                kernel_size,
+                stride,
+                padding,
+                dilation,
+                bias,
+                reduction=2):
+        super(CAMLayer, self).__init__()
+        self.linear_local = nn.Conv1d(bn_channels,
+                                 out_channels,
+                                 kernel_size,
+                                 stride=stride,
+                                 padding=padding,
+                                 dilation=dilation,
+                                 bias=bias)
+        self.linear1 = nn.Conv1d(bn_channels, bn_channels // reduction, 1)
+        self.relu = nn.ReLU(inplace=True)
+        self.linear2 = nn.Conv1d(bn_channels // reduction, out_channels, 1)
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, x):
+        y = self.linear_local(x)
+        context = x.mean(-1, keepdim=True)+self.seg_pooling(x)
+        context = self.relu(self.linear1(context))
+        m = self.sigmoid(self.linear2(context))
+        return y*m
+
+    def seg_pooling(self, x, seg_len=100, stype='avg'):
+        if stype == 'avg':
+            seg = F.avg_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True)
+        elif stype == 'max':
+            seg = F.max_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True)
+        else:
+            raise ValueError('Wrong segment pooling type.')
+        shape = seg.shape
+        seg = seg.unsqueeze(-1).expand(*shape, seg_len).reshape(*shape[:-1], -1)
+        seg = seg[..., :x.shape[-1]]
+        return seg
+
+
+class CAMDenseTDNNLayer(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 bn_channels,
+                 kernel_size,
+                 stride=1,
+                 dilation=1,
+                 bias=False,
+                 config_str='batchnorm-relu',
+                 memory_efficient=False):
+        super(CAMDenseTDNNLayer, self).__init__()
+        assert kernel_size % 2 == 1, 'Expect equal paddings, but got even kernel size ({})'.format(
+            kernel_size)
+        padding = (kernel_size - 1) // 2 * dilation
+        self.memory_efficient = memory_efficient
+        self.nonlinear1 = get_nonlinear(config_str, in_channels)
+        self.linear1 = nn.Conv1d(in_channels, bn_channels, 1, bias=False)
+        self.nonlinear2 = get_nonlinear(config_str, bn_channels)
+        self.cam_layer = CAMLayer(bn_channels,
+                                out_channels,
+                                kernel_size,
+                                stride=stride,
+                                padding=padding,
+                                dilation=dilation,
+                                bias=bias)
+
+    def bn_function(self, x):
+        return self.linear1(self.nonlinear1(x))
+
+    def forward(self, x):
+        if self.training and self.memory_efficient:
+            x = cp.checkpoint(self.bn_function, x)
+        else:
+            x = self.bn_function(x)
+        x = self.cam_layer(self.nonlinear2(x))
+        return x
+
+
+class CAMDenseTDNNBlock(nn.ModuleList):
+    def __init__(self,
+                 num_layers,
+                 in_channels,
+                 out_channels,
+                 bn_channels,
+                 kernel_size,
+                 stride=1,
+                 dilation=1,
+                 bias=False,
+                 config_str='batchnorm-relu',
+                 memory_efficient=False):
+        super(CAMDenseTDNNBlock, self).__init__()
+        for i in range(num_layers):
+            layer = CAMDenseTDNNLayer(in_channels=in_channels + i * out_channels,
+                                   out_channels=out_channels,
+                                   bn_channels=bn_channels,
+                                   kernel_size=kernel_size,
+                                   stride=stride,
+                                   dilation=dilation,
+                                   bias=bias,
+                                   config_str=config_str,
+                                   memory_efficient=memory_efficient)
+            self.add_module('tdnnd%d' % (i + 1), layer)
+
+    def forward(self, x):
+        for layer in self:
+            x = torch.cat([x, layer(x)], dim=1)
+        return x
+
+
+class TransitLayer(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 bias=True,
+                 config_str='batchnorm-relu'):
+        super(TransitLayer, self).__init__()
+        self.nonlinear = get_nonlinear(config_str, in_channels)
+        self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias)
+
+    def forward(self, x):
+        x = self.nonlinear(x)
+        x = self.linear(x)
+        return x
+
+
+class DenseLayer(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 bias=False,
+                 config_str='batchnorm-relu'):
+        super(DenseLayer, self).__init__()
+        self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias)
+        self.nonlinear = get_nonlinear(config_str, out_channels)
+
+    def forward(self, x):
+        if len(x.shape) == 2:
+            x = self.linear(x.unsqueeze(dim=-1)).squeeze(dim=-1)
+        else:
+            x = self.linear(x)
+        x = self.nonlinear(x)
+        return x
+
+
+class BasicResBlock(nn.Module):
+    expansion = 1
+
+    def __init__(self, in_planes, planes, stride=1):
+        super(BasicResBlock, self).__init__()
+        self.conv1 = nn.Conv2d(in_planes,
+                               planes,
+                               kernel_size=3,
+                               stride=(stride, 1),
+                               padding=1,
+                               bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.conv2 = nn.Conv2d(planes,
+                               planes,
+                               kernel_size=3,
+                               stride=1,
+                               padding=1,
+                               bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+
+        self.shortcut = nn.Sequential()
+        if stride != 1 or in_planes != self.expansion * planes:
+            self.shortcut = nn.Sequential(
+                nn.Conv2d(in_planes,
+                          self.expansion * planes,
+                          kernel_size=1,
+                          stride=(stride, 1),
+                          bias=False),
+                nn.BatchNorm2d(self.expansion * planes))
+
+    def forward(self, x):
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = self.bn2(self.conv2(out))
+        out += self.shortcut(x)
+        out = F.relu(out)
+        return out

fireredtts3/core.py

@@ -0,0 +1,555 @@
+"""FireRedTTS3 合成管线：在 FireRedTTS3Base / FireRedTTS3Instruct 上增加完整的文本前端处理。
+
+用法示例::
+
+    from fireredtts3.core import FireRedTTS3, FireRedTTS3Instruct
+
+    # ---- Base ----
+    tts = FireRedTTS3('pretrained_models')
+    gen_audio, gen_audio_sr = tts.generate(
+        language=None,          # None 表示自动判定语种
+        prompt_text='...',
+        prompt_audio=prompt_audio,
+        prompt_audio_sr=16000,
+        text='这是一段很长的文本，需要被自动切句。第二句。第三句。',
+    )
+    torchaudio.save('gen.wav', gen_audio.cpu(), gen_audio_sr)
+
+    # ---- Instruct ----
+    instruct = FireRedTTS3Instruct('pretrained_models')
+    gen_audio, gen_audio_sr, gen_text = instruct.generate_voice_design(
+        instruction='一个年轻女性的温柔嗓音，语速稍慢。',
+        text='今天天气很好，我们一起去公园散步吧。',
+    )
+    torchaudio.save('design.wav', gen_audio.cpu(), gen_audio_sr)
+"""
+
+import os
+import torch
+import torchaudio
+import numpy as np
+from typing import List, Optional, Callable
+
+from fireredtts3.llm.fireredtts3_base import FireRedTTS3Base
+from fireredtts3.llm.fireredtts3_instruct import FireRedTTS3Instruct as FireRedTTS3InstructBackend
+from fireredtts3.utils.text_normalize import (
+    clean_text,
+    clean_tn_spaces,
+    split_paragraph,
+    detect_language,
+    lang_tag_to_locale,
+    build_llm_normalizer,
+    build_wetext_normalizer,
+)
+from fireredtts3.utils.llm_tn.text_normalizer import TextNormalizer as LlmTextNormalizer
+
+
+def cross_fade(
+    seg_a: torch.Tensor,
+    seg_b: torch.Tensor,
+    fade_len: int,
+) -> torch.Tensor:
+    """将两段波形线性 cross-fading 拼接。"""
+    if fade_len <= 0:
+        return torch.cat([seg_a, seg_b], dim=1)
+    fade_len = int(min(fade_len, seg_a.shape[1], seg_b.shape[1]))
+    if fade_len <= 0:
+        return torch.cat([seg_a, seg_b], dim=1)
+
+    # 前段末尾 fade_len 个点做淡出，后段开头 fade_len 个点做淡入
+    ramp = torch.linspace(0.0, 1.0, fade_len, device=seg_a.device, dtype=seg_a.dtype).view(1, -1)
+
+    a_tail = seg_a[:, -fade_len:] * (1.0 - ramp)
+    b_head = seg_b[:, :fade_len] * ramp
+
+    overlap = a_tail + b_head
+
+    head = seg_a[:, :-fade_len]
+    tail = seg_b[:, fade_len:]
+    return torch.cat([head, overlap, tail], dim=1)
+
+
+class TextFrontendMixin:
+    """文本前端（text front-end）能力：清洗 + 语种判定 + TN + 拆句。
+
+    被 :class:`FireRedTTS3` 与 :class:`FireRedTTS3Instruct` 复用，保证
+    克隆 / 音色设计 / 编辑等任务共享同一套文本规整逻辑。
+    """
+
+    _WETEXT_LANGS = {"Chinese", "English"}  # wetext 仅支持中/英
+
+    def _init_frontend(
+        self,
+        use_fasttext: bool = True,
+        use_llm_tn: bool = False,
+        use_wetext: bool = True,
+        tn_api_url: Optional[str] = None,
+        tn_api_key: Optional[str] = None,
+        tn_model: Optional[str] = None,
+        tn_kwargs: Optional[dict] = None,
+    ):
+        """初始化文本前端组件：fasttext 语种检测 + llm_tn / wetext 归一化。"""
+        self.use_fasttext = use_fasttext
+        self._llm_tn = None
+        if use_fasttext:
+            try:
+                self._llm_tn = LlmTextNormalizer(use_fasttext=True)
+            except Exception as e:
+                print(f"[WARN] Failed to init llm_tn TextNormalizer for lang detect: {e}", flush=True)
+                self._llm_tn = None
+        self.use_wetext = use_wetext
+        self.wetext_normalizer = None
+        self.llm_normalizer = None
+        # 默认走 wetext（本地、无需 API key）。llm_tn 是可选增强：仅当显式
+        # use_llm_tn=True 且成功构建（即有 .env / 环境变量）时才启用。
+        if use_wetext:
+            self.wetext_normalizer = build_wetext_normalizer()
+        if use_llm_tn:
+            self.llm_normalizer = build_llm_normalizer(
+                api_url=tn_api_url,
+                api_key=tn_api_key,
+                model=tn_model,
+                **(tn_kwargs or {}),
+            )
+
+    def _detect_lang(self, text: str) -> str:
+        """判定文本语种，返回如 ``Chinese`` / ``English`` 的 lang tag。"""
+        return detect_language(
+            text,
+            fasttext_detector=self._llm_tn.detect_locale if self._llm_tn is not None else None,
+        )
+
+    def _normalize_text(self, text: str, language: str) -> str:
+        """TN 回退链：llm_tn → wetext（中/英/方言/粤语）→ 原文。"""
+        if self.llm_normalizer is not None:
+            locale = lang_tag_to_locale(language) if language else None
+            try:
+                return clean_tn_spaces(self.llm_normalizer(text, locale=locale))
+            except Exception as e:
+                print(f"[WARN] llm_tn normalization failed, fallback to raw text: {e}", flush=True)
+                return clean_tn_spaces(text)
+        _can_wetext = (
+            language in self._WETEXT_LANGS
+            or language == "Cantonese"
+            or language.startswith("ZH_")
+        )
+        if _can_wetext and self.wetext_normalizer is not None:
+            try:
+                return clean_tn_spaces(self.wetext_normalizer(text))
+            except Exception as e:
+                print(f"[WARN] wetext normalization failed, fallback to raw text: {e}", flush=True)
+                return clean_tn_spaces(text)
+        # 其他语种且无 llm_tn => 仅基础清洗（返回原文）
+        return clean_tn_spaces(text)
+
+    def _apply_frontend(
+        self,
+        text: str,
+        language: Optional[str] = None,
+        do_clean: bool = True,
+        do_tn: bool = True,
+        do_split: bool = True,
+        token_max_n: int = 80,
+        token_min_n: int = 60,
+        merge_len: int = 20,
+    ) -> "tuple[str, str, List[str]]":
+        """对文本执行清洗 / 语种判定 / 拆句 / TN，返回 ``(text, language, sentences)``。
+
+        注：返回的 ``text`` 已按 ``do_split`` 结果合并（未开启拆句则原样返回）。
+        """
+        if do_clean:
+            text = clean_text(text)
+        if not text:
+            raise ValueError("text is empty after cleaning")
+
+        if language is None:
+            language = self._detect_lang(text)
+
+        # zh 按字符分句，非 zh 按 token 分句
+        if do_split:
+            split_lang = "zh" if language == "Chinese" else "en"
+            tokenize = None
+            if split_lang != "zh":
+                tokenize = lambda s: self._tokenize_text(s).shape[1]
+            sentences = split_paragraph(
+                text,
+                tokenize=tokenize,
+                lang=split_lang,
+                token_max_n=token_max_n,
+                token_min_n=token_min_n,
+                merge_len=merge_len,
+            )
+        else:
+            sentences = [text]
+
+        if do_tn:
+            sentences = [self._normalize_text(s, language) for s in sentences]
+        sentences = [s for s in sentences if s and s.strip()]
+        if not sentences:
+            raise ValueError("all sentences are empty after normalization")
+
+        return "".join(sentences), language, sentences
+
+
+class FireRedTTS3(TextFrontendMixin, FireRedTTS3Base):
+    """带完整文本前端（清洗 + 语种判定 + 拆句 + 拼接）的零样本 TTS 管线。"""
+
+    def __init__(
+        self,
+        pretrained_model_dir: str,
+        use_fasttext: bool = True,
+        use_llm_tn: bool = False,
+        use_wetext: bool = True,
+        tn_api_url: Optional[str] = None,
+        tn_api_key: Optional[str] = None,
+        tn_model: Optional[str] = None,
+        tn_kwargs: Optional[dict] = None,
+    ):
+        super().__init__(pretrained_model_dir)
+        self._init_frontend(
+            use_fasttext=use_fasttext,
+            use_llm_tn=use_llm_tn,
+            use_wetext=use_wetext,
+            tn_api_url=tn_api_url,
+            tn_api_key=tn_api_key,
+            tn_model=tn_model,
+            tn_kwargs=tn_kwargs,
+        )
+
+    def _synthesize_one(
+        self,
+        text: str,
+        language: str,
+        prompt_text: str,
+        prompt_audio: torch.Tensor,
+        prompt_audio_sr: int,
+        **kwargs,
+    ):
+        """合成单句文本，返回 ``(gen_audio, gen_audio_sr)``。"""
+        return super().generate(
+            language=language,
+            prompt_text=prompt_text,
+            prompt_audio=prompt_audio,
+            prompt_audio_sr=prompt_audio_sr,
+            text=text,
+            **kwargs,
+        )
+
+    def generate(
+        self,
+        # Input
+        text: str,
+        language: Optional[str] = None,
+        prompt_text: str = "",
+        prompt_audio: Optional[torch.Tensor] = None,
+        prompt_audio_sr: Optional[int] = None,
+        # Inference
+        stop_threshold: float = 0.5,
+        n_timesteps: int = 10,
+        inference_cfg: float = 2.0,
+        seed: int = 1234,
+        # 文本前端
+        do_clean: bool = True,
+        do_tn: bool = True,
+        do_split: bool = True,
+        token_max_n: int = 80,
+        token_min_n: int = 60,
+        merge_len: int = 20,
+        # 拼接
+        cross_fade_ms: float = 50.0,
+        max_text_len: int = 300,
+    ):
+        """合成完整文本，自动清洗、切句、逐句生成并 cross-fading 拼接。"""
+        if not isinstance(text, str) or not text.strip():
+            raise ValueError("text must be a non-empty string")
+
+        text, language, sentences = self._apply_frontend(
+            text=text,
+            language=language,
+            do_clean=do_clean,
+            do_tn=do_tn,
+            do_split=do_split,
+            token_max_n=token_max_n,
+            token_min_n=token_min_n,
+            merge_len=merge_len,
+        )
+
+        gen_audio_sr = None
+        segments: List[torch.Tensor] = []
+        for i, sent in enumerate(sentences):
+            seg, seg_sr = self._synthesize_one(
+                text=sent,
+                language=language,
+                prompt_text=prompt_text,
+                prompt_audio=prompt_audio,
+                prompt_audio_sr=prompt_audio_sr,
+                stop_threshold=stop_threshold,
+                n_timesteps=n_timesteps,
+                inference_cfg=inference_cfg,
+                seed=seed,
+            )
+            gen_audio_sr = seg_sr
+            segments.append(seg.cpu())
+
+        gen_audio = segments[0]
+        if len(segments) > 1:
+            fade_len = int(cross_fade_ms / 1000.0 * gen_audio_sr)
+            for s in segments[1:]:
+                gen_audio = cross_fade(gen_audio, s, fade_len)
+        return gen_audio, gen_audio_sr
+
+
+class FireRedTTS3Instruct(TextFrontendMixin, FireRedTTS3InstructBackend):
+    """FireRedTTS3-Instruct：指令驱动的语音生成与编辑（含文本前端）。
+
+    在 :class:`fireredtts3.llm.fireredtts3_instruct.FireRedTTS3Instruct` 之上，
+    叠加与 :class:`FireRedTTS3` 一致的文本前端能力（清洗 / 语种判定 / TN /
+    拆句 + cross-fade 拼接），支持 4 类任务：
+
+    - ``generate_tts``              —— ICL 零样本语音克隆（参考音频 + 参考文本）
+    - ``generate_voice_design``     —— 音色设计：按自然语言音色描述生成新声音
+    - ``generate_semantic_edit``    —— 语义编辑：改词 / 插入 / 删除等内容级编辑
+    - ``generate_acoustic_edit``    —— 声学编辑：语速 / 音高 / 音量等声学属性编辑
+    """
+
+    def __init__(
+        self,
+        pretrained_model_dir: str,
+        use_fasttext: bool = True,
+        use_llm_tn: bool = False,
+        use_wetext: bool = True,
+        tn_api_url: Optional[str] = None,
+        tn_api_key: Optional[str] = None,
+        tn_model: Optional[str] = None,
+        tn_kwargs: Optional[dict] = None,
+    ):
+        super().__init__(pretrained_model_dir)  # 先加载 RedAE + InstructCore + tokenizer
+        self._init_frontend(
+            use_fasttext=use_fasttext,
+            use_llm_tn=use_llm_tn,
+            use_wetext=use_wetext,
+            tn_api_url=tn_api_url,
+            tn_api_key=tn_api_key,
+            tn_model=tn_model,
+            tn_kwargs=tn_kwargs,
+        )
+
+    # ------------------------------------------------------------------ #
+    # 任务 1: ICL 零样本语音克隆
+    # ------------------------------------------------------------------ #
+    def generate_tts(
+        self,
+        prompt_text: str,
+        prompt_audio: torch.Tensor,
+        prompt_audio_sr: int,
+        text: str,
+        language: Optional[str] = None,
+        # Inference
+        stop_threshold: float = 0.5,
+        n_timesteps: int = 10,
+        inference_cfg: float = 2.0,
+        seed: int = 1234,
+        # 文本前端
+        do_clean: bool = True,
+        do_tn: bool = True,
+        do_split: bool = True,
+        token_max_n: int = 80,
+        token_min_n: int = 60,
+        merge_len: int = 20,
+        cross_fade_ms: float = 50.0,
+    ):
+        """ICL 零样本语音克隆（Instruct 版）。
+
+        Args:
+            prompt_text: 参考音频对应的文本转写。
+            prompt_audio: 参考音频波形 (1, T) 或 (C, T)。
+            prompt_audio_sr: 参考音频采样率。
+            text: 待合成文本；可包含多句（自动拆句 + cross-fade 拼接）。
+            language: 可选语种 tag，为 None 时自动判定。
+        """
+        text, language, sentences = self._apply_frontend(
+            text=text,
+            language=language,
+            do_clean=do_clean,
+            do_tn=do_tn,
+            do_split=do_split,
+            token_max_n=token_max_n,
+            token_min_n=token_min_n,
+            merge_len=merge_len,
+        )
+
+        gen_audio_sr = None
+        segments: List[torch.Tensor] = []
+        for sent in sentences:
+            seg, seg_sr, _ = super().generate_tts(
+                prompt_text=prompt_text,
+                prompt_audio=prompt_audio,
+                prompt_audio_sr=prompt_audio_sr,
+                text=sent,
+                stop_threshold=stop_threshold,
+                n_timesteps=n_timesteps,
+                inference_cfg=inference_cfg,
+                seed=seed,
+            )
+            gen_audio_sr = seg_sr
+            segments.append(seg.cpu())
+
+        gen_audio = segments[0]
+        if len(segments) > 1:
+            fade_len = int(cross_fade_ms / 1000.0 * gen_audio_sr)
+            for s in segments[1:]:
+                gen_audio = cross_fade(gen_audio, s, fade_len)
+        return gen_audio, gen_audio_sr
+
+    # ------------------------------------------------------------------ #
+    # 任务 2: Voice Design —— 按音色描述生成新声音（无需参考音频）
+    # ------------------------------------------------------------------ #
+    def generate_voice_design(
+        self,
+        instruction: str,
+        text: str,
+        language: Optional[str] = None,
+        # Audio Inference Settings
+        n_timesteps: int = 10,
+        inference_cfg: float = 1.2,
+        # Random
+        seed: int = 2,
+        # 文本前端
+        do_clean: bool = True,
+        do_tn: bool = True,
+        do_split: bool = True,
+        token_max_n: int = 80,
+        token_min_n: int = 60,
+        merge_len: int = 20,
+        cross_fade_ms: float = 50.0,
+    ):
+        """Voice Design：从自然语言音色描述生成一段全新语音（无参考音频）。
+
+        模型先输出一段 CoT 语音属性规划（返回 ``gen_text``），再据此合成音频。
+
+        Args:
+            instruction: 音色描述，如“一个年轻女性的温柔嗓音，语速稍慢”。
+            text: 待合成文本；可包含多句（自动拆句 + 逐句生成 + cross-fade 拼接）。
+            language: 可选语种 tag，为 None 时自动判定（用于 TN）。
+        Returns:
+            ``(gen_audio, gen_audio_sr, gen_text)``。gen_text 为模型输出的
+            语音属性规划（CoT）。
+        """
+        text, language, sentences = self._apply_frontend(
+            text=text,
+            language=language,
+            do_clean=do_clean,
+            do_tn=do_tn,
+            do_split=do_split,
+            token_max_n=token_max_n,
+            token_min_n=token_min_n,
+            merge_len=merge_len,
+        )
+
+        gen_audio_sr = None
+        segments: List[torch.Tensor] = []
+        gen_text = None
+        for i, sent in enumerate(sentences):
+            seg, seg_sr, seg_text = super().generate_voice_design(
+                instruction=instruction,
+                text=sent,
+                n_timesteps=n_timesteps,
+                inference_cfg=inference_cfg,
+                seed=seed,
+            )
+            gen_audio_sr = seg_sr
+            segments.append(seg.cpu())
+            if i == 0:
+                gen_text = seg_text  # CoT 规划只需取第一次
+
+        gen_audio = segments[0]
+        if len(segments) > 1:
+            fade_len = int(cross_fade_ms / 1000.0 * gen_audio_sr)
+            for s in segments[1:]:
+                gen_audio = cross_fade(gen_audio, s, fade_len)
+        return gen_audio, gen_audio_sr, gen_text
+
+    # ------------------------------------------------------------------ #
+    # 任务 3: Semantic Edit —— 内容级编辑（改词 / 插入 / 删除）
+    # ------------------------------------------------------------------ #
+    def generate_semantic_edit(
+        self,
+        instruction: str,
+        audio_in: torch.Tensor,
+        audio_in_sr: torch.Tensor,
+        n_timesteps: int = 10,
+        inference_cfg: float = 1.2,
+        seed: int = 1234,
+    ):
+        """Semantic Edit：按自然语言指令对音频进行内容级编辑。
+
+        支持插入 / 删除 / 替换等语义编辑，返回编辑后音频与模型生成的
+        编辑后文本（CoT，即 ``<|sot|>{rewritten text}<|eot|>``）。
+
+        Args:
+            instruction: 编辑指令，如 "insert '简直' after the character or word at index 8."。
+            audio_in: 输入音频波形。
+            audio_in_sr: 输入音频采样率。
+        Returns:
+            ``(gen_audio, gen_audio_sr, gen_text)``。
+        """
+        return super().generate_semantic_edit(
+            instruction=instruction,
+            audio_in=audio_in,
+            audio_in_sr=audio_in_sr,
+            n_timesteps=n_timesteps,
+            inference_cfg=inference_cfg,
+            seed=seed,
+        )
+
+    # ------------------------------------------------------------------ #
+    # 任务 4: Acoustic Edit —— 声学属性编辑（语速 / 音高 / 音量）
+    # ------------------------------------------------------------------ #
+    def generate_acoustic_edit(
+        self,
+        instruction: str,
+        audio_in: torch.Tensor,
+        audio_in_sr: torch.Tensor,
+        n_timesteps: int = 10,
+        inference_cfg: float = 1.2,
+        seed: int = 1234,
+    ):
+        """Acoustic Edit：按自然语言指令对音频进行声学属性编辑。
+
+        支持调整语速 / 音高 / 音量（如 "adjust the speed to 0.5x"）。
+
+        Args:
+            instruction: 编辑指令，需使用模型训练模板，如
+                "adjust the speed to 0.5x" / "shift the pitch by 3 steps"。
+            audio_in: 输入音频波形。
+            audio_in_sr: 输入音频采样率。
+        Returns:
+            ``(gen_audio, gen_audio_sr)``。
+        """
+        return super().generate_acoustic_edit(
+            instruction=instruction,
+            audio_in=audio_in,
+            audio_in_sr=audio_in_sr,
+            n_timesteps=n_timesteps,
+            inference_cfg=inference_cfg,
+            seed=seed,
+        )
+
+
+if __name__ == '__main__':
+    tts = FireRedTTS3('pretrained_models')
+    print('[INFO] FireRedTTS3 (text front-end) loaded')
+
+    prompt_audio_path = 'tests/prompts/default_prompt.wav'
+    prompt_text = '在欧洲行走简直就是走进汽车博览馆博览会，'
+    prompt_audio, prompt_audio_sr = torchaudio.load(prompt_audio_path)
+
+    text = '法院与不动产登记部门加强沟通，并督促银行提前办理抵押预约登记。我们也需要关注后续的进展。'
+    gen_audio, gen_audio_sr = tts.generate(
+        language=None,          # 自动判定语种
+        prompt_text=prompt_text,
+        prompt_audio=prompt_audio,
+        prompt_audio_sr=prompt_audio_sr,
+        text=text,
+    )
+    torchaudio.save('gen.wav', gen_audio.cpu(), gen_audio_sr)

fireredtts3/llm/dit.py

@@ -0,0 +1,194 @@
+import math
+import torch
+import torch.nn.functional as F
+from torch import nn
+from fireredtts3.llm.modules import (
+    RotaryEmbedding,
+    TimestepEmbedder,
+    Attention,
+    FeedForward,
+    RMSNorm,
+)
+
+
+class Transpose(torch.nn.Module):
+    def __init__(self, dim0: int, dim1: int):
+        super().__init__()
+        self.dim0 = dim0
+        self.dim1 = dim1
+
+    def forward(self, x: torch.Tensor):
+        x = torch.transpose(x, self.dim0, self.dim1)
+        return x
+
+
+class ConvBlock(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: int = 3,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.kernel_size = kernel_size
+        self.block = torch.nn.Sequential(
+            nn.Conv1d(
+                in_channels, 
+                out_channels, 
+                kernel_size=kernel_size, 
+                padding=(kernel_size-1)//2
+            ),
+            nn.Mish(),
+            nn.Conv1d(
+                out_channels, 
+                out_channels, 
+                kernel_size=kernel_size, 
+                padding=(kernel_size-1)//2
+            ),
+        )
+    
+    def forward(self, x: torch.Tensor, mask: torch.Tensor = None):
+        """
+        Args:
+            x: shape (b, t, c)
+            mask: shape (b, t, 1), default to None
+        """
+        if mask is not None: x = x * mask
+        x = x.transpose(1, 2)
+        x = self.block(x)
+        x = x.transpose(1, 2)
+        if mask is not None: x = x * mask
+        return x
+
+
+def modulate(x, shift, scale):
+    return x * (1 + scale) + shift
+
+
+class FinalLayer(nn.Module):
+    """
+    The final layer of DiT.
+    """
+    def __init__(self, hidden_size, out_channels):
+        super().__init__()
+        self.adaLN_modulation = nn.Sequential(
+            nn.SiLU(),
+            nn.Linear(hidden_size, 2 * hidden_size, bias=True)
+        )
+        self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.linear = nn.Linear(hidden_size, out_channels, bias=True)
+
+    def forward(self, x, c):
+        shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1)
+        x = modulate(self.norm_final(x), shift, scale)
+        x = self.linear(x)
+        return x
+
+
+class DiTBlock(nn.Module):
+    def __init__(
+        self, 
+        hidden_size, 
+        num_heads, 
+        mlp_ratio=4.0, 
+        dropout=0.1, 
+        **kwargs
+    ):
+        super().__init__()
+        # Attn
+        self.norm1 = RMSNorm(hidden_size, eps=1e-6)
+        self.attn = Attention(
+            dim=hidden_size,
+            heads=num_heads,
+            dim_head=hidden_size // num_heads,
+            dropout=dropout
+        )
+        # Conv
+        self.norm2 = RMSNorm(hidden_size, eps=1e-6)
+        self.conv = ConvBlock(in_channels=hidden_size, out_channels=hidden_size, kernel_size=3)
+        # FFN 
+        self.norm3 = RMSNorm(hidden_size, eps=1e-6)
+        self.mlp = FeedForward(dim=hidden_size, mult=mlp_ratio, dropout=dropout, approximate="tanh")
+        # Time AdaLN condition
+        self.adaLN_modulation = nn.Sequential(
+            nn.SiLU(),
+            nn.Linear(hidden_size, 9 * hidden_size)
+        )
+
+    def forward(self, x: torch.Tensor, c: torch.Tensor, mask: torch.Tensor, rope: torch.Tensor):
+        """
+        Args:
+            x(torch.Tensor): shape (b, t, c)
+            c(torch.Tensor): shae (b, 1, c), time condition
+            mask(torch.Tensor): default to None, DO NOT USE THIS ARG.
+            rope(torch.Tensor): positional embedding.
+        """
+        # AdaLN for t
+        (
+            shift_msa, scale_msa, gate_msa, 
+            shift_mlp, scale_mlp, gate_mlp, 
+            shift_conv, scale_conv, gate_conv
+        ) = self.adaLN_modulation(c).chunk(9, dim=-1)
+        # Attn
+        x = x + gate_msa * self.attn(
+            modulate(self.norm1(x), shift_msa, scale_msa),
+            mask=mask, 
+            rope=rope
+        )
+        # Conv
+        x = x + gate_conv * self.conv(
+            modulate(self.norm2(x), shift_conv, scale_conv), 
+            mask=mask,
+        )
+        # FFN
+        x = x + gate_mlp * self.mlp(
+            modulate(self.norm3(x), shift_mlp, scale_mlp)
+        )
+        return x
+
+
+class DiT(nn.Module):
+    """
+    Diffusion model with a Transformer backbone.
+    """
+    def __init__(
+        self,
+        in_channels: int,
+        out_channels: int,
+        mlp_ratio: float = 4.0,
+        depth: int = 28,
+        num_heads: int = 8,
+        hidden_size: int = 256,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.in_proj = nn.Linear(in_channels, hidden_size)
+        self.t_embedder = TimestepEmbedder(hidden_size)
+        self.rotary_embed = RotaryEmbedding(hidden_size // num_heads)
+        self.blocks = nn.ModuleList([
+            DiTBlock(hidden_size, num_heads, mlp_ratio=mlp_ratio, dropout=0.0) 
+            for _ in range(depth)
+        ])
+        self.final_layer = FinalLayer(hidden_size, self.out_channels)
+
+    def forward(self, x:torch.Tensor, t:torch.Tensor):
+        """Full attention DiT head.
+
+        Args:
+            x: shape (b, t, c), including xt and other conditions
+            t: shape (b,).
+        Returns:
+            pred: shape (b, t, c)
+        """
+        # time
+        t = self.t_embedder(t.view(-1)).unsqueeze(1)  # (b, 1, c)
+        x = self.in_proj(x)
+        rope = self.rotary_embed.forward_from_seq_len(x.shape[1])
+        for block in self.blocks:
+            x = block(x, t, mask=None, rope=rope)
+        x = self.final_layer(x, t)
+        return x
+

fireredtts3/llm/fireredtts3_base.py

@@ -0,0 +1,346 @@
+import os
+import torch
+import torchaudio
+import torch.nn.functional as F
+from transformers import (
+    Qwen3Model, Qwen3Config,
+    PretrainedConfig, PreTrainedModel
+)
+from fireredtts3.llm.patch_encoder import PatchEncoder, RotaryEmbedding
+from fireredtts3.llm.dit import DiT
+from fireredtts3.redae.redae import RedAE
+from fireredtts3.campp.campp import CamppEmbedding
+from fireredtts3.utils.utils import fix_seed
+from fireredtts3.utils.text_tokenizer import (
+    load_text_tokenizer, 
+    MULTI_LANG_TAGS, MULTI_DIALECT_TAGS,
+)
+
+
+Qwen3_1_7B_ConfigDict = {
+    "architectures": [
+        "Qwen3ForCausalLM"
+    ],
+    "attention_bias": False,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151645,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "initializer_range": 0.02,
+    "intermediate_size": 6144,
+    "max_position_embeddings": 40960,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": None,
+    "rope_theta": 1000000,
+    "sliding_window": None,
+    "tie_word_embeddings": True,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "5.6.2",
+    "use_cache": True,
+    "use_sliding_window": False,
+    "vocab_size": 151936,
+    "attn_implementation": "flash_attention_2",
+}
+
+
+class FireRedTTS3BaseCoreConfig(PretrainedConfig):
+    def __init__(
+        self,
+        redae_dim: int = 64,
+        # Shared
+        num_history_patches: int = 2,
+        spk_in_dim: int = 512,
+        # PatchEncoder
+        patch_size: int = 4,
+        patch_encoder_hidden_size: int = 1024,
+        patch_encoder_mlp_ratio: int = 4,
+        patch_encoder_depth: int = 8,
+        patch_encoder_num_heads: int = 16,
+        # DiT
+        dit_mlp_ratio: int = 3,
+        dit_depth: int = 11,
+        dit_num_heads: int = 16,
+        dit_hidden_size: int = 1024,
+        # Other
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.redae_dim = redae_dim
+        # Shared
+        self.num_history_patches=num_history_patches
+        self.spk_in_dim=spk_in_dim
+        # PatchEncoder
+        self.patch_size=patch_size
+        self.patch_encoder_hidden_size=patch_encoder_hidden_size
+        self.patch_encoder_mlp_ratio=patch_encoder_mlp_ratio
+        self.patch_encoder_depth=patch_encoder_depth
+        self.patch_encoder_num_heads=patch_encoder_num_heads
+        # DiT
+        self.dit_mlp_ratio=dit_mlp_ratio
+        self.dit_depth=dit_depth
+        self.dit_num_heads=dit_num_heads
+        self.dit_hidden_size=dit_hidden_size
+
+
+class FireRedTTS3BaseCore(PreTrainedModel):
+    config_class = FireRedTTS3BaseCoreConfig
+    base_model_prefix = "fireredtts3_base_core"
+
+    _supports_flash_attn = True
+    _supports_sdpa = True
+
+    def __init__(self, config: FireRedTTS3BaseCoreConfig):
+        super().__init__(config)
+        # Backbone Transformer
+        self.backbone_llm_config = Qwen3Config.from_dict(Qwen3_1_7B_ConfigDict)
+        self.backbone_llm = Qwen3Model(self.backbone_llm_config)
+        # Speaker proj
+        self.spk_proj_llm = torch.nn.Linear(config.spk_in_dim, self.backbone_llm_config.hidden_size)
+        self.spk_proj_dit = torch.nn.Linear(config.spk_in_dim, config.spk_in_dim)
+        # PatchEncoder
+        self.patch_encoder = PatchEncoder(
+            in_dim=config.redae_dim,
+            out_dim=self.backbone_llm_config.hidden_size,
+            patch_size=config.patch_size,
+            hidden_size=config.patch_encoder_hidden_size,
+            mlp_ratio=config.patch_encoder_mlp_ratio,
+            depth=config.patch_encoder_depth,
+            num_heads=config.patch_encoder_num_heads,
+        )
+        # DiT
+        self.dit_head = torch.nn.Linear(self.backbone_llm_config.hidden_size, config.dit_hidden_size)
+        self.dit = DiT(
+            in_channels=(config.redae_dim+config.spk_in_dim+config.dit_hidden_size),
+            out_channels=config.redae_dim,
+            mlp_ratio=config.dit_mlp_ratio,
+            depth=config.dit_depth,
+            num_heads=config.dit_num_heads,
+            hidden_size=config.dit_hidden_size,
+        )
+        # Stop
+        self.stop_head = torch.nn.Linear(self.backbone_llm_config.hidden_size, 1)
+        # Shared
+        self.redae_dim = config.redae_dim
+        self.patch_size = self.patch_encoder.patch_size
+        self.history_patches = config.num_history_patches
+        self.history_length = config.num_history_patches * self.patch_size
+        self.post_init()
+    
+    # Manually init RotaryEmbedding buffers
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, RotaryEmbedding):
+            module.rope_init()
+    
+    # Backbone Transformer AR wrapper
+    @torch.autocast(device_type='cuda', dtype=torch.bfloat16)
+    def _backbone_one_step(self, input_embeds: torch.Tensor, cache = None):
+        outs = self.backbone_llm.forward(
+            inputs_embeds=input_embeds,
+            use_cache=True,
+            past_key_values=cache,
+        )
+        # Only take the last timestep
+        hidden_states = outs.last_hidden_state
+        new_cache = outs.past_key_values
+        return hidden_states, new_cache
+    
+    # Flow head wrapper
+    def _flow_one_step(
+        self, 
+        hist_latents: torch.Tensor, 
+        backbone_cond: torch.Tensor,
+        spk_cond: torch.Tensor,
+        t_span: torch.Tensor,
+        inference_cfg: float,
+    ):
+        # Compose input
+        x0 = torch.randn(1, self.patch_size, self.redae_dim, device=hist_latents.device)
+        
+        xt = torch.cat([hist_latents, x0], dim=1)   # History clean + current noise
+        cond = torch.cat([
+            backbone_cond.repeat_interleave(self.patch_size, dim=1),    # Correspond backbone cond
+            spk_cond.unsqueeze(1).repeat(1, self.history_length+self.patch_size, 1) # Spk 
+        ], dim=-1)
+        # Run flow inference
+        for ti, t in enumerate(t_span[:-1]):
+            dt = t_span[ti+1]-t
+            t_in = t.view(-1, 1, 1)
+            x_in = torch.cat([xt, cond], dim=2)
+            if inference_cfg > 0:
+                x_in_cfg = torch.cat([xt, cond * 0], dim=2)
+                x_in = torch.cat([x_in, x_in_cfg], dim=0)
+                t_in = t_in.expand(2, -1, -1)
+            vt = self.dit(x=x_in, t=t_in)
+            if inference_cfg > 0:
+                vt_cond, vt_cfg = vt.chunk(2, dim=0)
+                vt = (1.0 + inference_cfg) * vt_cond - inference_cfg * vt_cfg
+            # Only denoise current patch
+            xt[:, -self.patch_size:] = xt[:, -self.patch_size:] + dt.view(-1, 1, 1) * vt[:, -self.patch_size:]
+        # Remove history
+        x1 = xt[:, -self.patch_size:]        
+        return x1
+
+    # Core LLM-DiT AR Loop
+    @torch.no_grad()
+    def generate(
+        self, 
+        # Input
+        spk_emb: torch.Tensor,
+        text_tokens: torch.Tensor,
+        prompt_latents: torch.Tensor,
+        # Inference settings
+        n_timesteps: int = 10,
+        inference_cfg: float = 2.0,
+        stop_threshold: float = 0.5,
+        # Length control
+        min_gen_steps: int = 6,
+        max_gen_steps: int = None,
+    ):
+        device = text_tokens.device
+
+        # Compose input sequence
+        input_embeds: torch.Tensor = self.backbone_llm.embed_tokens(text_tokens)
+        patch_prompt_latents: torch.Tensor = self.patch_encoder(prompt_latents)
+        spk_embs_llm = self.spk_proj_llm(spk_emb)
+        input_embeds = torch.cat([spk_embs_llm.unsqueeze(1), input_embeds, patch_prompt_latents], dim=1)
+        
+        # Prepare DiT decode
+        t_span = torch.linspace(0, 1, n_timesteps + 1).to(device)
+        t_span = 1 - torch.cos(t_span * 0.5 * torch.pi) # (n_timesteps+1,)
+        latents_gen = F.pad(prompt_latents, (0, 0, self.history_length, 0))
+        
+        # Prepare DiT conditions
+        dit_spk_cond = self.spk_proj_dit(spk_emb) # (b=1, c)
+
+        # Prepare Backbone states
+        backbone_cond = input_embeds.new_zeros(1, self.history_patches, input_embeds.shape[-1])
+        backbone_cache = None
+
+        max_gen_steps = (400 if max_gen_steps is None else max_gen_steps)
+        for step_index in range(max_gen_steps):
+            # Backbone condition
+            backbone_out, backbone_cache = self._backbone_one_step(input_embeds, cache=backbone_cache)   # (b=1, t, c)
+
+            # Stop prediction
+            stop_logits = self.stop_head(backbone_out[:, -1]).squeeze(-1)
+            stop_score = torch.sigmoid(stop_logits).item()
+            if stop_score >= stop_threshold:
+                if min_gen_steps is not None:
+                    if step_index >= min_gen_steps: break
+                else:
+                    break
+            
+            # DiT decode
+            if step_index == 0:
+                one_backbone_out = backbone_out[:, -patch_prompt_latents.shape[1]:]
+            else:
+                one_backbone_out = backbone_out[:, -1:]
+            backbone_cond = torch.cat([backbone_cond, one_backbone_out], dim=1)
+
+            one_latents = self._flow_one_step(
+                hist_latents=latents_gen[:, -self.history_length:],
+                backbone_cond=self.dit_head((backbone_cond[:, -(self.history_patches+1):])),
+                spk_cond=dit_spk_cond,
+                t_span=t_span,
+                inference_cfg=inference_cfg,
+            )
+            
+            input_embeds = self.patch_encoder(one_latents)
+            latents_gen = torch.cat([latents_gen, one_latents], dim=1)
+        
+        # Remove dummy history
+        latents_gen = latents_gen[:, self.history_length:]
+        return latents_gen
+
+
+# RedAE + TextTokenizer + TTS3Core
+class FireRedTTS3Base(object):
+    def __init__(self, pretrained_model_dir: str):
+        self.device = torch.device('cuda')
+        # RedAE
+        redae_model_dir = os.path.join(pretrained_model_dir, 'redae')
+        assert os.path.exists(redae_model_dir), f'{redae_model_dir} not found'
+        self.redae = RedAE.from_pretrained(redae_model_dir)
+        self.redae.to(self.device)
+        # LLM-DiT 
+        tts_model_dir = os.path.join(pretrained_model_dir, 'fireredtts3_base')
+        assert os.path.exists(tts_model_dir), f'{tts_model_dir} not found'
+        self.tts_core = FireRedTTS3BaseCore.from_pretrained(tts_model_dir)
+        self.tts_core.to(self.device)
+        # Text Tokenizer
+        text_tok_dir = os.path.join(pretrained_model_dir, 'text_tokenizer')
+        assert os.path.exists(text_tok_dir), f'{text_tok_dir} not found'
+        self.text_tokenizer = load_text_tokenizer(text_tok_dir)
+        # Speaker
+        spk_ckpt_path = os.path.join(pretrained_model_dir, 'campp/campplus_voxceleb.bin')
+        assert os.path.exists(spk_ckpt_path), f'{spk_ckpt_path} not found'
+        self.spk_extractor = CamppEmbedding(spk_ckpt_path)
+        self.spk_extractor.to(self.device)
+    
+    def _tokenize_text(self, text:str):
+        tokens = self.text_tokenizer(
+            text, 
+            truncation=False, padding=False, add_special_tokens=False,
+        )["input_ids"]
+        tokens = torch.tensor([tokens], dtype=torch.long, device=self.device)
+        return tokens
+
+    @torch.inference_mode()
+    def generate(
+        self,
+        # Input
+        language: str,
+        prompt_text: str,
+        prompt_audio: torch.Tensor,
+        prompt_audio_sr: int,
+        text: str,
+        # Inference
+        stop_threshold: float = 0.5,
+        n_timesteps: int = 10,
+        inference_cfg: float = 2.0,
+        seed: int = 1234,
+    ):
+        # ICL Text
+        lang_tag = f'<|{language}|>'
+        assert lang_tag in MULTI_LANG_TAGS+MULTI_DIALECT_TAGS, f'invalid language: {language}'
+        input_text = f'{lang_tag}<|sot|>{prompt_text}{text}<|eot|>'
+        text_tokens = self._tokenize_text(input_text)
+        # Prompt audio
+        prompt_audio = prompt_audio[:1]
+        prompt_audio = torchaudio.functional.resample(prompt_audio, prompt_audio_sr, self.redae.sample_rate)
+        prompt_audio_sr = self.redae.sample_rate
+        prompt_audio = self.redae.pad_to_multiple_of(prompt_audio, self.redae.downsample_rate*self.tts_core.patch_size)
+        prompt_audio = prompt_audio.to(self.device)
+        prompt_latents = self.redae.encode(prompt_audio, prompt_audio_sr)
+        prompt_latents = prompt_latents.to(torch.float32)
+        # Spk emb
+        spk_emb = self.spk_extractor.forward(prompt_audio, prompt_audio_sr)
+        spk_emb = spk_emb.to(self.device)
+        # TTS
+        if seed is not None:
+            fix_seed(seed)
+        gen_latents = self.tts_core.generate(
+            # Input
+            spk_emb=spk_emb,
+            text_tokens=text_tokens,
+            prompt_latents=prompt_latents,
+            # Inference settings
+            n_timesteps=n_timesteps,
+            inference_cfg=inference_cfg,
+            stop_threshold=stop_threshold,
+            # Length control
+            min_gen_steps=6,
+            max_gen_steps=None,
+        )
+        gen_audio, gen_audio_sr = self.redae.decode(gen_latents)
+        # Remove prompts
+        gen_audio = gen_audio[:, prompt_audio.shape[1]:]
+        return gen_audio, gen_audio_sr

fireredtts3/llm/fireredtts3_instruct.py

@@ -0,0 +1,508 @@
+import os
+import torch
+import torchaudio
+import torch.nn.functional as F
+from transformers import (
+    Qwen3ForCausalLM, Qwen3Config,
+    PretrainedConfig, PreTrainedModel
+)
+from transformers.generation.logits_process import (
+    LogitsProcessorList,
+    RepetitionPenaltyLogitsProcessor,
+    TemperatureLogitsWarper,
+    TopKLogitsWarper,
+    TopPLogitsWarper,
+)
+from fireredtts3.llm.patch_encoder import PatchEncoder, RotaryEmbedding
+from fireredtts3.llm.dit import DiT
+from fireredtts3.redae.redae import RedAE
+from fireredtts3.utils.utils import fix_seed
+from fireredtts3.utils.text_tokenizer import load_text_tokenizer
+from fireredtts3.llm.fireredtts3_base import Qwen3_1_7B_ConfigDict
+from fireredtts3.utils.chatml import (
+    CHATML_LATENT_IN_PAD_ID,
+    CHATML_LATENT_OUT_PAD_ID,
+    compose_generate_input_tts,
+    compose_generate_input_voice_design,
+    compose_generate_input_semantic_edit,
+    compose_generate_input_acoustic_edit,
+)
+
+
+TEXT_EOT_ID: int = 151677
+AUDIO_SOS_ID: int = 151669
+REDAE_SCALE = 0.4
+
+
+def init_text_logits_processor(
+    repetition_penalty: float = None,
+    do_sample: bool = True,
+    temperature: float = None,
+    top_p: float = None,
+    top_k: int = None,
+):
+    text_logits_processor = LogitsProcessorList()
+    if repetition_penalty is not None and repetition_penalty != 1.0:
+        text_logits_processor.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty))
+    if do_sample:
+        if temperature is not None and temperature != 1.0:
+            text_logits_processor.append(TemperatureLogitsWarper(temperature=float(temperature)))
+        if top_k is not None and top_k > 0:
+            text_logits_processor.append(TopKLogitsWarper(top_k=top_k))
+        if top_p is not None and top_p < 1.0:
+            text_logits_processor.append(TopPLogitsWarper(top_p=top_p))
+    return text_logits_processor
+
+
+class FireRedTTS3InstructCoreConfig(PretrainedConfig):
+    def __init__(
+        self,
+        redae_dim: int = 64,
+        # Shared
+        num_history_patches: int = 2,
+        # PatchEncoder
+        patch_size: int = 4,
+        patch_encoder_hidden_size: int = 1024,
+        patch_encoder_mlp_ratio: int = 4,
+        patch_encoder_depth: int = 8,
+        patch_encoder_num_heads: int = 16,
+        # DiT
+        dit_mlp_ratio: int = 3,
+        dit_depth: int = 11,
+        dit_num_heads: int = 16,
+        dit_hidden_size: int = 1024,
+        # Other
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.redae_dim = redae_dim
+        # Shared
+        self.num_history_patches=num_history_patches
+        # PatchEncoder
+        self.patch_size=patch_size
+        self.patch_encoder_hidden_size=patch_encoder_hidden_size
+        self.patch_encoder_mlp_ratio=patch_encoder_mlp_ratio
+        self.patch_encoder_depth=patch_encoder_depth
+        self.patch_encoder_num_heads=patch_encoder_num_heads
+        # DiT
+        self.dit_mlp_ratio=dit_mlp_ratio
+        self.dit_depth=dit_depth
+        self.dit_num_heads=dit_num_heads
+        self.dit_hidden_size=dit_hidden_size
+
+
+class FireRedTTS3InstructCore(PreTrainedModel):
+    config_class = FireRedTTS3InstructCoreConfig
+    base_model_prefix = "fireredtts3_instruct_core"
+
+    _supports_flash_attn = True
+    _supports_sdpa = True
+
+    def __init__(self, config: FireRedTTS3InstructCoreConfig):
+        super().__init__(config)
+        # Backbone Transformer
+        self.backbone_llm_config = Qwen3Config.from_dict(Qwen3_1_7B_ConfigDict)
+        self.backbone_llm = Qwen3ForCausalLM(self.backbone_llm_config)
+        # PatchEncoder
+        self.patch_encoder = PatchEncoder(
+            in_dim=config.redae_dim,
+            out_dim=self.backbone_llm_config.hidden_size,
+            patch_size=config.patch_size,
+            hidden_size=config.patch_encoder_hidden_size,
+            mlp_ratio=config.patch_encoder_mlp_ratio,
+            depth=config.patch_encoder_depth,
+            num_heads=config.patch_encoder_num_heads,
+        )
+        # DiT
+        self.dit_head = torch.nn.Linear(self.backbone_llm_config.hidden_size, config.dit_hidden_size)
+        self.dit = DiT(
+            in_channels=(config.redae_dim+config.dit_hidden_size),
+            out_channels=config.redae_dim,
+            mlp_ratio=config.dit_mlp_ratio,
+            depth=config.dit_depth,
+            num_heads=config.dit_num_heads,
+            hidden_size=config.dit_hidden_size,
+        )
+        # Stop
+        self.stop_head = torch.nn.Linear(self.backbone_llm_config.hidden_size, 1)
+        # Shared
+        self.redae_dim = config.redae_dim
+        self.patch_size = self.patch_encoder.patch_size
+        self.history_patches = config.num_history_patches
+        self.history_length = config.num_history_patches * self.patch_size
+        self.post_init()
+    
+    # Manually init RotaryEmbedding buffers
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, RotaryEmbedding):
+            module.rope_init()
+    
+    # Backbone Transformer AR wrapper
+    @torch.autocast(device_type='cuda', dtype=torch.bfloat16)
+    def _backbone_one_step(self, input_embeds: torch.Tensor, cache = None):
+        outs = self.backbone_llm.model.forward(
+            inputs_embeds=input_embeds,
+            use_cache=True,
+            past_key_values=cache,
+        )
+        # Only take the last timestep
+        hidden_states = outs.last_hidden_state
+        new_cache = outs.past_key_values
+        return hidden_states, new_cache
+    
+    # Flow head wrapper
+    def _flow_one_step(
+        self, 
+        hist_latents: torch.Tensor, 
+        backbone_cond: torch.Tensor,
+        t_span: torch.Tensor,
+        inference_cfg: float,
+    ):
+        # Compose input
+        x0 = torch.randn(1, self.patch_size, self.redae_dim, device=hist_latents.device)
+        
+        xt = torch.cat([hist_latents, x0], dim=1)   # History clean + current noise
+        cond = backbone_cond.repeat_interleave(self.patch_size, dim=1)    # Correspond backbone cond
+        # Run flow inference
+        for ti, t in enumerate(t_span[:-1]):
+            dt = t_span[ti+1]-t
+            t_in = t.view(-1, 1, 1)
+            x_in = torch.cat([xt, cond], dim=2)
+            if inference_cfg > 0:
+                x_in_cfg = torch.cat([xt, cond * 0], dim=2)
+                x_in = torch.cat([x_in, x_in_cfg], dim=0)
+                t_in = t_in.expand(2, -1, -1)
+            vt = self.dit(x=x_in, t=t_in)
+            if inference_cfg > 0:
+                vt_cond, vt_cfg = vt.chunk(2, dim=0)
+                vt = (1.0 + inference_cfg) * vt_cond - inference_cfg * vt_cfg
+            # Only denoise current patch
+            xt[:, -self.patch_size:] = xt[:, -self.patch_size:] + dt.view(-1, 1, 1) * vt[:, -self.patch_size:]
+        # Remove history
+        x1 = xt[:, -self.patch_size:]        
+        return x1
+
+    # Core LLM-DiT AR Loop
+    @torch.no_grad()
+    def generate(
+        self, 
+        # Input
+        text_tokens: torch.Tensor,
+        latents_in: torch.Tensor = None,
+        latents_in_mask: torch.Tensor = None,   # For filling text_tokens
+        # Output (ICL)
+        latents_out: torch.Tensor = None,
+        latents_out_mask: torch.Tensor = None,  # For filling text_tokens
+        # Text inference settings
+        infer_text: bool = False,
+        text_repetition_penalty: float = None,
+        text_do_sample: bool = True,
+        text_temperature: float = None,
+        text_top_p: float = None,
+        text_top_k: int = None,
+        # Audio inference settings
+        n_timesteps: int = 10,
+        inference_cfg: float = 2.0,
+        stop_threshold: float = 0.5,
+        # Audio length control
+        min_gen_steps: int = 6,
+        max_gen_steps: int = None,
+    ):
+        device = text_tokens.device
+
+        # Compose input sequence
+        input_embeds: torch.Tensor = self.backbone_llm.model.embed_tokens(text_tokens)
+        # Any input audio
+        if latents_in is not None:
+            latents_patch_in: torch.Tensor = self.patch_encoder(latents_in)
+            input_embeds = input_embeds.masked_scatter(
+                latents_in_mask.unsqueeze(-1),
+                latents_patch_in.reshape(-1).to(input_embeds),
+            )
+        # Any output audio (ICL prompt)
+        if latents_out is not None:
+            latents_patch_out = self.patch_encoder.forward(latents_out)
+            input_embeds = input_embeds.masked_scatter(
+                latents_out_mask.unsqueeze(-1),
+                latents_patch_out.reshape(-1).to(input_embeds),
+            )
+        else:
+            latents_out = torch.zeros(1, 0, self.config.redae_dim, device=device)
+            latents_patch_out = None
+        
+        # Prepare DiT decode
+        t_span = torch.linspace(0, 1, n_timesteps + 1).to(device)
+        t_span = 1 - torch.cos(t_span * 0.5 * torch.pi) # (n_timesteps+1,)
+
+        # Init Backbone states
+        backbone_cache = None
+
+        # --- Infer text 
+        if infer_text:
+            # Text logits processor
+            text_logits_processor = init_text_logits_processor(
+                text_repetition_penalty, text_do_sample,
+                text_temperature, text_top_p, text_top_k,
+            )
+            text_gen_ids = torch.empty((1, 0), dtype=torch.long, device=device)
+            for text_step_index in range(200):
+                backbone_out, backbone_cache = self._backbone_one_step(input_embeds, cache=backbone_cache)   # (b=1, t, c)
+                # Sampling
+                logits = self.backbone_llm.lm_head(backbone_out[:, -1, :])  # (1, V)
+                scores = text_logits_processor(text_gen_ids, logits)
+                if text_do_sample:
+                    probs = torch.softmax(scores.float(), dim=-1)
+                    next_token = torch.multinomial(probs, num_samples=1)[:, 0]  # (1,)
+                else:
+                    next_token = scores.argmax(dim=-1)                          # (1,)
+                # Next step
+                input_embeds = self.backbone_llm.model.embed_tokens(next_token.unsqueeze(0))
+                # Whether stop
+                if next_token.item() == TEXT_EOT_ID:
+                    break
+                text_gen_ids = torch.cat([text_gen_ids, next_token.unsqueeze(0)], dim=1)
+            # Finalize text inference
+            _, backbone_cache = self._backbone_one_step(input_embeds, cache=backbone_cache)   # (b=1, t, c)
+            # Process <|sosp|> for audio start
+            next_token = next_token * 0 + AUDIO_SOS_ID
+            input_embeds = self.backbone_llm.model.embed_tokens(next_token.unsqueeze(0))
+            _, backbone_cache = self._backbone_one_step(input_embeds, cache=backbone_cache)   # (b=1, t, c)
+        
+        # --- Infer audio
+        latents_gen =  F.pad(latents_out, (0, 0, self.history_length, 0))
+        backbone_cond = input_embeds.new_zeros(1, self.history_patches, input_embeds.shape[-1])
+
+        max_gen_steps = (400 if max_gen_steps is None else max_gen_steps)
+        for step_index in range(max_gen_steps):
+            # Backbone condition
+            backbone_out, backbone_cache = self._backbone_one_step(input_embeds, cache=backbone_cache)   # (b=1, t, c)
+
+            # Stop prediction
+            stop_logits = self.stop_head(backbone_out[:, -1]).squeeze(-1)
+            stop_score = torch.sigmoid(stop_logits).item()
+            if stop_score >= stop_threshold:
+                if min_gen_steps is not None:
+                    if step_index >= min_gen_steps: break
+                else:
+                    break
+            
+            # DiT decode
+            if step_index == 0 and latents_patch_out is not None:
+                one_backbone_out = backbone_out[:, -latents_patch_out.shape[1]:]
+            else:
+                one_backbone_out = backbone_out[:, -1:]
+            backbone_cond = torch.cat([backbone_cond, one_backbone_out], dim=1)
+
+            one_latents = self._flow_one_step(
+                hist_latents=latents_gen[:, -self.history_length:],
+                backbone_cond=self.dit_head((backbone_cond[:, -(self.history_patches+1):])),
+                t_span=t_span,
+                inference_cfg=inference_cfg,
+            )
+            
+            input_embeds = self.patch_encoder(one_latents)
+            latents_gen = torch.cat([latents_gen, one_latents], dim=1)
+        
+        # Remove dummy history
+        latents_gen = latents_gen[:, self.history_length:]
+
+        if infer_text:
+            return latents_gen, text_gen_ids
+        else:
+            return latents_gen
+
+
+# RedAE + TextTokenizer + TTS3Core
+class FireRedTTS3Instruct(object):
+    def __init__(self, pretrained_model_dir: str):
+        self.device = torch.device('cuda')
+        # RedAE
+        redae_model_dir = os.path.join(pretrained_model_dir, 'redae')
+        assert os.path.exists(redae_model_dir), f'{redae_model_dir} not found'
+        self.redae = RedAE.from_pretrained(redae_model_dir)
+        self.redae.to(self.device)
+        # LLM-DiT 
+        tts_model_dir = os.path.join(pretrained_model_dir, 'fireredtts3_instruct')
+        assert os.path.exists(tts_model_dir), f'{tts_model_dir} not found'
+        self.tts_core = FireRedTTS3InstructCore.from_pretrained(tts_model_dir)
+        self.tts_core.to(self.device)
+        # Text Tokenizer
+        text_tok_dir = os.path.join(pretrained_model_dir, 'text_tokenizer')
+        assert os.path.exists(text_tok_dir), f'{text_tok_dir} not found'
+        self.text_tokenizer = load_text_tokenizer(text_tok_dir)
+    
+    def _tokenize_text(self, text:str):
+        tokens = self.text_tokenizer(
+            text, 
+            truncation=False, padding=False, add_special_tokens=False,
+        )["input_ids"]
+        tokens = torch.tensor([tokens], dtype=torch.long, device=self.device)
+        return tokens
+
+    def _tokenize_audio(self, audio: torch.Tensor, audio_sr: int):
+        audio = audio[:1]
+        audio = torchaudio.functional.resample(audio, audio_sr, self.redae.sample_rate)
+        audio_sr = self.redae.sample_rate
+        audio = self.redae.pad_to_multiple_of(audio, self.redae.downsample_rate*self.tts_core.patch_size)
+        audio = audio.to(self.device)
+        latents = self.redae.encode(audio, audio_sr) * REDAE_SCALE
+        latents = latents.to(torch.float32)
+        return latents
+
+    # --- Inference Interface
+    @torch.inference_mode()
+    def generate_tts(
+        self,
+        # Input
+        prompt_text: str,
+        prompt_audio: torch.Tensor,
+        prompt_audio_sr: int,
+        text: str,
+        # Inference
+        stop_threshold: float = 0.5,
+        n_timesteps: int = 10,
+        inference_cfg: float = 2.0,
+        seed: int = 1234,
+    ):
+        prompt_latents = self._tokenize_audio(prompt_audio, prompt_audio_sr)
+        text_in = compose_generate_input_tts(prompt_latents.shape[1]//self.tts_core.patch_size, prompt_text, text)
+        text_tokens = self._tokenize_text(text_in)
+        # AR generate
+        if seed is not None:
+            fix_seed(seed)
+        gen_latents = self.tts_core.generate(
+            # Input
+            text_tokens=text_tokens,
+            latents_out=prompt_latents,
+            latents_out_mask=(text_tokens==CHATML_LATENT_OUT_PAD_ID),
+            # Text inference settings
+            infer_text=False,
+            # Audio inference settings
+            n_timesteps=n_timesteps,
+            inference_cfg=inference_cfg,
+            stop_threshold=0.5,
+            # Audio length control
+            min_gen_steps=6,
+            max_gen_steps=None,
+        )
+        gen_audio, gen_audio_sr = self.redae.decode(gen_latents / REDAE_SCALE)
+        # Remove prompts
+        gen_audio = gen_audio[:, (self.redae.downsample_rate*prompt_latents.shape[1]):]
+        return gen_audio, gen_audio_sr
+
+    def generate_voice_design(
+        self,
+        # Input
+        instruction: str,
+        text: str,
+        # Audio Inference Settings
+        n_timesteps: int = 10,
+        inference_cfg: float = 1.2,
+        # Random
+        seed: int = 2,
+    ):
+        text_in = compose_generate_input_voice_design(instruction, text)
+        text_tokens = self._tokenize_text(text_in)
+
+        # AR generate
+        if seed is not None:
+            fix_seed(seed)
+        gen_latents, gen_text_ids = self.tts_core.generate(
+            # Input
+            text_tokens=text_tokens,
+            # Text inference settings
+            infer_text=True,
+            text_repetition_penalty=1.0,
+            text_do_sample=True,
+            text_temperature=0.7,
+            text_top_p=0.8,
+            text_top_k=20,
+            # Audio inference settings
+            n_timesteps=n_timesteps,
+            inference_cfg=inference_cfg,
+            stop_threshold=0.5,
+            # Audio length control
+            min_gen_steps=6,
+            max_gen_steps=None,
+        )
+        gen_audio, gen_audio_sr = self.redae.decode(gen_latents / REDAE_SCALE)
+        gen_text = self.text_tokenizer.decode(gen_text_ids.squeeze(0).cpu())
+        
+        return gen_audio, gen_audio_sr, gen_text
+
+    def generate_semantic_edit(
+        self,
+        # Input
+        instruction: str,
+        audio_in: torch.Tensor,
+        audio_in_sr: torch.Tensor,
+        # Audio Inference Settings
+        n_timesteps: int = 10,
+        inference_cfg: float = 1.2,
+        # Random
+        seed: int = 1234,
+    ):
+        latents_in = self._tokenize_audio(audio_in, audio_in_sr)
+        text_in = compose_generate_input_semantic_edit(instruction, latents_in.shape[1]//self.tts_core.patch_size)
+        text_tokens = self._tokenize_text(text_in)
+        # AR generate
+        if seed is not None:
+            fix_seed(seed)
+        gen_latents, gen_text_ids = self.tts_core.generate(
+            # Input
+            text_tokens=text_tokens,
+            latents_in=latents_in,
+            latents_in_mask=(text_tokens==CHATML_LATENT_IN_PAD_ID),
+            # Text inference settings
+            infer_text=True,
+            text_repetition_penalty=1.0,
+            text_do_sample=False,
+            # Audio inference settings
+            n_timesteps=n_timesteps,
+            inference_cfg=inference_cfg,
+            stop_threshold=0.5,
+            # Audio length control
+            min_gen_steps=6,
+            max_gen_steps=None,
+        )
+        gen_audio, gen_audio_sr = self.redae.decode(gen_latents / REDAE_SCALE)
+        gen_text = self.text_tokenizer.decode(gen_text_ids.squeeze(0).cpu())
+        
+        return gen_audio, gen_audio_sr, gen_text
+
+    def generate_acoustic_edit(
+        self,
+        # Input
+        instruction: str,
+        audio_in: torch.Tensor,
+        audio_in_sr: torch.Tensor,
+        # Audio Inference Settings
+        n_timesteps: int = 10,
+        inference_cfg: float = 1.2,
+        # Random
+        seed: int = 1234,
+    ):
+        latents_in = self._tokenize_audio(audio_in, audio_in_sr)
+        text_in = compose_generate_input_acoustic_edit(instruction, latents_in.shape[1]//self.tts_core.patch_size)
+        text_tokens = self._tokenize_text(text_in)
+        # AR generate
+        if seed is not None:
+            fix_seed(seed)
+        gen_latents = self.tts_core.generate(
+            # Input
+            text_tokens=text_tokens,
+            latents_in=latents_in,
+            latents_in_mask=(text_tokens==CHATML_LATENT_IN_PAD_ID),
+            # Text inference settings
+            infer_text=False,
+            # Audio inference settings
+            n_timesteps=n_timesteps,
+            inference_cfg=inference_cfg,
+            stop_threshold=0.5,
+            # Audio length control
+            min_gen_steps=6,
+            max_gen_steps=None,
+        )
+        gen_audio, gen_audio_sr = self.redae.decode(gen_latents / REDAE_SCALE)
+        return gen_audio, gen_audio_sr

fireredtts3/llm/modules.py

@@ -0,0 +1,185 @@
+import math
+import torch
+import torch.nn.functional as F
+from torch import nn
+from fireredtts3.llm.rotary_embedding import (
+    apply_rotary_pos_emb, 
+    RotaryEmbedding
+)
+
+
+class SinusPositionEmbedding(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.dim = dim
+
+    def forward(self, x, scale=1000):
+        device = x.device
+        half_dim = self.dim // 2
+        emb = math.log(10000) / (half_dim - 1)
+        emb = torch.exp(torch.arange(half_dim, device=device).float() * -emb)
+        emb = scale * x.unsqueeze(1) * emb.unsqueeze(0)
+        emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
+        return emb
+
+
+class TimestepEmbedder(nn.Module):
+    def __init__(self, dim, freq_embed_dim=256):
+        super().__init__()
+        self.time_embed = SinusPositionEmbedding(freq_embed_dim)
+        self.time_mlp = nn.Sequential(
+            nn.Linear(freq_embed_dim, dim), 
+            nn.SiLU(), 
+            nn.Linear(dim, dim)
+        )
+
+    def forward(self, timestep):
+        time_hidden = self.time_embed(timestep)
+        time_hidden = time_hidden.to(timestep.dtype)
+        time = self.time_mlp(time_hidden)  # b d
+        return time
+
+    
+class RMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.native_rms_norm = float(torch.__version__[:3]) >= 2.4
+
+    def forward(self, x):
+        if self.native_rms_norm:
+            if self.weight.dtype in [torch.float16, torch.bfloat16]:
+                x = x.to(self.weight.dtype)
+            x = F.rms_norm(x, normalized_shape=(x.shape[-1],), weight=self.weight, eps=self.eps)
+        else:
+            variance = x.to(torch.float32).pow(2).mean(-1, keepdim=True)
+            x = x * torch.rsqrt(variance + self.eps)
+            if self.weight.dtype in [torch.float16, torch.bfloat16]:
+                x = x.to(self.weight.dtype)
+            x = x * self.weight
+
+        return x
+
+
+class FeedForward(nn.Module):
+    def __init__(self, dim, dim_out=None, mult=4, dropout=0.0, approximate: str = "none"):
+        super().__init__()
+        inner_dim = int(dim * mult)
+        dim_out = dim_out if dim_out is not None else dim
+
+        activation = nn.GELU(approximate=approximate)
+        project_in = nn.Sequential(nn.Linear(dim, inner_dim), activation)
+        self.ff = nn.Sequential(project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out))
+
+    def forward(self, x):
+        return self.ff(x)
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        heads: int = 8,
+        dim_head: int = 64,
+        dropout: float = 0.0,
+    ):
+        super().__init__()
+
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("Attention equires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+
+        self.dim = dim
+        self.heads = heads
+        self.inner_dim = dim_head * heads
+        self.dropout = dropout
+        self.to_q = nn.Linear(dim, self.inner_dim)
+        self.to_k = nn.Linear(dim, self.inner_dim)
+        self.to_v = nn.Linear(dim, self.inner_dim)
+
+        self.to_out = nn.ModuleList([])
+        self.to_out.append(nn.Linear(self.inner_dim, dim))
+        self.to_out.append(nn.Dropout(dropout))
+
+    def forward(
+        self,
+        x: float,  # noised input x
+        mask=None,
+        rope=None,  # rotary position embedding for x
+    ) -> torch.Tensor:
+
+        batch_size = x.shape[0]
+
+        query = self.to_q(x)
+        key = self.to_k(x)
+        value = self.to_v(x)
+
+        # attention
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // self.heads
+        query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
+        key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
+
+        # apply rotary position embedding
+        if rope is not None:
+            freqs, xpos_scale = rope
+            q_xpos_scale, k_xpos_scale = (xpos_scale, xpos_scale**-1.0) if xpos_scale is not None else (1.0, 1.0)
+            query = apply_rotary_pos_emb(query, freqs, q_xpos_scale)
+            key = apply_rotary_pos_emb(key, freqs, k_xpos_scale)
+
+        # with sdpa_kernel(SDPBackend.EFFICIENT_ATTENTION):
+        x = F.scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False)
+        
+        x = x.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)
+        x = x.to(query.dtype)
+        x = self.to_out[0](x)
+        x = self.to_out[1](x)
+
+        if mask is not None:
+            mask = mask.unsqueeze(-1)
+            x = x.masked_fill(~mask, 0.0)
+
+        return x
+
+
+class DiTBlock(nn.Module):
+    def __init__(
+        self, 
+        hidden_size, 
+        num_heads, 
+        mlp_ratio=4.0, 
+        dropout=0.1, 
+        **kwargs
+    ):
+        super().__init__()
+        self.norm1 = RMSNorm(hidden_size, eps=1e-6)
+        self.attn = Attention(
+            dim=hidden_size,
+            heads=num_heads,
+            dim_head=hidden_size // num_heads,
+            dropout=dropout
+        )
+        self.norm2 = RMSNorm(hidden_size, eps=1e-6)
+        self.mlp = FeedForward(dim=hidden_size, mult=mlp_ratio, dropout=dropout, approximate="tanh")
+
+    def forward(self, x, mask, rope):
+        x = x + self.attn(self.norm1(x), mask=mask, rope=rope)
+        x = x + self.mlp(self.norm2(x))
+        return x
+
+
+class FinalLayer(nn.Module):
+    """
+    The final layer of DiT.
+    """
+    def __init__(self, hidden_size, out_channels):
+        super().__init__()
+        self.norm_final = RMSNorm(hidden_size, eps=1e-6)
+        self.linear = nn.Linear(hidden_size, out_channels, bias=True)
+
+    def forward(self, x):
+        x = self.norm_final(x)
+        x = self.linear(x)
+        return x
+

fireredtts3/llm/patch_encoder.py

@@ -0,0 +1,65 @@
+import torch
+from fireredtts3.llm.modules import (
+    DiTBlock,
+    FinalLayer,
+    RotaryEmbedding,
+)
+
+
+class PatchEncoder(torch.nn.Module):
+    def __init__(
+        self, 
+        # In & out
+        in_dim: int,
+        out_dim: int,
+        # Model config
+        patch_size: int = 4,
+        hidden_size: int = 1024,
+        mlp_ratio: int = 3,
+        depth: int = 8,
+        num_heads: int = 8,
+    ):
+        super().__init__()
+        self.in_dim = in_dim
+        self.patch_size = patch_size
+        self.hidden_size = hidden_size
+        self.out_dim = out_dim
+        # [CLS] token
+        self.cls_tok = torch.nn.Parameter(torch.zeros(1, 1, hidden_size))
+        self.rotary_embed = RotaryEmbedding(hidden_size // num_heads)
+        self.blocks = torch.nn.ModuleList([
+            DiTBlock(hidden_size, num_heads, mlp_ratio=mlp_ratio) for _ in range(depth)
+        ])
+        # Input & output proj
+        self.in_proj = (
+            torch.nn.Linear(in_dim, hidden_size)
+            if in_dim != hidden_size else 
+            torch.nn.Identity()
+        )
+        self.out_proj = FinalLayer(hidden_size, out_dim)
+
+    def forward(self, inputs_embeds: torch.Tensor):
+        """Patch encoder aggregating {patch_size} latents into one.
+
+        Args:
+            inputs_embeds(torch.Tensor): shape (b=1, t, c).
+        Returns:
+            hidden_states(torch.Tensor): shape (b=1, t//patch_size, c).
+        """
+        assert inputs_embeds.shape[1] % self.patch_size == 0, \
+            'inputs_embeds.shape={} patch_size={}'.format(inputs_embeds.shape, self.patch_size)
+        
+        inputs_embeds = self.in_proj(inputs_embeds)
+        # Patchify, (b=1, t, c) -> (t//patch_size, patch_size, c)
+        hidden_states = inputs_embeds.reshape(-1, self.patch_size, self.hidden_size)
+        cls_tok = self.cls_tok.expand(hidden_states.shape[0], -1, -1)   # (b*t//patch_size, 1, c)
+        hidden_states = torch.cat([cls_tok, hidden_states], dim=1)  # (b*t//patch_size, 1+patch_size, c)
+        # NOTE full attention
+        rope = self.rotary_embed.forward_from_seq_len(hidden_states.shape[1])
+        for block in self.blocks:
+            hidden_states = block(hidden_states, None, rope)
+        hidden_states = self.out_proj(hidden_states)
+        hidden_states = hidden_states[:, 0]    # (t//patch_size, c)
+        hidden_states = hidden_states.unsqueeze(0)
+        return hidden_states
+

fireredtts3/llm/rotary_embedding.py

@@ -0,0 +1,75 @@
+import torch
+from torch.nn import Module
+from torch.amp import autocast
+from torch import cat, stack, arange
+from einops import rearrange
+
+
+class RotaryEmbedding(Module):
+    def __init__(
+        self,
+        dim,
+        interpolation_factor = 1.,
+        base = 10000,
+        base_rescale_factor = 1.
+    ):
+        super().__init__()
+        # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning
+        # has some connection to NTK literature
+        # https://www.reddit.com/r/LocalLLaMA/comments/14lz7j5/ntkaware_scaled_rope_allows_llama_models_to_have/
+        base *= base_rescale_factor ** (dim / (dim - 2))
+
+        self._rope_dim = dim
+        self._rope_base = base
+
+        inv_freq = 1. / (base ** (arange(0, dim, 2).float() / dim))
+        self.register_buffer('inv_freq', inv_freq, persistent=False)
+
+        assert interpolation_factor >= 1.
+        self.interpolation_factor = interpolation_factor
+
+    def rope_init(self):
+        dim, base = self._rope_dim, self._rope_base
+        self.inv_freq = 1. / (base ** (arange(0, dim, 2).float() / dim))
+
+    def forward_from_seq_len(self, seq_len):
+        device = self.inv_freq.device
+
+        t = arange(seq_len, device = device)
+        return self.forward(t)
+
+    @autocast('cuda', enabled = False)
+    def forward(self, t, offset = 0):
+        if t.ndim == 1:
+            t = rearrange(t, 'n -> 1 n')
+
+        freqs = torch.einsum('b i , j -> b i j', t.type_as(self.inv_freq), self.inv_freq) / self.interpolation_factor
+        freqs = stack((freqs, freqs), dim = -1)
+        freqs = rearrange(freqs, '... d r -> ... (d r)')
+
+        return freqs, 1.
+
+
+def rotate_half(x):
+    x = rearrange(x, '... (d r) -> ... d r', r = 2)
+    x1, x2 = x.unbind(dim = -1)
+    x = stack((-x2, x1), dim = -1)
+    return rearrange(x, '... d r -> ... (d r)')
+
+
+@autocast('cuda', enabled = False)
+def apply_rotary_pos_emb(t, freqs, scale = 1):
+    rot_dim, seq_len, orig_dtype = freqs.shape[-1], t.shape[-2], t.dtype
+
+    freqs = freqs[:, -seq_len:, :]
+    scale = scale[:, -seq_len:, :] if isinstance(scale, torch.Tensor) else scale
+
+    if t.ndim == 4 and freqs.ndim == 3:
+        freqs = rearrange(freqs, 'b n d -> b 1 n d')
+
+    # partial rotary embeddings, Wang et al. GPT-J
+    t, t_unrotated = t[..., :rot_dim], t[..., rot_dim:]
+    t = (t * freqs.cos() * scale) + (rotate_half(t) * freqs.sin() * scale)
+    out = cat((t, t_unrotated), dim = -1)
+
+    return out.type(orig_dtype)

fireredtts3/redae/redae.py

@@ -0,0 +1,496 @@
+import math
+import torch
+import torchaudio
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import (
+    Qwen3Config, Qwen3Model, 
+    PretrainedConfig, PreTrainedModel
+)
+
+# --- Encoder
+class Qwen3ClsDownsample(torch.nn.Module):
+    def __init__(
+        self,
+        in_dim: int,
+        out_dim: int,
+        downsample_rate: int = 2,
+        # Qwen
+        hidden_size: int = 896,
+        intermediate_size: int = 896*4,
+        num_hidden_layers: int = 4,
+        max_position_embeddings: int = 32768,
+        num_attention_heads: int = 14,
+        num_key_value_heads: int = 2,
+    ):
+        super().__init__()
+        self.in_dim = in_dim
+        self.out_dim = out_dim
+        self.downsample_rate = downsample_rate
+        self.qwen3_config = Qwen3Config(
+            hidden_size=hidden_size,
+            intermediate_size=intermediate_size,
+            num_hidden_layers=num_hidden_layers,
+            max_position_embeddings=max_position_embeddings,
+            num_attention_heads=num_attention_heads,
+            num_key_value_heads=num_key_value_heads,
+            attn_implementation='flash_attention_2',
+        )
+        self.qwen3 = Qwen3Model(self.qwen3_config)
+        self.cls_tok = torch.nn.Parameter(torch.ones(1, 1, hidden_size))
+        # Input & output proj
+        self.in_proj = (
+            torch.nn.Linear(in_dim, hidden_size)
+            if in_dim != hidden_size else 
+            torch.nn.Identity()
+        )
+        self.out_proj = (
+            torch.nn.Linear(hidden_size, out_dim)
+            if out_dim != hidden_size else 
+            torch.nn.Identity()
+        )
+    
+    def forward(self, xs: torch.Tensor):
+        """Downsample latents as BERT [CLS] token
+
+        Args:
+            xs(torch.Tensor): shape (b, t, c).
+        Returns:
+            ys(torch.Tensor): shape (b, t//downsample_rate, c).
+        """
+        assert xs.shape[1] % self.downsample_rate == 0, xs.shape
+        b = xs.shape[0]
+        # Patchify, (b, t, c) -> (b*t/down, down, c)
+        xs = xs.reshape(-1, self.downsample_rate, self.in_dim)
+        xs = self.in_proj(xs)
+        cls_tok = self.cls_tok.expand(xs.shape[0], -1, -1)
+        xs = torch.cat([xs, cls_tok], dim=1)  # (b*t/down, down+1, c)
+        # Forward LLM
+        ys = []
+        for xs_chunk in torch.split(xs, 32768, dim=0):  # FlashAttn limits
+            outs_chunk = self.qwen3(inputs_embeds=xs_chunk)
+            ys_chunk = outs_chunk.last_hidden_state[:, -1]   # (b*t/down, c)
+            ys.append(ys_chunk)
+        ys = torch.cat(ys, dim=0)
+        # ---
+        ys = self.out_proj(ys)
+        ys = ys.reshape(b, -1, self.out_dim)
+        return ys
+
+
+class RedAEAudioEncoder(torch.nn.Module):
+    def __init__(
+        self,
+        # Output
+        out_dim: int = 1024,
+        # Input reshape
+        audio_patch_size: int = 480,    # 50Hz
+        audio_sample_rate: int = 24000,
+        # Qwen
+        hidden_size: int = 896,
+        intermediate_size: int = 896*4,
+        num_hidden_layers: int = 24,
+        max_position_embeddings: int = 32768,
+        max_window_layers: int = 0,
+        num_attention_heads: int = 14,
+        num_key_value_heads: int = 2,
+        sliding_window: int = 64,
+        use_sliding_window: bool = True,
+        # Extra downsample
+        extra_downsample_rate: int = 2,   # 50Hz -> 25Hz
+        downsample_num_hidden_layers: int = 4,
+    ):
+        super().__init__()
+        self.audio_patch_size = audio_patch_size
+        self.audio_sample_rate = audio_sample_rate
+        self.extra_downsample_rate = extra_downsample_rate
+        self.out_dim = out_dim
+        self.in_proj = nn.Sequential(
+            nn.Linear(self.audio_patch_size, hidden_size),
+            nn.Linear(hidden_size, hidden_size),
+        )
+        self.qwen3_config = Qwen3Config(
+            hidden_size=hidden_size,
+            intermediate_size=intermediate_size,
+            num_hidden_layers=num_hidden_layers,
+            max_position_embeddings=max_position_embeddings,
+            max_window_layers=max_window_layers,
+            num_attention_heads=num_attention_heads,
+            num_key_value_heads=num_key_value_heads,
+            sliding_window=sliding_window,
+            use_sliding_window=use_sliding_window,
+            attn_implementation='flash_attention_2',
+        )
+        self.qwen3 = Qwen3Model(self.qwen3_config)
+        if self.extra_downsample_rate > 1:
+            self.downsample = Qwen3ClsDownsample(
+                in_dim=hidden_size,
+                out_dim=hidden_size,
+                downsample_rate=extra_downsample_rate,
+                hidden_size=hidden_size,
+                intermediate_size=intermediate_size,
+                num_hidden_layers=downsample_num_hidden_layers,
+                max_position_embeddings=max_position_embeddings,
+                num_attention_heads=num_attention_heads,
+                num_key_value_heads=num_key_value_heads,
+            )
+        self.out_proj = nn.Linear(hidden_size, out_dim)
+
+    # --- Some property
+    @property
+    def downsample_rate(self):
+        return int(self.audio_patch_size * self.extra_downsample_rate)
+    
+    @property
+    def sample_rate(self):
+        return self.audio_sample_rate
+    
+    @property
+    def hidden_size(self):
+        return self.out_dim
+    
+    # --- Forward
+    def forward(self, audio: torch.Tensor):
+        """
+        Args:
+            audio(torch.Tensor): shape (b, t)
+        Returns:
+            xs(torch.Tensor): shape (b, t//self.downsample_rate, self.hidden_size)
+        """ 
+        assert audio.shape[1] % self.downsample_rate == 0, audio.shape
+        # Patchify
+        xs = audio.unfold(
+            dimension=1,
+            size=self.audio_patch_size,
+            step=self.audio_patch_size,
+        )   # (b, num_patch, patch_size) ~ (b, t, c)
+        # LLM
+        xs = self.in_proj(xs)
+        outs = self.qwen3(
+            inputs_embeds=xs,
+            attention_mask=None,
+        )
+        xs = outs.last_hidden_state # (b, t, c)
+        # Downsample
+        if self.extra_downsample_rate > 1:
+            xs = self.downsample(xs)
+        xs = self.out_proj(xs)
+        return xs
+
+
+# --- Decoder
+class ISTFT(nn.Module):
+    def __init__(
+        self, 
+        n_fft: int, 
+        hop_length: int, 
+        win_length: int, 
+        padding: str = "same"
+    ):
+        super().__init__()
+        assert padding in ["center", "same"], "Padding must be 'center' or 'same'."
+        self.padding = padding
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+        self.win_length = win_length
+        window = torch.hann_window(win_length)
+        self.register_buffer("window", window)
+
+    def forward(self, spec: torch.Tensor) -> torch.Tensor:
+        if self.padding == "center":
+            # Fallback to pytorch native implementation
+            return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
+        elif self.padding == "same":
+            pad = (self.win_length - self.hop_length) // 2
+        else:
+            raise ValueError("Padding must be 'center' or 'same'.")
+
+        assert spec.dim() == 3, "Expected a 3D tensor as input"
+        B, N, T = spec.shape
+
+        # Inverse FFT
+        ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
+        ifft = ifft * self.window[None, :, None]
+
+        # Overlap and Add
+        output_size = (T - 1) * self.hop_length + self.win_length
+        y = torch.nn.functional.fold(
+            ifft, output_size=(1, output_size), kernel_size=(1, self.win_length), stride=(1, self.hop_length),
+        )[:, 0, 0, pad:-pad]
+
+        # Window envelope
+        window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
+        window_envelope = torch.nn.functional.fold(
+            window_sq, output_size=(1, output_size), kernel_size=(1, self.win_length), stride=(1, self.hop_length),
+        ).squeeze()[pad:-pad]
+
+        # Normalize
+        assert (window_envelope > 1e-11).all()
+        y = y / window_envelope
+
+        return y
+
+
+class ISTFTHead(nn.Module):
+    def __init__(
+        self, 
+        dim: int, 
+        n_fft: int, 
+        hop_length: int, 
+        padding: str = "same"
+    ):
+        super().__init__()
+        self.hop_length = hop_length
+        out_dim = n_fft + 2
+        self.out = torch.nn.Linear(dim, out_dim)
+        self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x_pred = self.out(x)
+        x_pred = x_pred.transpose(1, 2)
+        mag, p = x_pred.chunk(2, dim=1)
+        mag = torch.exp(mag)
+        mag = torch.clip(mag, max=1e2)  # safeguard to prevent excessively large magnitudes
+        # wrapping happens here. These two lines produce real and imaginary value
+        x = torch.cos(p)
+        y = torch.sin(p)
+        # recalculating phase here does not produce anything new
+        # only costs time
+        # phase = torch.atan2(y, x)
+        # S = mag * torch.exp(phase * 1j)
+        # better directly produce the complex value 
+        S = mag * (x + 1j * y)
+        audio = self.istft(S)
+        return audio
+
+
+class RedAEAudioDecoder(torch.nn.Module):
+    def __init__(
+        self,
+        in_dim: int = 64,
+        upsample_rate: int = 2, # 25Hz -> 50Hz
+        # Output reshape
+        audio_patch_size: int = 480,    # 50Hz
+        audio_sample_rate: int = 24000,
+        # Qwen(mirrors encoder)
+        hidden_size: int = 896,
+        intermediate_size: int = 896*4,
+        num_hidden_layers: int = 18,
+        max_position_embeddings: int = 32768,
+        max_window_layers: int = 0,
+        num_attention_heads: int = 14,
+        num_key_value_heads: int = 2,
+        sliding_window: int = 64,
+        use_sliding_window: bool = True,
+    ):
+        super().__init__()
+        self.upsample_rate = upsample_rate
+        self.audio_patch_size = audio_patch_size
+        self.audio_sample_rate = audio_sample_rate
+        # Upsample MLP
+        self.in_proj = nn.Linear(in_dim, upsample_rate * hidden_size)
+        # Qwen3
+        self.qwen3_config = Qwen3Config(
+            hidden_size=hidden_size,
+            intermediate_size=intermediate_size,
+            num_hidden_layers=num_hidden_layers,
+            max_position_embeddings=max_position_embeddings,
+            max_window_layers=max_window_layers,
+            num_attention_heads=num_attention_heads,
+            num_key_value_heads=num_key_value_heads,
+            sliding_window=sliding_window,
+            use_sliding_window=use_sliding_window,
+        )
+        self.qwen3 = Qwen3Model(self.qwen3_config)
+
+        self.istft_head = ISTFTHead(
+            dim=hidden_size, 
+            n_fft=audio_patch_size*4,
+            hop_length=audio_patch_size,
+            padding='same',
+        )
+
+    def forward(
+        self,
+        xs: torch.Tensor,
+    ):
+        """
+        Args:
+            xs(torch.Tensor): shape (b, t, c)
+        Returns:
+            audio(torch.Tensor): shape (b, t*upsample_rate*patch_size)
+            audio_len(torch.Tensor): shape (b,)
+        """
+        # Upsample
+        xs = self.in_proj(xs)   # (b, t, upsample_rate, c)
+        xs = xs.reshape(xs.shape[0], -1, self.qwen3_config.hidden_size) # (b, t*upsample_rate, c)
+        # Qwen3
+        outs = self.qwen3(
+            inputs_embeds=xs,
+            attention_mask=None,
+        )
+        xs = outs.last_hidden_state # (b, t*upsample_rate, c)
+        audio = self.istft_head(xs)
+        return audio
+
+
+# --- RedAE
+class RedAEConfig(PretrainedConfig):
+    model_type = "redae"
+
+    def __init__(
+        self,
+        # Shared
+        audio_patch_size: int = 480,
+        audio_sample_rate: int = 24000,
+        # Encoder
+        enc_hidden_size: int = 896,
+        enc_intermediate_size: int = 3584,
+        enc_num_hidden_layers: int = 18,
+        enc_max_position_embeddings: int = 32768,
+        enc_max_window_layers: int = 0,
+        enc_num_attention_heads: int = 14,
+        enc_num_key_value_heads: int = 2,
+        enc_sliding_window: int = 64,
+        enc_use_sliding_window: bool = True,
+        enc_extra_downsample_rate: int = 2,   # 50Hz -> 25Hz
+        enc_downsample_num_hidden_layers: int = 4,
+        # Intermediate
+        bottleneck_dim: int = 64,
+        # Decoder
+        dec_hidden_size: int = 896,
+        dec_intermediate_size: int = 3584,
+        dec_num_hidden_layers: int = 18,
+        dec_max_position_embeddings: int = 32768,
+        dec_max_window_layers: int = 0,
+        dec_num_attention_heads: int = 14,
+        dec_num_key_value_heads: int = 2,
+        dec_sliding_window: int = 64,
+        dec_use_sliding_window: bool = True,
+        # Other
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        # Shared
+        self.audio_patch_size=audio_patch_size
+        self.audio_sample_rate=audio_sample_rate
+        # Encoder
+        self.enc_hidden_size=enc_hidden_size
+        self.enc_intermediate_size=enc_intermediate_size
+        self.enc_num_hidden_layers=enc_num_hidden_layers
+        self.enc_max_position_embeddings=enc_max_position_embeddings
+        self.enc_max_window_layers=enc_max_window_layers
+        self.enc_num_attention_heads=enc_num_attention_heads
+        self.enc_num_key_value_heads=enc_num_key_value_heads
+        self.enc_sliding_window=enc_sliding_window
+        self.enc_use_sliding_window=enc_use_sliding_window
+        self.enc_extra_downsample_rate=enc_extra_downsample_rate
+        self.enc_downsample_num_hidden_layers=enc_downsample_num_hidden_layers
+        # Intermediate
+        self.bottleneck_dim=bottleneck_dim
+        # Decoder
+        self.dec_hidden_size=dec_hidden_size
+        self.dec_intermediate_size=dec_intermediate_size
+        self.dec_num_hidden_layers=dec_num_hidden_layers
+        self.dec_max_position_embeddings=dec_max_position_embeddings
+        self.dec_max_window_layers=dec_max_window_layers
+        self.dec_num_attention_heads=dec_num_attention_heads
+        self.dec_num_key_value_heads=dec_num_key_value_heads
+        self.dec_sliding_window=dec_sliding_window
+        self.dec_use_sliding_window=dec_use_sliding_window
+
+        
+class RedAE(PreTrainedModel):
+    config_class = RedAEConfig
+    base_model_prefix = "redae"
+
+    _supports_flash_attn = True
+    _supports_sdpa = True
+
+    def __init__(self, config: RedAEConfig):
+        super().__init__(config)
+
+        self.encoder = RedAEAudioEncoder(
+            out_dim=config.bottleneck_dim,
+            # Input reshape
+            audio_patch_size=config.audio_patch_size,    # 50Hz
+            audio_sample_rate=config.audio_sample_rate,
+            # Qwen
+            hidden_size=config.enc_hidden_size,
+            intermediate_size=config.enc_intermediate_size,
+            num_hidden_layers=config.enc_num_hidden_layers,
+            max_position_embeddings=config.enc_max_position_embeddings,
+            max_window_layers=config.enc_max_window_layers,
+            num_attention_heads=config.enc_num_attention_heads,
+            num_key_value_heads=config.enc_num_key_value_heads,
+            sliding_window=config.enc_sliding_window,
+            use_sliding_window=config.enc_use_sliding_window,
+            # Extra downsample
+            extra_downsample_rate=config.enc_extra_downsample_rate,   # 50Hz -> 25Hz
+            downsample_num_hidden_layers=config.enc_downsample_num_hidden_layers,
+        )
+        self.decoder = RedAEAudioDecoder(
+            in_dim=config.bottleneck_dim,
+            upsample_rate=config.enc_extra_downsample_rate,
+            audio_patch_size=config.audio_patch_size,
+            audio_sample_rate=config.audio_sample_rate,
+            hidden_size=config.dec_hidden_size,
+            intermediate_size=config.dec_intermediate_size,
+            num_hidden_layers=config.dec_num_hidden_layers,
+            max_position_embeddings=config.dec_max_position_embeddings,
+            max_window_layers=config.dec_max_window_layers,
+            num_attention_heads=config.dec_num_attention_heads,
+            num_key_value_heads=config.dec_num_key_value_heads,
+            sliding_window=config.dec_sliding_window,
+            use_sliding_window=config.dec_use_sliding_window,
+        )
+        self.post_init()
+    
+    # --- Shared property
+    @property
+    def downsample_rate(self):
+        return self.encoder.downsample_rate
+    
+    @property
+    def sample_rate(self):
+        return self.decoder.audio_sample_rate
+    
+    @property
+    def hidden_size(self):
+        return self.encoder.out_dim
+    
+    @staticmethod
+    def pad_to_multiple_of(audio: torch.Tensor, multiple_of: int):
+        target_samples = math.ceil(audio.shape[-1] / multiple_of) * multiple_of
+        pad_len = target_samples - audio.shape[-1]
+        if pad_len > 0:
+            audio = F.pad(audio, (pad_len, 0))  # NOTE left pad
+        return audio
+
+    @torch.autocast(device_type='cuda', dtype=torch.bfloat16)
+    @torch.no_grad()
+    def encode(self, audio: torch.Tensor, audio_sr:int):
+        """
+        Args:
+            audio: shape (b, t)
+            audio_sr: int
+        Returns:
+            latents: shape (b, l=t//960, c=64)
+        """
+        audio = audio[:1]
+        audio = torchaudio.functional.resample(audio, audio_sr, self.sample_rate)
+        audio = self.pad_to_multiple_of(audio, self.downsample_rate)
+        latents = self.encoder.forward(audio)
+        return latents
+    
+    @torch.no_grad()
+    def decode(self, latents: torch.Tensor):
+        """
+        Args:
+            latents: shape (b, l=t//960, c=64)
+        Returns:
+            audio: shape (b, t)
+            audio_sr: int
+        """
+        audio = self.decoder.forward(latents)
+        return audio, self.sample_rate

fireredtts3/utils/chatml.py

@@ -0,0 +1,118 @@
+from typing import List, Dict
+
+
+CHATML_LATENT_IN_PAD_SYM = '<|image_pad|>'
+CHATML_LATENT_IN_PAD_ID = 151655
+CHATML_LATENT_OUT_PAD_SYM = '<|video_pad|>'
+CHATML_LATENT_OUT_PAD_ID = 151656
+
+
+def convert_to_chatml(
+    # User input
+    text_in: str,   # Required
+    latent_in_len: int = 0,
+    # Assistant output
+    text_out: str = "",
+    latent_out_len: int = 0,
+    # Placeholder(will be replaced with audio latents)
+    latent_in_pad: str = CHATML_LATENT_IN_PAD_SYM,
+    latent_out_pad:str = CHATML_LATENT_OUT_PAD_SYM,
+):
+    # System prompt
+    messages = [{'role': 'system', 'content': 'You are a helpful assistant.'}]
+    # Input 
+    input_message = {
+        "role": "user", 
+        "content": [
+            {
+                "type": "text", 
+                "text": text_in + ' /no_think', # NOTE should add /no_think for Qwen3(not Qwen3.5)
+            }
+        ]
+    }
+    if latent_in_len>0:
+        input_message['content'].insert(0, 
+            {
+                "type": "audio", 
+                "audio": latent_in_pad * latent_in_len,
+            }
+        )
+    messages.append(input_message)
+    # Output 
+    output_message = {
+        "role": "assistant", 
+        "content": [
+            {
+                "type": "text", 
+                "text": f"<think>\n\n</think>\n\n" + text_out,  # text_out should be wrapped in <|sot|><|eot|>
+            }
+        ]
+    }
+    if latent_out_len>0:
+        output_message['content'].append(
+            {
+                "type": "audio", 
+                "audio": latent_out_pad * latent_out_len,
+            }
+        )
+    messages.append(output_message)
+    # Convert to chatml string
+    chatml_str_list: List[str] = []
+    for msg in messages:
+        if isinstance(msg['content'], str):
+            chatml_str_list.append(
+                f'<|im_start|>{msg["role"]}\n{msg["content"]}<|im_end|>\n'
+            )
+        else:
+            chatml_str_list.append(f'<|im_start|>{msg["role"]}\n')
+            for content in msg["content"]:
+                if content["type"] == "text":
+                    chatml_str_list.append(content["text"])
+                elif content["type"] == "audio":
+                    chatml_str_list.append(
+                        f"<|sosp|>{content['audio']}<|eosp|>\n"
+                    )
+            chatml_str_list.append(f"<|im_end|>\n")
+    chatml_str = ''.join(chatml_str_list)
+    return chatml_str
+
+
+def compose_generate_input_tts(
+    prompt_latent_len: int,
+    prompt_text: str,
+    text: str,
+):
+    text_in = "Convert text to speech.\n{}".format(prompt_text+text)
+    chatml_str = convert_to_chatml(text_in=text_in, latent_out_len=prompt_latent_len)
+    chatml_str = chatml_str.removesuffix('<|eosp|>\n<|im_end|>\n')  # Remove ending tags
+    return chatml_str
+
+
+def compose_generate_input_voice_design(
+    instruction: str,
+    text: str,
+):
+    text_in = "{}\n\n根据上述音色描述，首先整理成语音属性，再合成以下文本对应的音频：\n{}".format(instruction, text)
+    chatml_str = convert_to_chatml(text_in=text_in, text_out="<|sot|>")
+    chatml_str = chatml_str.removesuffix('<|im_end|>\n')  # Remove ending tags
+    return chatml_str
+
+
+def compose_generate_input_semantic_edit(
+    instruction: str,
+    audio_in_latent_len: int,
+):
+    text_in = 'Identify the content of the audio. {}'.format(instruction.strip())
+    chatml_str = convert_to_chatml(text_in=text_in, latent_in_len=audio_in_latent_len, text_out="<|sot|>")
+    chatml_str = chatml_str.removesuffix('<|im_end|>\n')  # Remove ending tags
+    return chatml_str
+
+
+def compose_generate_input_acoustic_edit(
+    instruction: str,
+    audio_in_latent_len: int,
+):
+    chatml_str = convert_to_chatml(text_in=instruction, latent_in_len=audio_in_latent_len, latent_out_len=1)
+    chatml_str = chatml_str.removesuffix('<|video_pad|><|eosp|>\n<|im_end|>\n')  # Remove ending tags
+    return chatml_str
+

fireredtts3/utils/llm_tn/templates/ar-SA.json

@@ -0,0 +1,13 @@
+{
+  "locale": "ar-SA",
+  "language_name": "Arabic",
+  "system": "You are an accurate text normalizer for ar-SA (Modern Standard Arabic). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Arabic (2023 -> ألفان وثلاثة وعشرون; 12 أكتوبر -> الثاني عشر من أكتوبر).\n- Decimals use فاصلة (2.5 -> اثنان فاصلة خمسة); percentages use بالمئة (15% -> خمسة عشر بالمئة).\n- Phone numbers, ISBNs and codes are read digit-by-digit (050-1234567 -> صفر خمسة صفر واحد اثنان ثلاثة أربعة خمسة ستة سبعة).\n- Roman numerals are converted to Arabic numbers (XVIII -> الثامن عشر).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Arabic names (ر.س -> ريال سعودي; $ -> دولار).\n- Math operators become words (= -> يساوي, + -> زائد, - -> ناقص).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "سيُعقد الاجتماع في 12 أكتوبر 2023.", "output": "سيُعقد الاجتماع في الثاني عشر من أكتوبر ألفين وثلاثة وعشرين."},
+    {"input": "سعر المنتج هو 1,500 ريال.", "output": "سعر المنتج هو ألف وخمسمائة ريال."},
+    {"input": "بلغ التضخم 7.5% هذا العام.", "output": "بلغ التضخم سبعة فاصلة خمسة بالمئة هذا العام."},
+    {"input": "رقم جوالي هو 050-1234567.", "output": "رقم جوالي هو صفر خمسة صفر واحد اثنان ثلاثة أربعة خمسة ستة سبعة."},
+    {"input": "الجهاز يستخدم GPS وصيغة NaOH.", "output": "الجهاز يستخدم G P S وصيغة N a O H."},
+    {"input": "الصيغة (a+b)² بسيطة.", "output": "الصيغة a زائد b في المربع بسيطة."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/cs-CZ.json

@@ -0,0 +1,13 @@
+{
+  "locale": "cs-CZ",
+  "language_name": "Czech",
+  "system": "You are an accurate text normalizer for cs-CZ (Czech). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Czech (2023 -> dva tisíce dvacet tři; 12. října -> dvanáctého října).\n- Decimals use čárka (2.5 -> dvě čárka pět); percentages use procent (15% -> patnáct procent).\n- Dates use genitive case for day/month (10. 12. 2023 -> desátého prosince dva tisíce dvacet tři).\n- Phone numbers, ISBNs and codes are read digit-by-digit (603-123456 -> šest nula tři jedna dva tři čtyři pět šest).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Czech names (Kč -> korun; $ -> dolarů; € -> eur).\n- Math operators become words (= -> se rovná, + -> plus, - -> mínus).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Schůzka se uskuteční 12. října 2023.", "output": "Schůzka se uskuteční dvanáctého října dva tisíce dvacet tři."},
+    {"input": "Produkt stojí 1 500 korun.", "output": "Produkt stojí tisíc pět set korun."},
+    {"input": "Inflace dosáhla 7,5 % letos.", "output": "Inflace dosáhla sedm čárka pět procent letos."},
+    {"input": "Mé telefonní číslo je 603-123456.", "output": "Mé telefonní číslo je šest nula tři jedna dva tři čtyři pět šest."},
+    {"input": "Zařízení používá GPS a NaOH.", "output": "Zařízení používá G P S a N a O H."},
+    {"input": "Vzorec (a+b)² je jednoduchý.", "output": "Vzorec a plus b na druhou je jednoduchý."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/de-DE.json

@@ -0,0 +1,21 @@
+{
+  "locale": "de-DE",
+  "language_name": "German",
+  "system": "You are an accurate text normalizer for de-DE (German). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Spell out all numbers, dates, times, currencies and units the way they are read aloud in German (e.g. 2024 -> zweitausendvierundzwanzig; dates use ordinal day + ordinal month, e.g. 12.10.23 -> zwölften zehnten dreiundzwanzig).\n- Compound German numbers are written as a single word (e.g. 25 -> fünfundzwanzig).\n- When normalizing acronyms, spell out the individual letters as pronounced in German, except when the acronym is a widely recognized and pronounceable name.\n- Convert punctuation that is spoken aloud into words (e.g., write 'Punkt' instead of a period in URLs, 'at' for '@' in emails, 'Bindestrich' for hyphens).\n- Segment compound words, websites, and file names into recognizable component words.\n- Common file extensions should be spoken out. Uncommon file extensions should be spelled out.\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Die Veranstaltung ist am 12.10.23.", "output": "Die Veranstaltung ist am zwölften zehnten dreiundzwanzig."},
+    {"input": "Das Auto kostet 25.000€ und fährt maximal 120 km/h.", "output": "Das Auto kostet fünfundzwanzigtausend Euro und fährt maximal hundertzwanzig Kilometer pro Stunde."},
+    {"input": "Etwa 3/4 der Befragten stimmten zu.", "output": "Etwa drei Viertel der Befragten stimmten zu."},
+    {"input": "Schreiben Sie an info@firma.de oder besuchen Sie www.firma.de.", "output": "Schreiben Sie an info at firma Punkt d e oder besuchen Sie w w w Punkt firma Punkt d e."},
+    {"input": "Der Zug ICE 123 fährt um 18:45 Uhr ab.", "output": "Der Zug I C E hundertdreiundzwanzig fährt um achtzehn Uhr fünfundvierzig ab."},
+    {"input": "H₂SO₄ ist Schwefelsäure und C₆H₁₂O₆ ist Glukose.", "output": "H zwei S O vier ist Schwefelsäure und C sechs H zwölf O sechs ist Glukose."},
+    {"input": "Die ISBN 978-3-492-28656-1 ist gültig.", "output": "Die I S B N neun sieben acht drei vier neun zwei zwei acht sechs fünf sechs eins ist gültig."},
+    {"input": "Die Matrikelnummer ist 1234567.", "output": "Die Matrikelnummer ist eins zwei drei vier fünf sechs sieben."},
+    {"input": "Die NASA und die USA kooperieren.", "output": "Die Nasa und die U S A kooperieren."},
+    {"input": "Das MacBook Pro M1 ist schnell.", "output": "Das MacBook Pro M eins ist schnell."},
+    {"input": "Gemäß § 13 StGB ist das strafbar.", "output": "Gemäß Paragraf dreizehn S T G B ist das strafbar."},
+    {"input": "Die UNESCO hat das Denkmal geschützt.", "output": "Die Unesco hat das Denkmal geschützt."},
+    {"input": "George VI. war ein beliebter König.", "output": "George der Sechste war ein beliebter König."},
+    {"input": "Schreiben Sie an admin@server.local.", "output": "Schreiben Sie an admin at server Punkt local."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/el-GR.json

@@ -0,0 +1,13 @@
+{
+  "locale": "el-GR",
+  "language_name": "Greek",
+  "system": "You are an accurate text normalizer for el-GR (Greek). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Greek (2023 -> δύο χιλιάδες είκοσι τρία; 12 Οκτωβρίου -> δώδεκα Οκτωβρίου).\n- Decimals use κόμμα (2.5 -> δύο κόμμα πέντε); percentages use τοις εκατό (15% -> δεκαπέντε τοις εκατό).\n- Ordinal uses -ος/-ο suffix (XVIII -> δέκατος όγδοος).\n- Phone numbers, ISBNs and codes are read digit-by-digit (210-1234567 -> δύο ένα μηδέν ένα δύο τρία τέσσερα πέντε έξι επτά).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Greek names (€ -> ευρώ; $ -> δολάρια).\n- Math operators become words (= -> ισούται με, + -> συν, - -> πλην).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Η συνάντηση θα γίνει στις 12 Οκτωβρίου 2023.", "output": "Η συνάντηση θα γίνει στις δώδεκα Οκτωβρίου δύο χιλιάδες είκοσι τρία."},
+    {"input": "Το προϊόν κοστίζει 1.500 ευρώ.", "output": "Το προϊόν κοστίζει χίλια πεντακόσια ευρώ."},
+    {"input": "Ο πληθωρισμός έφτασε το 7,5% φέτος.", "output": "Ο πληθωρισμός έφτασε το επτά κόμμα πέντε τοις εκατό φέτος."},
+    {"input": "Το τηλέφωνό μου είναι 210-1234567.", "output": "Το τηλέφωνό μου είναι δύο ένα μηδέν ένα δύο τρία τέσσερα πέντε έξι επτά."},
+    {"input": "Η συσκευή χρησιμοποιεί GPS και NaOH.", "output": "Η συσκευή χρησιμοποιεί G P S και N a O H."},
+    {"input": "Ο τύπος (a+b)² είναι απλός.", "output": "Ο τύπος a συν b στο τετράγωνο είναι απλός."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/en-US.json

@@ -0,0 +1,20 @@
+{
+  "locale": "en-US",
+  "language_name": "English",
+  "system": "You are an accurate text normalizer for en-US (English). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Spell out all numbers, dates, times, currencies and units the way they are read aloud in English (e.g. 2023 -> twenty twenty three; $5.99 -> five dollars and ninety nine cents; 120 km/h -> one hundred and twenty kilometers per hour).\n- When normalizing acronyms, spell out the individual letters separated by spaces, except when the acronym is a widely recognized and pronounceable name (e.g. \"NASA\").\n- Convert punctuation that is spoken aloud into words (e.g., write 'dot' instead of a period in URLs, 'at' for '@' in emails, 'dash' or 'hyphen' for '-').\n- Segment compound words, websites, and file names into recognizable component words.\n- Common file extensions should be spoken out (e.g. .jpg -> jpeg). Uncommon file extensions should be spelled out letter by letter.\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "He bought 2.5 kg of apples for $5.99 on 10/12/2023.", "output": "He bought two point five kilograms of apples for five dollars and ninety nine cents on October twelfth twenty twenty three."},
+    {"input": "Send the image.jpg and the config.yaml to info@nasa.gov.", "output": "Send the image dot jpeg and the config dot y a m l to info at nasa dot gov."},
+    {"input": "The LA Lakers won 102-98 against the Miami Heat.", "output": "The L A Lakers won one hundred and two to ninety eight against the Miami Heat."},
+    {"input": "Dr. Smith's clinic is at 123 W. 5th St.", "output": "Doctor Smith's clinic is at one twenty three West fifth street."},
+    {"input": "It requires >= 8GB of RAM.", "output": "It requires greater than or equal to eight gigabytes of RAM."},
+    {"input": "Glucose is C₆H₁₂O₆ and water is H₂O.", "output": "Glucose is C six H twelve O six and water is H two O."},
+    {"input": "The deadline is 2024-12-31.", "output": "The deadline is December thirty first twenty twenty four."},
+    {"input": "The building was constructed in MDCCCXLV.", "output": "The building was constructed in eighteen forty five."},
+    {"input": "Reply with #YesOrNo to win €3.1M.", "output": "Reply with hashtag yes or no to win three point one million euros."},
+    {"input": "The event is sponsored by NASA and the strain is SARS-CoV-2.", "output": "The event is sponsored by Nasa and the strain is Sars Cov two."},
+    {"input": "The bus arrives at 14:50.", "output": "The bus arrives at fourteen fifty."},
+    {"input": "He tested positive for COVID.", "output": "He tested positive for covid."},
+    {"input": "The book's ISBN is 978-0-13-468599-1.", "output": "The book's I S B N is nine seven eight zero one three four six eight five nine nine one."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/es-MX.json

@@ -0,0 +1,23 @@
+{
+  "locale": "es-MX",
+  "language_name": "Spanish",
+  "system": "You are an accurate text normalizer for es-MX (Mexican Spanish). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Spell out all numbers, dates, times, currencies and units the way they are read aloud in Mexican Spanish. Years are read as a full cardinal number (e.g. 2023 -> dos mil veintitrés; 1990 -> mil novecientos noventa).\n- Dates use the format 'day de month de year' (e.g. 15/10/2023 -> quince de octubre de dos mil veintitrés). The first day of the month is read as 'primero'.\n- When normalizing acronyms, spell out the individual letters, except when the acronym is a widely recognized and pronounceable name (e.g. \"RENFE\").\n- Convert punctuation that is spoken aloud into words (e.g., write 'punto' for URLs, 'arroba' for '@' in emails, 'guion' for hyphens).\n- Segment compound words, websites, and file names into recognizable component words.\n- Common file extensions should be spoken out. Uncommon file extensions should be spelled out.\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "La reunión es el 15/10/2023.", "output": "La reunión es el quince de octubre de dos mil veintitrés."},
+    {"input": "Nací el 3 de mayo de 1990.", "output": "Nací el tres de mayo de mil novecientos noventa."},
+    {"input": "Su email es contacto@empresa.mx.", "output": "Su email es contacto arroba empresa punto e me equis."},
+    {"input": "La velocidad máx. en la autopista es de 120 km/h.", "output": "La velocidad máxima en la autopista es de ciento veinte kilómetros por hora."},
+    {"input": "El resultado fue 3/4 partes del total.", "output": "El resultado fue tres cuartas partes del total."},
+    {"input": "Es medianoche (00:00).", "output": "Es medianoche (cero horas)."},
+    {"input": "El Siglo XXI y la Sección IX.", "output": "El Siglo veintiuno y la Sección nueve."},
+    {"input": "El producto cuesta US$25.99.", "output": "El producto cuesta veinticinco dólares con noventa y nueve centavos."},
+    {"input": "La UNESCO declaró patrimonio y la PROFECO protege al consumidor.", "output": "La UNESCO declaró patrimonio y la PROFECO protege al consumidor."},
+    {"input": "Usa Python 3.10.4 en el servidor.", "output": "Usa Python tres punto diez punto cuatro en el servidor."},
+    {"input": "META (Facebook) subió en la bolsa.", "output": "M E T A (Facebook) subió en la bolsa."},
+    {"input": "La reunión es a las 12:00 en punto.", "output": "La reunión es a las doce en punto."},
+    {"input": "Participa en el concurso #Verano y menciona a @amigo.", "output": "Participa en el concurso hashtag Verano y menciona a arroba amigo."},
+    {"input": "La acción GOOG subió hoy.", "output": "La acción G O O G subió hoy."},
+    {"input": "El código es SKU: PRD-567-890.", "output": "El código es S K U: P R D guión cinco seis siete guión ocho nueve cero."},
+    {"input": "El libro tiene ISBN 978-607-16-1234-5.", "output": "El libro tiene I S B N nueve siete ocho, guión seis cero siete, guión uno seis, guión uno dos tres cuatro, guión cinco."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/fi-FI.json

@@ -0,0 +1,13 @@
+{
+  "locale": "fi-FI",
+  "language_name": "Finnish",
+  "system": "You are an accurate text normalizer for fi-FI (Finnish). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Finnish (2023 -> kaksituhattakaksikymmentäkolme; 12. lokakuuta -> kahdestoista lokakuuta).\n- Decimals use pilkku (2.5 -> kaksi pilkku viisi); percentages use prosenttia (15% -> viisitoista prosenttia).\n- Ordinal uses -s/-nna suffix (XVIII -> kahdeksastoista).\n- Phone numbers, ISBNs and codes are read digit-by-digit (040-1234567 -> nolla neljä nolla yksi kaksi kolme neljä viisi kuusi seitsemän).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Finnish names (€ -> euroa; $ -> dollaria).\n- Math operators become words (= -> on yhtä kuin, + -> plus, - -> miinus).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Kokous pidetään 12. lokakuuta 2023.", "output": "Kokous pidetään kahdestoista lokakuuta kaksituhattakaksikymmentäkolme."},
+    {"input": "Tuote maksaa 1 500 euroa.", "output": "Tuote maksaa tuhat viisisataa euroa."},
+    {"input": "Inflaatio oli 7,5 prosenttia tänä vuonna.", "output": "Inflaatio oli seitsemän pilkku viisi prosenttia tänä vuonna."},
+    {"input": "Puhelinnumeroni on 040-1234567.", "output": "Puhelinnumeroni on nolla neljä nolla yksi kaksi kolme neljä viisi kuusi seitsemän."},
+    {"input": "Laite käyttää GPS:ää ja NaOH:ta.", "output": "Laite käyttää G P S:ää ja N a O H:ta."},
+    {"input": "Kaava (a+b)² on yksinkertainen.", "output": "Kaava a plus b toiseen on yksinkertainen."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/fr-FR.json

@@ -0,0 +1,23 @@
+{
+  "locale": "fr-FR",
+  "language_name": "French",
+  "system": "You are an accurate text normalizer for fr-FR (French). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Spell out all numbers, dates, times, currencies and units the way they are read aloud in French.\n- When normalizing acronyms, spell out the individual letters separated by spaces (e.g. \"SNCF\" -> \"S N C F\"), except when the acronym is a widely recognized and pronounceable name (e.g. \"OTAN\").\n- Convert punctuation that is spoken aloud into words (e.g., write 'point' instead of a period in URLs, 'arobase' for '@', 'tiret' for hyphens).\n- Segment compound words, websites, and file names into recognizable component words.\n- Common file extensions should be spoken out. Uncommon file extensions should be spelled out.\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Le taux de change est de 1€ = 1,10$.", "output": "Le taux de change est de un euro égale un virgule dix dollars."},
+    {"input": "M. Dupont habite au 15, av. des Champs-Élysées.", "output": "Monsieur Dupont habite au quinze, avenue des Champs-Élysées."},
+    {"input": "Le vol AF123 partira le 1er juin à 18h45.", "output": "Le vol A F cent vingt-trois partira le premier juin à dix-huit heures quarante-cinq."},
+    {"input": "Son site est www.boutique-mode.fr.", "output": "Son site est w w w point boutique tiret mode point f r."},
+    {"input": "Louis XIV est mort en 1715.", "output": "Louis quatorze est mort en mille sept cent quinze."},
+    {"input": "L'éthanol est C₂H₅OH.", "output": "L'éthanol est C deux H cinq O H."},
+    {"input": "Le virus SARS-CoV-2 et la bactérie E. coli sont étudiés.", "output": "Le virus Sars Cov deux et la bactérie E coli sont étudiés."},
+    {"input": "L'UNICEF aide les enfants et l'OMS a publié un rapport.", "output": "L'Unicef aide les enfants et l'O M S a publié un rapport."},
+    {"input": "L'action BAC a baissé et le cours de XOM fluctue.", "output": "L'action BAC a baissé et le cours de XOM fluctue."},
+    {"input": "a² - b² = (a + b)(a - b).", "output": "a au carré moins b au carré égale a plus b fois a moins b."},
+    {"input": "La NASA a lancé une fusée.", "output": "La NASA a lancé une fusée."},
+    {"input": "La formule H₂SO₄ est un acide fort.", "output": "La formule H deux S O quatre est un acide fort."},
+    {"input": "Contactez admin@server.local.", "output": "Contactez admin arobase server point local."},
+    {"input": "L'UNICEF aide les enfants.", "output": "L'Unicef aide les enfants."},
+    {"input": "L'ISBN est 3-598-21508-8.", "output": "L'I S B N est trois, cinq cent quatre-vingt-dix-huit, vingt et un mille cinq cent huit, huit."},
+    {"input": "8 + (6 × 2) = 20.", "output": "Huit plus, entre parenthèses, six fois deux, égale vingt."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/hi-IN.json

@@ -0,0 +1,13 @@
+{
+  "locale": "hi-IN",
+  "language_name": "Hindi",
+  "system": "You are an accurate text normalizer for hi-IN (Hindi). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Hindi (2023 -> दो हज़ार तेईस; 12 अक्टूबर -> बारह अक्टूबर).\n- Use Indian numbering system: 1000 -> हज़ार, 100000 -> लाख, 10000000 -> करोड़.\n- Decimals use दशमलव (2.5 -> दो दशमलव पाँच); percentages use प्रतिशत (15% -> पंद्रह प्रतिशत).\n- Phone numbers, ISBNs and codes are read digit-by-digit (981-1234567 -> नौ आठ एक एक दो तीन चार पाँच छह सात).\n- Roman numerals are converted to Hindi numbers (XVIII -> अठारह).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Hindi/Indian names (₹ -> रुपये; $ -> डॉलर).\n- Math operators become words (= -> बराबर है, + -> जोड़, - -> घटा).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "बैठक 12 अक्टूबर 2023 को होगी।", "output": "बैठक बारह अक्टूबर दो हज़ार तेईस को होगी।"},
+    {"input": "उत्पाद की कीमत ₹1,50,000 है।", "output": "उत्पाद की कीमत डेढ़ लाख रुपये है।"},
+    {"input": "मुद्रास्फीति इस साल 7.5% तक पहुँच गई।", "output": "मुद्रास्फीति इस साल सात दशमलव पाँच प्रतिशत तक पहुँच गई।"},
+    {"input": "मेरा फ़ोन नंबर 981-1234567 है।", "output": "मेरा फ़ोन नंबर नौ आठ एक एक दो तीन चार पाँच छह सात है।"},
+    {"input": "डिवाइस GPS और NaOH का उपयोग करता है।", "output": "डिवाइस G P S और N a O H का उपयोग करता है।"},
+    {"input": "सूत्र (a+b)² सरल है।", "output": "सूत्र a जोड़ b का वर्ग सरल है।"}
+  ]
+}

fireredtts3/utils/llm_tn/templates/id-ID.json

@@ -0,0 +1,13 @@
+{
+  "locale": "id-ID",
+  "language_name": "Indonesian",
+  "system": "You are an accurate text normalizer for id-ID (Indonesian). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Indonesian (2023 -> dua ribu dua puluh tiga; 12 Oktober -> dua belas Oktober).\n- Decimals use koma (2.5 -> dua koma lima); percentages use persen (15% -> lima belas persen).\n- Ordinals use ke- prefix (XVIII -> kedelapan belas).\n- Phone numbers, ISBNs and codes are read digit-by-digit (0812-345678 -> nol delapan satu dua tiga empat lima enam tujuh delapan).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Indonesian names (Rp -> rupiah; $ -> dolar).\n- Math operators become words (= -> sama dengan, + -> tambah, - -> kurang).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Rapat akan diadakan pada 12 Oktober 2023.", "output": "Rapat akan diadakan pada dua belas Oktober dua ribu dua puluh tiga."},
+    {"input": "Harga produk adalah Rp 1.500.000.", "output": "Harga produk adalah satu juta lima ratus ribu rupiah."},
+    {"input": "Inflasi mencapai 7,5% tahun ini.", "output": "Inflasi mencapai tujuh koma lima persen tahun ini."},
+    {"input": "Nomor telepon saya 0812-345678.", "output": "Nomor telepon saya nol delapan satu dua tiga empat lima enam tujuh delapan."},
+    {"input": "Perangkat ini menggunakan GPS dan NaOH.", "output": "Perangkat ini menggunakan G P S dan N a O H."},
+    {"input": "Rumus (a+b)² sederhana.", "output": "Rumus a tambah b kuadrat sederhana."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/it-IT.json

@@ -0,0 +1,23 @@
+{
+  "locale": "it-IT",
+  "language_name": "Italian",
+  "system": "You are an accurate text normalizer for it-IT (Italian). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Spell out all numbers, dates, times, currencies and units the way they are read aloud in Italian (e.g. 2024 -> duemilaventiquattro, 120 km/h -> centoventi chilometri orari).\n- When normalizing acronyms, spell out the individual letters as pronounced in Italian, except when the acronym is a widely recognized and pronounceable name.\n- Convert punctuation that is spoken aloud into words (e.g., write 'punto' instead of a period in URLs, 'chiocciola' for '@' in emails, 'trattino' for hyphens).\n- Segment compound words, websites, and file names into recognizable component words.\n- Common file extensions should be spoken out. Uncommon file extensions should be spelled out.\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "La riunione è fissata per il 15/07/2024.", "output": "La riunione è fissata per il quindici luglio duemilaventiquattro."},
+    {"input": "Il prezzo dell'auto è di 25.000€ e la velocità massima è 120 km/h.", "output": "Il prezzo dell'auto è di venticinquemila euro e la velocità massima è centoventi chilometri orari."},
+    {"input": "Il 3/4 dei partecipanti ha votato a favore.", "output": "I tre quarti dei partecipanti hanno votato a favore."},
+    {"input": "Scrivimi a info@azienda.it o visita www.azienda.it.", "output": "Scrivimi a info chiocciola azienda punto i t o visita vu vu vu punto azienda punto i t."},
+    {"input": "Papa Giovanni XXIII regnò dal 1958.", "output": "Papa Giovanni ventitreesimo regnò dal millenovecentocinquantotto."},
+    {"input": "Il centralino è 800 123456.", "output": "Il centralino è otto zero zero, uno due tre quattro cinque sei."},
+    {"input": "L'UNICEF aiuta i bambini e l'IVA è al 22%.", "output": "L'unicef aiuta i bambini e l'iva è al ventidue percento."},
+    {"input": "Vedi D.Lgs. 81/2008.", "output": "Vedi decreto legislativo ottantuno del duemilaotto."},
+    {"input": "La S.p.A. è quotata in borsa.", "output": "La società per azioni è quotata in borsa."},
+    {"input": "Il glucosio è C₆H₁₂O₆.", "output": "Il glucosio è C sei H dodici O sei."},
+    {"input": "La versione è la 8.1.4.", "output": "La versione è la otto punto uno punto quattro."},
+    {"input": "Il mio numero è 345 678 9012.", "output": "Il mio numero è tre quattro cinque, sei sette otto, nove zero uno due."},
+    {"input": "Menziona @admin nel post.", "output": "Menziona at admin nel post."},
+    {"input": "La formula H₂SO₄ è un acido.", "output": "La formula H due S O quattro è un acido."},
+    {"input": "Il modello è XJ-5000.", "output": "Il modello è X J, cinquemila."},
+    {"input": "L'ISBN è 978-88-430-1234-5.", "output": "L'I S B N è nove sette otto, otto otto, quattro tre zero, uno due tre quattro, cinque."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/ja-JP.json

@@ -0,0 +1,25 @@
+{
+  "locale": "ja-JP",
+  "language_name": "Japanese",
+  "system": "You are an accurate text normalizer for ja-JP (Japanese). Your task is to rewrite the input into the way it is actually read aloud, normalizing numbers, dates, symbols, currencies, units, acronyms and addresses. CRITICAL: Keep the original Japanese writing EXACTLY as-is — every kanji must stay kanji, every hiragana stays hiragana, every katakana word stays the same, and keep the original punctuation 。、？「」. NEVER convert existing kanji/hiragana into other scripts (no romaji, no turning kanji into hiragana). Only convert the specific tokens listed below; leave everything else untouched.\n\nRules:\n- Numbers are written as Chinese numerals (kanji): 2024 -> 二千二十四, 1500 -> 千五百, 100 -> 百, 250000 -> 二十五万, 0 -> 〇 (only in phone numbers/postal codes/serial codes). Decimals use 点 (2.5 -> 二点五).\n- Dates: 5月10日 -> 五月十日; 24/06/15 -> 二十四年六月十五日; 10-30 -> 十月三十日; 2023-07-14 -> 二千二十三年七月十四日; years are read as a full number followed by 年.\n- Times: 3時15分 -> 三時十五分.\n- Fractions are read as '<denominator>分の<numerator>': 4/9 -> 九分の四.\n- Sports scores use 対: 11-9 -> 十一対九.\n- Roman numerals become kanji numbers: II -> 二.\n- Phone numbers and postal codes are read digit by digit in kanji, 0 -> 〇, '+' -> プラス, keeping the original grouping separated by spaces: 03-1234-5678 -> 〇三 一二三四 五六七八.\n- Version numbers: read each dot as 点 and each number as kanji (0 -> 〇): 18.2.0 -> 十八点二点〇.\n- License plates / serial numbers: the kanji place name stays as kanji, numbers are read digit by digit in kanji: 札幌100よ1234 -> 札幌百よ一二三四.\n- ISBN: read 'ISBN' as アイエスビーエヌ, replace each '-' with a space, and read every digit group one digit at a time: ISBN 978-4-00-123456-1 -> アイエスビーエヌ 九七八 四 〇〇 一二三四五六 一.\n- Currencies: read the amount in kanji then the unit word — ¥ -> 円, $ -> ドル(cents -> セント), € -> ユーロ(cents -> セント), £ -> ポンド(pence -> ペンス), ￦ -> ウォン, ₹ -> ルピー. e.g. €75.50 -> 七十五ユーロ五十セント.\n- Units: spell the unit as a katakana word — km -> キロメートル, kg -> キログラム, mL -> ミリリットル, °C -> 度, ㎡ -> 平方メートル, km/h -> キロメートル毎時, kW -> キロワット.\n- Acronyms/initialisms: spell each letter out in katakana joined without spaces (CPU -> シーピーユー, AI -> エーアイ), EXCEPT widely pronounceable names which become their katakana reading (NATO -> ネイトー, UNESCO -> ユネスコ, IKEA -> イケア, Wi-Fi -> ワイファイ, UFO -> ユーフォー).\n- Chemical formulas: read each letter and subscript number in katakana joined without spaces (subscripts 2->ツー 3->スリー 4->フォー 5->ファイブ 6->シックス): H₂O -> エイチツーオー, NaCl -> エヌエーシーエル, CO₂ -> シーオーツー.\n- URLs and emails: read every component in katakana joined without spaces — '@' -> アット, '.' -> ドット, '/' -> スラッシュ, ':' -> コロン, 'http' -> エイチティーティーピー, 'https' -> エイチティーティーピーエス, 'www' -> ダブリューダブリューダブリュー. e.g. user@domain.com -> ユーザーアットドメインドットコム.\n- Addresses with hyphens use 丁目/番地/号: 1-2-3 -> 一丁目二番地三号.\n- Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "会議は2024年5月10日です。", "output": "会議は二千二十四年五月十日です。"},
+    {"input": "電話番号は03-1234-5678です。", "output": "電話番号は〇三 一二三四 五六七八です。"},
+    {"input": "費用は€75.50。", "output": "費用は七十五ユーロ五十セント。"},
+    {"input": "速度は100km/h。", "output": "速度は百キロメートル毎時。"},
+    {"input": "4/9の割合で11-9の大接戦。", "output": "九分の四の割合で十一対九の大接戦。"},
+    {"input": "CPUは重要だが、NATOも大切。", "output": "シーピーユーは重要だが、ネイトーも大切。"},
+    {"input": "水はH₂O、CO₂を排出。", "output": "水はエイチツーオー、シーオーツーを排出。"},
+    {"input": "メールはuser@domain.com。", "output": "メールはユーザーアットドメインドットコム。"},
+    {"input": "住所は東京都渋谷区神南1-2-3です。", "output": "住所は東京都渋谷区神南一丁目二番地三号です。"},
+    {"input": "ISBN 978-4-00-123456-1。", "output": "アイエスビーエヌ 九七八 四 〇〇 一二三四五六 一。"},
+    {"input": "国税通則法第65条、React 18.2.0を使う。", "output": "国税通則法第六十五条、リアクト十八点二点〇を使う。"},
+    {"input": "計算すると5 + 3 = 8、π ≈ 3.14。", "output": "計算すると五足す三は八、パイは約三点一四。"},
+    {"input": "√9 = 3で、8 ≠ 5。", "output": "九の平方根は三で、八 ノットイコール 五。"},
+    {"input": "H₂SO₄は硫酸です。", "output": "エイチツーエスオーフォーは硫酸です。"},
+    {"input": "VIII代目社長が就任した。", "output": "八代目社長が就任した。"},
+    {"input": "コード進行はC→G→Am→F。", "output": "コード進行はシーからジーからエーマイナーからエフ。"},
+    {"input": "VINは1HGCM82633A123456です。", "output": "ブイアイエヌ 一エイチジーシーエム八二六三三エー一二三四五六です。"},
+    {"input": "三菱UFJ(8306)の株価。", "output": "三菱ユーエフジェー八三〇六の株価。"}
+  ]
+}

fireredtts3/utils/llm_tn/templates/ko-KR.json

@@ -0,0 +1,13 @@
+{
+  "locale": "ko-KR",
+  "language_name": "Korean",
+  "system": "You are an accurate text normalizer for ko-KR (Korean). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Korean (2023년 -> 이천이십삼년, 12월 -> 십이월).\n- Use Korean units: 1000 -> 천, 10000 -> 만, 100000000 -> 억.\n- Decimals use 점 (2.5 -> 이 점 오), percentages use 퍼센트 (15% -> 십오 퍼센트).\n- Phone numbers, ISBNs and codes are read digit-by-digit (010-1234-5678 -> 공일공 일이삼사 오육칠팔).\n- Roman numerals are converted to Korean numbers (XVIII -> 십팔).\n- Latin acronyms/initialisms are spelled letter-by-letter with spaces (GPS -> G P S, NaOH -> N a O H). Recognizeable pronounceable acronyms are kept as-is (NASA, Wi-Fi).\n- Currency amounts use Korean currency names (₩, KRW -> 원; $ -> 달러).\n- Math operators become words (= -> 같다, + -> 더하기, - -> 빼기).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "2023년 10월 12일에 새로운 제품이 출시되었습니다.", "output": "이천이십삼년 십월 십이일에 새로운 제품이 출시되었습니다."},
+    {"input": "이 제품의 가격은 25,000원 입니다.", "output": "이 제품의 가격은 이만 오천원 입니다."},
+    {"input": "지지율은 75.5%에 도달했습니다.", "output": "지지율은 칠십오 점 오퍼센트에 도달했습니다."},
+    {"input": "제 전화번호는 010-1234-5678 입니다.", "output": "제 전화번호는 공일공 일이삼사 오육칠팔 입니다."},
+    {"input": "제18차 개정안에서는 GPS를 사용합니다.", "output": "제십팔차 개정안에서는 G P S를 사용합니다."},
+    {"input": "이 공식은 (a+b)² 입니다.", "output": "이 공식은 a 더하기 b의 제곱 입니다."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/lt-LT.json

@@ -0,0 +1,29 @@
+{
+  "locale": "lt-LT",
+  "language_name": "Lithuanian",
+  "system": "You are an accurate text normalizer for lt-LT (Lithuanian). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nGuiding principle (Lithuanian TTS practice): expand only what is genuinely spoken aloud. Do NOT over-normalize. When in doubt, either read a token as an ordinary word or spell out its letters, and never rewrite an abbreviation into its full wording or reinterpret a code semantically.\n\nRules:\n- Spell out numbers, dates, times, currencies, decimals and units the way they are read aloud in Lithuanian, respecting the grammatical case required by the surrounding context (prepositions and the following noun).\n- Decimals: read the fractional part as a fraction, e.g. 0,3 -> trys dešimtosios, 0,25 -> dvidešimt penkios šimtosios, 0,02 -> dvi šimtąsias, 0,026 -> dvidešimt šešios tūkstantosios, 5,2 -> penki ir dvi dešimtąsias; 3,14 -> trys ir keturiolika šimtųjų.\n- Fractions follow Lithuanian conventions (1/2 -> pusę, 1/3 -> trečdalį, 1/4 -> ketvirtadalis, 3/5 -> tris penktadalius), inflected to agree with the sentence.\n- Mathematical operators are read as words: + plius, - minus, × kart, ÷ padalinti iš, = lygu, ≠ nelygu, > daugiau nei, < mažiau nei, ≈ apytiksliai lygu, √ kvadratinė šaknis iš, ∛ kubinė šaknis iš. A number squared -> 'pakelta kvadratu' (3² -> trys pakelta kvadratu); a variable or expression squared -> 'kvadratu' (a² -> a kvadratu). Read variable letters by their Lithuanian names (x -> iks, y -> ygrek, a -> a, b -> bė, c -> cė, r -> er, f -> ef). Do NOT read brackets or parentheses aloud.\n- Letters and acronyms: spell out the letters using their Lithuanian letter names written together without spaces (a, bė, cė, čė, dė, e, ef, gė, haš, i, jot, ka, el, em, en, o, pė, er, es, eš, tė, u, vė, iks, ygrek, zė, žė; W -> vė). E.g. WMT -> vėemtė, SN -> esen, LR -> eler, BK -> bėka. Do NOT use English letter names. Read an acronym as a word only when it is a widely recognized, pronounceable name (e.g. SARS-CoV-2 -> sars kov du). Never expand an abbreviation or acronym into its full wording (keep BK as bėka, not 'Baudžiamojo kodekso').\n- Chemical formulas: spell each element symbol with its Lithuanian letter name and read subscript digits as cardinals, e.g. H₂O -> haš du o, CH₄ -> cė haš keturi, CO₂ -> cė o du.\n- Codes, serial numbers, license plates, product codes, stock tickers and biological or gene codes: read digits one by one (not as a whole number), spell non-pronounceable letter groups by their Lithuanian letter names, read pronounceable parts as words, and do NOT read connecting hyphens aloud (separate the parts by a pause), e.g. AEI 250 -> AEI, du penki nulis.\n- Legal references: read section, part and point numbers as ordinals agreeing with the following noun (154 straipsnio -> šimtas penkiasdešimt ketvirto straipsnio; 2 dalis -> antra dalis). Read dotted article numbers digit by digit with 'taškas' for the dot and do NOT interpret them semantically (CK 1.5 -> cėka vienas taškas penki, never 'pirmo straipsnio penkta dalis'). In reference codes read '-' as 'brūkšnelis' and '/' as 'pasvirasis brūkšnys'.\n- Convert punctuation that is spoken aloud into words in URLs and emails ('taškas' for dots, 'eta' for '@', 'pasvirasis brūkšnys' for '/').\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Suvalgiau 1/2 picos.", "output": "Suvalgiau pusę picos."},
+    {"input": "Nuėjome 3/5 kelio.", "output": "Nuėjome tris penktadalius kelio."},
+    {"input": "Jis nubėgo 1/7 maršruto.", "output": "Jis nubėgo septintadalį maršruto."},
+    {"input": "0,3 yra daugiau nei 0,25.", "output": "Trys dešimtosios yra daugiau nei dvidešimt penkios šimtosios."},
+    {"input": "Paklaida siekė 0,02 mm.", "output": "Paklaida siekė dvi šimtąsias milimetro."},
+    {"input": "Jis nubėgo 5,2 km.", "output": "Jis nubėgo penkis ir dvi dešimtąsias kilometro."},
+    {"input": "2 + 3 = 5.", "output": "Du plius trys lygu penki."},
+    {"input": "20 ÷ 5 = 4.", "output": "Dvidešimt padalinti iš penkių lygu keturi."},
+    {"input": "3² = 9.", "output": "Trys pakelta kvadratu lygu devyni."},
+    {"input": "(x − a)² + (y − b)² = r²", "output": "Iks minus a kvadratu plius ygrek minus bė kvadratu lygu er kvadratu."},
+    {"input": "Konkurse jis užėmė III vietą.", "output": "Konkurse jis užėmė trečią vietą."},
+    {"input": "Vandens cheminė formulė yra H₂O.", "output": "Vandens cheminė formulė yra haš du o."},
+    {"input": "Metanas žymimas CH₄.", "output": "Metanas žymimas cė haš keturi."},
+    {"input": "CK 1.5 straipsnis nustato civilinės teisės principus.", "output": "Cėka vienas taškas penki straipsnis nustato civilinės teisės principus."},
+    {"input": "BK 154 straipsnio 2 dalis taikoma šmeižto atvejams.", "output": "Bėka šimtas penkiasdešimt ketvirto straipsnio antra dalis taikoma šmeižto atvejams."},
+    {"input": "AMZN užsidarė pakilus akcijų vertei.", "output": "aemzėen užsidarė pakilus akcijų vertei."},
+    {"input": "Šio viruso atmaina yra SARS-CoV-2.", "output": "Šio viruso atmaina yra sars kov du."},
+    {"input": "Geno kodas yra BRCA1.", "output": "Geno kodas yra bėercėa vienas."},
+    {"input": "Automobilio numeris yra AEI 250.", "output": "Automobilio numeris yra AEI, du penki nulis."},
+    {"input": "Susitikimas vyks 2025-04-20.", "output": "Susitikimas vyks du tūkstančiai dvidešimt penktų metų balandžio dvidešimtą dieną."},
+    {"input": "Sekite @oficialuspuslapis ir naudokite #vacations2024.", "output": "Sekite eta oficialuspuslapis ir naudokite grotažymė vacations du tūkstančiai dvidešimt keturi."},
+    {"input": "Įdiegiau 8.6.1 versiją.", "output": "Įdiegiau aštuoni taškas šeši taškas vienas versiją."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/nl-NL.json

@@ -0,0 +1,13 @@
+{
+  "locale": "nl-NL",
+  "language_name": "Dutch",
+  "system": "You are an accurate text normalizer for nl-NL (Dutch). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Dutch (2023 -> tweeduizend drieëntwintig; 12 oktober -> twaalf oktober).\n- Decimals use komma (2.5 -> twee komma vijf); percentages use procent (15% -> vijftien procent).\n- Ordinal uses -de/-ste suffix (XVIII -> achttiende).\n- Phone numbers, ISBNs and codes are read digit-by-digit (06-12345678 -> nul zes een twee drie vier vijf zes zeven acht).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Dutch names (€ -> euro; $ -> dollar).\n- Math operators become words (= -> is gelijk aan, + -> plus, - -> min).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "De vergadering is op 12 oktober 2023.", "output": "De vergadering is op twaalf oktober tweeduizend drieëntwintig."},
+    {"input": "Het product kost € 1.500,00.", "output": "Het product kost vijftienhonderd euro."},
+    {"input": "De inflatie bereikte 7,5% dit jaar.", "output": "De inflatie bereikte zeven komma vijf procent dit jaar."},
+    {"input": "Mijn telefoonnummer is 06-12345678.", "output": "Mijn telefoonnummer is nul zes een twee drie vier vijf zes zeven acht."},
+    {"input": "Het apparaat gebruikt GPS en NaOH.", "output": "Het apparaat gebruikt G P S en N a O H."},
+    {"input": "De formule (a+b)² is eenvoudig.", "output": "De formule a plus b in het kwadraat is eenvoudig."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/pl-PL.json

@@ -0,0 +1,13 @@
+{
+  "locale": "pl-PL",
+  "language_name": "Polish",
+  "system": "You are an accurate text normalizer for pl-PL (Polish). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Polish (2023 -> dwa tysiące dwadzieścia trzy; 12 października -> dwunastego października).\n- Decimals use przecinek (2.5 -> dwa przecinek pięć); percentages use procent (15% -> piętnaście procent).\n- Ordinal uses -ty/-ta suffix (XVIII -> osiemnasty).\n- Phone numbers, ISBNs and codes are read digit-by-digit (600-123456 -> sześć zero zero jeden dwa trzy cztery pięć sześć).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Polish names (zł -> złotych; $ -> dolarów).\n- Math operators become words (= -> równa się, + -> plus, - -> minus).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Spotkanie odbędzie się 12 października 2023.", "output": "Spotkanie odbędzie się dwunastego października dwa tysiące dwadzieścia trzy."},
+    {"input": "Produkt kosztuje 1 500 złotych.", "output": "Produkt kosztuje tysiąc pięćset złotych."},
+    {"input": "Inflacja osiągnęła 7,5% w tym roku.", "output": "Inflacja osiągnęła siedem przecinek pięć procent w tym roku."},
+    {"input": "Mój numer telefonu to 600-123456.", "output": "Mój numer telefonu to sześć zero zero jeden dwa trzy cztery pięć sześć."},
+    {"input": "Urządzenie używa GPS i NaOH.", "output": "Urządzenie używa G P S i N a O H."},
+    {"input": "Wzór (a+b)² jest prosty.", "output": "Wzór a plus b do kwadratu jest prosty."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/pt-BR.json

@@ -0,0 +1,13 @@
+{
+  "locale": "pt-BR",
+  "language_name": "Portuguese (Brazil)",
+  "system": "You are an accurate text normalizer for pt-BR (Brazilian Portuguese). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Portuguese (2023 -> dois mil e vinte e três; 10/12/2023 -> dez de dezembro de dois mil e vinte e três).\n- Decimals use vírgula (2.5 -> dois vírgula cinco); percentages use por cento (15% -> quinze por cento).\n- Ordinals use -º suffix (XVIII -> décimo oitavo).\n- Phone numbers, ISBNs and codes are read digit-by-digit (11-91234-5678 -> um um nove um dois três quatro cinco seis sete oito).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Brazilian names (R$ -> real/reais; $ -> dólares).\n- Math operators become words (= -> é igual a, + -> mais, - -> menos).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "A reunião será em 10 de dezembro de 2023.", "output": "A reunião será em dez de dezembro de dois mil e vinte e três."},
+    {"input": "O produto custa R$ 1.500,00.", "output": "O produto custa mil e quinhentos reais."},
+    {"input": "A inflação atingiu 7,5% este ano.", "output": "A inflação atingiu sete vírgula cinco por cento este ano."},
+    {"input": "Meu telefone é 11-91234-5678.", "output": "Meu telefone é um um nove um dois três quatro cinco seis sete oito."},
+    {"input": "O dispositivo usa GPS e NaOH.", "output": "O dispositivo usa G P S e N a O H."},
+    {"input": "A fórmula (a+b)² é simples.", "output": "A fórmula a mais b ao quadrado é simples."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/ro-RO.json

@@ -0,0 +1,13 @@
+{
+  "locale": "ro-RO",
+  "language_name": "Romanian",
+  "system": "You are an accurate text normalizer for ro-RO (Romanian). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Romanian (2023 -> două mii douăzeci și trei; 12 octombrie -> doisprezece octombrie).\n- Decimals use virgulă (2.5 -> două virgulă cinci); percentages use la sută (15% -> cincisprezece la sută).\n- Ordinal uses -lea/-a suffix (XVIII -> al optsprezecelea).\n- Phone numbers, ISBNs and codes are read digit-by-digit (07-1234567 -> zero șapte unu doi trei patru cinci șase șapte).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Romanian names (lei -> lei; $ -> dolari).\n- Math operators become words (= -> este egal cu, + -> plus, - -> minus).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Întâlnirea va avea loc pe 12 octombrie 2023.", "output": "Întâlnirea va avea loc pe doisprezece octombrie două mii douăzeci și trei."},
+    {"input": "Produsul costă 1.500 de lei.", "output": "Produsul costă o mie cinci sute de lei."},
+    {"input": "Inflația a ajuns la 7,5% anul acesta.", "output": "Inflația a ajuns la șapte virgulă cinci la sută anul acesta."},
+    {"input": "Numărul meu de telefon este 07-1234567.", "output": "Numărul meu de telefon este zero șapte unu doi trei patru cinci șase șapte."},
+    {"input": "Dispozitivul folosește GPS și NaOH.", "output": "Dispozitivul folosește G P S și N a O H."},
+    {"input": "Formula (a+b)² este simplă.", "output": "Formula a plus b la pătrat este simplă."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/ru-RU.json

@@ -0,0 +1,13 @@
+{
+  "locale": "ru-RU",
+  "language_name": "Russian",
+  "system": "You are an accurate text normalizer for ru-RU (Russian). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers, dates and times into spoken Russian (2023 год -> две тысячи двадцать третий год, 12 октября -> двенадцатого октября).\n- Decimals use запятая/целых (2.5 -> две целых пять десятых); percentages use процентов (15% -> пятнадцать процентов).\n- Dates in Russian use genitive case for the day month (10/12/2023 -> десять декабря две тысячи двадцать третьего года).\n- Phone numbers, ISBNs and codes are read in groups (8-900-123-45-67 -> восемь девятьсот двадцать три или цифрами).\n- Roman numerals are converted to Russian numbers (XVIII -> восемнадцатый).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable produceable acronyms kept as-is (NASA).\n- Currency amounts use Russian names (₽ -> рублей; $ -> долларов; € -> евро).\n- Math operators become words (= -> равно, + -> плюс, - -> минус).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Встреча состоится 10 декабря 2023 года.", "output": "Встреча состоится десятого декабря две тысячи двадцать третьего года."},
+    {"input": "Цена продукта составляет 2,500 рублей.", "output": "Цена продукта составляет две тысячи пятьсот рублей."},
+    {"input": "Инфляция достигла 7.5% в этом году.", "output": "Инфляция достигла семи целых пяти десятых процента в этом году."},
+    {"input": "Мой номер телефона 8-900-123-45-67.", "output": "Мой номер телефона восемь девятьсот сто двадцать три сорок пять шестьдесят семь."},
+    {"input": "Аппарат использует GPS и формулу NaOH.", "output": "Аппарат использует G P S и формулу N a O H."},
+    {"input": "Формула (a+b)² проста.", "output": "Формула a плюс b в квадрате проста."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/th-TH.json

@@ -0,0 +1,13 @@
+{
+  "locale": "th-TH",
+  "language_name": "Thai",
+  "system": "You are an accurate text normalizer for th-TH (Thai). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Thai (2023 -> สองพันยี่สิบสาม; 12 ตุลาคม -> สิบสองตุลาคม).\n- Decimals use จุด (2.5 -> สองจุดห้า); percentages use เปอร์เซ็นต์ (15% -> สิบห้าเปอร์เซ็นต์).\n- Phone numbers, ISBNs and codes are read digit-by-digit (081-2345678 -> ศูนย์แปดหนึ่งสองสามสี่ห้าหกเจ็ดแปด).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Thai names (บาท -> บาท; ดอลลาร์ -> ดอลลาร์).\n- Math operators become words (= -> เท่ากับ, + -> บวก, - -> ลบ).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "การประชุมจะจัดขึ้นในวันที่ 12 ตุลาคม 2023", "output": "การประชุมจะจัดขึ้นในวันที่ สิบสอง ตุลาคม สองพันยี่สิบสาม"},
+    {"input": "สินค้าราคา 1,500 บาท", "output": "สินค้าราคา หนึ่งพันห้าร้อยบาท"},
+    {"input": "อัตราเงินเฟ้อถึง 7.5% ในปีนี้", "output": "อัตราเงินเฟ้อถึง เจ็ดจุดห้าเปอร์เซ็นต์ ในปีนี้"},
+    {"input": "เบอร์โทรศัพท์ของฉันคือ 081-2345678", "output": "เบอร์โทรศัพท์ของฉันคือ ศูนย์แปดหนึ่งสองสามสี่ห้าหกเจ็ดแปด"},
+    {"input": "อุปกรณ์ใช้ GPS และ NaOH", "output": "อุปกรณ์ใช้ G P S และ N a O H"},
+    {"input": "สูตร (a+b)² ง่ายมาก", "output": "สูตร a บวก b กำลังสอง ง่ายมาก"}
+  ]
+}

fireredtts3/utils/llm_tn/templates/tr-TR.json

@@ -0,0 +1,13 @@
+{
+  "locale": "tr-TR",
+  "language_name": "Turkish",
+  "system": "You are an accurate text normalizer for tr-TR (Turkish). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Turkish (2023 -> iki bin yirmi üç; 12 Ekim -> on iki Ekim).\n- Decimals use virgül (2.5 -> iki virgül beş); percentages use yüzde (15% -> yüzde on beş).\n- Ordinals use -ıncı/-nci suffix (XVIII -> on sekizinci).\n- Phone numbers, ISBNs and codes are read digit-by-digit (0532-1234567 -> sıfır beş üç iki bir iki üç dört beş altı yedi).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Turkish names (₺ -> lira; $ -> dolar; € -> avro).\n- Math operators become words (= -> eşittir, + -> artı, - -> eksi).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Toplantı 12 Ekim 2023 tarihinde yapılacak.", "output": "Toplantı on iki Ekim iki bin yirmi üç tarihinde yapılacak."},
+    {"input": "Ürünün fiyatı 1.500 liradır.", "output": "Ürünün fiyatı bin beş yüz liradır."},
+    {"input": "Enflasyon bu yıl %7,5'e ulaştı.", "output": "Enflasyon bu yıl yüzde yedi virgül beşe ulaştı."},
+    {"input": "Telefon numaram 0532-1234567.", "output": "Telefon numaram sıfır beş üç iki bir iki üç dört beş altı yedi."},
+    {"input": "Cihaz GPS ve NaOH formülünü kullanır.", "output": "Cihaz G P S ve N a O H formülünü kullanır."},
+    {"input": "(a+b)² formülü basittir.", "output": "a artı b'nin karesi formülü basittir."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/uk-UA.json

@@ -0,0 +1,13 @@
+{
+  "locale": "uk-UA",
+  "language_name": "Ukrainian",
+  "system": "You are an accurate text normalizer for uk-UA (Ukrainian). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Ukrainian (2023 -> дві тисячі двадцять три; 12 жовтня -> дванадцятого жовтня).\n- Decimals use кома (2.5 -> дві кома п'ять); percentages use відсотків (15% -> п'ятнадцять відсотків).\n- Dates use genitive case for day/month (10/12/2023 -> десяте грудня дві тисячі двадцять третього року).\n- Phone numbers, ISBNs and codes are read in groups (8-900-123-45-67 -> вісім дев'ятьсот двадцять три сорок п'ять шістдесят сім).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Ukrainian names (₴ -> гривень; $ -> доларів; € -> євро).\n- Math operators become words (= -> дорівнює, + -> плюс, - -> мінус).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Зустріч відбудеться 12 жовтня 2023 року.", "output": "Зустріч відбудеться дванадцятого жовтня дві тисячі двадцять третього року."},
+    {"input": "Ціна продукту становить 1 500 гривень.", "output": "Ціна продукту становить одну тисячу п'ятсот гривень."},
+    {"input": "Інфляція досягла 7,5% цього року.", "output": "Інфляція досягла семи цілих п'яти десятих відсотка цього року."},
+    {"input": "Мій номер телефону 8-900-123-45-67.", "output": "Мій номер телефону вісім дев'ятсот двадцять три сорок п'ять шістдесят сім."},
+    {"input": "Пристрій використовує GPS та NaOH.", "output": "Пристрій використовує G P S та N a O H."},
+    {"input": "Формула (a+b)² проста.", "output": "Формула a плюс b у квадраті проста."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/vi-VN.json

@@ -0,0 +1,13 @@
+{
+  "locale": "vi-VN",
+  "language_name": "Vietnamese",
+  "system": "You are an accurate text normalizer for vi-VN (Vietnamese). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers and dates into spoken Vietnamese (2023 -> hai nghìn không trăm hai mươi ba / hai ngàn hai mươi ba; 12/10/2023 -> ngày mười hai tháng mười năm hai nghìn không trăm hai mươi ba).\n- Decimals use phẩy (2.5 -> hai phẩy năm); percentages use phần trăm (15% -> mười lăm phần trăm).\n- Ordinal uses thứ prefix (XVIII -> thứ mười tám).\n- Phone numbers, ISBNs and codes are read digit-by-digit (090-1234567 -> không chín không một hai ba bốn năm sáu bảy).\n- Latin acronyms/initialisms are spelled letter-by-letter (GPS -> G P S, NaOH -> N a O H). Recognizable pronounceable acronyms kept as-is (NASA).\n- Currency amounts use Vietnamese names (₫ -> đồng; $ -> đô la).\n- Math operators become words (= -> bằng, + -> cộng, - -> trừ).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "Cuộc họp diễn ra vào ngày 12/10/2023.", "output": "Cuộc họp diễn ra vào ngày mười hai tháng mười năm hai nghìn không trăm hai mươi ba."},
+    {"input": "Sản phẩm có giá 1.500.000 đồng.", "output": "Sản phẩm có giá một triệu năm trăm nghìn đồng."},
+    {"input": "Lạm phát đạt 7,5% trong năm nay.", "output": "Lạm phát đạt bảy phẩy năm phần trăm trong năm nay."},
+    {"input": "Số điện thoại của tôi là 090-1234567.", "output": "Số điện thoại của tôi là không chín không một hai ba bốn năm sáu bảy."},
+    {"input": "Thiết bị dùng GPS và NaOH.", "output": "Thiết bị dùng G P S và N a O H."},
+    {"input": "Công thức (a+b)² rất đơn giản.", "output": "Công thức a cộng b bình phương rất đơn giản."}
+  ]
+}

fireredtts3/utils/llm_tn/templates/zh-CN.json

@@ -0,0 +1,25 @@
+{
+  "locale": "zh-CN",
+  "language_name": "Chinese",
+  "system": "You are an accurate text normalizer for zh-CN (Simplified Mandarin Chinese). Your task is to normalize unstandardized text from the following categories into the way it is actually spoken aloud, based on the context: [Cardinal, Date, Decimal, Ordinal, Fraction, Time, Currency, Unit, Electronic Address, Initialism/Acronym, ISBN, Roman Numeral, Telephone, Sports Score, Mathematical Expression, Symbol, Abbreviation, Chemical Formula, Legal Reference, Vehicle/Product Code, Geographic Coordinates, Version Number, License Plate, Musical Notation, Stock Ticker, Biological Classification, Address].\n\nRules:\n- Convert numbers, symbols and dates into spoken Chinese characters (Hanzi).\n- Years in dates are read digit-by-digit (2023年 -> 二零二三年); plain cardinal numbers are read as a whole (6000 -> 六千); decimals use 点 (2.5 -> 二点五).\n- Phone numbers, ISBNs and account numbers are read digit-by-digit (138-1234-5678 -> 一三八一二三四五六七八).\n- Percentages and fractions follow Chinese conventions (15% -> 百分之十五, 3/4 -> 四分之三, 3:1 -> 三比一).\n- Roman numerals are converted to Chinese numbers (XVIII -> 十八, II型 -> 二型).\n- Strings of Latin letters (acronyms, initialisms, chemical formulas, ISBN labels, technical unit abbreviations) are read letter-by-letter: insert a single space between every letter and KEEP the original capitalization (GPS -> G P S, NaOH -> N a O H, ISBN -> I S B N, 1TB -> 一T B, iOS -> i O S). Exception: a widely recognized pronounceable English word is kept as-is (NASA, Wi-Fi).\n- Latin abbreviations/titles are expanded to their full ENGLISH form, not translated (Dr. -> Doctor, Mr. -> Mister).\n- Convert spoken punctuation into words (URLs: '.' -> 点; emails: '@' -> 艾特 or 'at'). Math operators become words (|x| -> 绝对值, = -> 等于, + -> 加, - -> 减).\n- Keep all other words unchanged. Output ONLY the normalized sentence, with no explanations, no quotes, and no extra text.",
+  "examples": [
+    {"input": "2023年10月12日，苹果公司发布了iOS 17.1。", "output": "二零二三年十月十二日，苹果公司发布了I O S十七点一。"},
+    {"input": "这辆车的售价为¥250,000，最高时速120km/h。", "output": "这辆车的售价为二十五万元，最高时速一百二十千米每小时。"},
+    {"input": "μ's支持率达到了75.5%，比分是3:1。", "output": "mu's支持率达到了百分之七十五点五，比分是三比一。"},
+    {"input": "我每天上班都使用GPS导航，氢氧化钠是NaOH。", "output": "我每天上班都使用G P S导航，氢氧化钠是N a O H。"},
+    {"input": "第XVIII修正案规定数据量为1TB。", "output": "第十八修正案规定数据量为一T B。"},
+    {"input": "我的电话号码是 138-1234-5678。", "output": "我的电话号码是一三八一二三四五六七八。"},
+    {"input": "请拨打10086。", "output": "请拨打一零零八六。"},
+    {"input": "我的邮箱是happyzhang@hotmail.com", "output": "我的邮箱是 happy zhang at hot mail 点 com。"},
+    {"input": "Dr. Zhang的诊断是II型糖尿病。", "output": "Doctor Zhang的诊断是二型糖尿病。"},
+    {"input": "车架号LFV3A28K9E3012345。", "output": "车架号L F V三A二八K九E三零一二三四五。"},
+    {"input": "MMXXIII年，超级碗LVI在这里举办。", "output": "二零二三年，第五十六届超级碗在这里举办。"},
+    {"input": "机票花了AUD100，酒店CAD$25.75。", "output": "机票花了一百澳元，酒店二十五点七五加元。"},
+    {"input": "这个公式是(σ+b)²。", "output": "这个公式是sigma加B的平方。"},
+    {"input": "软件更新到v2023.12.31。", "output": "软件更新到V二零二三点一二点三一。"},
+    {"input": "这是一首5/4拍的爵士乐。", "output": "这是一首五四拍的爵士乐。"},
+    {"input": "他开车速度在6.5～7.5m/s。", "output": "他开车速度在六点五到七点五米每秒。"},
+    {"input": "罗马的坐标是41.90°N 12.50°E。", "output": "罗马的坐标是北纬四十一点九零度东经十二点五零度。"},
+    {"input": "这块手表售价HK$1000。", "output": "这块手表售价一千港币。"}
+  ]
+}

fireredtts3/utils/llm_tn/text_normalizer.py

@@ -0,0 +1,428 @@
+"""LLM-based few-shot Text Normalizer (TN) for TTS front-ends."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+import unicodedata
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Dict, List, Optional
+
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass
+
+DEFAULT_API_URL = os.environ.get("LLM_TN_API_URL")
+DEFAULT_API_KEY = os.environ.get("LLM_TN_API_KEY")
+DEFAULT_MODEL = os.environ.get("LLM_TN_MODEL")
+
+TEMPLATES_DIR = Path(__file__).resolve().parent / "templates"
+
+SUPPORTED_LOCALES = [
+    "ar-SA", "cs-CZ", "de-DE", "el-GR", "en-US", "es-MX",
+    "fi-FI", "fr-FR", "hi-IN", "id-ID", "it-IT", "ja-JP",
+    "ko-KR", "lt-LT", "nl-NL", "pl-PL", "pt-BR", "ro-RO", "ru-RU",
+    "th-TH", "tr-TR", "uk-UA", "vi-VN", "zh-CN",
+]
+
+# Meta FastText language-id model (lid.176).
+# Download: curl -L -o .../models/lid.176.ftz https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz
+FASTTEXT_MODEL_PATH = Path(__file__).resolve().parent / "models" / "lid.176.ftz"
+
+# Map FastText ISO-639 language codes to our locale codes.
+_FT_LANG_TO_LOCALE = {
+    "ar": "ar-SA", "cs": "cs-CZ", "de": "de-DE", "el": "el-GR",
+    "en": "en-US", "es": "es-MX", "fi": "fi-FI", "fr": "fr-FR",
+    "hi": "hi-IN", "id": "id-ID", "it": "it-IT", "ja": "ja-JP",
+    "ko": "ko-KR", "lt": "lt-LT", "nl": "nl-NL", "pl": "pl-PL",
+    "pt": "pt-BR", "ro": "ro-RO", "ru": "ru-RU", "th": "th-TH",
+    "tr": "tr-TR", "uk": "uk-UA", "vi": "vi-VN", "zh": "zh-CN",
+}
+
+# Characters indicative of a specific Latin-script language.
+_LT_CHARS = set("ąčęėįšųūž")   # Lithuanian
+_DE_CHARS = set("äöüß")         # German
+_FR_CHARS = set("àâæçéèêëîïôœùûÿ")  # French
+_ES_CHARS = set("ñ¿¡")          # Spanish
+_IT_CHARS = set("àèéìíîòóùú")   # Italian
+
+# Stop-words to disambiguate Latin-script languages.
+_STOPWORDS: Dict[str, set] = {
+    "de-DE": {"und", "der", "die", "das", "ist", "ein", "nicht", "mit", "den",
+              "von", "sie", "ich", "auf", "für", "auch", "wird", "im", "am"},
+    "en-US": {"the", "and", "is", "are", "of", "to", "in", "for", "on", "with",
+              "this", "that", "you", "it", "have", "was", "please", "at"},
+    "es-MX": {"el", "la", "los", "las", "de", "que", "es", "un", "una", "por",
+              "con", "para", "su", "se", "no", "más", "está", "del"},
+    "fr-FR": {"le", "la", "les", "des", "est", "et", "un", "une", "vous", "que",
+              "pour", "dans", "qui", "pas", "sur", "au", "du", "ce"},
+    "it-IT": {"il", "la", "le", "di", "che", "è", "un", "una", "per", "con",
+              "non", "sono", "del", "della", "gli", "nel", "ha", "si"},
+}
+
+
+class TextNormalizer:
+    """Few-shot, LLM-driven multilingual text normalizer."""
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        templates_dir: Path = TEMPLATES_DIR,
+        temperature: float = 0.1,
+        max_tokens: int = 1024,
+        timeout: int = 30,
+        max_retries: int = 3,
+        default_locale: str = "zh-CN",
+        use_fasttext: bool = True,
+        fasttext_path: Path = FASTTEXT_MODEL_PATH,
+    ) -> None:
+        self.model = model or DEFAULT_MODEL
+        self.api_url = api_url or DEFAULT_API_URL
+        self.api_key = api_key or DEFAULT_API_KEY
+        if not self.api_url:
+            raise ValueError(
+                "TextNormalizer requires an LLM API URL. Set LLM_TN_API_URL in "
+                "your environment / .env file, or pass api_url explicitly."
+            )
+        if not self.api_key:
+            raise ValueError(
+                "TextNormalizer requires an LLM API key. Set LLM_TN_API_KEY in "
+                "your environment / .env file, or pass api_key explicitly."
+            )
+        self.templates_dir = Path(templates_dir)
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.default_locale = default_locale
+        self.stop = []
+        self.use_fasttext = use_fasttext
+        self.fasttext_path = Path(fasttext_path)
+        self._ft_model = None
+        self._ft_loaded = False
+        self._ft_lock = threading.Lock()
+        self._templates: Dict[str, Dict] = {}
+        self._load_templates()
+
+    def _get_fasttext(self):
+        """Lazily load FastText lid.176 model (thread-safe)."""
+        if not self.use_fasttext:
+            return None
+        if self._ft_loaded:
+            return self._ft_model
+        with self._ft_lock:
+            if self._ft_loaded:
+                return self._ft_model
+            self._ft_loaded = True
+            try:
+                import fasttext
+                if self.fasttext_path.exists():
+                    self._ft_model = fasttext.load_model(str(self.fasttext_path))
+                else:
+                    self._ft_model = None
+            except Exception:
+                self._ft_model = None
+        return self._ft_model
+
+    def _fasttext_locale(self, text: str):
+        """Predict a supported locale via FastText, or None if unavailable."""
+        model = self._get_fasttext()
+        if model is None:
+            return None
+        snippet = text.replace("\n", " ").strip()
+        if not snippet:
+            return None
+        try:
+            # Use low-level predictor to avoid numpy-2.0 incompatibility in predict().
+            with self._ft_lock:
+                preds = model.f.predict(snippet, 1, 0.0, "strict")
+        except Exception:
+            return None
+        if not preds:
+            return None
+        _prob, label = preds[0]
+        lang = label.replace("__label__", "")
+        return _FT_LANG_TO_LOCALE.get(lang)
+
+    def _load_templates(self) -> None:
+        for loc in SUPPORTED_LOCALES:
+            path = self.templates_dir / f"{loc}.json"
+            if path.exists():
+                with path.open(encoding="utf-8") as f:
+                    self._templates[loc] = json.load(f)
+
+    def available_locales(self) -> List[str]:
+        return sorted(self._templates.keys())
+
+    _TN_SYMBOLS = set("0123456789@#$£€¥%&<>=^`~/\\+*°²³×÷")
+
+    def needs_normalization(self, text: str) -> bool:
+        """Return False for plain prose (no digits/symbols needing TN)."""
+        if not text or not text.strip():
+            return False
+        for ch in text:
+            if ch in self._TN_SYMBOLS:
+                return True
+        if re.search(r"https?://|www\.|[\w.+-]+@[\w-]+\.", text):
+            return True
+        if re.search(r"\b[A-Z]{2,}\b", text):
+            return True
+        if re.search(r"\b[IVXLCDM]{2,}\b", text):
+            return True
+        if re.search(r"\b[A-Za-zÀ-ÿ]{1,4}\.", text):
+            return True
+        return False
+
+    @staticmethod
+    def _script_stats(text: str) -> Dict[str, int]:
+        stats = {"han": 0, "kana": 0, "latin": 0, "hangul": 0, "cyrillic": 0,
+                 "arabic": 0, "devanagari": 0, "thai": 0, "greek": 0}
+        for ch in text:
+            code = ord(ch)
+            if 0x3040 <= code <= 0x30FF:  # kana
+                stats["kana"] += 1
+            elif (0x4E00 <= code <= 0x9FFF or
+                  0x3400 <= code <= 0x4DBF or
+                  0xF900 <= code <= 0xFAFF or
+                  0x20000 <= code <= 0x2FA1F):
+                stats["han"] += 1
+            elif 0xAC00 <= code <= 0xD7AF:  # hangul
+                stats["hangul"] += 1
+            elif 0x0400 <= code <= 0x04FF or 0x0500 <= code <= 0x052F:  # cyrillic
+                stats["cyrillic"] += 1
+            elif 0x0600 <= code <= 0x06FF or 0x0750 <= code <= 0x077F:  # arabic
+                stats["arabic"] += 1
+            elif 0x0900 <= code <= 0x097F:  # devanagari
+                stats["devanagari"] += 1
+            elif 0x0E00 <= code <= 0x0E7F:  # thai
+                stats["thai"] += 1
+            elif 0x0370 <= code <= 0x03FF:  # greek
+                stats["greek"] += 1
+            elif ch.isalpha() and code < 0x250:  # latin
+                stats["latin"] += 1
+        return stats
+
+    def _detect_latin_locale(self, text: str) -> Optional[str]:
+        """Disambiguate Latin text among de/en/es/fr/it/lt; None if ambiguous."""
+        low = text.lower()
+        chars = set(low)
+        if chars & _LT_CHARS:
+            return "lt-LT"
+        if "ß" in low or (chars & _DE_CHARS):
+            de_signal = "ß" in low
+        else:
+            de_signal = False
+        if "ñ" in low or "¿" in low or "¡" in low:
+            return "es-MX"
+        words = re.findall(r"[a-zà-ÿ]+", low)
+        if not words:
+            return None
+        wordset = set(words)
+        scores = {loc: len(wordset & sw) for loc, sw in _STOPWORDS.items()}
+        if chars & _DE_CHARS:
+            scores["de-DE"] += 1
+        if de_signal:
+            scores["de-DE"] += 2
+        if chars & _FR_CHARS:
+            scores["fr-FR"] += 1
+        if chars & _IT_CHARS:
+            scores["it-IT"] += 1
+        best = max(scores, key=scores.get)
+        if scores[best] == 0:
+            return None
+        ranked = sorted(scores.values(), reverse=True)
+        if len(ranked) > 1 and ranked[0] == ranked[1]:
+            return None
+        return best
+
+    def detect_locale(self, text: str, use_llm_fallback: bool = True,
+                      fallback_locale: Optional[str] = None) -> str:
+        """Detect the locale of *text* among SUPPORTED_LOCALES."""
+        fb = fallback_locale or self.default_locale
+        stats = self._script_stats(text)
+        if stats["kana"] > 0:
+            return "ja-JP"
+        if stats["han"] > 0:
+            return "zh-CN"
+        if re.search(r"[\u3000-\u303F\uFF01-\uFF0F\uFF1A-\uFF20]", text):
+            return "zh-CN"
+        if any(stats[k] > 0 for k in ("hangul", "cyrillic", "arabic",
+                                      "devanagari", "thai", "greek")):
+            ft_guess = self._fasttext_locale(text)
+            if ft_guess is not None:
+                return ft_guess
+            if use_llm_fallback:
+                llm_guess = self._detect_locale_llm(text)
+                if llm_guess in SUPPORTED_LOCALES:
+                    return llm_guess
+            return fb
+        if stats["latin"] == 0 or not re.search(r"[A-Za-zÀ-ÿ]{3,}", text):
+            return fb
+        ft_guess = self._fasttext_locale(text)
+        if ft_guess is not None:
+            return ft_guess
+        guess = self._detect_latin_locale(text)
+        if guess is not None:
+            return guess
+        if use_llm_fallback:
+            llm_guess = self._detect_locale_llm(text)
+            if llm_guess in SUPPORTED_LOCALES:
+                return llm_guess
+        return fb
+
+    def _detect_locale_llm(self, text: str) -> Optional[str]:
+        prompt = (
+            "Identify the language of the text below and answer with EXACTLY one "
+            "of these locale codes and nothing else: "
+            + ", ".join(SUPPORTED_LOCALES)
+            + f".\n\nText: {text}\nLocale:"
+        )
+        messages = [
+            {"role": "system", "content": "You are a precise language identifier."},
+            {"role": "user", "content": prompt},
+        ]
+        try:
+            out = self._chat(messages, max_tokens=16)
+        except Exception:
+            return None
+        out = out.strip()
+        for loc in SUPPORTED_LOCALES:
+            if loc.lower() in out.lower():
+                return loc
+        lang = out.lower()[:2]
+        for loc in SUPPORTED_LOCALES:
+            if loc.lower().startswith(lang):
+                return loc
+        return None
+
+    def _build_messages(self, text: str, locale: str) -> List[Dict[str, str]]:
+        tmpl = self._templates.get(locale)
+        if tmpl is None:
+            raise ValueError(f"No template available for locale '{locale}'")
+        messages: List[Dict[str, str]] = [{"role": "system", "content": tmpl["system"]}]
+        for ex in tmpl.get("examples", []):
+            messages.append({"role": "user", "content": ex["input"]})
+            messages.append({"role": "assistant", "content": ex["output"]})
+        messages.append({"role": "user", "content": text})
+        return messages
+
+    @staticmethod
+    def _clean_output(text: str, raw_input: str) -> str:
+        text = text.split("\n\n", 1)[0]
+        out = text.strip()
+        if len(out) >= 2 and out[0] in "\"'“”「" and out[-1] in "\"'“”」":
+            out = out[1:-1].strip()
+        return out or raw_input
+
+    def normalize(
+        self,
+        text: str,
+        locale: Optional[str] = None,
+        auto_detect: bool = True,
+        force: bool = False,
+        fallback_locale: Optional[str] = None,
+    ) -> str:
+        """Normalize *text* into its spoken form."""
+        if text is None:
+            return text
+        if not text.strip():
+            return text
+        if not force and not self.needs_normalization(text):
+            return text
+        if locale is None:
+            if auto_detect:
+                locale = self.detect_locale(text, fallback_locale=fallback_locale)
+            else:
+                locale = fallback_locale or self.default_locale
+        if locale not in self._templates:
+            locale = fallback_locale or self.default_locale
+        messages = self._build_messages(text, locale)
+        result = self._chat(messages)
+        return self._clean_output(result, text)
+
+    def _build_request(self, messages: List[Dict[str, str]], max_tokens: int):
+        """Build OpenAI-compatible chat/completions request.
+
+        CoT disabled based on model name:
+          ``deepseek`` -> ``thinking: {"type": "disabled"}``
+          ``qwen``     -> ``chat_template_kwargs: {"enable_thinking": False}``
+        """
+        model = (self.model or "").lower()
+        payload = {
+            "model": self.model,
+            "messages": messages,
+            "stream": False,
+            "max_tokens": max_tokens,
+            "temperature": self.temperature,
+        }
+        if "deepseek" in model:
+            payload["thinking"] = {"type": "disabled"}
+        elif "qwen" in model:
+            payload["chat_template_kwargs"] = {"enable_thinking": False}
+        if self.stop:
+            payload["stop"] = self.stop
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
+        return payload, headers
+
+    def _parse_response(self, obj: Dict) -> str:
+        msg = obj["choices"][0]["message"]
+        content = msg.get("content")
+        if content:
+            return content
+        reasoning = msg.get("reasoning_content")
+        if reasoning:
+            return reasoning
+        raise RuntimeError("empty completion content")
+
+    def _chat(self, messages: List[Dict[str, str]], max_tokens: Optional[int] = None) -> str:
+        payload, headers = self._build_request(messages, max_tokens or self.max_tokens)
+        data = json.dumps(payload).encode("utf-8")
+        last_err: Optional[Exception] = None
+        for attempt in range(self.max_retries):
+            try:
+                req = urllib.request.Request(
+                    self.api_url, data=data, headers=headers, method="POST"
+                )
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    body = resp.read().decode("utf-8")
+                obj = json.loads(body)
+                return self._parse_response(obj)
+            except (urllib.error.URLError, urllib.error.HTTPError, RuntimeError,
+                    KeyError, IndexError, json.JSONDecodeError) as exc:
+                last_err = exc
+                time.sleep(min(2 ** attempt, 8))
+        raise RuntimeError(f"LLM request failed after {self.max_retries} retries: {last_err}")
+
+
+if __name__ == "__main__":
+    import sys
+    tn = TextNormalizer()
+    if len(sys.argv) > 1:
+        sample = " ".join(sys.argv[1:])
+        loc = tn.detect_locale(sample)
+        print(f"[locale={loc}] {tn.normalize(sample)}")
+    else:
+        demos = [
+            "He bought 2.5 kg of apples for $5.99 on 10/12/2023.",
+            "会议定于2023年5月20日举行，电话138-1234-5678。",
+            "这是一句不需要归一化的普通中文。",
+            "Le vol AF123 partira le 1er juin à 18h45.",
+        ]
+        for d in demos:
+            loc = tn.detect_locale(d)
+            need = tn.needs_normalization(d)
+            print(f"\nINPUT : {d}")
+            print(f"locale={loc}  needs_tn={need}")
+            print(f"OUTPUT: {tn.normalize(d)}")

fireredtts3/utils/text_normalize.py

@@ -0,0 +1,600 @@
+"""
+TTS 文本前端（text front-end）工具。
+
+some functions are adapted from https://github.com/FunAudioLLM/CosyVoice/blob/main/cosyvoice/utils/frontend_utils.py
+some functions are adapted from https://github.com/OpenBMB/VoxCPM/blob/main/src/voxcpm/utils/text_normalize.py
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Callable, List, Optional
+
+try:
+    import regex
+except ImportError:  # pragma: no cover
+    regex = None
+
+# --------------------------------------------------------------------------- #
+# 1) 文本初步清洗
+# --------------------------------------------------------------------------- #
+def preprocess_text(sentence: str) -> str:
+    """对文本做最基础的 utf-8 级清洗（参考 FireRedTTS.preprocess_text）。"""
+    if not sentence:
+        return ""
+
+    # utf-8 编码/解码，忽略无法解码的字节
+    sentence = bytes(sentence, "utf-8").decode("utf-8", "ignore")
+
+    if regex is not None:
+        # 去除零宽字符（保留零宽连接符 \u200d）
+        sentence = regex.sub(r"[\p{Cf}--[\u200d]]", "", sentence, flags=regex.V1)
+        # 去除私有区字符
+        sentence = regex.sub(r"\p{Co}", "", sentence)
+    else:
+        sentence = re.sub(r"[\u200b-\u200f\u2028-\u202f\u2060-\u206f\ufeff]", "", sentence)
+        sentence = re.sub(r"[\ue000-\uf8ff]", "", sentence)
+
+    sentence = sentence.replace("\u00a0", " ")  # 不换行空格 -> 普通空格
+    sentence = sentence.replace("\ufffd", "")   # 替换字符 -> 空
+    # 行分隔符 / 段分隔符 -> 换行（U+2028 LINE SEPARATOR, U+2029 PARAGRAPH SEPARATOR）
+    sentence = sentence.replace("\u2028", "\n")
+    sentence = sentence.replace("\u2029", "\n")
+
+    return sentence
+
+
+# --------------------------------------------------------------------------- #
+# 2) Markdown 清洗 + 去除表情符号（参考 VoxCPM.utils.text_normalize）
+# --------------------------------------------------------------------------- #
+def clean_markdown(md_text: str) -> str:
+    """去除 Markdown 语法，保留纯文本内容。"""
+    if not md_text:
+        return md_text
+    # 去除图片语法 ![alt](url)
+    md_text = re.sub(r"!\[[^\]]*\]\([^\)]+\)", "", md_text)
+    # 去除链接但保留文本 [text](url) -> text
+    md_text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", md_text)
+    # 替换无序列表符号
+    md_text = re.sub(r"^(\s*)-\s+", r"\1", md_text, flags=re.MULTILINE)
+    # 去除标题符号（#）
+    md_text = re.sub(r"^#{1,6}\s*", "", md_text, flags=re.MULTILINE)
+    # 只处理句首的加粗/斜体/删除线等强调标记（**text**、*text*、__text__、~~text~~），
+    # 行首的 * / ~ 几乎只可能是 markdown 标记；正文中的素 * / ~ （乘法、波浪号）不受影响。
+    md_text = re.sub(r"^\s*\*\*([^*]+)\*\*", r"\1", md_text, flags=re.MULTILINE)
+    md_text = re.sub(r"^\s*__([^_]+)__", r"\1", md_text, flags=re.MULTILINE)
+    md_text = re.sub(r"^\s*\*([^*]+)\*", r"\1", md_text, flags=re.MULTILINE)
+    md_text = re.sub(r"^\s*~~([^~]+)~~", r"\1", md_text, flags=re.MULTILINE)
+    # 行首多余的离散 * / ~（例如独立的 * item）
+    md_text = re.sub(r"^\s*[*~]\s+", "", md_text, flags=re.MULTILINE)
+    # 去除多余空行
+    md_text = re.sub(r"\n\s*\n", "\n", md_text)
+    md_text = md_text.strip()
+    return md_text
+
+
+def remove_emoji(text: str) -> str:
+    """去除表情符号（Emoji_Presentation 与带变体选择符的 Emoji）。"""
+    if regex is None:
+        return text
+    return regex.compile(
+        r"\p{Emoji_Presentation}|\p{Emoji}\uFE0F", flags=regex.UNICODE
+    ).sub("", text)
+
+
+_CJK = r"\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff"
+_RE_CJK_SPACE_CJK = re.compile(r"([{}])\s+([{}])".format(_CJK, _CJK))
+_RE_CJK_SPACE_LATIN = re.compile(r"([{0}])\s+([a-zA-Z0-9])".format(_CJK))
+_RE_LATIN_SPACE_CJK = re.compile(r"([a-zA-Z0-9])\s+([{0}])".format(_CJK))
+
+
+def clean_tn_spaces(text: str) -> str:
+    """清理 TN 后在汉字之间、汉字与英文/数字之间多余的空格。
+
+    TN（尤其 LLM-based）经常在汉字/数字/符号间插入不必要的空格，如：
+    "今天 天气 很好"  → "今天天气很好"
+    "苹果 AI 助手"     → "苹果AI助手"
+    """
+    if not text:
+        return text
+    text = _RE_CJK_SPACE_CJK.sub(r"\1\2", text)
+    text = _RE_CJK_SPACE_LATIN.sub(r"\1\2", text)
+    text = _RE_LATIN_SPACE_CJK.sub(r"\1\2", text)
+    return text
+
+
+# 符号化简（参考 FireRedTTS utils.py 的 symbol_reduction）。
+# 只保留"输出会被后续 _SYMBOL_TO_SPACE / _SYMBOL_TO_COMMA 处理"的映射：
+#   - 映射到 ~（→ 逗号）、·（→ 空格）、...（→ 逗号）的条目。
+# 其余映射的目的地（如 " ( ) . : ; ! ? - % + = 等）本身就在 _WETEXT_KEEP
+# 允许集中，原字符也会被保留，映射纯属多余，故全部去掉。
+_SYMBOL_REDUCTION = {
+    # 波浪/破折线 → ASCII ~（随后被 _SYMBOL_TO_COMMA 化简为逗号）
+    "〜": "~", "～": "~",
+    # 间隔点/��点 → ·（随后被 _SYMBOL_TO_SPACE 化简为空格）
+    "・": "·", "•": "·", "‧": "·",
+    # 省略号变体 → "..."（避免被 _SYMBOL_TO_COMMA 吞成一个逗号）
+    "…": "...", "⋯": "...", "〰": "...", "﹏": "...",
+}
+_SYMBOL_TO_SPACE = re.compile(r"[·•‧│|¦/\\]")
+_SYMBOL_TO_COMMA = re.compile(r"[…~&*%$#^:;/\\|]+")
+_WETEXT_KEEP = re.compile(
+    r"[^"
+    r"\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff"   # CJK 汉字
+    r"0-9A-Za-z"
+    r"，。、？！：；…—～·"
+    r".,:;!?()\[\]'\"\-_"
+    r"\s"
+    r"]"
+)
+
+
+def _apply_symbol_reduction(text: str) -> str:
+    """把全角/异体符号化简为对应的半角标准符号（参考 symbol_reduction）。"""
+    return "".join(_SYMBOL_REDUCTION.get(ch, ch) for ch in text)
+
+
+def clean_wetext_output(text: str) -> str:
+    # 清洗 wetext 归一化后的中/英文输出，把无法读出的符号化简为逗号或空格。
+    if not text:
+        return text
+    text = _apply_symbol_reduction(text)
+    text = _SYMBOL_TO_SPACE.sub(" ", text)
+    text = _SYMBOL_TO_COMMA.sub(",", text)
+    text = _WETEXT_KEEP.sub("", text)
+    # 规整空白，连续逗号合并
+    text = re.sub(r"[ \t]+", " ", text).strip()
+    text = re.sub(r"[,，]{2,}", ",", text)
+    text = re.sub(r"\s+[,，]", ",", text)
+    text = re.sub(r"[,，]+\s*$", "", text)
+    return text
+
+
+def clean_text(text: str, llm_normalizer: Optional[Callable[[str], str]] = None) -> str:
+    """文本初步清洗：utf-8 清洗 + markdown 清洗 + 去除表情符号 + 规整空白。
+
+    Args:
+        text: 输入文本。
+        llm_normalizer: 可选，一个可调用的文本归一化器（通常为
+            ``llm_tn`` 的 ``TextNormalizer.normalize``）。若提供，则在基础
+            清洗之后调用它以支持多语种 TN（例如日语、韩语等 wetext
+            不支持的语种）。该调用方需自行保证模板齐全。
+    """
+    if not text:
+        return text
+    text = preprocess_text(text)
+    text = clean_markdown(text)
+    text = remove_emoji(text)
+    text = re.sub(r"\s+", " ", text)
+    text = text.strip()
+    if llm_normalizer is not None:
+        try:
+            text = llm_normalizer(text)
+        except Exception as e:  # 失败时回退为清洗后的原文
+            print(f"[WARN] llm_tn normalization failed, fallback to raw text: {e}", flush=True)
+        text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+# --------------------------------------------------------------------------- #
+# 3) 分句分段（参考 VoxCPM.utils.text_normalize.split_paragraph）
+# --------------------------------------------------------------------------- #
+def _is_decimal_dot(text: str, i: int) -> bool:
+    """判断位置 *i* 上的 ``.`` / ``:`` 是否为数字内部标点（而非句号/冒号）。
+
+    例如 ``€1.2M`` 中的小数点 (前后是数字)、``19:30`` 中的冒号 (前后是数字)。
+    用于分句时不切断这些数字内部的标点。
+    """
+    return (i > 0 and i + 1 < len(text)
+            and text[i - 1].isdigit()
+            and text[i + 1].isdigit())
+
+
+def split_paragraph(
+    text: str,
+    tokenize: Optional[Callable[[str], List[str]]] = None,
+    lang: str = "zh",
+    token_max_n: int = 80,
+    token_min_n: int = 60,
+    merge_len: int = 20
+) -> List[str]:
+    """将段落按句号/标点拆分为若干句，并合并过短的句子。
+
+    分句逻辑：
+    1. 每个句子最大长度 ``token_max_n``、最小长度 ``token_min_n``；
+       若末尾句子长度小于 ``merge_len`` 则并入前一句。
+    2. 按语种计算句子长度（zh 按字符数，其余按 token 数）。
+    3. 按标点切分句子。
+
+    注意：紧邻两边都是数字的 ``.``（如 ``€1.2M``、``0.85``、``v1.8.3``）
+    及 ``:``（如 ``19:30``）被视为数字内部标点，不做切分。
+    """
+    def _measure(_text: str) -> int:
+        if lang == "zh":
+            return len(_text)
+        if tokenize is not None:
+            n = tokenize(_text)
+            if isinstance(n, int):
+                return n
+            return len(n)
+        return len(_text)
+
+    def calc_utt_length(_text: str) -> int:
+        return _measure(_text)
+
+    def should_merge(_text: str) -> bool:
+        return _measure(_text) < merge_len
+
+    if lang == "zh":
+        pounc = ["。", "？", "！", "；", "、", ".", "?", "!", ";"]
+    else:
+        pounc = [".", "?", "!", ";"]
+
+    # 按标点切分（跳过数字内部的小数点 ``.`` ）
+    st = 0
+    utts: List[str] = []
+    for i, c in enumerate(text):
+        if c in pounc:
+            # 对 . 和 : 检查是否在数字内部（如 €1.2M）
+            if c == "." and _is_decimal_dot(text, i):
+                continue
+            if len(text[st:i]) > 0:
+                utts.append(text[st:i] + c)
+            if i + 1 < len(text) and text[i + 1] in ['"', "”"]:
+                tmp = utts.pop(-1)
+                utts.append(tmp + text[i + 1])
+                st = i + 2
+            else:
+                st = i + 1
+    trailing = text[st:] if st < len(text) else ""
+    if trailing:
+        utts.append(trailing)
+    elif len(utts) == 0:
+        utts.append(text + ("。" if lang == "zh" else ""))
+    # 合并过短 / 超长处理
+    final_utts: List[str] = []
+    cur_utt = ""
+    for utt in utts:
+        if calc_utt_length(cur_utt + utt) > token_max_n and calc_utt_length(cur_utt) > token_min_n:
+            final_utts.append(cur_utt)
+            cur_utt = ""
+        cur_utt = cur_utt + utt
+    if len(cur_utt) > 0:
+        if should_merge(cur_utt) and len(final_utts) != 0:
+            final_utts[-1] = final_utts[-1] + cur_utt
+        else:
+            final_utts.append(cur_utt)
+
+    return final_utts
+
+
+# --------------------------------------------------------------------------- #
+# 4) 语种自动判定（复用 llm_tn/text_normalizer.py 的 fasttext）
+# --------------------------------------------------------------------------- #
+# 将 llm_tn 的 locale（如 "zh-CN"、"en-US"）映射到 fireredtts3 的 lang tag。
+_LOCALE_TO_LANG_TAG = {
+    "zh-CN": "Chinese",
+    "en-US": "English",
+    "ja-JP": "Japanese",
+    "ko-KR": "Korean",
+    "es-MX": "Spanish",
+    "fr-FR": "French",
+    "ru-RU": "Russian",
+    "ar-SA": "Arabic",
+    "tr-TR": "Turkish",
+    "id-ID": "Indonesian",
+    "pt-BR": "Portuguese",
+    "it-IT": "Italian",
+    "nl-NL": "Dutch",
+    "vi-VN": "Vietnamese",
+    "de-DE": "German",
+    "uk-UA": "Ukrainian",
+    "th-TH": "Thai",
+    "pl-PL": "Polish",
+    "ro-RO": "Romanian",
+    "el-GR": "Greek",
+    "cs-CZ": "Czech",
+    "fi-FI": "Finnish",
+    "hi-IN": "Hindi",
+}
+
+# lang tag -> llm_tn locale（反向映射）。当上层已指定语种时，用它把已知语种
+# 传给 llm_tn，从而覆盖 llm_tn 内部的自动检测，避免误判（如俄语被当成中文）。
+_LANG_TAG_TO_LOCALE = {v: k for k, v in _LOCALE_TO_LANG_TAG.items()}
+# 中文方言统一走 zh-CN 的 llm_tn 模板（方言无独立模板）。
+_LANG_DIALECT_TO_LOCALE = {
+    "Cantonese": "zh-CN",
+    "ZH_Anhui": "zh-CN", "ZH_Fujian": "zh-CN", "ZH_Gansu": "zh-CN",
+    "ZH_Guizhou": "zh-CN", "ZH_Hebei": "zh-CN", "ZH_Henan": "zh-CN",
+    "ZH_Hubei": "zh-CN", "ZH_Hunan": "zh-CN", "ZH_Jiangxi": "zh-CN",
+    "ZH_Liaoning": "zh-CN", "ZH_Minnan": "zh-CN", "ZH_Ningxia": "zh-CN",
+    "ZH_Shaanxi": "zh-CN", "ZH_Shandong": "zh-CN", "ZH_Shanghai": "zh-CN",
+    "ZH_Shanxi": "zh-CN", "ZH_Sichuan": "zh-CN", "ZH_Tianjin": "zh-CN",
+    "ZH_Wenzhou": "zh-CN", "ZH_Wu": "zh-CN", "ZH_Yunnan": "zh-CN",
+}
+
+
+def lang_tag_to_locale(lang_tag: str) -> Optional[str]:
+    """把 lang tag 映射为 llm_tn 的 locale。
+
+    支持标准语言 tag（如 ``"Russian"``）与中文方言 tag（如 ``"ZH_Sichuan"``、
+    ``"Cantonese"``）。方言无独立 llm_tn 模板，统一映射到 ``zh-CN``。
+    无法识别时返回 ``None``。
+    """
+    if not lang_tag:
+        return None
+    if lang_tag in _LANG_TAG_TO_LOCALE:
+        return _LANG_TAG_TO_LOCALE[lang_tag]
+    if lang_tag in _LANG_DIALECT_TO_LOCALE:
+        return _LANG_DIALECT_TO_LOCALE[lang_tag]
+    return None
+
+
+def detect_language(
+    text: str,
+    fasttext_detector: Optional[Callable[[str], Optional[str]]] = None,
+    default_locale: str = "zh-CN",
+) -> str:
+    """自动判定文本语种
+    内部使用 Meta FastText ``lid.176.ftz`` 模型。
+    """
+    if fasttext_detector is not None:
+        try:
+            locale = fasttext_detector(text)
+            if locale:
+                tag = _LOCALE_TO_LANG_TAG.get(locale)
+                if tag:
+                    return tag
+        except Exception:
+            pass
+
+    # 回退：含汉字 -> 中文；含日文假名 -> 日文；否则英文
+    if re.search(r"[\u4e00-\u9fff]", text):
+        return "Chinese"
+    if re.search(r"[\u3040-\u30ff\u31f0-\u31ff]", text):  # 平假名/片假名
+        return "Japanese"
+    return "English"
+
+
+# --------------------------------------------------------------------------- #
+# 5) 归一化器工厂
+# --------------------------------------------------------------------------- #
+def build_wetext_normalizer() -> Optional[Callable[[str], str]]:
+    """构建 wetext 归一化器，仅处理中文和英文。
+
+    实现参考 VoxCPM 的
+    ``voxcpm/utils/text_normalize.py``：使用 ``wetext.Normalizer``（zh / en）
+    做文本归一化，并用 ``contains_chinese`` 自动判定中/英。仅支持中文和英文，
+    其他语种需使用 ``llm_tn``。
+
+    Returns:
+        一个以 ``text`` 为输入、返回归一化文本的可调用对象；若 wetext 不可用
+        返回 ``None``。
+    """
+    try:
+        from wetext import Normalizer
+
+        zh_tn_model = Normalizer(lang="zh", operator="tn", remove_erhua=True)
+        en_tn_model = Normalizer(lang="en", operator="tn")
+
+        chinese_char_pattern = re.compile(r"[\u4e00-\u9fff]+")
+
+        def _contains_chinese(text: str) -> bool:
+            return bool(chinese_char_pattern.search(text))
+
+        def _normalize(text: str) -> str:
+            if not text:
+                return text
+            is_zh = _contains_chinese(text)
+            if is_zh:
+                out = zh_tn_model.normalize(text)
+            else:
+                out = en_tn_model.normalize(text)
+            # 对 wetext TN 输出做强力清洗，去掉所有不应保留的特殊符号
+            return clean_wetext_output(out)
+
+        return _normalize
+    except Exception as e:
+        print(f"[WARN] Failed to build wetext normalizer: {e}", flush=True)
+        return None
+
+
+def build_llm_normalizer(
+    api_url: Optional[str] = None,
+    api_key: Optional[str] = None,
+    model: Optional[str] = None,
+    **kwargs,
+) -> Optional[Callable[[str], str]]:
+    """根据用户提供的 API 配置构建一个 llm_tn 归一化器（可调用对象）。
+
+    llm_tn（``llm_tn/text_normalizer.py``）通过 LLM 实现多语种 TN，只要
+    templates 齐全即可处理 wetext（仅中/英）不支持的语种（日、韩、俄等）。
+
+    Args:
+        api_url: LLM API 地址。为 ``None`` 时从环境变量 / ``.env`` 读取
+            （``LLM_TN_API_URL``，llm_tn 不内置默认地址）。
+        api_key: LLM API 密钥。为 ``None`` 时从环境变量 / ``.env`` 读取
+            （``LLM_TN_API_KEY``，llm_tn 不内置默认密钥）。
+        model: 使用的模型名。为 ``None`` 时从环境变量 / ``.env`` 读取
+            （``LLM_TN_MODEL``，llm_tn 不内置默认模型）。
+        **kwargs: 透传给 ``llm_tn.TextNormalizer`` 的其他参数。
+
+    Returns:
+        一个以 ``text`` 为输入、返回归一化文本的可调用对象；若初始化失败
+        返回 ``None``。
+    """
+    try:
+        from fireredtts3.utils.llm_tn.text_normalizer import (
+            TextNormalizer as LlmTextNormalizer,
+        )
+
+        init_kwargs = dict(kwargs)
+        if api_url is not None:
+            init_kwargs["api_url"] = api_url
+        if api_key is not None:
+            init_kwargs["api_key"] = api_key
+        if model is not None:
+            init_kwargs["model"] = model
+        tn = LlmTextNormalizer(**init_kwargs)
+
+        def _normalize(text: str, locale: Optional[str] = None) -> str:
+            """调用 llm_tn.normalize；当提供了 locale 时覆盖其内部自动检测。"""
+            if locale:
+                return tn.normalize(text, locale=locale, auto_detect=False)
+            return tn.normalize(text)
+
+        return _normalize
+    except Exception as e:
+        print(f"[WARN] Failed to build llm_tn normalizer: {e}", flush=True)
+        return None
+
+
+# --------------------------------------------------------------------------- #
+# 统一入口
+# --------------------------------------------------------------------------- #
+class TextNormalizer:
+    """封装 初步清洗 -> 分句 -> 逐句按需 TN 的文本处理管线。
+
+    处理顺序：
+    1. ``clean``    -- 仅做初步清洗（utf-8 / markdown / emoji / 空白规整），
+                        不做任何 TN。
+    2. ``split``     -- 清洗后按语种分句。
+    3. 逐句 TN      -- 对每个句子做文本归一化：
+
+        * llm_tn（``llm_normalizer``）优先，可处理全部语种；其内部通过
+          ``needs_normalization`` 判断，仅当句子确实需要 TN 时才发起 LLM 调用。
+        * wetext（``wetext_normalizer``）作为中/英回退；耗时极短，不做
+          "是否需要 TN" 的判断，直接归一化。
+    """
+
+    def __init__(
+        self,
+        fasttext_detector: Optional[Callable[[str], Optional[str]]] = None,
+        llm_normalizer: Optional[Callable[[str], str]] = None,
+        wetext_normalizer: Optional[Callable[[str], str]] = None,
+    ):
+        """初始化文本处理管线。
+
+        Args:
+            fasttext_detector: 语种判定器，接收文本返回 ``llm_tn`` 的 locale。
+            llm_normalizer: 多语种文本归一化器（``llm_tn`` 的 ``normalize``）。
+                内部自带 ``needs_normalization`` 判断，按需调用 LLM。
+            wetext_normalizer: 中/英文本归一化器（``wetext`` 的 ``Normalizer``）。
+                当 llm_normalizer 不可用时，对中/英文本直接归一化。
+        """
+        self.fasttext_detector = fasttext_detector
+        self.llm_normalizer = llm_normalizer
+        self.wetext_normalizer = wetext_normalizer
+
+    def clean(self, text: str) -> str:
+        """仅做初步清洗，不做任何 TN。"""
+        return clean_text(text)
+
+    def _tn(self, text: str, lang_tag: Optional[str] = None) -> str:
+        """对单个句子做 TN。
+
+        TN 回退链：llm_tn（全部语种） → wetext（仅中/英）→ 原文。
+
+        当提供了 ``lang_tag``（如 ``"Russian"``），wetext 分支只会对中/英语种
+        调用 wetext；非中/英时直接返回原文（只做空格清理），从而避免无
+        llm_tn 时俄��、日语等被错误地送进英文 TN。
+
+        TN 之后统一清理汉字之间、汉字与英文/数字之间的多余空格。
+        """
+        if self.llm_normalizer is not None:
+            locale = lang_tag_to_locale(lang_tag) if lang_tag else None
+            try:
+                return clean_tn_spaces(self.llm_normalizer(text, locale=locale))
+            except Exception as e:
+                print(f"[WARN] llm_tn normalization failed, fallback to raw text: {e}", flush=True)
+                return clean_tn_spaces(text)
+        # 没有 llm_tn 时，只有中/英/方言/粤语才走 wetext
+        if self.wetext_normalizer is not None:
+            if lang_tag is None:
+                # 未提供语种 -> 按文本内容自动判定
+                lang_tag = detect_language(text, self.fasttext_detector)
+            _can_wetext = (
+                lang_tag in ("Chinese", "English")
+                or lang_tag == "Cantonese"
+                or lang_tag.startswith("ZH_")
+            )
+            if _can_wetext:
+                try:
+                    return clean_tn_spaces(self.wetext_normalizer(text))
+                except Exception as e:
+                    print(f"[WARN] wetext normalization failed, fallback to raw text: {e}", flush=True)
+                    return clean_tn_spaces(text)
+        return clean_tn_spaces(text)
+
+    def split(
+        self,
+        text: str,
+        lang: str = "zh",
+        tokenize: Optional[Callable[[str], List[str]]] = None,
+        do_tn: bool = True,
+        lang_tag: Optional[str] = None,
+        **kwargs,
+    ) -> List[str]:
+        """清洗后分句，并对每个句子逐句做 TN（默认开启）。
+
+        Args:
+            text: 输入文本。
+            lang: 分句所用的语种（zh 按字符，其他按 token）。
+            tokenize: 分句长度计算用的 tokenizer（非 zh 时使用）。
+            do_tn: 是否对每个句子执行 TN。为 ``False`` 时仅分句不做归一化。
+            lang_tag: 若已知语种 tag（如 ``"Russian"``），传给 llm_tn 覆盖自动检测。
+            **kwargs: 透传给 ``split_paragraph`` 的其他参数。
+        """
+        text = self.clean(text)
+        if not text:
+            return []
+        utts = split_paragraph(text, tokenize=tokenize, lang=lang, **kwargs)
+        if not do_tn:
+            return utts
+        return [self._tn(u, lang_tag=lang_tag) for u in utts]
+
+    def detect_lang(self, text: str) -> str:
+        """判定文本语种（返回 lang tag）。"""
+        return detect_language(text, self.fasttext_detector)
+
+
+if __name__ == "__main__":
+    wetext_normalizer = build_wetext_normalizer()
+    llm_normalizer = None # build_llm_normalizer()
+    tn = TextNormalizer(
+        wetext_normalizer=wetext_normalizer,
+        llm_normalizer=llm_normalizer,
+    )
+    samples = [
+        "**你好，世界！** 今天天气很好。我们一起去公园散步吧。",
+        "バスケがしたいいです",
+        # ---- Qwen-Audio-3.0-TTS Long-text Generation 样例 ----
+        "As she passed through the crowd of squires and yeomen who already filled the lower end of the vast apartment a scrap of paper was thrust into her hand which she received almost unconsciously and continued to hold without examining its contents. The assurance that she possessed some friend in this awful assembly gave her courage to look around and to mark into whose presence she had been conducted. She gazed accordingly upon a scene which might well have struck terror into a bolder heart than hers. On an elevated seat at the upper end of the great hall directly before the accused sat the grand master of the temple in full and ample robes of flowing white holding in his hand the mystic staff which bore the symbol of the order. At his feet was placed a table occupied by two scribes whose duty it was to record the proceedings of the day. Their chairs were black and formed a marked contrast to the warlike appearance of the knights who attended the solemn gathering. The preceptors of whom there were four present occupied seats behind their superiors and behind them stood the esquires of the order robed in white",
+        "天色渐渐暗下来的时候，我把最后一件旧毛衣叠好，轻轻放进那只褪色的帆布包里。门锁轻轻一扣，楼道的灯光跟着亮了又灭，像极了这些年里头起起落落的日常。我站在台阶上深吸了口气，冷空气顺着鼻腔往里钻，带着点泥土与枯叶的腥气。其实心里头早就有了准备，只是真到了要转身的那一刻，脚步还是忍不住慢了半拍。风掠过树梢发出沙沙的响动，像是在劝我别回头。我知道这一走，有些人就再也碰不着了，可生活原本就是这样，聚散从来不讲道理，只能顺着岁月的河水流向该去的地方。 往前走的时候，脑海里不断闪过些零碎画面，那间住了好多年的小屋，窗台上总是晒着半干的衣裳，厨房的排烟罩一到做饭时分就会嗡���作响。那时候总觉得日子长得很，怎么熬也熬不到头，如今真要离开了，反倒觉得光阴像手里的细沙，攥得越紧漏得越快。我伸手摸了摸口袋里那张没带走的旧车票，纸边已经起了毛。其实送行的人并不多，大家都默契地没有多问什么，只说了句保重。我也学着他们的样子笑了笑，把那些没说出口的牵挂都咽进了肚子里。人嘛，总要学会一个人把情绪消化掉，等到夜深人静的时候再慢慢拿出来晾干。 拐过街角，远处的路灯渐渐亮起来，映在潮湿的地面上泛起一层柔和的光晕。",
+        # ---- Qwen-Audio-3.0-TTS Text Normalization 样例 ----
+        # EN
+        "Sales promo: Buy 3 items, get Chapter XII free!",
+        "The shadow puppet show begins promptly at 19:30 and ends at 21:15.",
+        "Ventilation rate must stay between 0.85 and 1.25 air changes per hour.",
+        "Grant awarded: €1.2M for cryo-EM infrastructure",
+        "The grant covers $12K for lab materials and $3.5M for equipment.",
+        "Stir gently until pH = 7.0 ± 0.2, then cap.",
+        # ZH
+        "今日体测数据：静息心率62.4次/分，体脂率18.7%，深蹲最大负重95.3公斤，跑步机耐力测试成绩12.8分钟。",
+        "置信区间估计中，μ的95%CI为12.3～15.7，σ²的90%CI是8.1—11.4，而p值范围设为0.01→0.05。",
+        "本次支付成功率对比：微信 97:3，支付宝 96:4，云闪付 95:5，三者比例差异需重点关注。",
+        "固件升级包已发布，兼容版本号v1.8.3与v2.1.0，需确认设备当前运行build 7892。",
+        "特种设备检验员需持TSG Z8002证书上岗，现场须佩戴N95口罩及PPE防护装备。",
+        "本次实验中，样品A的纯度为99.95%，样品B含杂质1.2‰，两者差值达98.75%。",
+    ]
+    for s in samples:
+        print("=" * 60)
+        print("INPUT    :", s)
+        print("LANG     :", tn.detect_lang(s))
+        cleaned = tn.clean(s)
+        print("CLEAN    :", cleaned)
+        # 分句（含逐句 TN）
+        parts_tn = tn.split(s, do_tn=True)
+        print("SPLIT+TN:")
+        for i, u in enumerate(parts_tn):
+            print(f"  [{i}] {u}")

fireredtts3/utils/text_tokenizer.py

@@ -0,0 +1,252 @@
+from typing import Dict
+from transformers import AutoTokenizer
+
+
+MULTI_LANG_TAGS = [
+    "<|Chinese|>", "<|English|>", "<|Cantonese|>",
+    "<|Japanese|>", "<|Korean|>", "<|Spanish|>",
+    "<|French|>", "<|Russian|>", "<|Arabic|>",
+    "<|Turkish|>", "<|Indonesian|>", "<|Portuguese|>",
+    "<|Italian|>", "<|Dutch|>", "<|Vietnamese|>",
+    "<|German|>", "<|Ukrainian|>", "<|Thai|>", 
+    "<|Polish|>", "<|Romanian|>", "<|Greek|>",
+    "<|Czech|>", "<|Finnish|>", "<|Hindi|>", 
+]
+
+MULTI_DIALECT_TAGS = [
+    "<|ZH_Anhui|>", "<|ZH_Fujian|>", "<|ZH_Gansu|>", 
+    "<|ZH_Guizhou|>", "<|ZH_Hebei|>", "<|ZH_Henan|>", 
+    "<|ZH_Hubei|>", "<|ZH_Hunan|>", "<|ZH_Jiangxi|>", 
+    "<|ZH_Liaoning|>", "<|ZH_Minnan|>", "<|ZH_Ningxia|>", 
+    "<|ZH_Shaanxi|>", "<|ZH_Shandong|>", "<|ZH_Shanghai|>", 
+    "<|ZH_Shanxi|>", "<|ZH_Sichuan|>", "<|ZH_Tianjin|>", 
+    "<|ZH_Wenzhou|>", "<|ZH_Wu|>", "<|ZH_Yunnan|>", 
+]
+
+
+def load_text_tokenizer(tokenizer_dir:str):
+    text_tokenizer = AutoTokenizer.from_pretrained(
+        pretrained_model_name_or_path=tokenizer_dir
+    )
+    vocab: Dict = text_tokenizer.get_vocab()
+    # Add special tokens
+    special_tokens = [
+        "<|sosp|>",
+        "<|eosp|>",
+        "<|empty|>",
+        "<|Human|>",
+        "<|SpeechLM|>",
+        "<|sostm|>",
+        "<|eostm|>",
+        "<|sot|>",
+        "<|eot|>",
+        "<|TEXT_ONLY|>",
+        "<|AUDIO_ONLY|>",
+        "<|ASR|>",
+        "<|TTS|>",
+        "<|INTERLEAVE|>",
+        "<|UNDERSTANDING|>",
+        # Placeholders
+        "<|placeholder_001|>",
+        "<|placeholder_002|>",
+        "<|placeholder_003|>",
+        "<|placeholder_004|>",
+        "<|placeholder_005|>",
+        "<|placeholder_006|>",
+        "<|placeholder_007|>",
+        "<|placeholder_008|>",
+        "<|placeholder_009|>",
+        "<|placeholder_010|>",
+        "<|placeholder_011|>",
+        "<|placeholder_012|>",
+        "<|placeholder_013|>",
+        "<|placeholder_014|>",
+        "<|placeholder_015|>",
+        "<|placeholder_016|>",
+        "<|placeholder_017|>",
+        "<|placeholder_018|>",
+        "<|placeholder_019|>",
+        "<|placeholder_020|>",
+        "<|placeholder_021|>",
+        "<|placeholder_022|>",
+        "<|placeholder_023|>",
+        "<|placeholder_024|>",
+        "<|placeholder_025|>",
+        "<|placeholder_026|>",
+        "<|placeholder_027|>",
+        "<|placeholder_028|>",
+        "<|placeholder_029|>",
+        "<|placeholder_030|>",
+        "<|placeholder_031|>",
+        "<|placeholder_032|>",
+        "<|placeholder_033|>",
+        "<|placeholder_034|>",
+        "<|placeholder_035|>",
+        "<|placeholder_036|>",
+        "<|placeholder_037|>",
+        "<|placeholder_038|>",
+        "<|placeholder_039|>",
+        "<|placeholder_040|>",
+        "<|placeholder_041|>",
+        "<|placeholder_042|>",
+        "<|placeholder_043|>",
+        "<|placeholder_044|>",
+        "<|placeholder_045|>",
+        "<|placeholder_046|>",
+        "<|placeholder_047|>",
+        "<|placeholder_048|>",
+        "<|placeholder_049|>",
+        "<|placeholder_050|>",
+        "<|placeholder_051|>",
+        "<|placeholder_052|>",
+        "<|placeholder_053|>",
+        "<|placeholder_054|>",
+        "<|placeholder_055|>",
+        "<|placeholder_056|>",
+        "<|placeholder_057|>",
+        "<|placeholder_058|>",
+        "<|placeholder_059|>",
+        "<|placeholder_060|>",
+        "<|placeholder_061|>",
+        "<|placeholder_062|>",
+        "<|placeholder_063|>",
+        "<|placeholder_064|>",
+        "<|placeholder_065|>",
+        "<|placeholder_066|>",
+        "<|placeholder_067|>",
+        "<|placeholder_068|>",
+        "<|placeholder_069|>",
+        "<|placeholder_070|>",
+        "<|placeholder_071|>",
+        "<|placeholder_072|>",
+        "<|placeholder_073|>",
+        "<|placeholder_074|>",
+        "<|placeholder_075|>",
+        "<|placeholder_076|>",
+        "<|placeholder_077|>",
+        "<|placeholder_078|>",
+        "<|placeholder_079|>",
+        "<|placeholder_080|>",
+        "<|placeholder_081|>",
+        "<|placeholder_082|>",
+        "<|placeholder_083|>",
+        "<|placeholder_084|>",
+        "<|placeholder_085|>",
+        "<|placeholder_086|>",
+        "<|placeholder_087|>",
+        "<|placeholder_088|>",
+        "<|placeholder_089|>",
+        "<|placeholder_090|>",
+        "<|placeholder_091|>",
+        "<|placeholder_092|>",
+        "<|placeholder_093|>",
+        "<|placeholder_094|>",
+        "<|placeholder_095|>",
+        "<|placeholder_096|>",
+        "<|placeholder_097|>",
+        "<|placeholder_098|>",
+        "<|placeholder_099|>",
+        "<|placeholder_100|>",
+        "<|placeholder_101|>",
+        "<|placeholder_102|>",
+        "<|placeholder_103|>",
+        "<|placeholder_104|>",
+        "<|placeholder_105|>",
+        "<|placeholder_106|>",
+        "<|placeholder_107|>",
+        "<|placeholder_108|>",
+        "<|placeholder_109|>",
+        "<|placeholder_110|>",
+        "<|placeholder_111|>",
+        "<|placeholder_112|>",
+        "<|placeholder_113|>",
+        "<|placeholder_114|>",
+        "<|placeholder_115|>",
+        "<|placeholder_116|>",
+        "<|placeholder_117|>",
+        "<|placeholder_118|>",
+        "<|placeholder_119|>",
+        "<|placeholder_120|>",
+        "<|placeholder_121|>",
+        "<|placeholder_122|>",
+        "<|placeholder_123|>",
+        "<|placeholder_124|>",
+        "<|placeholder_125|>",
+        "<|placeholder_126|>",
+        "<|placeholder_127|>",
+        "<|placeholder_128|>",
+        "<|placeholder_129|>",
+        "<|placeholder_130|>",
+        "<|placeholder_131|>",
+        "<|placeholder_132|>",
+        "<|placeholder_133|>",
+        "<|placeholder_134|>",
+        "<|placeholder_135|>",
+        "<|placeholder_136|>",
+        "<|placeholder_137|>",
+        "<|placeholder_138|>",
+        "<|placeholder_139|>",
+        "<|placeholder_140|>",
+        "<|placeholder_141|>",
+        "<|placeholder_142|>",
+        "<|placeholder_143|>",
+        "<|placeholder_144|>",
+        "<|placeholder_145|>",
+        "<|placeholder_146|>",
+        "<|placeholder_147|>",
+        "<|placeholder_148|>",
+        "<|placeholder_149|>",
+        "<|placeholder_150|>",
+        "<|placeholder_151|>",
+        "<|placeholder_152|>",
+        "<|placeholder_153|>",
+        "<|placeholder_154|>",
+        "<|placeholder_155|>",
+        "<|placeholder_156|>",
+        "<|placeholder_157|>",
+        "<|placeholder_158|>",
+        "<|placeholder_159|>",
+        "<|placeholder_160|>",
+        "<|placeholder_161|>",
+        "<|placeholder_162|>",
+        "<|placeholder_163|>",
+        "<|placeholder_164|>",
+        "<|placeholder_165|>",
+        "<|placeholder_166|>",
+        "<|placeholder_167|>",
+        "<|placeholder_168|>",
+        "<|placeholder_169|>",
+        "<|placeholder_170|>",
+        "<|placeholder_171|>",
+        "<|placeholder_172|>",
+        "<|placeholder_173|>",
+        "<|placeholder_174|>",
+        "<|placeholder_175|>",
+        "<|placeholder_176|>",
+        "<|placeholder_177|>",
+        "<|placeholder_178|>",
+        "<|placeholder_179|>",
+        "<|placeholder_180|>",
+        "<|placeholder_181|>",
+        "<|placeholder_182|>",
+        "<|placeholder_183|>",
+        "<|placeholder_184|>",
+        "<|placeholder_185|>",
+        "<|placeholder_186|>",
+        "<|placeholder_187|>",
+        "<|placeholder_188|>",
+        "<|placeholder_189|>",
+        "<|placeholder_190|>",
+        "<|placeholder_191|>",
+        "<|placeholder_192|>",
+        # Multi-lang tag
+        *MULTI_LANG_TAGS,
+        # Multi-dialect tag
+        *MULTI_DIALECT_TAGS,
+        # For SpeechEdit CoT
+        "<|edit|>", "<|frame_patch|>", "<|end_edit|>",
+    ]
+    for token in special_tokens:
+        if token not in vocab:
+            text_tokenizer.add_tokens([token], special_tokens=True)
+    return text_tokenizer

fireredtts3/utils/utils.py

@@ -0,0 +1,9 @@
+import random
+import torch
+
+
+def fix_seed(seed: int):
+    random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)

requirements.txt

@@ -0,0 +1,11 @@
+torch==2.8.0
+torchaudio==2.8.0
+https://huggingface.co/datasets/multimodalart/zerogpu-blackwell-wheels/resolve/main/wheels/pt28-cu128-cp312/flash_attn-2.8.3-cp312-cp312-linux_x86_64.whl
+transformers==5.6.2
+einops==0.8.2
+python-dotenv
+regex
+wetext
+fasttext
+numpy
+soundfile