Dynamic Accent Flash: add Flash-DPO (CosyVoice3) as --version flash (default), diacritic front-end, Flash results, samples
e4ae2bb verified | #!/usr/bin/env python | |
| """Dynamic Accent: local Gradio demo (single GPU). | |
| pip install -r requirements.txt # includes gradio | |
| python app.py --cosyvoice_dir ../CosyVoice # then open http://127.0.0.1:7860 | |
| Text box -> optional auto-tag (editable preview of the tagged text) -> version + voice + rate -> Synthesize, or | |
| Compare to hear Phase 3 and Flash on the same text, seed and voice. Loaded versions stay resident (Phase 2/3 share one | |
| CosyVoice2 base; Flash has its own CosyVoice3 base), so a GPU with >= 24 GB is recommended. Flash needs CUDA. | |
| """ | |
| import argparse, json, os, sys, types | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| sys.path.insert(0, HERE) | |
| import infer # noqa: E402 | |
| from tagger import Tagger, parse_map, strip_tags, TAGGED, LANGS # noqa: E402 | |
| EXAMPLES = [ | |
| "Omo, I'm very excited to announce this new model, ọ na-eme m obi ụtọ, I no go lie give you", | |
| "Ẹ káàbọ̀ sí ìpàdé wa òní. Today we will talk about the new school calendar.", | |
| "Barka da yamma, yaya kuke? The weather report says it will rain tomorrow morning.", | |
| "Kedu ka ị mere? Abeg no vex, I go call you for evening.", | |
| ] | |
| class Engine: | |
| def __init__(self, a): | |
| self.a = a | |
| self.models = {} | |
| self.cosy = self.tok = None | |
| self.taggers = {} | |
| def get(self, version, voice): | |
| if version not in self.models: | |
| ns = types.SimpleNamespace(**vars(self.a)) | |
| ns.voice = voice | |
| S = infer.build(version, ns, self.cosy, self.tok, copy_llm=self.cosy is not None) | |
| if S.cosy is not None: # flash (CosyVoice3) keeps its own base | |
| self.cosy, self.tok = S.cosy, S.tok | |
| S._rate0 = S.rate | |
| self.models[version] = S | |
| S = self.models[version] | |
| if getattr(S, "_voice_name", None) != voice: | |
| S.set_voice(voice, self.a.voices); S._voice_name = voice | |
| return S | |
| def tag(self, text, lang_map, use_lid): | |
| key = (lang_map, use_lid) | |
| if key not in self.taggers: | |
| self.taggers[key] = Tagger("glotlid" if use_lid else None, parse_map(lang_map)) | |
| return self.taggers[key].tag(strip_tags(text)) | |
| def say(self, text, version, voice, rate, seed): | |
| S = self.get(version, voice) | |
| S.rate = S._rate0 if rate == "default" else (infer.RATES.index(rate) if rate in infer.RATES else 0) | |
| w = S.say(text, seed=int(seed)) | |
| return S.sr, w | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--cosyvoice_dir", default=os.environ.get("COSYVOICE_REPO", os.path.join(HERE, "CosyVoice"))) | |
| ap.add_argument("--model_dir", default=os.environ.get("COSYVOICE2_DIR", os.path.join(HERE, "pretrained", "CosyVoice2-0.5B"))) | |
| ap.add_argument("--host", default="127.0.0.1") | |
| ap.add_argument("--port", type=int, default=7860) | |
| ap.add_argument("--share", action="store_true") | |
| a = ap.parse_args() | |
| for k, v in dict(lm="", flow="", prompts=os.path.join(HERE, "prompts", "prompts.json"), | |
| voices=os.path.join(HERE, "prompts", "voices.json"), dtype="auto", stitch=False, rate="default", | |
| frontend="codeswitch", frontend_lang="eng", no_diacritics=False, experimental_translate=False, | |
| cv3_dir=os.environ.get("CV3_DIR", os.path.join(HERE, "pretrained", "Fun-CosyVoice3-0.5B-2512"))).items(): | |
| setattr(a, k, v) | |
| if not os.path.isdir(a.model_dir): | |
| from huggingface_hub import snapshot_download | |
| snapshot_download("FunAudioLLM/CosyVoice2-0.5B", local_dir=a.model_dir) | |
| import gradio as gr | |
| E = Engine(a) | |
| S = infer.SETTINGS | |
| voices = S.get("voices", ["anchor_male", "waxal_ibo_5_female", "waxal_yor_5_female"]) | |
| rec = S.get("recommended", {}) | |
| rates = ["default"] + S.get("rates", ["none", "very slow", "slow", "normal", "fast", "very fast"]) | |
| def prepare(text, auto, lang_map, use_lid, force): | |
| if not text.strip(): | |
| return "" | |
| if force and force != "off": | |
| return f"<{force}>{strip_tags(text)}</{force}>" | |
| if auto and not TAGGED.search(text): | |
| return E.tag(text, lang_map or "", bool(use_lid)) | |
| return text | |
| def hint(version, voice): | |
| r = rec.get(version, []) | |
| return "" if voice in r else f"Tip: {version} is recommended with {', '.join(r)}. {rec.get('note', '')}" | |
| def synth(text, tagged, auto, lang_map, use_lid, force, version, voice, rate, seed): | |
| t = tagged.strip() if tagged.strip() else prepare(text, auto, lang_map, use_lid, force) | |
| sr, w = E.say(t, version, voice, rate, seed) | |
| return t, (sr, w), hint(version, voice) | |
| def compare(text, tagged, auto, lang_map, use_lid, force, voice, rate, seed): | |
| t = tagged.strip() if tagged.strip() else prepare(text, auto, lang_map, use_lid, force) | |
| a3 = E.say(t, "phase3", voice, rate, seed) | |
| af = E.say(t, "flash", voice, rate, seed) | |
| return t, a3, af | |
| with gr.Blocks(title="Dynamic Accent: code-switching TTS") as demo: | |
| gr.Markdown("## Dynamic Accent: code-switching TTS\nEnglish, Nigerian Pidgin, Igbo, Yoruba and Hausa, mixed in one " | |
| "sentence. Tag spans with `<eng> <pcm> <ibo> <yor> <hau>`, or turn on auto-tag and edit its preview.") | |
| text = gr.Textbox(label="Text", lines=3, value=EXAMPLES[0]) | |
| with gr.Row(): | |
| auto = gr.Checkbox(label="Auto-tag languages", value=True) | |
| use_lid = gr.Checkbox(label="Use GlotLID (1.7 GB download, needs fasttext)", value=False) | |
| force = gr.Dropdown(["off"] + LANGS, value="off", label="Force one language") | |
| lang_map = gr.Textbox(label="Word overrides (e.g. omo=pcm,sha=pcm)", value="") | |
| tag_btn = gr.Button("Preview tags") | |
| tagged = gr.Textbox(label="Tagged text (edit before synthesis; leave empty to tag automatically)", lines=3) | |
| with gr.Row(): | |
| version = gr.Dropdown(sorted(infer.VERSIONS), value=S.get("default_version", "flash"), label="Version") | |
| voice = gr.Dropdown(voices, value=S.get("default_voice", "anchor_male"), label="Voice") | |
| rate = gr.Dropdown(rates, value="default", label="Rate (phase3/flash)") | |
| seed = gr.Number(value=0, precision=0, label="Seed") | |
| tip = gr.Markdown() | |
| with gr.Row(): | |
| go = gr.Button("Synthesize", variant="primary") | |
| cmp_btn = gr.Button("Compare Phase 3 vs Flash") | |
| out = gr.Audio(label="Output", type="numpy") | |
| with gr.Row(): | |
| out2 = gr.Audio(label="Phase 3", type="numpy") | |
| out3 = gr.Audio(label="Flash", type="numpy") | |
| gr.Examples(EXAMPLES, inputs=text) | |
| tag_btn.click(prepare, [text, auto, lang_map, use_lid, force], tagged) | |
| version.change(hint, [version, voice], tip) | |
| voice.change(hint, [version, voice], tip) | |
| go.click(synth, [text, tagged, auto, lang_map, use_lid, force, version, voice, rate, seed], [tagged, out, tip]) | |
| cmp_btn.click(compare, [text, tagged, auto, lang_map, use_lid, force, voice, rate, seed], [tagged, out2, out3]) | |
| demo.queue(max_size=8).launch(server_name=a.host, server_port=a.port, share=a.share) | |
| if __name__ == "__main__": | |
| main() | |