Add application file

app.py

@@ -0,0 +1,519 @@
+import os
+import traceback
+
+import gradio as gr
+import numpy as np
+import pyrootutils
+import torch
+from loguru import logger
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams, TokensPrompt
+from funasr_onnx import Paraformer
+from huggingface_hub import snapshot_download
+
+from tools.wer import compute_wers
+
+os.environ["EINX_FILTER_TRACEBACK"] = "false"
+os.environ["VLLM_USE_V1"] = "0"
+
+pyrootutils.setup_root(__file__, indicator=".project-root", pythonpath=True)
+from i18n import i18n
+from text.chn_text_norm.text import Text as ChnNormedText
+from xcodec2.modeling_xcodec2 import XCodec2Model
+
+
+TEXTBOX_PLACEHOLDER = i18n("Put your text here.")
+
+# ===== Hugging Face Model IDs =====
+LLASA_MODEL_ID = "ASLP-lab/VoiceSculptor"
+LLASA_SUBFOLDER = "LLaSA-Instruct-3B"
+XCODEC_MODEL_ID = "HKUSTAudio/xcodec2"
+PARAFORMER_REPO_ID = "funasr/Paraformer-large"
+
+# logo
+LOGO_URL = "https://raw.githubusercontent.com/ASLP-lab/VoiceSculptor/main/assets/logo.png"
+
+
+def normalize_text_final(user_input: str) -> str:
+    return ChnNormedText(raw_text=user_input).normalize()
+
+
+def extract_speech_ids(speech_tokens_str):
+    speech_ids = []
+    for token_str in speech_tokens_str:
+        if token_str.startswith("<|s_") and token_str.endswith("|>"):
+            num_str = token_str[4:-2]
+            speech_ids.append(int(num_str))
+        else:
+            logger.warning(f"Unexpected token: {token_str}")
+    return speech_ids
+
+
+def get_asr(asr_model: Paraformer, wav_list: list[np.ndarray]) -> list[str]:
+    """wav_list: list of 1D numpy waveform (16k)"""
+    try:
+        result = asr_model(wav_list)
+        if isinstance(result, dict):
+            result = [result]
+
+        texts = []
+        for res in result:
+            preds = res.get("preds", None)
+            if preds is None:
+                texts.append(res.get("text", ""))
+            else:
+                texts.append(preds[0] if len(preds) > 0 else "")
+
+        # 容错：batch 返回数量不一致 -> fallback
+        if len(texts) != len(wav_list):
+            logger.warning(f"[ASR] batch返回数量不一致: got {len(texts)} expect {len(wav_list)}，fallback逐条补齐")
+            texts = []
+            for w in wav_list:
+                try:
+                    r = asr_model(w)
+                    if isinstance(r, list) and len(r) > 0:
+                        r0 = r[0]
+                        preds = r0.get("preds", None)
+                        texts.append(preds[0] if preds else r0.get("text", ""))
+                    elif isinstance(r, dict):
+                        preds = r.get("preds", None)
+                        texts.append(preds[0] if preds else r.get("text", ""))
+                    else:
+                        texts.append("")
+                except Exception:
+                    texts.append("")
+        return texts
+
+    except Exception as e:
+        logger.warning(f"[ASR] batch失败，fallback逐条: {e}")
+        texts = []
+        for w in wav_list:
+            try:
+                r = asr_model(w)
+                if isinstance(r, list) and len(r) > 0:
+                    r0 = r[0]
+                    preds = r0.get("preds", None)
+                    texts.append(preds[0] if preds else r0.get("text", ""))
+                elif isinstance(r, dict):
+                    preds = r.get("preds", None)
+                    texts.append(preds[0] if preds else r.get("text", ""))
+                else:
+                    texts.append("")
+            except Exception:
+                texts.append("")
+        return texts
+
+
+def inference_batch(
+    model: LLM,
+    codec_model: XCodec2Model,
+    device: str,
+    tokenizer: AutoTokenizer,
+    refined_text: str,
+    instruct_text: str,
+    control_tags: str,
+    batch_size: int = 5,
+) -> list[tuple[int, np.ndarray]]:
+    refined_text_norm = normalize_text_final(refined_text)
+    instruct_text_norm = normalize_text_final(instruct_text)
+
+    if len(refined_text_norm) < 5:
+        raise ValueError("输入文本长度不能少于5个字符")
+    if len(refined_text_norm) > 150:
+        raise ValueError("输入文本长度不能超过150个字符")
+
+    target_text = instruct_text_norm + "<|endofprompt|>" + control_tags + refined_text_norm
+    formatted_text = f"<|TEXT_UNDERSTANDING_START|>{target_text}<|TEXT_UNDERSTANDING_END|>"
+    chat = [
+        {"role": "user", "content": "Convert the text to speech:" + formatted_text},
+        {"role": "assistant", "content": "<|SPEECH_GENERATION_START|>"},
+    ]
+
+    with torch.no_grad():
+        input_ids = tokenizer.apply_chat_template(
+            chat,
+            tokenize=True,
+            return_tensors="pt",
+            continue_final_message=True,
+        ).to(device)
+
+        speech_end_id = tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_END|>")
+        prompt_ids = input_ids.squeeze(0).tolist()
+        prompts = [TokensPrompt(prompt_token_ids=prompt_ids) for _ in range(batch_size)]
+
+        base_seed = int.from_bytes(os.urandom(4), "little")
+
+        try:
+            sampling_params_list = [
+                SamplingParams(
+                    temperature=0.9,
+                    top_p=0.95,
+                    top_k=15,
+                    max_tokens=2048,
+                    repetition_penalty=1.05,
+                    stop_token_ids=[speech_end_id],
+                    seed=base_seed + i,
+                )
+                for i in range(batch_size)
+            ]
+            outputs = model.generate(prompts=prompts, sampling_params=sampling_params_list)
+        except TypeError:
+            logger.warning("[vLLM] 当前版本不支持 SamplingParams(seed=...)，将不带 seed 生成")
+            sampling_params = SamplingParams(
+                temperature=0.9,
+                top_p=0.95,
+                top_k=15,
+                max_tokens=2048,
+                repetition_penalty=1.05,
+                stop_token_ids=[speech_end_id],
+            )
+            outputs = model.generate(prompts=prompts, sampling_params=sampling_params)
+
+        audios: list[tuple[int, np.ndarray]] = []
+        for out in outputs:
+            token_ids = out.outputs[0].token_ids
+            if len(token_ids) > 0 and token_ids[-1] == speech_end_id:
+                token_ids = token_ids[:-1]
+
+            speech_tokens = tokenizer.batch_decode(token_ids, skip_special_tokens=True)
+            speech_tokens = extract_speech_ids(speech_tokens)
+
+            speech_tokens_t = torch.tensor(speech_tokens, device=device).unsqueeze(0).unsqueeze(0)
+            wav = codec_model.decode_code(speech_tokens_t)
+            wav = wav.squeeze(0).squeeze(0).detach().cpu().numpy().astype(np.float32)
+            audios.append((16000, wav))
+
+        return audios
+
+
+def build_app():
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    logger.info(f"✅ Loading models on device={device}")
+
+    # ===== LLaSA =====
+    tokenizer = AutoTokenizer.from_pretrained(LLASA_MODEL_ID, subfolder=LLASA_SUBFOLDER, trust_remote_code=True)
+
+    model = LLM(
+        model=LLASA_MODEL_ID,
+        gpu_memory_utilization=0.90,
+        max_model_len=2048,
+        enable_prefix_caching=True,
+        dtype="auto",
+        quantization=None,
+        enforce_eager=False,
+        kv_cache_dtype="auto",
+        trust_remote_code=True,
+        hf_model_subfolder=LLASA_SUBFOLDER,
+    )
+
+    # ===== XCodec2 =====
+    codec_model = XCodec2Model.from_pretrained(XCODEC_MODEL_ID).eval().to(device)
+
+    # ===== Paraformer =====
+    paraformer_dir = snapshot_download(
+        repo_id=PARAFORMER_REPO_ID,
+        local_dir="checkpoints/Paraformer-large",
+        local_dir_use_symlinks=False,
+    )
+    asr_model = Paraformer(paraformer_dir, batch_size=5, quantize=True)
+
+    logger.info("✅ Models loaded: VoiceSculptor + xcodec2 + Paraformer")
+
+    INSTRUCT_TEMPLATES = {
+        "自定义": "",
+        "default": "这是一位男性评书表演者，用传统说唱腔调，以变速节奏和韵律感极强的语速讲述江湖故事，音量时高时低，充满江湖气。",
+        "幼儿园女教师-温柔甜美": "这是一位幼儿园女教师，用甜美明亮的嗓音，以极慢且富有耐心的语速，带着温柔鼓励的情感，用标准普通话给小朋友讲睡前故事，音量轻柔适中，咬字格外清晰。",
+        "电台主播-平静温柔": "深夜电台主播，男性、音调偏低、语速偏慢、音量小；情绪平静带点忧伤，语气温柔；音色微哑",
+        "成熟御姐-冷静坚定": "成熟御姐风格，音调偏低、语速正常、音量中等；情绪冷静，语气不容置疑的坚定；音色偏磁性，吐字清晰",
+        "年轻妈妈-温暖安抚": "年轻妈妈哄孩子入睡，女性、音调柔和偏低、语速偏慢、音量偏小但清晰；情绪温暖安抚、充满耐心与爱意，语气轻柔哄劝、像贴近耳边低声说话；音色软糯，吐字清晰、节奏舒缓。",
+        "小女孩-尖锐清脆": "一位7岁的小女孩，用天真高亢的童声，以不稳定的快节奏，充满兴奋和炫耀地背诵乘法口诀，音调忽高忽低，带着儿童特有的尖锐清脆。",
+        "老奶奶-沙哑低沉": "一位慈祥的老奶奶，用沙哑低沉的嗓音，以极慢而温暖的语速讲述民间传说，音量微弱但清晰，带着怀旧和神秘的情感。",
+        "诗歌朗诵-雄浑有力": "一位男性现代诗朗诵者，用深沉磁性的低音，以顿挫有力的节奏演绎艾青诗歌，音量洪亮，情感激昂澎湃。",
+        "童话风格-甜美夸张": "这是一位女性童话旁白朗诵者，用甜美夸张的童声，以跳跃变化的语速讲述《安徒生童话》，音调偏高，充满奇幻色彩。",
+        "评书风格-抑扬顿挫": "这是一位男性评书表演者，用传统说唱腔调，以变速节奏和韵律感极强的语速讲述江湖故事，音量时高时低，充满江湖气。",
+        "新闻风格-平静专业": "这是一位女性新闻主播，用标准普通话以清晰明亮的中高音，以平稳专业的语速播报时事新闻，音量洪亮，情感客观中立。",
+        "相声风格-夸张幽默": "这是一位男性相声表演者，用夸张幽默的嗓音，以时快时慢的节奏抖包袱，音调起伏大，充满喜感和节奏感。",
+        "游戏直播-亢奋激昂": "这是一位男性游戏解说，用亢奋激昂的嗓音，以极快且情绪化的语速直播电竞比赛，音量突然爆发，充满悬念和热血。",
+        "悬疑小说-低沉神秘": "一位男性悬疑小说演播者，用低沉神秘的嗓音，以时快时慢的变速节奏营造紧张氛围，音量忽高忽低，充满悬念感。",
+        "戏剧表演-夸张戏剧": "这是一位男性戏剧表演者，用夸张戏剧化的嗓音，以忽高忽低的音调和时快时慢的语速表演独白，充满张力。",
+        "法治节目-庄严庄重": "这是一位男性法治节目主持人，用严肃庄重的嗓音，以平稳有力的语速讲述案件，音量适中，体现法律的威严。",
+        "纪录片旁白-低沉磁性": "这是一位男性纪录片旁白，用深沉磁性的嗓音，以缓慢而富有画面感的语速讲述自然奇观，音量适中，充满敬畏和诗意。",
+        "广告配音-沧桑浑厚": "这是一位男性白酒品牌广告配音，用沧桑浑厚的嗓音，以缓慢而豪迈的语速，音量洪亮，传递历史底蕴和男人情怀。",
+        "冥想引导师-空灵悠长": "一位女性冥想引导师，用空灵悠长的气声，以极慢而飘渺的语速，配合环境音效，音量轻柔，营造禅意空间。",
+        "ASMR-气声耳语": "一位女性ASMR主播，用气声耳语，以极慢而细腻的语速，配合唇舌音，音量极轻，营造极度放松的氛围。",
+    }
+
+    TEXT_REQUIREMENTS = {
+        "自定义": "",
+        "default": "话说那武松，提着哨棒，直奔景阳冈。天色将晚，酒劲上头，只听一阵狂风，老虎来啦！",
+        "幼儿园女教师-温柔甜美": "月亮婆婆升上天空啦，星星宝宝都困啦。小白兔躺在床上，盖好小被子，闭上眼睛。兔妈妈轻轻地唱着摇篮曲：睡吧睡吧，我亲爱的宝贝。",
+        "电台主播-平静温柔": "大家好，欢迎收听你的月亮我的心，好男人就是我，我就是：曾小贤。",
+        "成熟御姐-冷静坚定": "别担心，我不会让你输，把那些乱七八糟的念头先收起来，姐姐带你赢。",
+        "年轻妈妈-温暖安抚": "从前有座山，山里有座庙，庙里面有个小和尚，小和尚在给老和尚讲故事，他说：从前有座山，山里有座庙，庙里面有个小和尚。",
+        "小女孩-尖锐清脆": "一一得一！一二得二！一三得三！我会背乘法口诀啦！老师今天表扬我啦！妈妈说我最棒！",
+        "老奶奶-沙哑低沉": "很久很久以前，在山的那边，住着一只会说话的狐狸。它常常在月圆之夜，变成美丽的姑娘，来到村子里。",
+        "诗歌朗诵-雄浑有力": "为什么我的眼里常含泪水？因为我对这土地爱得深沉。这土地，这河流，这吹刮着的暴风。",
+        "童话风格-甜美夸张": "在一个很冷很冷的夜晚，小女孩擦亮了一根火柴。突然，温暖的火炉出现了！她觉得自己好像坐在火炉旁。",
+        "评书风格-抑扬顿挫": "话说那武松，提着哨棒，直奔景阳冈。天色将晚，酒劲上头，只听一阵狂风，老虎来啦！",
+        "新闻风格-平静专业": "本台讯，今日凌晨，我国成功发射新一代载人飞船试验船。此次任务验证了多项关键技术，为后续空间站建设奠定基础。",
+        "相声风格-夸张幽默": "我这个人啊，最大的优点就是太谦虚。谦虚到什么程度？连谦虚本身都觉得我太谦虚了！",
+        "游戏直播-亢奋激昂": "大招！大招好了！开团了！ACE！团灭！这波操作神了！冠军相尽显无疑！",
+        "悬疑小说-低沉神秘": "深夜，他独自走在空无一人的小巷。脚步声，回声，还有……另一个人的呼吸声。他猛地回头——什么也没有。",
+        "戏剧表演-夸张戏剧": "我疯了！彻底疯了！你们都说我疯了！可疯的是这个世界！清醒的人反而被当成疯子！",
+        "法治节目-庄严庄重": "天网恢恢，疏而不漏。任何触犯法律的行为，终将受到公正的审判。正义或许会迟到，但绝不会缺席。",
+        "纪录片旁白-低沉磁性": "在这片广袤的非洲草原上，生命与死亡每天都在上演。猎豹的速度，羚羊的敏捷，都是生存的代价。",
+        "广告配音-沧桑浑厚": "一杯敬过往，一杯敬远方。传承千年的酿造工艺，只在每一滴醇香。老朋友，值得好酒。",
+        "冥想引导师-空灵悠长": "想象你是一片叶子，随风飘落。没有牵挂，没有重量。只有呼吸，只有当下，只有宁静。",
+        "ASMR-气声耳语": "现在，让我在你耳边轻声细语。听到我的声音了吗？放松你的头皮，感受每一个毛孔都在呼吸。",
+    }
+
+    def build_control_tags(age, gender, pitch, pitch_var, volume, speed, emo):
+        tag_map = {
+            "小孩": "<|小孩|>", "青年": "<|青年|>", "中年": "<|中年|>", "老年": "<|老年|>",
+            "男性": "<|男性|>", "女性": "<|女性|>",
+            "音调很高": "<|音调很高|>", "音调较高": "<|音调较高|>", "音调中等": "<|音调中等|>",
+            "音调较低": "<|音调较低|>", "音调很低": "<|音调很低|>",
+            "音调变化很强": "<|音调变化很强|>", "音调变化较强": "<|音调变化较强|>", "音调变化一般": "<|音调变化一般|>",
+            "音调变化较弱": "<|音调变化较弱|>", "音调变化很弱": "<|音调变化很弱|>",
+            "音量很大": "<|音量很大|>", "音量较大": "<|音量较大|>", "音量中等": "<|音量中等|>",
+            "音量较小": "<|音量较小|>", "音量很小": "<|音量很小|>",
+            "语速很快": "<|语速很快|>", "语速较快": "<|语速较快|>", "语速中等": "<|语速中等|>",
+            "语速较慢": "<|语速较慢|>", "语速很慢": "<|语速很慢|>",
+            "开心": "<|开心|>", "生气": "<|生气|>", "难过": "<|难过|>", "惊讶": "<|惊讶|>", "厌恶": "<|厌恶|>", "害怕": "<|害怕|>",
+        }
+        tags = []
+        for v in [gender, age, speed, volume, pitch, pitch_var, emo]:
+            if v != "不指定":
+                tags.append(tag_map[v])
+        return "".join(tags)
+
+    def inference_select_best3(refined_text, instruct_text, age, gender, pitch, pitch_var, volume, speed, emo):
+        control_tags = build_control_tags(age, gender, pitch, pitch_var, volume, speed, emo)
+        try:
+            audios5 = inference_batch(
+                model=model,
+                codec_model=codec_model,
+                device=device,
+                tokenizer=tokenizer,
+                refined_text=refined_text,
+                instruct_text=instruct_text,
+                control_tags=control_tags,
+                batch_size=5,
+            )
+            wav_list = [wav for (_, wav) in audios5]
+            asr_texts = get_asr(asr_model, wav_list)
+
+            refined_text_norm = normalize_text_final(refined_text)
+            gt_texts = [refined_text_norm] * len(asr_texts)
+            wers = compute_wers(gt_texts, asr_texts, lang="zh")
+
+            for i, (hyp, w) in enumerate(zip(asr_texts, wers)):
+                logger.info(f"[ASR/WER] idx={i} wer={w:.4f}  gt='{refined_text_norm}'  asr='{hyp}'")
+
+            best_idx = np.argsort(np.array(wers))[:3].tolist()
+            logger.info(f"[ASR/WER] best_idx={best_idx} best_wers={[float(wers[i]) for i in best_idx]}")
+            best3 = [audios5[i] for i in best_idx]
+            return best3[0], best3[1], best3[2]
+        except Exception as e:
+            logger.error(f"推理/ASR/WER 失败: {e}", exc_info=True)
+            logger.error("错误详细信息:\n" + traceback.format_exc())
+            return None, None, None
+
+    THEME = gr.themes.Soft(
+                primary_hue="orange",
+                secondary_hue="cyan",
+                neutral_hue="slate",
+                )
+
+    CUSTOM_CSS = """
+    /* layout */
+    #vs-root {max-width: 1180px; margin: 0 auto;}
+    #vs-header {padding: 14px 14px 4px 14px;}
+    #vs-card {border-radius: 14px; padding: 14px; border: 1px solid rgba(0,0,0,0.08);}
+
+    /* ===== VoiceSculptor palette (from logo) ===== */
+    :root, .gradio-container {
+    --vs-orange: #FF6A00;
+    --vs-orange2:#FFB000;
+    --vs-teal:   #00A6C6;
+    --vs-blue:   #0B2E8A;
+    --vs-teal-a: rgba(0,166,198,.18);
+    }
+
+    /* primary button */
+    .gr-button-primary, button.primary {
+    background: linear-gradient(90deg, var(--vs-orange), var(--vs-orange2)) !important;
+    border: none !important;
+    color: white !important;
+    }
+    .gr-button-primary:hover, button.primary:hover {
+    filter: brightness(1.03);
+    }
+    .gr-button-primary:active, button.primary:active {
+    filter: brightness(0.98);
+    }
+
+    /* links */
+    .gradio-container a {
+    color: var(--vs-teal) !important;
+    }
+    .gradio-container a:hover {
+    text-decoration: underline;
+    }
+
+    /* focus ring / active border for inputs */
+    textarea:focus, input:focus {
+    border-color: var(--vs-teal) !important;
+    box-shadow: 0 0 0 3px var(--vs-teal-a) !important;
+    outline: none !important;
+    }
+    /* some gradio versions wrap inputs in these */
+    .gr-input:focus-within, .gr-text-input:focus-within, .gr-box:focus-within {
+    border-color: var(--vs-teal) !important;
+    box-shadow: 0 0 0 3px var(--vs-teal-a) !important;
+    }
+
+    /* accordion highlight */
+    .gr-accordion .label, .gr-accordion summary {
+    color: var(--vs-blue) !important;
+    }
+    """
+
+    DEFAULT_STYLE = "评书风格-抑扬顿挫"
+    template_choices = [k for k in INSTRUCT_TEMPLATES.keys() if k not in ("default",)]
+
+    BEST_PRACTICE_MD = """
+## Best Practice Guide（音色设计）
+
+完整指南请见：Voice Design README  
+https://github.com/ASLP-lab/VoiceSculptor/blob/main/docs/voice_design.md  
+
+### 关键约束
+- **voice_prompt ≤ 200 字**
+- **当前仅支持中文**
+- **待合成文本长度 ≥ 5 个字**
+
+### 写法建议
+- **具体**：用可感知特质词（低沉/清脆/沙哑/明亮、语速快慢、音量大小等），避免“好听/不错”。  
+- **完整**：建议覆盖 **3–4 个维度**（人设/场景 + 性别/年龄 + 音调/语速 + 音质/情绪）。  
+- **客观**：描述声音特征与表达方式，避免“我喜欢/很棒”。  
+- **不做模仿**：禁止“像某明星/某演员”，只描述声音特质本身。  
+- **尽量精炼**：每个词都承载信息，避免重复强调（如“非常非常”）。  
+
+### 参考模板
+> - 这是一位男性评书表演者，用传统说唱腔调，以变速节奏和韵律感极强的语速讲述江湖故事，音量时高时低，充满江湖气。  
+> - 深夜电台主播，男性、音调偏低、语速偏慢、音量小；情绪平静带点忧伤，语气温柔；音色微哑。
+> - 成熟御姐风格，音调偏低、语速正常、音量中等；情绪冷静，语气不容置疑的坚定；音色偏磁性，吐字清晰。
+
+
+### 细粒度控制提示
+- 细粒度控制（年龄/性别/音调/语速/音量/情感等）**建议与指令描述保持一致**，尽量避免相互矛盾（如指令写“低沉慢速”，细粒度却选“音调很高/语速很快”）。
+"""
+
+    with gr.Blocks(theme=THEME, css=CUSTOM_CSS) as app:
+        with gr.Column(elem_id="vs-root"):
+            with gr.Row(elem_id="vs-header"):
+                gr.HTML(f"""
+                <div style="display:flex; align-items:center; gap:16px;">
+                  <img src="{LOGO_URL}"
+                       alt="Voice Sculptor Logo"
+                       style="width:360px; max-height:130px; object-fit:contain; display:block;" />
+                  <div>
+                    <div style="font-size:32px; font-weight:700; line-height:1;">Voice Sculptor</div>
+                    <div style="opacity:.85; margin-top:6px;">
+                      {i18n('An instruct text-to-speech solution based on LLaSA and CosyVoice2 developed by the ASLP lab and collaborators.')}
+                    </div>
+                  </div>
+                </div>
+                """)
+
+            with gr.Row():
+                # Left: Controls + Guide
+                with gr.Column(scale=5, elem_id="vs-card"):
+                    gr.Markdown("### 🪄 Voice Design（捏音色）")
+
+                    with gr.Accordion("🎭 风格与文本", open=True):
+                        instruct_template = gr.Dropdown(
+                            choices=template_choices,
+                            value=DEFAULT_STYLE,
+                            label=i18n("指令风格（必选）"),
+                            interactive=True,
+                        )
+
+                        instruct_text = gr.Textbox(
+                            label=i18n("指令文本"),
+                            placeholder=TEXTBOX_PLACEHOLDER,
+                            lines=4,
+                            value=INSTRUCT_TEMPLATES.get(DEFAULT_STYLE, INSTRUCT_TEMPLATES["default"]),
+                        )
+
+                        text = gr.Textbox(
+                            label=i18n("待合成文本"),
+                            placeholder=TEXTBOX_PLACEHOLDER,
+                            lines=4,
+                            value=TEXT_REQUIREMENTS.get(DEFAULT_STYLE, TEXT_REQUIREMENTS["default"]),
+                        )
+
+                    with gr.Accordion("🎛️ 细粒度声音控制（可选）", open=False):
+                        with gr.Row():
+                            age_ctrl = gr.Dropdown(label="年龄", choices=["不指定", "小孩", "青年", "中年", "老年"], value="不指定")
+                            gender_ctrl = gr.Dropdown(label="性别", choices=["不指定", "男性", "女性"], value="不指定")
+
+                        with gr.Row():
+                            pitch_ctrl = gr.Dropdown(
+                                label="音调高度",
+                                choices=["不指定", "音调很高", "音调较高", "音调中等", "音调较低", "音调很低"],
+                                value="不指定",
+                            )
+                            pitch_var_ctrl = gr.Dropdown(
+                                label="音调变化",
+                                choices=["不指定", "音调变化很强", "音调变化较强", "音调变化一般", "音调变化较弱", "音调变化很弱"],
+                                value="不指定",
+                            )
+
+                        with gr.Row():
+                            volume_ctrl = gr.Dropdown(
+                                label="音量",
+                                choices=["不指定", "音量很大", "音量较大", "音量中等", "音量较小", "音量很小"],
+                                value="不指定",
+                            )
+                            speed_ctrl = gr.Dropdown(
+                                label="语速",
+                                choices=["不指定", "语速很快", "语速较快", "语速中等", "语速较慢", "语速很慢"],
+                                value="不指定",
+                            )
+
+                        emo_ctrl = gr.Dropdown(
+                            label="情感",
+                            choices=["不指定", "开心", "生气", "难过", "惊讶", "厌恶", "害怕"],
+                            value="不指定",
+                        )
+
+                    with gr.Accordion("📚 Best Practice Guide", open=False):
+                        gr.Markdown(BEST_PRACTICE_MD)
+
+                    def apply_template(tpl_name):
+                        return INSTRUCT_TEMPLATES.get(tpl_name, ""), TEXT_REQUIREMENTS.get(tpl_name, "")
+
+                    instruct_template.change(apply_template, inputs=[instruct_template], outputs=[instruct_text, text])
+
+                # Right: Results + Generate
+                with gr.Column(scale=5, elem_id="vs-card"):
+                    gr.Markdown("### 🎵 Results")
+                    generate = gr.Button("🎧 Generate", variant="primary")
+                    audio_output1 = gr.Audio(label=i18n("Generated Audio 1"), type="numpy", interactive=False)
+                    audio_output2 = gr.Audio(label=i18n("Generated Audio 2"), type="numpy", interactive=False)
+                    audio_output3 = gr.Audio(label=i18n("Generated Audio 3"), type="numpy", interactive=False)
+
+        generate.click(
+            fn=inference_select_best3,
+            inputs=[text, instruct_text, age_ctrl, gender_ctrl, pitch_ctrl, pitch_var_ctrl, volume_ctrl, speed_ctrl, emo_ctrl],
+            outputs=[audio_output1, audio_output2, audio_output3],
+        )
+
+    return app
+
+
+if __name__ == "__main__":
+    demo = build_app()
+    demo.launch()

requirements.txt

@@ -0,0 +1,14 @@
+gradio
+torch
+transformers
+vllm
+funasr-onnx
+huggingface_hub
+jiwer
+zhon
+loguru
+pyrootutils
+jieba
+torchtune
+torchao
+vector_quantize_pytorch

tools/wer.py

@@ -0,0 +1,59 @@
+# tools/wer.py
+from __future__ import annotations
+
+from typing import List, Tuple
+import string
+
+from jiwer import process_words
+from zhon.hanzi import punctuation as zh_punctuation
+
+# 中文标点 + 英文标点 + '-'
+_PUNCTUATION_ALL = zh_punctuation + string.punctuation + "-"
+
+
+def _normalize_pair(gt: str, gen: str, lang: str) -> Tuple[str, str]:
+    gt = "" if gt is None else str(gt)
+    gen = "" if gen is None else str(gen)
+
+    # 去标点（保留 "'"）
+    for x in _PUNCTUATION_ALL:
+        if x == "'":
+            continue
+        gt = gt.replace(x, "")
+        gen = gen.replace(x, "")
+
+    # 统一空格与连字符
+    gt = gt.replace("  ", " ").replace("-", " ")
+    gen = gen.replace("  ", " ").replace("-", " ")
+
+    if lang == "zh":
+        # 把“字”当作 token
+        gt = " ".join([ch for ch in gt])
+        gen = " ".join([ch for ch in gen])
+    elif lang == "en":
+        gt = gt.lower()
+        gen = gen.lower()
+    else:
+        raise NotImplementedError("lang must be 'zh' or 'en'")
+
+    return gt, gen
+
+
+def compute_wers(gt_texts: List[str], gen_texts: List[str], lang: str = "zh") -> List[float]:
+    if len(gt_texts) != len(gen_texts):
+        raise ValueError(f"Length mismatch: {len(gt_texts)} != {len(gen_texts)}")
+
+    wers: List[float] = []
+    for gt_raw, gen_raw in zip(gt_texts, gen_texts):
+        gt_norm, gen_norm = _normalize_pair(gt_raw, gen_raw, lang=lang)
+        measures = process_words(reference=gt_norm, hypothesis=gen_norm)
+        wers.append(float(measures.wer))
+    return wers
+
+
+
+if __name__ == "__main__":
+    gt = ["你好世界啊", "今天天气不对", "abc-def"]
+    gen = ["你好，世界！", "今天 天气 不错", "abc def"]
+    print(compute_wers(gt, gen, lang="zh"))
+    print(compute_wers(["Hello World"], ["hello, world!"], lang="en"))

xcodec2/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

xcodec2/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": []
+}

xcodec2/README.md

@@ -0,0 +1,69 @@
+---
+license: cc-by-nc-4.0
+tags:
+- audio-to-audio
+pipeline_tag: audio-to-audio
+---
+
+[![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2502.04128)
+**Update (2025-02-13):** Add [Llasa finetune instruction](https://github.com/zhenye234/LLaSA_training/tree/main/finetune).
+
+**Update (2025-02-07):** Our paper has been released!
+
+
+## Paper
+ 
+LLaSA: Scaling Train Time and Inference Time Compute for LLaMA based Speech Synthesis  
+
+Codec Does Matter: Exploring the Semantic Shortcoming of Codec for Audio Language Model (AAAI 2025, xcodec 1.0)
+
+
+# Getting Started with XCodec2 on Hugging Face
+XCodec2 is a speech tokenizer that offers the following key features:
+
+1. **Single Vector Quantization**
+2. **50 Tokens per Second**
+3. **Multilingual Speech Semantic Support and High-Quality Speech Reconstruction**
+
+
+To use `xcodec2`, ensure you have it installed. You can install it using the following command:
+
+```bash
+conda create -n xcodec2 python=3.9
+conda activate xcodec2
+pip install xcodec2  (Use `xcodec2==0.1.5` for codec inference and llasa fine-tuning. I’ve removed unnecessary dependencies, and it works fine in my testing. However,  I’m not sure if other problems may arise. If you prefer more stability, I recommend using `xcodec2==0.1.3` which accurately aligns during my codec training.)
+ 
+```
+Then,
+```python
+import torch
+import soundfile as sf
+from transformers import AutoConfig
+
+ 
+from xcodec2.modeling_xcodec2 import XCodec2Model
+ 
+model_path = "HKUSTAudio/xcodec2"  
+ 
+model = XCodec2Model.from_pretrained(model_path)
+model.eval().cuda()   
+
+ 
+wav, sr = sf.read("test.wav")   
+wav_tensor = torch.from_numpy(wav).float().unsqueeze(0)  # Shape: (1, T)
+
+ 
+with torch.no_grad():
+   # Only 16khz speech
+   # Only supports single input. For batch inference, please refer to the link below.
+    vq_code = model.encode_code(input_waveform=wav_tensor)
+    print("Code:", vq_code )  
+
+    recon_wav = model.decode_code(vq_code).cpu()       # Shape: (1, 1, T')
+
+ 
+sf.write("reconstructed.wav", recon_wav[0, 0, :].numpy(), sr)
+print("Done! Check reconstructed.wav")
+```
+
+# If you want to train your own xcodec2, batch inference, or large-scale code extraction, the code is released [here](https://github.com/zhenye234/X-Codec-2.0).

xcodec2/config.json

@@ -0,0 +1,11 @@
+{
+    "model_type": "xcodec2",
+    "semantic_hidden_size": 1024,
+    "codec_encoder_hidden_size": 1024,
+    "codec_decoder_hidden_size": 1024,
+    "use_vocos": true,
+    "architectures": [
+      "XCodec2Model"
+    ]
+  }
+  

xcodec2/configuration_bigcodec.py

@@ -0,0 +1,19 @@
+from transformers import PretrainedConfig
+
+class BigCodecConfig(PretrainedConfig):
+    model_type = "bigcodec"
+
+    def __init__(
+        self,
+        # 下面这些只是示例超参
+        semantic_hidden_size=1024,
+        codec_encoder_hidden_size=1024,
+        codec_decoder_hidden_size=1024,
+        use_vocos=True,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.semantic_hidden_size = semantic_hidden_size
+        self.codec_encoder_hidden_size = codec_encoder_hidden_size
+        self.codec_decoder_hidden_size = codec_decoder_hidden_size
+        self.use_vocos = use_vocos

xcodec2/modeling_xcodec2.py

@@ -0,0 +1,164 @@
+import torch
+import torch.nn as nn
+from transformers import PreTrainedModel
+from xcodec2.configuration_bigcodec import BigCodecConfig
+
+from xcodec2.vq.codec_encoder import CodecEncoder_Transformer
+from xcodec2.vq.codec_decoder_vocos import CodecDecoderVocos
+from xcodec2.vq.module import SemanticEncoder
+from transformers import AutoFeatureExtractor, Wav2Vec2BertModel
+
+class XCodec2Model(PreTrainedModel):
+    config_class = BigCodecConfig
+
+    def __init__(self, config: BigCodecConfig):
+        super().__init__(config)
+
+        # 1) 语义模型
+        self.semantic_model = Wav2Vec2BertModel.from_pretrained(
+            "facebook/w2v-bert-2.0",
+            output_hidden_states=True
+        )
+        self.semantic_model.eval()
+
+        self.SemanticEncoder_module = SemanticEncoder(
+             config.semantic_hidden_size,
+             config.semantic_hidden_size,
+             config.semantic_hidden_size
+        )
+
+        # 2) Codec Encoder
+        self.CodecEnc = CodecEncoder_Transformer()
+
+        # 3) Codec Decoder
+        self.generator = CodecDecoderVocos()  
+
+        # 4) 两个全连接层
+        self.fc_prior = nn.Linear(2048, 2048)
+        self.fc_post_a = nn.Linear(2048, 1024)
+        feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/w2v-bert-2.0")
+        self.feature_extractor = feature_extractor
+
+    def forward(self, input_waveform, sample_rate=16000):
+        """
+        这里的 forward 不一定要叫 forward，也可以拆成别的方法；
+        但是如果想兼容 pipeline，需要在 forward 里给出核心逻辑。
+        
+        参数：
+          input_waveform: [batch_size, waveform_length]
+          sample_rate: 默认 16000
+        返回：
+          重构后的语音音频 (Tensor)
+        """
+        # 1) 特征提取
+        # 如果需要 padding，可以在这里做
+        input_features = self.feature_extractor(
+            input_waveform, 
+            sampling_rate=sample_rate, 
+            return_tensors="pt"
+        ).input_features.to(self.device)  # [batch, frames, feat_dim]
+
+        # 2) 语义层
+        semantic_output = self.semantic_model(input_features)
+        semantic_hidden_16 = semantic_output.hidden_states[16]  # 取第16层
+        semantic_hidden_16 = semantic_hidden_16.transpose(1, 2)  # [batch, hidden_dim, frames]
+        semantic_encoded = self.SemanticEncoder_module(semantic_hidden_16)
+
+        # 3) codec encoder
+        wav = input_waveform.unsqueeze(1).to(self.device)  # shape: [batch, 1, time]
+        vq_emb = self.CodecEnc(wav)  # [batch, time//down, 1024] 只是示例
+        vq_emb = vq_emb.transpose(1, 2)  # -> [batch, 1024, frames]
+
+        # 对齐语义向量的时间帧数，这里只做示例处理
+        # 真实做法里可能要先对齐维度
+        if vq_emb.shape[-1] != semantic_encoded.shape[-1]:
+            # 简单强行截断或补零都行，需要你自己决定
+            min_len = min(vq_emb.shape[-1], semantic_encoded.shape[-1])
+            vq_emb = vq_emb[:, :, :min_len]
+            semantic_encoded = semantic_encoded[:, :, :min_len]
+
+        # 4) 拼接
+        concat_emb = torch.cat([semantic_encoded, vq_emb], dim=1)  # [batch, 1024 + 1024, frames]
+
+        # 5) fc_prior
+        concat_emb = self.fc_prior(concat_emb.transpose(1, 2)).transpose(1, 2)
+
+        # 6) decoder 的量化部分
+        _, vq_code, _ = self.generator(concat_emb, vq=True)
+        vq_post_emb = self.generator.quantizer.get_output_from_indices(vq_code.transpose(1, 2))
+        vq_post_emb = vq_post_emb.transpose(1, 2)
+
+        # 7) fc_post_a
+        vq_post_emb = self.fc_post_a(vq_post_emb.transpose(1, 2)).transpose(1, 2)
+
+        # 8) 最后解码成波形
+        recon_audio = self.generator(vq_post_emb.transpose(1, 2), vq=False)[0]
+        # recon_audio: [batch, time]
+        return recon_audio
+
+    def encode_code(self, input_waveform, sample_rate=16000):
+        """
+        将输入的音频编码为代码表示。
+
+        参数：
+          input_waveform: [batch_size, waveform_length]
+          sample_rate: 默认 16000
+        返回：
+          编码后的代码 (Tensor)
+        """
+        with torch.no_grad():
+            # 1) 特征提取
+            input_features = self.feature_extractor(
+                input_waveform, 
+                sampling_rate=sample_rate, 
+                return_tensors="pt"
+            ).input_features.to(self.device)  # [batch, frames, feat_dim]
+
+            # 2) 语义层
+            semantic_output = self.semantic_model(input_features)
+            semantic_hidden_16 = semantic_output.hidden_states[16]  # 取第16层
+            semantic_hidden_16 = semantic_hidden_16.transpose(1, 2)  # [batch, hidden_dim, frames]
+            semantic_encoded = self.SemanticEncoder_module(semantic_hidden_16)
+
+            # 3) codec encoder
+            wav = input_waveform.unsqueeze(1).to(self.device)  # shape: [batch, 1, time]
+            vq_emb = self.CodecEnc(wav)  # [batch, time//down, 1024] 只是示例
+            vq_emb = vq_emb.transpose(1, 2)  # -> [batch, 1024, frames]
+
+            # 对齐语义向量的时间帧数，这里只做示例处理
+            if vq_emb.shape[-1] != semantic_encoded.shape[-1]:
+                min_len = min(vq_emb.shape[-1], semantic_encoded.shape[-1])
+                vq_emb = vq_emb[:, :, :min_len]
+                semantic_encoded = semantic_encoded[:, :, :min_len]
+
+            # 4) 拼接
+            concat_emb = torch.cat([semantic_encoded, vq_emb], dim=1)  # [batch, 2048, frames]
+
+            # 5) fc_prior
+            concat_emb = self.fc_prior(concat_emb.transpose(1, 2)).transpose(1, 2)
+
+            # 6) decoder 的量化部分，获取code
+            _, vq_code, _ = self.generator(concat_emb, vq=True)
+            # vq_code: [batch, frames]
+            return vq_code
+
+    def decode_code(self, vq_code):
+        """
+        将编码后的代码解码回音频。
+
+        参数：
+          vq_code: 编码后的代码 (Tensor) [batch, frames]
+        返回：
+          解码后的音频 (Tensor) [batch, waveform_length]
+        """
+        with torch.no_grad():
+            # 获取量化后的嵌入
+            vq_post_emb = self.generator.quantizer.get_output_from_indices(vq_code.transpose(1, 2))
+            vq_post_emb = vq_post_emb.transpose(1, 2)  # [batch, 1024, frames]
+
+            # 7) fc_post_a
+            vq_post_emb = self.fc_post_a(vq_post_emb.transpose(1, 2)).transpose(1, 2)  # [batch, 1024, frames]
+
+            # 8) 最后解码成波形
+            recon_audio = self.generator(vq_post_emb.transpose(1, 2), vq=False)[0]  # [batch, time]
+            return recon_audio

xcodec2/vq/__init__.py

@@ -0,0 +1,4 @@
+from xcodec2.vq.codec_encoder import CodecEncoder
+from xcodec2.vq.codec_decoder import CodecDecoder
+from xcodec2.vq.codec_decoder_vocos import CodecDecoderVocos
+from xcodec2.vq.codec_encoder import CodecEncoder_Transformer,CodecEncoder_only_Transformer