提交代码

.gitignore

@@ -0,0 +1,5 @@
+/.idea/misc.xml
+/.idea/modules.xml
+/.idea/inspectionProfiles/profiles_settings.xml
+/.idea/inspectionProfiles/Project_Default.xml
+/.idea/vcs.xml

.idea/.gitignore

@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+/.idea/
+/chat-tts.iml

README.md

@@ -5,7 +5,7 @@ colorFrom: blue
 colorTo: purple
 sdk: gradio
 sdk_version: 4.41.0
-app_file: app.py
+app_file: web\app.py
 pinned: false
 ---
 

requirements.txt

@@ -0,0 +1,29 @@
+# PyTorch and related libraries
+torch
+torchvision
+torchaudio
+
+# Hugging Face transformers library
+transformers
+
+# Configuration management with OmegaConf
+omegaconf
+
+# Interactive widgets for Jupyter Notebooks
+ipywidgets
+
+# Gradio for creating web UIs
+gradio
+
+# Vector quantization for PyTorch
+vector_quantize_pytorch
+# Hugging Face Hub client
+huggingface_hub
+
+vocos
+
+spaces
+
+ChatTTS
+
+av

test/api.py

@@ -0,0 +1,72 @@
+# Import necessary libraries and configure settings
+
+import torch
+import ChatTTS
+import os,sys
+from common_test import  *
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+from tool.logger import get_logger
+
+
+torch._dynamo.config.cache_size_limit = 64
+torch._dynamo.config.suppress_errors = True
+torch.set_float32_matmul_precision('high')
+
+logger= get_logger("api")
+# Initialize and load the model:
+chat = ChatTTS.Chat()
+if chat.load(source="custom", custom_path="D:\\chenjgspace\\ai-model\\chattts",coef=None):
+    print("Models loaded successfully.")
+else:
+    print("Models load failed.")
+    sys.exit(1)
+
+# Define the text input for inference (Support Batching)
+texts = [
+    "我真的不敢相信，他那么年轻武功居然这么好",
+    ]
+
+
+#使用随机种子数,会导致每次生成的音频文件都是随机的音色
+rand_spk = chat.sample_random_speaker()
+print(rand_spk) # save it for later timbre recovery
+
+params_infer_code = ChatTTS.Chat.InferCodeParams(
+    spk_emb = rand_spk, # add sampled speaker
+    temperature = .3,   # using custom temperature
+    top_P = 0.7,        # top P decode
+    top_K = 20,         # top K decode
+)
+
+###################################
+# For sentence level manual control.
+
+# use oral_(0-9), laugh_(0-2), break_(0-7)
+# to generate special token in text to synthesize.
+params_refine_text = ChatTTS.Chat.RefineTextParams(
+    prompt='[oral_2][laugh_0][break_6]',
+)
+
+wavs = chat.infer(
+    texts,
+    params_refine_text=params_refine_text,
+    params_infer_code=params_infer_code,
+)
+
+
+# Perform inference and play the generated audio
+#wavs = chat.infer(texts)
+#Audio(wavs[0], rate=24_000, autoplay=True)
+
+# Save the generated audio
+#torchaudio.save("D:\\Download\\output.wav", torch.from_numpy(wavs[0]), 24000)
+prefix_name = "D:\\Download\\" + get_date_time()
+
+for index, wav in enumerate(wavs):
+    save_mp3_file(wav, index, prefix_name)
+
+
+
+

test/common_test.py

@@ -0,0 +1,24 @@
+import datetime
+import time
+import os,sys
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+from tool.logger import get_logger
+
+logger=get_logger("common-test")
+def save_mp3_file(wav, index, prefix_name):
+    from tool.pcm import pcm_arr_to_mp3_view
+    data = pcm_arr_to_mp3_view(wav)
+    mp3_filename = prefix_name + "_" + str(index) + ".mp3"
+    with open(mp3_filename, "wb") as f:
+        f.write(data)
+    logger.info(f"Audio saved to {mp3_filename}")
+
+
+def get_date_time():
+    # 获取当前时间戳（秒级别）
+    current_timestamp = int(time.time())
+    # 将时间戳转换为datetime对象
+    current_datetime = datetime.datetime.fromtimestamp(current_timestamp)
+    return current_datetime.strftime("%Y-%m-%d-%H-%M-%S")

tool/__init__.py

@@ -0,0 +1,4 @@
+from .av import load_audio
+from .pcm import pcm_arr_to_mp3_view
+from .np import float_to_int16
+from .ctx import TorchSeedContext

tool/av.py

@@ -0,0 +1,79 @@
+from io import BufferedWriter, BytesIO
+from pathlib import Path
+from typing import Dict
+
+import av
+from av.audio.resampler import AudioResampler
+import numpy as np
+
+
+video_format_dict: Dict[str, str] = {
+    "m4a": "mp4",
+}
+
+audio_format_dict: Dict[str, str] = {
+    "ogg": "libvorbis",
+    "mp4": "aac",
+}
+
+
+def wav2(i: BytesIO, o: BufferedWriter, format: str):
+    """
+    https://github.com/fumiama/Retrieval-based-Voice-Conversion-WebUI/blob/412a9950a1e371a018c381d1bfb8579c4b0de329/infer/lib/audio.py#L20
+    """
+    inp = av.open(i, "r")
+    format = video_format_dict.get(format, format)
+    out = av.open(o, "w", format=format)
+    format = audio_format_dict.get(format, format)
+
+    ostream = out.add_stream(format)
+
+    for frame in inp.decode(audio=0):
+        for p in ostream.encode(frame):
+            out.mux(p)
+
+    for p in ostream.encode(None):
+        out.mux(p)
+
+    out.close()
+    inp.close()
+
+
+def load_audio(file: str, sr: int) -> np.ndarray:
+    """
+    https://github.com/fumiama/Retrieval-based-Voice-Conversion-WebUI/blob/412a9950a1e371a018c381d1bfb8579c4b0de329/infer/lib/audio.py#L39
+    """
+
+    if not Path(file).exists():
+        raise FileNotFoundError(f"File not found: {file}")
+
+    try:
+        container = av.open(file)
+        resampler = AudioResampler(format="fltp", layout="mono", rate=sr)
+
+        # Estimated maximum total number of samples to pre-allocate the array
+        # AV stores length in microseconds by default
+        estimated_total_samples = int(container.duration * sr // 1_000_000)
+        decoded_audio = np.zeros(estimated_total_samples + 1, dtype=np.float32)
+
+        offset = 0
+        for frame in container.decode(audio=0):
+            frame.pts = None  # Clear presentation timestamp to avoid resampling issues
+            resampled_frames = resampler.resample(frame)
+            for resampled_frame in resampled_frames:
+                frame_data = resampled_frame.to_ndarray()[0]
+                end_index = offset + len(frame_data)
+
+                # Check if decoded_audio has enough space, and resize if necessary
+                if end_index > decoded_audio.shape[0]:
+                    decoded_audio = np.resize(decoded_audio, end_index + 1)
+
+                decoded_audio[offset:end_index] = frame_data
+                offset += len(frame_data)
+
+        # Truncate the array to the actual size
+        decoded_audio = decoded_audio[:offset]
+    except Exception as e:
+        raise RuntimeError(f"Failed to load audio: {e}")
+
+    return decoded_audio

tool/ctx.py

@@ -0,0 +1,14 @@
+import torch
+
+
+class TorchSeedContext:
+    def __init__(self, seed):
+        self.seed = seed
+        self.state = None
+
+    def __enter__(self):
+        self.state = torch.random.get_rng_state()
+        torch.manual_seed(self.seed)
+
+    def __exit__(self, type, value, traceback):
+        torch.random.set_rng_state(self.state)

tool/func.py

@@ -0,0 +1,35 @@
+
+import gradio as gr
+import random
+
+seed_min = 1
+seed_max = 4294967295
+
+seeds = {
+    "旁白": {"seed": 4444},
+    "中年女性": {"seed": 7869},
+    "年轻女性": {"seed": 6615},
+    "中年男性": {"seed": 4099},
+    "年轻男性": {"seed": 6653},
+}
+
+# 音色选项：用于预置合适的音色
+voices = {
+    "旁白": {"seed": 2},
+    "Timbre1": {"seed": 1111},
+    "Timbre2": {"seed": 2222},
+    "Timbre3": {"seed": 3333},
+    "Timbre4": {"seed": 4444},
+    "Timbre5": {"seed": 5555},
+    "Timbre6": {"seed": 6666},
+    "Timbre7": {"seed": 7777},
+    "Timbre8": {"seed": 8888},
+    "Timbre9": {"seed": 9999},
+}
+
+def on_voice_change(vocie_selection):
+    return voices.get(vocie_selection)["seed"]
+
+
+def generate_seed():
+    return gr.update(value=random.randint(seed_min, seed_max))

tool/logger/__init__.py

@@ -0,0 +1 @@
+from .log import get_logger

tool/logger/log.py

@@ -0,0 +1,73 @@
+import platform, sys
+import logging
+from datetime import datetime, timezone
+
+logging.getLogger("numba").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("wetext-zh_normalizer").setLevel(logging.WARNING)
+logging.getLogger("NeMo-text-processing").setLevel(logging.WARNING)
+
+# from https://github.com/FloatTech/ZeroBot-Plugin/blob/c70766a989698452e60e5e48fb2f802a2444330d/console/console_windows.go#L89-L96
+colorCodePanic = "\x1b[1;31m"
+colorCodeFatal = "\x1b[1;31m"
+colorCodeError = "\x1b[31m"
+colorCodeWarn = "\x1b[33m"
+colorCodeInfo = "\x1b[37m"
+colorCodeDebug = "\x1b[32m"
+colorCodeTrace = "\x1b[36m"
+colorReset = "\x1b[0m"
+
+log_level_color_code = {
+    logging.DEBUG: colorCodeDebug,
+    logging.INFO: colorCodeInfo,
+    logging.WARN: colorCodeWarn,
+    logging.ERROR: colorCodeError,
+    logging.FATAL: colorCodeFatal,
+}
+
+log_level_msg_str = {
+    logging.DEBUG: "DEBU",
+    logging.INFO: "INFO",
+    logging.WARN: "WARN",
+    logging.ERROR: "ERRO",
+    logging.FATAL: "FATL",
+}
+
+
+class Formatter(logging.Formatter):
+    def __init__(self, color=platform.system().lower() != "windows"):
+        # https://stackoverflow.com/questions/2720319/python-figure-out-local-timezone
+        self.tz = datetime.now(timezone.utc).astimezone().tzinfo
+        self.color = color
+
+    def format(self, record: logging.LogRecord):
+        logstr = "[" + datetime.now(self.tz).strftime("%z %Y%m%d %H:%M:%S") + "] ["
+        if self.color:
+            logstr += log_level_color_code.get(record.levelno, colorCodeInfo)
+        logstr += log_level_msg_str.get(record.levelno, record.levelname)
+        if self.color:
+            logstr += colorReset
+        if sys.version_info >= (3, 9):
+            fn = record.filename.removesuffix(".py")
+        elif record.filename.endswith(".py"):
+            fn = record.filename[:-3]
+        logstr += f"] {str(record.name)} | {fn} | {str(record.msg)%record.args}"
+        return logstr
+
+
+def get_logger(name: str, lv=logging.INFO, remove_exist=False, format_root=False):
+    logger = logging.getLogger(name)
+    logger.setLevel(lv)
+    if remove_exist and logger.hasHandlers():
+        logger.handlers.clear()
+    if not logger.hasHandlers():
+        syslog = logging.StreamHandler()
+        syslog.setFormatter(Formatter())
+        logger.addHandler(syslog)
+    else:
+        for h in logger.handlers:
+            h.setFormatter(Formatter())
+    if format_root:
+        for h in logger.root.handlers:
+            h.setFormatter(Formatter())
+    return logger

tool/np.py

@@ -0,0 +1,11 @@
+import math
+
+import numpy as np
+from numba import jit
+
+
+@jit
+def float_to_int16(audio: np.ndarray) -> np.ndarray:
+    am = int(math.ceil(float(np.abs(audio).max())) * 32768)
+    am = 32767 * 32768 // am
+    return np.multiply(audio, am).astype(np.int16)

tool/pcm.py

@@ -0,0 +1,21 @@
+import wave
+from io import BytesIO
+
+import numpy as np
+
+from .np import float_to_int16
+from .av import wav2
+
+
+def pcm_arr_to_mp3_view(wav: np.ndarray):
+    buf = BytesIO()
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)  # Mono channel
+        wf.setsampwidth(2)  # Sample width in bytes
+        wf.setframerate(24000)  # Sample rate in Hz
+        wf.writeframes(float_to_int16(wav))
+    buf.seek(0, 0)
+    buf2 = BytesIO()
+    wav2(buf, buf2, "mp3")
+    buf.seek(0, 0)
+    return buf2.getbuffer()

web/app.py

@@ -0,0 +1,246 @@
+import os, sys
+
+if sys.platform == "darwin":
+    os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+
+from tool.logger import get_logger
+import ChatTTS
+import argparse
+import gradio as gr
+from tool.func import *
+from tool.ctx import TorchSeedContext
+from tool.np import *
+
+logger = get_logger("app")
+
+# Initialize and load the model:
+chat = ChatTTS.Chat()
+
+
+def init_chat(args):
+    global chat
+    # 获取启动模式
+    MODEL = os.getenv('MODEL')
+    logger.info("loading ChatTTS model..., start MODEL:" + str(MODEL))
+    source = "custom"
+    # huggingface 部署模式下，模型则直接使用hf的模型数据
+    if MODEL == "HF":
+        source = "huggingface"
+
+    if chat.load(source=source, custom_path="D:\\chenjgspace\\ai-model\\chattts", coef=None):
+        print("Models loaded successfully.")
+    else:
+        print("Models load failed.")
+        sys.exit(1)
+
+
+def main(args):
+    with gr.Blocks() as demo:
+        gr.Markdown("# ChatTTS demo")
+        with gr.Row():
+            with gr.Column(scale=1):
+                text_input = gr.Textbox(
+                    label="转换内容",
+                    lines=4,
+                    max_lines=4,
+                    placeholder="Please Input Text...",
+                    value="柔柔的，浓浓的，痴痴的风，牵引起心底灵动的思潮；情愫悠悠，思情绵绵，风里默坐，红尘中的浅醉，诗词中的优柔，任那自在飞花轻似梦的情怀，裁一束霓衣，织就清浅淡薄的安寂。",
+                    interactive=True,
+                )
+        with gr.Row():
+            refine_text_checkBox = gr.Checkbox(
+                label="是否优化文本,如是则先对文本内容做优化分词",
+                interactive=True,
+                value=True
+            )
+            temperature_slider = gr.Slider(
+                minimum=0.00001,
+                maximum=1.0,
+                step=0.00001,
+                value=0.3,
+                interactive=True,
+                label="模型 Temperature 参数设置"
+            )
+            top_p_slider = gr.Slider(
+                minimum=0.1,
+                maximum=0.9,
+                step=0.05,
+                value=0.7,
+                label="模型 top_P 参数设置",
+                interactive=True,
+            )
+            top_k_slider = gr.Slider(
+                minimum=1,
+                maximum=20,
+                step=1,
+                value=20,
+                label="模型 top_K 参数设置",
+                interactive=True,
+            )
+        with gr.Row():
+            voice_selection = gr.Dropdown(
+                label="Timbre",
+                choices=voices.keys(),
+                value="旁白",
+                interactive=True,
+                show_label=True
+            )
+            audio_seed_input = gr.Number(
+                value=2,
+                label="音色种子",
+                interactive=True,
+                minimum=seed_min,
+                maximum=seed_max,
+            )
+            generate_audio_seed = gr.Button("随机生成音色种子", interactive=True)
+            text_seed_input = gr.Number(
+                value=42,
+                label="文本种子",
+                interactive=True,
+                minimum=seed_min,
+                maximum=seed_max,
+            )
+            generate_text_seed = gr.Button("随机生成文本种子", interactive=True)
+
+        with gr.Row():
+            spk_emb_text = gr.Textbox(
+                label="Speaker Embedding",
+                max_lines=3,
+                show_copy_button=True,
+                interactive=False,
+                scale=2,
+
+            )
+            reload_chat_button = gr.Button("Reload", scale=1, interactive=True)
+
+        with gr.Row():
+            generate_button = gr.Button("生成音频文件", scale=1, interactive=True)
+
+        with gr.Row():
+            text_output = gr.Textbox(
+                label="输出文本",
+                interactive=False,
+                show_copy_button=True,
+            )
+
+            audio_output = gr.Audio(
+                label="输出音频",
+                value=None,
+                format="wav",
+                autoplay=False,
+                streaming=False,
+                interactive=False,
+                show_label=True,
+                waveform_options=gr.WaveformOptions(
+                    sample_rate=24000,
+                ),
+            )
+        # 针对页面元素新增 监听事件
+        voice_selection.change(fn=on_voice_change, inputs=voice_selection, outputs=audio_seed_input)
+
+        audio_seed_input.change(fn=on_audio_seed_change, inputs=audio_seed_input, outputs=spk_emb_text)
+
+        generate_audio_seed.click(fn=generate_seed, outputs=audio_seed_input)
+
+        generate_text_seed.click(fn=generate_seed,outputs=text_seed_input)
+
+        # reload_chat_button.click()
+
+        generate_button.click(fn=get_chat_infer_text,
+                              inputs=[text_input,
+                                                       text_seed_input,
+                                                       refine_text_checkBox
+                                                       ],
+                              outputs=[text_output]
+                              ).then(fn=get_chat_infer_audio,
+                                     inputs=[text_output,
+                                                       temperature_slider,
+                                                       top_p_slider,
+                                                       top_k_slider,
+                                                       audio_seed_input,
+                                                       spk_emb_text
+                                                       ],
+                                     outputs=[audio_output])
+        # 初始化 spk_emb_text 数值
+        spk_emb_text.value = on_audio_seed_change(audio_seed_input.value)
+        logger.info("元素初始化完成,启动gradio服务=======")
+
+        # 运行gradio服务
+        demo.launch(
+            server_name=args.server_name,
+            server_port=args.server_port,
+            inbrowser=True,
+            show_api=False)
+
+
+
+def get_chat_infer_audio(chat_txt,
+                 temperature_slider,
+                 top_p_slider,
+                 top_k_slider,
+                 audio_seed_input,
+                 spk_emb_text):
+    logger.info("========开始生成音频文件=====")
+    #音频参数设置
+    params_infer_code = ChatTTS.Chat.InferCodeParams(
+        spk_emb=spk_emb_text,  # add sampled speaker
+        temperature=temperature_slider,  # using custom temperature
+        top_P=top_p_slider,  # top P decode
+        top_K=top_k_slider,  # top K decode
+    )
+
+    with TorchSeedContext(audio_seed_input):
+        wav = chat.infer(
+            text=chat_txt,
+            skip_refine_text=True, #跳过文本优化
+            params_infer_code=params_infer_code,
+        )
+        yield 24000, float_to_int16(wav[0]).T
+
+def get_chat_infer_text(text,seed,refine_text_checkBox):
+
+    logger.info("========开始优化文本内容=====")
+    global chat
+    if not refine_text_checkBox:
+        logger.info("========文本内容无需优化=====")
+        return  text
+
+    params_refine_text = ChatTTS.Chat.RefineTextParams(
+        prompt='[oral_2][laugh_0][break_6]',
+    )
+
+    with TorchSeedContext(seed):
+        chat_text = chat.infer(
+            text=text,
+            skip_refine_text=False,
+            refine_text_only=True,  #仅返回优化后文本内容
+            params_refine_text=params_refine_text,
+        )
+
+    return chat_text[0] if isinstance(chat_text, list) else chat_text
+
+def on_audio_seed_change(audio_seed_input):
+    global chat
+    with TorchSeedContext(audio_seed_input):
+        rand_spk = chat.sample_random_speaker()
+    return rand_spk
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="ChatTTS demo Launch")
+    parser.add_argument(
+        "--server_name", type=str, default="0.0.0.0", help="server name"
+    )
+    parser.add_argument("--server_port", type=int, default=8080, help="server port")
+    parser.add_argument(
+        "--custom_path", type=str, default="D:\\chenjgspace\\ai-model\\chattts", help="custom model path"
+    )
+    parser.add_argument(
+        "--coef", type=str, default=None, help="custom dvae coefficient"
+    )
+    args = parser.parse_args()
+    init_chat(args)
+    main(args)