VITS/XTTS.py

import os
import subprocess
import uuid
import time
import torch
import torchaudio

# langid is used to detect language for longer text
# Most users expect text to be their own language, there is checkbox to disable it
import langid
import re
import gradio as gr
from TTS.tts.configs.xtts_config import XttsConfig
from TTS.tts.models.xtts import Xtts
from TTS.utils.generic_utils import get_user_data_dir

class XTTS():
    def __init__(self):
        model_name = "tts_models/multilingual/multi-dataset/xtts_v2"
        # ModelManager().download_model(model_name)
        model_path = os.path.join(get_user_data_dir("tts"), model_name.replace("/", "--"))
        # print("XTTS downloaded")

        config = XttsConfig()
        config.load_json(os.path.join(model_path, "config.json"))

        self.model = Xtts.init_from_config(config)
        self.model.load_checkpoint(
            config,
            checkpoint_path=os.path.join(model_path, "model.pth"),
            vocab_path=os.path.join(model_path, "vocab.json"),
            eval=True,
            use_deepspeed=True,
        )
        self.model.cuda()

        self.supported_languages = config.languages

    def predict(self, 
        prompt,
        language,
        audio_file_pth,
        voice_cleanup,
    ):
        # 模型不支持语言
        if language not in self.supported_languages:
            gr.Warning(
                f"Language you put {language} in is not in is not in our Supported Languages, please choose from dropdown"
            )

            return (
                None,
                None,
                None,
                None,
            )

        language_predicted = langid.classify(prompt)[
            0
        ].strip()  # strip need as there is space at end!

        # tts expects chinese as zh-cn
        if language_predicted == "zh":
            # we use zh-cn
            language_predicted = "zh-cn"

        print(f"Detected language:{language_predicted}, Chosen language:{language}")

        speaker_wav = audio_file_pth

        # Filtering for microphone input, as it has BG noise, maybe silence in beginning and end
        # This is fast filtering not perfect

        # Apply all on demand
        lowpassfilter = denoise = trim = loudness = True

        if lowpassfilter:
            lowpass_highpass = "lowpass=8000,highpass=75,"
        else:
            lowpass_highpass = ""

        if trim:
            # better to remove silence in beginning and end for microphone
            trim_silence = "areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02,areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02,"
        else:
            trim_silence = ""

        if voice_cleanup:
            try:
                out_filename = (
                    speaker_wav + str(uuid.uuid4()) + ".wav"
                )  # ffmpeg to know output format

                # we will use newer ffmpeg as that has afftn denoise filter
                shell_command = f"ffmpeg -y -i {speaker_wav} -af {lowpass_highpass}{trim_silence} {out_filename}".split(
                    " "
                )

                command_result = subprocess.run(
                    [item for item in shell_command],
                    capture_output=False,
                    text=True,
                    check=True,
                )
                speaker_wav = out_filename
                print("Filtered microphone input")
            except subprocess.CalledProcessError:
                # There was an error - command exited with non-zero code
                print("Error: failed filtering, use original microphone input")
        else:
            speaker_wav = speaker_wav

        if len(prompt) < 2:
            gr.Warning("Please give a longer prompt text")
            return (
                None,
                None,
                None,
                None,
            )
        
        metrics_text = ""
        t_latent = time.time()

        # note diffusion_conditioning not used on hifigan (default mode), it will be empty but need to pass it to model.inference
        try:
            (
                gpt_cond_latent,
                speaker_embedding,
            ) = self.model.get_conditioning_latents(audio_path=speaker_wav, gpt_cond_len=30, gpt_cond_chunk_len=4, max_ref_length=60)
        except Exception as e:
            print("Speaker encoding error", str(e))
            gr.Warning(
                "It appears something wrong with reference, did you unmute your microphone?"
            )
            return (
                None,
                None,
                None,
            )

        latent_calculation_time = time.time() - t_latent
        # metrics_text=f"Embedding calculation time: {latent_calculation_time:.2f} seconds\n"

        # temporary comma fix
        prompt= re.sub("([^\x00-\x7F]|\w)(\.|\。|\?)",r"\1 \2\2",prompt)

        wav_chunks = []
        ## Direct mode
        
        print("I: Generating new audio...")
        t0 = time.time()
        out = self.model.inference(
            prompt,
            language,
            gpt_cond_latent,
            speaker_embedding,
            repetition_penalty=5.0,
            temperature=0.75,
        )
        inference_time = time.time() - t0
        print(f"I: Time to generate audio: {round(inference_time*1000)} milliseconds")
        metrics_text+=f"Time to generate audio: {round(inference_time*1000)} milliseconds\n"
        real_time_factor= (time.time() - t0) / out['wav'].shape[-1] * 24000
        print(f"Real-time factor (RTF): {real_time_factor}")
        metrics_text+=f"Real-time factor (RTF): {real_time_factor:.2f}\n"
        torchaudio.save("output.wav", torch.tensor(out["wav"]).unsqueeze(0), 24000)
        return (
            "output.wav",
            metrics_text,
            speaker_wav,
        )

title = "语音克隆 Coqui🐸 XTTS"

examples = [
    [
        "Once when I was six years old I saw a magnificent picture",
        "en",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Lorsque j'avais six ans j'ai vu, une fois, une magnifique image",
        "fr",
        "examples/male.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Als ich sechs war, sah ich einmal ein wunderbares Bild",
        "de",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Cuando tenía seis años, vi una vez una imagen magnífica",
        "es",
        "examples/male.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Quando eu tinha seis anos eu vi, uma vez, uma imagem magnífica",
        "pt",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Kiedy miałem sześć lat, zobaczyłem pewnego razu wspaniały obrazek",
        "pl",
        "examples/male.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Un tempo lontano, quando avevo sei anni, vidi un magnifico disegno",
        "it",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Bir zamanlar, altı yaşındayken, muhteşem bir resim gördüm",
        "tr",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Когда мне было шесть лет, я увидел однажды удивительную картинку",
        "ru",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Toen ik een jaar of zes was, zag ik op een keer een prachtige plaat",
        "nl",
        "examples/male.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "Když mi bylo šest let, viděl jsem jednou nádherný obrázek",
        "cs",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "当我还只有六岁的时候， 看到了一副精彩的插画",
        "zh-cn",
        "examples/female.wav",
        None,
        False,
        False,
        False,
        True,
    ],
    [
        "かつて 六歳のとき、素晴らしい絵を見ました",
        "ja",
        "examples/female.wav",
        None,
        False,
        True,
        False,
        True,
    ],
    [
        "한번은 내가 여섯 살이었을 때 멋진 그림을 보았습니다.",
        "ko",
        "examples/female.wav",
        None,
        False,
        True,
        False,
        True,
    ],
        [
        "Egyszer hat éves koromban láttam egy csodálatos képet",
        "hu",
        "examples/male.wav",
        None,
        False,
        True,
        False,
        True,
    ],
]



def main():
    with gr.Blocks(analytics_enabled=False) as demo:
        with gr.Row():
            with gr.Column():
                gr.Markdown(
                    """
                    ## 语音克隆 (Coqui TTS)
                    """
                )
            with gr.Column():
                # 用于对齐图像的占位符
                pass

        with gr.Row():
            with gr.Column():
                gr.Markdown("文本提示: 一次一两个句子最好。最多200个文本字符。")
            with gr.Column():
                gr.Markdown("语言: 选择用于合成语音的输出语言。")

        with gr.Row():
            with gr.Column():
                input_text_gr = gr.Textbox(
                    label="文本提示",
                    info="一次一两个句子最好。最多200个文本字符。",
                    value="嗨，我是你的新语音克隆。请尽量上传高质量的音频。",
                )
                language_gr = gr.Dropdown(
                    label="语言",
                    info="选择用于合成语音的输出语言",
                    choices=[
                        "en",
                        "es",
                        "fr",
                        "de",
                        "it",
                        "pt",
                        "pl",
                        "tr",
                        "ru",
                        "nl",
                        "cs",
                        "ar",
                        "zh-cn",
                        "ja",
                        "ko",
                        "hu",
                        "hi"
                    ],
                    max_choices=1,
                    value="en",
                )
                ref_gr = gr.Audio(
                    label="参考音频",
                    type="filepath",
                    value="examples/female.wav",
                    sources=["microphone", "upload"],
                )
                clean_ref_gr = gr.Checkbox(
                    label="清理参考语音",
                    value=False,
                    info="如果您的麦克风或参考语音有噪音，此选项可以改善输出。",
                )
                tts_button = gr.Button("发送", elem_id="send-btn", visible=True)


            with gr.Column():
                audio_gr = gr.Audio(label="合成音频", autoplay=True)
                out_text_gr = gr.Text(label="指标")
                ref_audio_gr = gr.Audio(label="使用的参考音频")
        with gr.Row():
            gr.Examples(examples,
                        label="示例",
                        inputs=[input_text_gr, language_gr, ref_gr, clean_ref_gr],
                        outputs=[audio_gr, out_text_gr, ref_audio_gr],
                        fn=XTTS().predict,
                        cache_examples=False,)
        tts_button.click(XTTS().predict, [input_text_gr, language_gr, ref_gr, clean_ref_gr], outputs=[audio_gr, out_text_gr, ref_audio_gr])
    return demo

if __name__ == "__main__":  
    demo = main()  
    demo.launch()