Upload 5 files

Data/Taffy/config.json

@@ -0,0 +1,97 @@
+{
+  "train": {
+    "log_interval": 100,
+    "eval_interval": 100,
+    "seed": 42,
+    "epochs": 10000,
+    "learning_rate": 0.0002,
+    "betas": [
+      0.8,
+      0.99
+    ],
+    "eps": 1e-09,
+    "batch_size": 12,
+    "fp16_run": false,
+    "lr_decay": 0.99995,
+    "segment_size": 16384,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0,
+    "skip_optimizer": true,
+    "keep_ckpts": 30
+  },
+  "data": {
+    "training_files": "Data/Taffy/filelists/train.list",
+    "validation_files": "Data/Taffy/filelists/val.list",
+    "max_wav_value": 32768.0,
+    "sampling_rate": 44100,
+    "filter_length": 2048,
+    "hop_length": 512,
+    "win_length": 2048,
+    "n_mel_channels": 128,
+    "mel_fmin": 0.0,
+    "mel_fmax": null,
+    "add_blank": true,
+    "n_speakers": 700,
+    "cleaned_text": true,
+    "spk2id": {
+      "永雏塔菲": 0
+    }
+  },
+  "model": {
+    "use_spk_conditioned_encoder": true,
+    "use_noise_scaled_mas": true,
+    "use_mel_posterior_encoder": false,
+    "use_duration_discriminator": true,
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0.1,
+    "resblock": "1",
+    "resblock_kernel_sizes": [
+      3,
+      7,
+      11
+    ],
+    "resblock_dilation_sizes": [
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ]
+    ],
+    "upsample_rates": [
+      8,
+      8,
+      2,
+      2,
+      2
+    ],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [
+      16,
+      16,
+      8,
+      2,
+      2
+    ],
+    "n_layers_q": 3,
+    "use_spectral_norm": false,
+    "gin_channels": 256
+  },
+  "version": "2.0"
+}

Data/Taffy/models/G_7600.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88ebc1c92d7981f45325106cc40b81524f38585082e1072c2118ed72a5a2c93f
+size 705938526

app.py

@@ -3,7 +3,7 @@ import os
 import logging
 
 import re_matching
-from tools.sentence import split_by_language, sentence_split
+from tools.sentence import split_by_language
 
 logging.getLogger("numba").setLevel(logging.WARNING)
 logging.getLogger("markdown_it").setLevel(logging.WARNING)
@@ -17,16 +17,13 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 import torch
-import ssl
-ssl._create_default_https_context = ssl._create_unverified_context
-import nltk
-nltk.download('cmudict')
 import utils
-from infer import infer, latest_version, get_net_g
+from infer import infer, latest_version, get_net_g, infer_multilang
 import gradio as gr
 import webbrowser
 import numpy as np
 from config import config
+from tools.translate import translate
 
 net_g = None
 
@@ -43,11 +40,15 @@ def generate_audio(
     length_scale,
     speaker,
     language,
+    skip_start=False,
+    skip_end=False,
 ):
     audio_list = []
-    silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
+    # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
     with torch.no_grad():
-        for piece in slices:
+        for idx, piece in enumerate(slices):
+            skip_start = (idx != 0) and skip_start
+            skip_end = (idx != len(slices) - 1) and skip_end
             audio = infer(
                 piece,
                 sdp_ratio=sdp_ratio,
@@ -59,10 +60,49 @@ def generate_audio(
                 hps=hps,
                 net_g=net_g,
                 device=device,
+                skip_start=skip_start,
+                skip_end=skip_end,
             )
             audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
             audio_list.append(audio16bit)
-            audio_list.append(silence)  # 将静音添加到列表中
+            # audio_list.append(silence)  # 将静音添加到列表中
+    return audio_list
+
+
+def generate_audio_multilang(
+    slices,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    speaker,
+    language,
+    skip_start=False,
+    skip_end=False,
+):
+    audio_list = []
+    # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
+    with torch.no_grad():
+        for idx, piece in enumerate(slices):
+            skip_start = (idx != 0) and skip_start
+            skip_end = (idx != len(slices) - 1) and skip_end
+            audio = infer_multilang(
+                piece,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+                sid=speaker,
+                language=language[idx],
+                hps=hps,
+                net_g=net_g,
+                device=device,
+                skip_start=skip_start,
+                skip_end=skip_end,
+            )
+            audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
+            audio_list.append(audio16bit)
+            # audio_list.append(silence)  # 将静音添加到列表中
     return audio_list
 
 
@@ -85,7 +125,9 @@ def tts_split(
     para_list = re_matching.cut_para(text)
     audio_list = []
     if not cut_by_sent:
-        for p in para_list:
+        for idx, p in enumerate(para_list):
+            skip_start = idx != 0
+            skip_end = idx != len(para_list) - 1
             audio = infer(
                 p,
                 sdp_ratio=sdp_ratio,
@@ -97,16 +139,22 @@ def tts_split(
                 hps=hps,
                 net_g=net_g,
                 device=device,
+                skip_start=skip_start,
+                skip_end=skip_end,
             )
             audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
             audio_list.append(audio16bit)
             silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
             audio_list.append(silence)
     else:
-        for p in para_list:
+        for idx, p in enumerate(para_list):
+            skip_start = idx != 0
+            skip_end = idx != len(para_list) - 1
             audio_list_sent = []
             sent_list = re_matching.cut_sent(p)
-            for s in sent_list:
+            for idx, s in enumerate(sent_list):
+                skip_start = (idx != 0) and skip_start
+                skip_end = (idx != len(sent_list) - 1) and skip_end
                 audio = infer(
                     s,
                     sdp_ratio=sdp_ratio,
@@ -118,6 +166,8 @@ def tts_split(
                     hps=hps,
                     net_g=net_g,
                     device=device,
+                    skip_start=skip_start,
+                    skip_end=skip_end,
                 )
                 audio_list_sent.append(audio)
                 silence = np.zeros((int)(44100 * interval_between_sent))
@@ -152,40 +202,116 @@ def tts_fn(
                 hps.data.sampling_rate,
                 np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
             )
-        result = re_matching.text_matching(text)
-        for one in result:
+        result = []
+        for slice in re_matching.text_matching(text):
+            _speaker = slice.pop()
+            temp_contant = []
+            temp_lang = []
+            for lang, content in slice:
+                if "|" in content:
+                    temp = []
+                    temp_ = []
+                    for i in content.split("|"):
+                        if i != "":
+                            temp.append([i])
+                            temp_.append([lang])
+                        else:
+                            temp.append([])
+                            temp_.append([])
+                    temp_contant += temp
+                    temp_lang += temp_
+                else:
+                    if len(temp_contant) == 0:
+                        temp_contant.append([])
+                        temp_lang.append([])
+                    temp_contant[-1].append(content)
+                    temp_lang[-1].append(lang)
+            for i, j in zip(temp_lang, temp_contant):
+                result.append([*zip(i, j), _speaker])
+        for i, one in enumerate(result):
+            skip_start = i != 0
+            skip_end = i != len(result) - 1
             _speaker = one.pop()
-            for lang, content in one:
+            idx = 0
+            while idx < len(one):
+                text_to_generate = []
+                lang_to_generate = []
+                while True:
+                    lang, content = one[idx]
+                    temp_text = [content]
+                    if len(text_to_generate) > 0:
+                        text_to_generate[-1] += [temp_text.pop(0)]
+                        lang_to_generate[-1] += [lang]
+                    if len(temp_text) > 0:
+                        text_to_generate += [[i] for i in temp_text]
+                        lang_to_generate += [[lang]] * len(temp_text)
+                    if idx + 1 < len(one):
+                        idx += 1
+                    else:
+                        break
+                skip_start = (idx != 0) and skip_start
+                skip_end = (idx != len(one) - 1) and skip_end
+                print(text_to_generate, lang_to_generate)
                 audio_list.extend(
-                    generate_audio(
-                        content.split("|"),
+                    generate_audio_multilang(
+                        text_to_generate,
                         sdp_ratio,
                         noise_scale,
                         noise_scale_w,
                         length_scale,
                         _speaker,
-                        lang,
+                        lang_to_generate,
+                        skip_start,
+                        skip_end,
                     )
                 )
+                idx += 1
     elif language.lower() == "auto":
-        sentences_list = split_by_language(text, target_languages=["zh", "ja", "en"])
-        for sentences, lang in sentences_list:
-            lang = lang.upper()
-            if lang == "JA":
-                lang = "JP"
-            sentences = sentence_split(sentences, max=250)
-            for content in sentences:
+        for idx, slice in enumerate(text.split("|")):
+            if slice == "":
+                continue
+            skip_start = idx != 0
+            skip_end = idx != len(text.split("|")) - 1
+            sentences_list = split_by_language(
+                slice, target_languages=["zh", "ja", "en"]
+            )
+            idx = 0
+            while idx < len(sentences_list):
+                text_to_generate = []
+                lang_to_generate = []
+                while True:
+                    content, lang = sentences_list[idx]
+                    temp_text = [content]
+                    lang = lang.upper()
+                    if lang == "JA":
+                        lang = "JP"
+                    if len(text_to_generate) > 0:
+                        text_to_generate[-1] += [temp_text.pop(0)]
+                        lang_to_generate[-1] += [lang]
+                    if len(temp_text) > 0:
+                        text_to_generate += [[i] for i in temp_text]
+                        lang_to_generate += [[lang]] * len(temp_text)
+                    if idx + 1 < len(sentences_list):
+                        idx += 1
+                    else:
+                        break
+                skip_start = (idx != 0) and skip_start
+                skip_end = (idx != len(sentences_list) - 1) and skip_end
+                print(text_to_generate, lang_to_generate)
                 audio_list.extend(
-                    generate_audio(
-                        content.split("|"),
+                    generate_audio_multilang(
+                        text_to_generate,
                         sdp_ratio,
                         noise_scale,
                         noise_scale_w,
                         length_scale,
                         speaker,
-                        lang,
+                        lang_to_generate,
+                        skip_start,
+                        skip_end,
                     )
                 )
+                idx += 1
     else:
         audio_list.extend(
             generate_audio(
@@ -220,10 +346,9 @@ if __name__ == "__main__":
         with gr.Row():
             with gr.Column():
                 gr.Markdown(value="""
-               【AI星瞳①】在线语音合成（Bert-Vits2 2.0中日英）\n
+               【AI塔菲】在线语音合成（Bert-Vits2 2.0中日英）\n
                 作者：Xz乔希 https://space.bilibili.com/5859321\n
-                声音归属：星瞳_Official https://space.bilibili.com/401315430\n
-                【AI星瞳②】https://huggingface.co/spaces/XzJosh/Star-Bert-VITS2\n
+                声音归属：永雏塔菲 https://space.bilibili.com/1265680561\n
                 【AI合集】https://www.modelscope.cn/studios/xzjosh/Bert-VITS2\n
                 Bert-VITS2项目：https://github.com/Stardust-minus/Bert-VITS2\n
                 使用本模型请严格遵守法律法规！\n
@@ -304,6 +429,11 @@ if __name__ == "__main__":
             outputs=[text_output, audio_output],
         )
 
+        trans.click(
+            translate,
+            inputs=[text],
+            outputs=[text],
+        )
         slicer.click(
             tts_split,
             inputs=[

config.yml

@@ -4,10 +4,10 @@
 # 拟提供通用路径配置，统一存放数据，避免数据放得很乱
 # 每个数据集与其对应的模型存放至统一路径下，后续所有的路径配置均为相对于datasetPath的路径
 # 不填或者填空则路径为相对于项目根目录的路径
-dataset_path: "Data/XingTong"
+dataset_path: "Data/Taffy"
 
 # 模型镜像源，默认huggingface，使用openi镜像源需指定openi_token
-mirror: "openi"
+mirror: ""
 openi_token: ""  # openi token
 
 # resample 音频重采样配置
@@ -26,7 +26,7 @@ resample:
 # 注意， “:” 后需要加空格
 preprocess_text:
   # 原始文本文件路径，文本格式应为{wav_path}|{speaker_name}|{language}|{text}。
-  transcription_path: "filelists/XingTong.list"
+  transcription_path: "filelists/Taffy.list"
   # 数据清洗后文本路径，可以不填。不填则将在原始文本目录生成
   cleaned_path: ""
   # 训练集路径
@@ -36,7 +36,7 @@ preprocess_text:
   # 配置文件路径
   config_path: "config.json"
   # 每个speaker的验证集条数
-  val_per_spk: 5
+  val_per_spk: 4
   # 验证集最大条数，多于的会被截断并放到训练集中
   max_val_total: 8
   # 是否进行数据清洗
@@ -68,12 +68,12 @@ train_ms:
     WORLD_SIZE: 1
     RANK: 0
     # 可以填写任意名的环境变量
-    THE_ENV_VAR_YOU_NEED_TO_USE: "1234567"
+    # THE_ENV_VAR_YOU_NEED_TO_USE: "1234567"
   # 底模设置
   base:
     use_base_model: false
     repo_id: "Stardust_minus/Bert-VITS2"
-    model_image: "Bert-VITS2中日底模" # openi网页的模型名
+    model_image: "Bert-VITS2中日英底模-fix" # openi网页的模型名
   # 训练模型存储目录：与旧版本的区别，原先数据集是存放在logs/model_name下的，现在改为统一存放在Data/你的数据集/models下
   model: "models"
   # 配置文件路径
@@ -84,9 +84,9 @@ train_ms:
 # 注意， “:” 后需要加空格
 webui:
   # 推理设备
-  device: "cpu"
+  device: "cuda"
   # 模型路径
-  model: "models/G_8000.pth"
+  model: "models/G_7600.pth"
   # 配置文件路径
   config_path: "config.json"
   # 端口号
@@ -111,9 +111,9 @@ server:
   # 注意，所有模型都必须正确配置model与config的路径，空路径会导致加载错误。
   models:
     - # 模型的路径
-      model: "Data/XingTong/models/G_8000.pth"
+      model: "Data/Taffy/models/G_8000.pth"
       # 模型config.json的路径
-      config: "Data/XingTong/config.json"
+      config: "Data/Taffy/config.json"
       # 模型使用设备，若填写则会覆盖默认配置
       device: "cuda"
       # 模型默认使用的语言
@@ -138,9 +138,9 @@ server:
           noise_scale_w: 0.8
           length_scale: 1.2
     - # 模型的路径
-      model: "Data/XingTong/models/G_8000.pth"
+      model: "Data/Taffy/models/G_8000.pth"
       # 模型config.json的路径
-      config: "Data/XingTong/config.json"
+      config: "Data/Taffy/config.json"
       # 模型使用设备，若填写则会覆盖默认配置
       device: "cpu"
       # 模型默认使用的语言
@@ -149,3 +149,12 @@ server:
       # 不必填写所有人物，不填的使用默认值
       speakers: [ ] # 也可以不填
 
+
+# 百度翻译开放平台 api配置
+# api接入文档 https://api.fanyi.baidu.com/doc/21
+# 请不要在github等网站公开分享你的app id 与 key
+translate:
+  # 你的APPID
+  "app_key": ""
+  # 你的密钥
+  "secret_key": ""

infer.py

@@ -204,3 +204,93 @@ def infer(
         del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers
         torch.cuda.empty_cache()
         return audio
+        
+
+def infer_multilang(
+    text,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    language,
+    hps,
+    net_g,
+    device,
+    skip_start=False,
+    skip_end=False,
+):
+    bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
+    # bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
+    #     text, language, hps, device
+    # )
+    for idx, (t, l) in enumerate(zip(text, language)):
+        skip_start = (idx != 0) or (skip_start and idx == 0)
+        skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
+        (
+            temp_bert,
+            temp_ja_bert,
+            temp_en_bert,
+            temp_phones,
+            temp_tones,
+            temp_lang_ids,
+        ) = get_text(t, l, hps, device)
+        if skip_start:
+            temp_bert = temp_bert[:, 1:]
+            temp_ja_bert = temp_ja_bert[:, 1:]
+            temp_en_bert = temp_en_bert[:, 1:]
+            temp_phones = temp_phones[1:]
+            temp_tones = temp_tones[1:]
+            temp_lang_ids = temp_lang_ids[1:]
+        if skip_end:
+            temp_bert = temp_bert[:, :-1]
+            temp_ja_bert = temp_ja_bert[:, :-1]
+            temp_en_bert = temp_en_bert[:, :-1]
+            temp_phones = temp_phones[:-1]
+            temp_tones = temp_tones[:-1]
+            temp_lang_ids = temp_lang_ids[:-1]
+        bert.append(temp_bert)
+        ja_bert.append(temp_ja_bert)
+        en_bert.append(temp_en_bert)
+        phones.append(temp_phones)
+        tones.append(temp_tones)
+        lang_ids.append(temp_lang_ids)
+    bert = torch.concatenate(bert, dim=1)
+    ja_bert = torch.concatenate(ja_bert, dim=1)
+    en_bert = torch.concatenate(en_bert, dim=1)
+    phones = torch.concatenate(phones, dim=0)
+    tones = torch.concatenate(tones, dim=0)
+    lang_ids = torch.concatenate(lang_ids, dim=0)
+    with torch.no_grad():
+        x_tst = phones.to(device).unsqueeze(0)
+        tones = tones.to(device).unsqueeze(0)
+        lang_ids = lang_ids.to(device).unsqueeze(0)
+        bert = bert.to(device).unsqueeze(0)
+        ja_bert = ja_bert.to(device).unsqueeze(0)
+        en_bert = en_bert.to(device).unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+        del phones
+        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+        audio = (
+            net_g.infer(
+                x_tst,
+                x_tst_lengths,
+                speakers,
+                tones,
+                lang_ids,
+                bert,
+                ja_bert,
+                en_bert,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+            )[0][0, 0]
+            .data.cpu()
+            .float()
+            .numpy()
+        )
+        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio