feature: Support off-key analysis

app.py

@@ -39,7 +39,9 @@ class YukieGradio:
                         在使用此模型前请阅读[AI雪绘Yukie模型使用协议](https://huggingface.co/spaces/yukie/yukie-sovits3/edit/main/terms.md)
                         
                         # start！
-                        上传一段**纯人声**干音（60s以内），然后点击提交即可开始推理！
+                        上传一段**纯人声**干音（推荐60s以内），或者直接使用网站录音（二者只能选其一，优先使用上传音频）
+                        
+                        然后点击提交即可开始推理！
 
                         **请使用无bgm，无混响的人声来进行生成推理，否则效果可能会较差**
                         """)
@@ -47,6 +49,7 @@ class YukieGradio:
                                            "唱歌特化", "杂谈特化"], value="唱歌特化", interactive=True)
                     self.dev = gr.Dropdown(label="设备（云端一般请勿切换，使用默认值即可）", choices=[
                                            "cuda", "cpu"], value="cpu", interactive=True)
+                    self.inMic = gr.Microphone(label="录音")
                     self.inAudio = gr.Audio(label="上传音频")
                     self.needLogmmse = gr.Checkbox(label="是否使用自带降噪")
                     self.slice_db = gr.Slider(label="切片阈值(较嘈杂时-30，保留呼吸声时-50，一般默认-40)",
@@ -54,39 +57,45 @@ class YukieGradio:
                     self.vcTransform = gr.Number(
                         label="升降调（整数，可以正负，半音数量，升高八度就是12）", value=0)
                     self.vcSubmit = gr.Button("转换", variant="primary")
-                    self.outVcText = gr.Textbox(label="Output Message")
+                    self.outVcText = gr.Textbox(
+                        label="音高平均偏差半音数量，体现转换音频的跑调情况（一般小于0.5）")
                     self.outAudio = gr.Audio(
                         source="upload", type="numpy", label="Output Audio")
+                    self.f0_image = gr.Image(
+                        label="f0曲线，蓝色为输入音高，橙色为合成音频的音高（代码有误差）")
                     gr.Markdown(value="""
                         ## 注意
                         如果要在本地使用该demo，请使用  `git lfs clone https://huggingface.co/spaces/yukie/yukie-sovits3`克隆该仓库([简单教程](https://huggingface.co/spaces/yukie/yukie-sovits3/edit/main/local.md))
                     """)
-                    self.vcSubmit.click(infer, inputs=[self.inAudio, self.vcTransform, self.slice_db, self.needLogmmse, self.sid, self.dev], outputs=[
-                        self.outVcText, self.outAudio])
+                    self.vcSubmit.click(infer, inputs=[self.inMic, self.inAudio, self.vcTransform, self.slice_db, self.needLogmmse, self.sid, self.dev], outputs=[
+                        self.outVcText, self.outAudio, self.f0_image])
 
 
-def infer(inAudio, transform, slice_db, lm, sid, dev):
+def infer(inMic, inAudio, transform, slice_db, lm, sid, dev):
     if inAudio != None:
-        sampling_rate, audio = inAudio
+        sampling_rate, inaudio = inAudio
     else:
-        return "请上传一段音频后再次尝试", None
+        if inMic != None:
+            sampling_rate, inaudio = inMic
+        else:
+            return "请上传一段音频后再次尝试", None
 
     print("start inference")
     start_time = time.time()
     # 预处理，重编码
-    audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
-    if len(audio.shape) > 1:
-        audio = librosa.to_mono(audio.transpose(1, 0))
+    inaudio = (inaudio / np.iinfo(inaudio.dtype).max).astype(np.float32)
+    if len(inaudio.shape) > 1:
+        inaudio = librosa.to_mono(inaudio.transpose(1, 0))
     if sampling_rate != 32000:
-        audio = librosa.resample(
-            audio, orig_sr=sampling_rate, target_sr=32000)
+        inaudio = librosa.resample(
+            inaudio, orig_sr=sampling_rate, target_sr=32000)
     if lm:
-        audio = logmmse(audio, 32000)
+        inaudio = logmmse(inaudio, 32000)
 
-    out_wav_path = "tmp.wav"
-    soundfile.write(out_wav_path, audio, 32000, format="wav")
-    chunks = slicer.cut(out_wav_path, db_thresh=slice_db)
-    audio_data, audio_sr = slicer.chunks2audio(out_wav_path, chunks)
+    ori_wav_path = "tmp_ori.wav"
+    soundfile.write(ori_wav_path, inaudio, 32000, format="wav")
+    chunks = slicer.cut(ori_wav_path, db_thresh=slice_db)
+    audio_data, audio_sr = slicer.chunks2audio(ori_wav_path, chunks)
 
     audio = []
     sid = sid_map[sid]
@@ -108,8 +117,16 @@ def infer(inAudio, transform, slice_db, lm, sid, dev):
         audio.extend(list(_audio))
     audio = (np.array(audio) * 32768.0).astype('int16')
     used_time = time.time() - start_time
-    out_str = ("Success! total use time:{}s".format(used_time))
-    return out_str, (32000, audio)
+
+    out_wav_path = "tmp.wav"
+    soundfile.write(out_wav_path, audio, 32000, format="wav")
+
+    mistake, var = svc_model.calc_error(ori_wav_path, out_wav_path, transform)
+    out_picture = svc_model.f0_plt(ori_wav_path, out_wav_path, transform)
+    out_str = ("Success! total use time:{}s\n半音偏差:{}\n半音方差:{}".format(
+        used_time, mistake, var))
+
+    return out_str, (32000, audio), gr.Image.update("temp.jpg")
 
 
 if __name__ == "__main__":

inference/infer_tool.py

@@ -13,10 +13,13 @@ import parselmouth
 import soundfile
 import torch
 import torchaudio
+import pyworld
 
 from hubert import hubert_model
 import utils
+# from preprocess_hubert_f0 import compute_f0
 from models import SynthesizerTrn
+import matplotlib.pyplot as plt
 
 logging.getLogger('matplotlib').setLevel(logging.WARNING)
 
@@ -190,7 +193,7 @@ class Svc(object):
                 self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
                 **self.hps_ms.model)
             _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
-        if "half" in self.net_g_path and self.dev == "cuda":
+        if "half" in self.net_g_path and self.dev == torch.device("cuda"):
             _ = self.net_g_ms.half().eval().to(self.dev)
         else:
             _ = self.net_g_ms.eval().to(self.dev)
@@ -234,6 +237,47 @@ class Svc(object):
             print("vits use time:{}".format(use_time))
         return audio, audio.shape[-1]
 
+    def f0_plt(self, in_path, out_path, tran):
+        s1, input_pitch = self.get_unit_pitch(in_path, tran)
+        s2, output_pitch = self.get_unit_pitch(out_path, 0)
+        plt.clf()
+        plt.plot(plt_pitch(input_pitch), color="#66ccff")
+        plt.plot(plt_pitch(output_pitch), color="orange")
+        plt.savefig("temp.jpg")
+
+    def calc_error(self, in_path, out_path, tran):
+        input_pitch = compute_f0(in_path)
+        output_pitch = compute_f0(out_path)
+        sum_y = []
+        if np.sum(input_pitch == 0) / len(input_pitch) > 0.9:
+            mistake, var_take = 0, 0
+        else:
+            for i in range(min(len(input_pitch), len(output_pitch))):
+                if input_pitch[i] > 0 and output_pitch[i] > 0:
+                    sum_y.append(
+                        abs(f0_to_pitch(output_pitch[i]) - (f0_to_pitch(input_pitch[i]) + tran)))
+            num_y = 0
+            for x in sum_y:
+                num_y += x
+            len_y = len(sum_y) if len(sum_y) else 1
+            mistake = round(float(num_y / len_y), 2)
+            var_take = round(float(np.std(sum_y, ddof=1)), 2)
+        return mistake, var_take
+
+
+def compute_f0(path):
+    x, sr = librosa.load(path, sr=32000)
+    assert sr == 32000
+    f0, t = pyworld.dio(
+        x.astype(np.double),
+        fs=sr,
+        f0_ceil=800,
+        frame_period=1000 * 320 / sr,
+    )
+    f0 = pyworld.stonemask(x.astype(np.double), f0, t, 32000)
+    for index, pitch in enumerate(f0):
+        f0[index] = round(pitch, 1)
+    return f0
 
 # class SvcONNXInferModel(object):
 #     def __init__(self, hubert_onnx, vits_onnx, config_path):