src/rvc.py

from multiprocessing import cpu_count
from pathlib import Path
import torch
from scipy.io import wavfile
from infer_pack.models import (
    SynthesizerTrnMs256NSFsid,
    SynthesizerTrnMs256NSFsid_nono,
    SynthesizerTrnMs768NSFsid,
    SynthesizerTrnMs768NSFsid_nono,
)
from my_utils import load_audio
from vc_infer_pipeline import VC
import os
import torch.nn as nn
from transformers import HubertConfig, HubertModel

BASE_DIR = Path(__file__).resolve().parent.parent


# config cpu
def use_fp32_config():
    for config_file in [
        "32k.json",
        "40k.json",
        "48k.json",
        "48k_v2.json",
        "32k_v2.json",
    ]:
        with open(f"src/configs/{config_file}", "r") as f:
            strr = f.read().replace("true", "false")
        with open(f"src/configs/{config_file}", "w") as f:
            f.write(strr)

class Config:
    def __init__(self, device, is_half):
        self.device = device
        self.is_half = is_half
        self.n_cpu = 2 # set cpu cores
        self.gpu_name = None
        self.gpu_mem = None
        self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config()

    def device_config(self) -> tuple:
        if torch.cuda.is_available():
            i_device = int(self.device.split(":")[-1])
            self.gpu_name = torch.cuda.get_device_name(i_device)
            if (
                    ("16" in self.gpu_name and "V100" not in self.gpu_name.upper())
                    or "P40" in self.gpu_name.upper()
                    or "1060" in self.gpu_name
                    or "1070" in self.gpu_name
                    or "1080" in self.gpu_name
            ):
                print("16 series/10 series P40 forced single precision")
                self.is_half = False
                for config_file in ["32k.json", "40k.json", "48k.json"]:
                    with open(BASE_DIR / "src" / "configs" / config_file, "r") as f:
                        strr = f.read().replace("true", "false")
                    with open(BASE_DIR / "src" / "configs" / config_file, "w") as f:
                        f.write(strr)
                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "r") as f:
                    strr = f.read().replace("3.7", "3.0")
                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "w") as f:
                    f.write(strr)
            else:
                self.gpu_name = None
            self.gpu_mem = int(
                torch.cuda.get_device_properties(i_device).total_memory
                / 1024
                / 1024
                / 1024
                + 0.4
            )
            if self.gpu_mem <= 4:
                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "r") as f:
                    strr = f.read().replace("3.7", "3.0")
                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "w") as f:
                    f.write(strr)
        elif torch.backends.mps.is_available():
            print("No supported N-card found, use MPS for inference")
            self.device = "mps"
        else:
            print("No supported N-card found, use CPU for inference")
            self.device = "cpu"
            self.is_half = False
            use_fp32_config() # cpu config

        if self.n_cpu == 0:
            self.n_cpu = cpu_count()

        if self.is_half:
            # 6G memory config
            x_pad = 3
            x_query = 10
            x_center = 60
            x_max = 65
        else:
            # 5G memory config
            x_pad = 1
            x_query = 6
            x_center = 38
            x_max = 41

        if self.gpu_mem != None and self.gpu_mem <= 4:
            x_pad = 1
            x_query = 5
            x_center = 30
            x_max = 32

        return x_pad, x_query, x_center, x_max


class HubertModelWithFinalProj(HubertModel):
    def __init__(self, config):
        super().__init__(config)
        self.final_proj = nn.Linear(config.hidden_size, config.classifier_proj_size)


class FairseqHubertWrapper(nn.Module):
    def __init__(self, model_path_or_name="r3gm/hubert_base"):
        super().__init__()
        try:
            self.model = HubertModelWithFinalProj.from_pretrained(model_path_or_name)
        except Exception:
            self.model = HubertModel.from_pretrained(model_path_or_name)

    def extract_features(self, source, padding_mask=None, output_layer=12, **kwargs):
        param_dtype = next(self.model.parameters()).dtype
        if source.dtype != param_dtype:
            source = source.to(param_dtype)

        if source.dim() == 1:
            source = source.unsqueeze(0)

        with torch.no_grad():
            outputs = self.model(source, output_hidden_states=True)

        # 9 for v1, 12 for v2/ContentVec
        if output_layer is None or output_layer >= len(outputs.hidden_states):
            hidden_state = outputs.hidden_states[-1]
        else:
            hidden_state = outputs.hidden_states[output_layer]

        return (hidden_state, None)

    def final_proj(self, x):
        if hasattr(self.model, "final_proj"):
            return self.model.final_proj(x)
        return x


def load_hubert(device, is_half, model_path=None):
    if model_path and (os.path.exists(model_path) or os.path.isdir(model_path)):
        target_path = model_path
    else:
        target_path = "r3gm/hubert_base"

    hubert = FairseqHubertWrapper(target_path)
    hubert = hubert.to(device)

    if is_half and torch.device(device).type != "cpu":
        hubert = hubert.half()
    else:
        hubert = hubert.float()

    hubert.eval()
    return hubert


def get_vc(device, is_half, config, model_path):
    cpt = torch.load(model_path, map_location='cpu')
    if "config" not in cpt or "weight" not in cpt:
        raise ValueError(f'Incorrect format for {model_path}. Use a voice model trained using RVC v2 instead.')

    tgt_sr = cpt["config"][-1]
    cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0]
    if_f0 = cpt.get("f0", 1)
    version = cpt.get("version", "v1")

    if version == "v1":
        if if_f0 == 1:
            net_g = SynthesizerTrnMs256NSFsid(*cpt["config"], is_half=is_half)
        else:
            net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"])
    elif version == "v2":
        if if_f0 == 1:
            net_g = SynthesizerTrnMs768NSFsid(*cpt["config"], is_half=is_half)
        else:
            net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"])

    del net_g.enc_q
    print(net_g.load_state_dict(cpt["weight"], strict=False))
    net_g.eval().to(device)

    if is_half:
        net_g = net_g.half()
    else:
        net_g = net_g.float()

    vc = VC(tgt_sr, config)
    return cpt, version, net_g, tgt_sr, vc


def rvc_infer(index_path, index_rate, input_path, output_path, pitch_change, f0_method, cpt, version, net_g, filter_radius, tgt_sr, rms_mix_rate, protect, crepe_hop_length, vc, hubert_model, steps):
    times = [0, 0, 0]
    if_f0 = cpt.get('f0', 1)

    working_path = input_path

    for step in range(steps):
        audio = load_audio(working_path, 16000)

        audio_opt = vc.pipeline(
            hubert_model, net_g, step, audio, working_path, times, pitch_change,
            f0_method, index_path, index_rate, if_f0, filter_radius, tgt_sr,
            0, rms_mix_rate, version, protect, crepe_hop_length
        )

        wavfile.write(output_path, tgt_sr, audio_opt)
        working_path = output_path