Upload 20 files

F0Extractor.py

@@ -0,0 +1,100 @@
+import dataclasses
+import pathlib
+import libf0
+import librosa
+import numpy as np
+import resampy
+import torch
+import torchcrepe
+import torchfcpe
+import os
+
+# from tools.anyf0.rmvpe import RMVPE
+from rvc_cli.rvc.lib.predictors.RMVPE import RMVPE0Predictor
+from rvc_cli.rvc.configs.config import Config
+
+config = Config()
+
+
+@dataclasses.dataclass
+class F0Extractor:
+    wav_path: pathlib.Path
+    sample_rate: int = 44100
+    hop_length: int = 512
+    f0_min: int = 50
+    f0_max: int = 1600
+    method: str = "rmvpe"
+    x: np.ndarray = dataclasses.field(init=False)
+
+    def __post_init__(self):
+        self.x, self.sample_rate = librosa.load(self.wav_path, sr=self.sample_rate)
+
+    @property
+    def hop_size(self) -> float:
+        return self.hop_length / self.sample_rate
+
+    @property
+    def wav16k(self) -> np.ndarray:
+        return resampy.resample(self.x, self.sample_rate, 16000)
+
+    def extract_f0(self) -> np.ndarray:
+        f0 = None
+        method = self.method
+        if method == "crepe":
+            wav16k_torch = torch.FloatTensor(self.wav16k).unsqueeze(0).to(config.device)
+            f0 = torchcrepe.predict(
+                wav16k_torch,
+                sample_rate=16000,
+                hop_length=160,
+                batch_size=512,
+                fmin=self.f0_min,
+                fmax=self.f0_max,
+                device=config.device,
+            )
+            f0 = f0[0].cpu().numpy()
+        elif method == "fcpe":
+            audio = librosa.to_mono(self.x)
+            audio_length = len(audio)
+            f0_target_length = (audio_length // self.hop_length) + 1
+            audio = (
+                torch.from_numpy(audio)
+                .float()
+                .unsqueeze(0)
+                .unsqueeze(-1)
+                .to(config.device)
+            )
+            model = torchfcpe.spawn_bundled_infer_model(device=config.device)
+
+            f0 = model.infer(
+                audio,
+                sr=self.sample_rate,
+                decoder_mode="local_argmax",
+                threshold=0.006,
+                f0_min=self.f0_min,
+                f0_max=self.f0_max,
+                interp_uv=False,
+                output_interp_target_length=f0_target_length,
+            )
+            f0 = f0.squeeze().cpu().numpy()
+        elif method == "rmvpe":
+            model_rmvpe = RMVPE0Predictor(
+                os.path.join("rvc_cli", "rvc", "models", "predictors", "rmvpe.pt"),
+                is_half=config.is_half,
+                device=config.device,
+                # hop_length=80
+            )
+            f0 = model_rmvpe.infer_from_audio(self.wav16k, thred=0.03)
+
+        else:
+            raise ValueError(f"Unknown method: {self.method}")
+        return libf0.hz_to_cents(f0, librosa.midi_to_hz(0))
+
+    def plot_f0(self, f0):
+        from matplotlib import pyplot as plt
+
+        plt.figure(figsize=(10, 4))
+        plt.plot(f0)
+        plt.title(self.method)
+        plt.xlabel("Time (frames)")
+        plt.ylabel("F0 (cents)")
+        plt.show()

FCPE.py

@@ -0,0 +1,920 @@
+from typing import Union
+
+import torch.nn.functional as F
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.nn.utils.parametrizations import weight_norm
+from torchaudio.transforms import Resample
+import os
+import librosa
+import soundfile as sf
+import torch.utils.data
+from librosa.filters import mel as librosa_mel_fn
+import math
+from functools import partial
+
+from einops import rearrange, repeat
+from local_attention import LocalAttention
+from torch import nn
+
+os.environ["LRU_CACHE_CAPACITY"] = "3"
+
+
+def load_wav_to_torch(full_path, target_sr=None, return_empty_on_exception=False):
+    """Loads wav file to torch tensor."""
+    try:
+        data, sample_rate = sf.read(full_path, always_2d=True)
+    except Exception as error:
+        print(f"An error occurred loading {full_path}: {error}")
+        if return_empty_on_exception:
+            return [], sample_rate or target_sr or 48000
+        else:
+            raise
+
+    data = data[:, 0] if len(data.shape) > 1 else data
+    assert len(data) > 2
+
+    # Normalize data
+    max_mag = (
+        -np.iinfo(data.dtype).min
+        if np.issubdtype(data.dtype, np.integer)
+        else max(np.amax(data), -np.amin(data))
+    )
+    max_mag = (
+        (2**31) + 1 if max_mag > (2**15) else ((2**15) + 1 if max_mag > 1.01 else 1.0)
+    )
+    data = torch.FloatTensor(data.astype(np.float32)) / max_mag
+
+    # Handle exceptions and resample
+    if (torch.isinf(data) | torch.isnan(data)).any() and return_empty_on_exception:
+        return [], sample_rate or target_sr or 48000
+    if target_sr is not None and sample_rate != target_sr:
+        data = torch.from_numpy(
+            librosa.core.resample(
+                data.numpy(), orig_sr=sample_rate, target_sr=target_sr
+            )
+        )
+        sample_rate = target_sr
+
+    return data, sample_rate
+
+
+def dynamic_range_compression(x, C=1, clip_val=1e-5):
+    return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)
+
+
+def dynamic_range_decompression(x, C=1):
+    return np.exp(x) / C
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+    return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def dynamic_range_decompression_torch(x, C=1):
+    return torch.exp(x) / C
+
+
+class STFT:
+    def __init__(
+        self,
+        sr=22050,
+        n_mels=80,
+        n_fft=1024,
+        win_size=1024,
+        hop_length=256,
+        fmin=20,
+        fmax=11025,
+        clip_val=1e-5,
+    ):
+        self.target_sr = sr
+        self.n_mels = n_mels
+        self.n_fft = n_fft
+        self.win_size = win_size
+        self.hop_length = hop_length
+        self.fmin = fmin
+        self.fmax = fmax
+        self.clip_val = clip_val
+        self.mel_basis = {}
+        self.hann_window = {}
+
+    def get_mel(self, y, keyshift=0, speed=1, center=False, train=False):
+        sample_rate = self.target_sr
+        n_mels = self.n_mels
+        n_fft = self.n_fft
+        win_size = self.win_size
+        hop_length = self.hop_length
+        fmin = self.fmin
+        fmax = self.fmax
+        clip_val = self.clip_val
+
+        factor = 2 ** (keyshift / 12)
+        n_fft_new = int(np.round(n_fft * factor))
+        win_size_new = int(np.round(win_size * factor))
+        hop_length_new = int(np.round(hop_length * speed))
+
+        # Optimize mel_basis and hann_window caching
+        mel_basis = self.mel_basis if not train else {}
+        hann_window = self.hann_window if not train else {}
+
+        mel_basis_key = str(fmax) + "_" + str(y.device)
+        if mel_basis_key not in mel_basis:
+            mel = librosa_mel_fn(
+                sr=sample_rate, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax
+            )
+            mel_basis[mel_basis_key] = torch.from_numpy(mel).float().to(y.device)
+
+        keyshift_key = str(keyshift) + "_" + str(y.device)
+        if keyshift_key not in hann_window:
+            hann_window[keyshift_key] = torch.hann_window(win_size_new).to(y.device)
+
+        # Padding and STFT
+        pad_left = (win_size_new - hop_length_new) // 2
+        pad_right = max(
+            (win_size_new - hop_length_new + 1) // 2,
+            win_size_new - y.size(-1) - pad_left,
+        )
+        mode = "reflect" if pad_right < y.size(-1) else "constant"
+        y = torch.nn.functional.pad(y.unsqueeze(1), (pad_left, pad_right), mode=mode)
+        y = y.squeeze(1)
+
+        spec = torch.stft(
+            y,
+            n_fft_new,
+            hop_length=hop_length_new,
+            win_length=win_size_new,
+            window=hann_window[keyshift_key],
+            center=center,
+            pad_mode="reflect",
+            normalized=False,
+            onesided=True,
+            return_complex=True,
+        )
+        spec = torch.sqrt(spec.real.pow(2) + spec.imag.pow(2) + (1e-9))
+
+        # Handle keyshift and mel conversion
+        if keyshift != 0:
+            size = n_fft // 2 + 1
+            resize = spec.size(1)
+            spec = (
+                F.pad(spec, (0, 0, 0, size - resize))
+                if resize < size
+                else spec[:, :size, :]
+            )
+            spec = spec * win_size / win_size_new
+        spec = torch.matmul(mel_basis[mel_basis_key], spec)
+        spec = dynamic_range_compression_torch(spec, clip_val=clip_val)
+        return spec
+
+    def __call__(self, audiopath):
+        audio, sr = load_wav_to_torch(audiopath, target_sr=self.target_sr)
+        spect = self.get_mel(audio.unsqueeze(0)).squeeze(0)
+        return spect
+
+
+stft = STFT()
+
+
+def softmax_kernel(
+    data, *, projection_matrix, is_query, normalize_data=True, eps=1e-4, device=None
+):
+    b, h, *_ = data.shape
+
+    # Normalize data
+    data_normalizer = (data.shape[-1] ** -0.25) if normalize_data else 1.0
+
+    # Project data
+    ratio = projection_matrix.shape[0] ** -0.5
+    projection = repeat(projection_matrix, "j d -> b h j d", b=b, h=h)
+    projection = projection.type_as(data)
+    data_dash = torch.einsum("...id,...jd->...ij", (data_normalizer * data), projection)
+
+    # Calculate diagonal data
+    diag_data = data**2
+    diag_data = torch.sum(diag_data, dim=-1)
+    diag_data = (diag_data / 2.0) * (data_normalizer**2)
+    diag_data = diag_data.unsqueeze(dim=-1)
+
+    # Apply softmax
+    if is_query:
+        data_dash = ratio * (
+            torch.exp(
+                data_dash
+                - diag_data
+                - torch.max(data_dash, dim=-1, keepdim=True).values
+            )
+            + eps
+        )
+    else:
+        data_dash = ratio * (torch.exp(data_dash - diag_data + eps))
+
+    return data_dash.type_as(data)
+
+
+def orthogonal_matrix_chunk(cols, qr_uniform_q=False, device=None):
+    unstructured_block = torch.randn((cols, cols), device=device)
+    q, r = torch.linalg.qr(unstructured_block.cpu(), mode="reduced")
+    q, r = map(lambda t: t.to(device), (q, r))
+
+    if qr_uniform_q:
+        d = torch.diag(r, 0)
+        q *= d.sign()
+    return q.t()
+
+
+def exists(val):
+    return val is not None
+
+
+def empty(tensor):
+    return tensor.numel() == 0
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+def cast_tuple(val):
+    return (val,) if not isinstance(val, tuple) else val
+
+
+class PCmer(nn.Module):
+    def __init__(
+        self,
+        num_layers,
+        num_heads,
+        dim_model,
+        dim_keys,
+        dim_values,
+        residual_dropout,
+        attention_dropout,
+    ):
+        super().__init__()
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.dim_model = dim_model
+        self.dim_values = dim_values
+        self.dim_keys = dim_keys
+        self.residual_dropout = residual_dropout
+        self.attention_dropout = attention_dropout
+
+        self._layers = nn.ModuleList([_EncoderLayer(self) for _ in range(num_layers)])
+
+    def forward(self, phone, mask=None):
+        for layer in self._layers:
+            phone = layer(phone, mask)
+        return phone
+
+
+class _EncoderLayer(nn.Module):
+    def __init__(self, parent: PCmer):
+        super().__init__()
+        self.conformer = ConformerConvModule(parent.dim_model)
+        self.norm = nn.LayerNorm(parent.dim_model)
+        self.dropout = nn.Dropout(parent.residual_dropout)
+        self.attn = SelfAttention(
+            dim=parent.dim_model, heads=parent.num_heads, causal=False
+        )
+
+    def forward(self, phone, mask=None):
+        phone = phone + (self.attn(self.norm(phone), mask=mask))
+        phone = phone + (self.conformer(phone))
+        return phone
+
+
+def calc_same_padding(kernel_size):
+    pad = kernel_size // 2
+    return (pad, pad - (kernel_size + 1) % 2)
+
+
+class Swish(nn.Module):
+    def forward(self, x):
+        return x * x.sigmoid()
+
+
+class Transpose(nn.Module):
+    def __init__(self, dims):
+        super().__init__()
+        assert len(dims) == 2, "dims must be a tuple of two dimensions"
+        self.dims = dims
+
+    def forward(self, x):
+        return x.transpose(*self.dims)
+
+
+class GLU(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.dim = dim
+
+    def forward(self, x):
+        out, gate = x.chunk(2, dim=self.dim)
+        return out * gate.sigmoid()
+
+
+class DepthWiseConv1d(nn.Module):
+    def __init__(self, chan_in, chan_out, kernel_size, padding):
+        super().__init__()
+        self.padding = padding
+        self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, groups=chan_in)
+
+    def forward(self, x):
+        x = F.pad(x, self.padding)
+        return self.conv(x)
+
+
+class ConformerConvModule(nn.Module):
+    def __init__(
+        self, dim, causal=False, expansion_factor=2, kernel_size=31, dropout=0.0
+    ):
+        super().__init__()
+
+        inner_dim = dim * expansion_factor
+        padding = calc_same_padding(kernel_size) if not causal else (kernel_size - 1, 0)
+
+        self.net = nn.Sequential(
+            nn.LayerNorm(dim),
+            Transpose((1, 2)),
+            nn.Conv1d(dim, inner_dim * 2, 1),
+            GLU(dim=1),
+            DepthWiseConv1d(
+                inner_dim, inner_dim, kernel_size=kernel_size, padding=padding
+            ),
+            Swish(),
+            nn.Conv1d(inner_dim, dim, 1),
+            Transpose((1, 2)),
+            nn.Dropout(dropout),
+        )
+
+    def forward(self, x):
+        return self.net(x)
+
+
+def linear_attention(q, k, v):
+    if v is None:
+        out = torch.einsum("...ed,...nd->...ne", k, q)
+        return out
+    else:
+        k_cumsum = k.sum(dim=-2)
+        D_inv = 1.0 / (torch.einsum("...nd,...d->...n", q, k_cumsum.type_as(q)) + 1e-8)
+        context = torch.einsum("...nd,...ne->...de", k, v)
+        out = torch.einsum("...de,...nd,...n->...ne", context, q, D_inv)
+        return out
+
+
+def gaussian_orthogonal_random_matrix(
+    nb_rows, nb_columns, scaling=0, qr_uniform_q=False, device=None
+):
+    nb_full_blocks = int(nb_rows / nb_columns)
+    block_list = []
+
+    for _ in range(nb_full_blocks):
+        q = orthogonal_matrix_chunk(
+            nb_columns, qr_uniform_q=qr_uniform_q, device=device
+        )
+        block_list.append(q)
+
+    remaining_rows = nb_rows - nb_full_blocks * nb_columns
+    if remaining_rows > 0:
+        q = orthogonal_matrix_chunk(
+            nb_columns, qr_uniform_q=qr_uniform_q, device=device
+        )
+        block_list.append(q[:remaining_rows])
+
+    final_matrix = torch.cat(block_list)
+
+    if scaling == 0:
+        multiplier = torch.randn((nb_rows, nb_columns), device=device).norm(dim=1)
+    elif scaling == 1:
+        multiplier = math.sqrt((float(nb_columns))) * torch.ones(
+            (nb_rows,), device=device
+        )
+    else:
+        raise ValueError(f"Invalid scaling {scaling}")
+
+    return torch.diag(multiplier) @ final_matrix
+
+
+class FastAttention(nn.Module):
+    def __init__(
+        self,
+        dim_heads,
+        nb_features=None,
+        ortho_scaling=0,
+        causal=False,
+        generalized_attention=False,
+        kernel_fn=nn.ReLU(),
+        qr_uniform_q=False,
+        no_projection=False,
+    ):
+        super().__init__()
+        nb_features = default(nb_features, int(dim_heads * math.log(dim_heads)))
+
+        self.dim_heads = dim_heads
+        self.nb_features = nb_features
+        self.ortho_scaling = ortho_scaling
+
+        self.create_projection = partial(
+            gaussian_orthogonal_random_matrix,
+            nb_rows=self.nb_features,
+            nb_columns=dim_heads,
+            scaling=ortho_scaling,
+            qr_uniform_q=qr_uniform_q,
+        )
+        projection_matrix = self.create_projection()
+        self.register_buffer("projection_matrix", projection_matrix)
+
+        self.generalized_attention = generalized_attention
+        self.kernel_fn = kernel_fn
+        self.no_projection = no_projection
+        self.causal = causal
+
+    @torch.no_grad()
+    def redraw_projection_matrix(self):
+        projections = self.create_projection()
+        self.projection_matrix.copy_(projections)
+        del projections
+
+    def forward(self, q, k, v):
+        device = q.device
+
+        if self.no_projection:
+            q = q.softmax(dim=-1)
+            k = torch.exp(k) if self.causal else k.softmax(dim=-2)
+        else:
+            create_kernel = partial(
+                softmax_kernel, projection_matrix=self.projection_matrix, device=device
+            )
+            q = create_kernel(q, is_query=True)
+            k = create_kernel(k, is_query=False)
+
+        attn_fn = linear_attention if not self.causal else self.causal_linear_fn
+
+        if v is None:
+            out = attn_fn(q, k, None)
+            return out
+        else:
+            out = attn_fn(q, k, v)
+            return out
+
+
+class SelfAttention(nn.Module):
+    def __init__(
+        self,
+        dim,
+        causal=False,
+        heads=8,
+        dim_head=64,
+        local_heads=0,
+        local_window_size=256,
+        nb_features=None,
+        feature_redraw_interval=1000,
+        generalized_attention=False,
+        kernel_fn=nn.ReLU(),
+        qr_uniform_q=False,
+        dropout=0.0,
+        no_projection=False,
+    ):
+        super().__init__()
+        assert dim % heads == 0, "dimension must be divisible by number of heads"
+        dim_head = default(dim_head, dim // heads)
+        inner_dim = dim_head * heads
+        self.fast_attention = FastAttention(
+            dim_head,
+            nb_features,
+            causal=causal,
+            generalized_attention=generalized_attention,
+            kernel_fn=kernel_fn,
+            qr_uniform_q=qr_uniform_q,
+            no_projection=no_projection,
+        )
+
+        self.heads = heads
+        self.global_heads = heads - local_heads
+        self.local_attn = (
+            LocalAttention(
+                window_size=local_window_size,
+                causal=causal,
+                autopad=True,
+                dropout=dropout,
+                look_forward=int(not causal),
+                rel_pos_emb_config=(dim_head, local_heads),
+            )
+            if local_heads > 0
+            else None
+        )
+
+        self.to_q = nn.Linear(dim, inner_dim)
+        self.to_k = nn.Linear(dim, inner_dim)
+        self.to_v = nn.Linear(dim, inner_dim)
+        self.to_out = nn.Linear(inner_dim, dim)
+        self.dropout = nn.Dropout(dropout)
+
+    @torch.no_grad()
+    def redraw_projection_matrix(self):
+        self.fast_attention.redraw_projection_matrix()
+
+    def forward(
+        self,
+        x,
+        context=None,
+        mask=None,
+        context_mask=None,
+        name=None,
+        inference=False,
+        **kwargs,
+    ):
+        _, _, _, h, gh = *x.shape, self.heads, self.global_heads
+
+        cross_attend = exists(context)
+        context = default(context, x)
+        context_mask = default(context_mask, mask) if not cross_attend else context_mask
+        q, k, v = self.to_q(x), self.to_k(context), self.to_v(context)
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (q, k, v))
+        (q, lq), (k, lk), (v, lv) = map(lambda t: (t[:, :gh], t[:, gh:]), (q, k, v))
+
+        attn_outs = []
+        if not empty(q):
+            if exists(context_mask):
+                global_mask = context_mask[:, None, :, None]
+                v.masked_fill_(~global_mask, 0.0)
+            if cross_attend:
+                pass  # TODO: Implement cross-attention
+            else:
+                out = self.fast_attention(q, k, v)
+            attn_outs.append(out)
+
+        if not empty(lq):
+            assert (
+                not cross_attend
+            ), "local attention is not compatible with cross attention"
+            out = self.local_attn(lq, lk, lv, input_mask=mask)
+            attn_outs.append(out)
+
+        out = torch.cat(attn_outs, dim=1)
+        out = rearrange(out, "b h n d -> b n (h d)")
+        out = self.to_out(out)
+        return self.dropout(out)
+
+
+def l2_regularization(model, l2_alpha):
+    l2_loss = []
+    for module in model.modules():
+        if type(module) is nn.Conv2d:
+            l2_loss.append((module.weight**2).sum() / 2.0)
+    return l2_alpha * sum(l2_loss)
+
+
+class FCPE(nn.Module):
+    def __init__(
+        self,
+        input_channel=128,
+        out_dims=360,
+        n_layers=12,
+        n_chans=512,
+        use_siren=False,
+        use_full=False,
+        loss_mse_scale=10,
+        loss_l2_regularization=False,
+        loss_l2_regularization_scale=1,
+        loss_grad1_mse=False,
+        loss_grad1_mse_scale=1,
+        f0_max=1975.5,
+        f0_min=32.70,
+        confidence=False,
+        threshold=0.05,
+        use_input_conv=True,
+    ):
+        super().__init__()
+        if use_siren is True:
+            raise ValueError("Siren is not supported yet.")
+        if use_full is True:
+            raise ValueError("Full model is not supported yet.")
+
+        self.loss_mse_scale = loss_mse_scale if (loss_mse_scale is not None) else 10
+        self.loss_l2_regularization = (
+            loss_l2_regularization if (loss_l2_regularization is not None) else False
+        )
+        self.loss_l2_regularization_scale = (
+            loss_l2_regularization_scale
+            if (loss_l2_regularization_scale is not None)
+            else 1
+        )
+        self.loss_grad1_mse = loss_grad1_mse if (loss_grad1_mse is not None) else False
+        self.loss_grad1_mse_scale = (
+            loss_grad1_mse_scale if (loss_grad1_mse_scale is not None) else 1
+        )
+        self.f0_max = f0_max if (f0_max is not None) else 1975.5
+        self.f0_min = f0_min if (f0_min is not None) else 32.70
+        self.confidence = confidence if (confidence is not None) else False
+        self.threshold = threshold if (threshold is not None) else 0.05
+        self.use_input_conv = use_input_conv if (use_input_conv is not None) else True
+
+        self.cent_table_b = torch.Tensor(
+            np.linspace(
+                self.f0_to_cent(torch.Tensor([f0_min]))[0],
+                self.f0_to_cent(torch.Tensor([f0_max]))[0],
+                out_dims,
+            )
+        )
+        self.register_buffer("cent_table", self.cent_table_b)
+
+        # conv in stack
+        _leaky = nn.LeakyReLU()
+        self.stack = nn.Sequential(
+            nn.Conv1d(input_channel, n_chans, 3, 1, 1),
+            nn.GroupNorm(4, n_chans),
+            _leaky,
+            nn.Conv1d(n_chans, n_chans, 3, 1, 1),
+        )
+
+        # transformer
+        self.decoder = PCmer(
+            num_layers=n_layers,
+            num_heads=8,
+            dim_model=n_chans,
+            dim_keys=n_chans,
+            dim_values=n_chans,
+            residual_dropout=0.1,
+            attention_dropout=0.1,
+        )
+        self.norm = nn.LayerNorm(n_chans)
+
+        # out
+        self.n_out = out_dims
+        self.dense_out = weight_norm(nn.Linear(n_chans, self.n_out))
+
+    def forward(
+        self, mel, infer=True, gt_f0=None, return_hz_f0=False, cdecoder="local_argmax"
+    ):
+        if cdecoder == "argmax":
+            self.cdecoder = self.cents_decoder
+        elif cdecoder == "local_argmax":
+            self.cdecoder = self.cents_local_decoder
+
+        x = (
+            self.stack(mel.transpose(1, 2)).transpose(1, 2)
+            if self.use_input_conv
+            else mel
+        )
+        x = self.decoder(x)
+        x = self.norm(x)
+        x = self.dense_out(x)
+        x = torch.sigmoid(x)
+
+        if not infer:
+            gt_cent_f0 = self.f0_to_cent(gt_f0)
+            gt_cent_f0 = self.gaussian_blurred_cent(gt_cent_f0)
+            loss_all = self.loss_mse_scale * F.binary_cross_entropy(x, gt_cent_f0)
+            if self.loss_l2_regularization:
+                loss_all = loss_all + l2_regularization(
+                    model=self, l2_alpha=self.loss_l2_regularization_scale
+                )
+            x = loss_all
+        if infer:
+            x = self.cdecoder(x)
+            x = self.cent_to_f0(x)
+            x = (1 + x / 700).log() if not return_hz_f0 else x
+
+        return x
+
+    def cents_decoder(self, y, mask=True):
+        B, N, _ = y.size()
+        ci = self.cent_table[None, None, :].expand(B, N, -1)
+        rtn = torch.sum(ci * y, dim=-1, keepdim=True) / torch.sum(
+            y, dim=-1, keepdim=True
+        )
+        if mask:
+            confident = torch.max(y, dim=-1, keepdim=True)[0]
+            confident_mask = torch.ones_like(confident)
+            confident_mask[confident <= self.threshold] = float("-INF")
+            rtn = rtn * confident_mask
+        return (rtn, confident) if self.confidence else rtn
+
+    def cents_local_decoder(self, y, mask=True):
+        B, N, _ = y.size()
+        ci = self.cent_table[None, None, :].expand(B, N, -1)
+        confident, max_index = torch.max(y, dim=-1, keepdim=True)
+        local_argmax_index = torch.arange(0, 9).to(max_index.device) + (max_index - 4)
+        local_argmax_index = torch.clamp(local_argmax_index, 0, self.n_out - 1)
+        ci_l = torch.gather(ci, -1, local_argmax_index)
+        y_l = torch.gather(y, -1, local_argmax_index)
+        rtn = torch.sum(ci_l * y_l, dim=-1, keepdim=True) / torch.sum(
+            y_l, dim=-1, keepdim=True
+        )
+        if mask:
+            confident_mask = torch.ones_like(confident)
+            confident_mask[confident <= self.threshold] = float("-INF")
+            rtn = rtn * confident_mask
+        return (rtn, confident) if self.confidence else rtn
+
+    def cent_to_f0(self, cent):
+        return 10.0 * 2 ** (cent / 1200.0)
+
+    def f0_to_cent(self, f0):
+        return 1200.0 * torch.log2(f0 / 10.0)
+
+    def gaussian_blurred_cent(self, cents):
+        mask = (cents > 0.1) & (cents < (1200.0 * np.log2(self.f0_max / 10.0)))
+        B, N, _ = cents.size()
+        ci = self.cent_table[None, None, :].expand(B, N, -1)
+        return torch.exp(-torch.square(ci - cents) / 1250) * mask.float()
+
+
+class FCPEInfer:
+    def __init__(self, model_path, device=None, dtype=torch.float32):
+        if device is None:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.device = device
+        ckpt = torch.load(model_path, map_location=torch.device(self.device))
+        self.args = DotDict(ckpt["config"])
+        self.dtype = dtype
+        model = FCPE(
+            input_channel=self.args.model.input_channel,
+            out_dims=self.args.model.out_dims,
+            n_layers=self.args.model.n_layers,
+            n_chans=self.args.model.n_chans,
+            use_siren=self.args.model.use_siren,
+            use_full=self.args.model.use_full,
+            loss_mse_scale=self.args.loss.loss_mse_scale,
+            loss_l2_regularization=self.args.loss.loss_l2_regularization,
+            loss_l2_regularization_scale=self.args.loss.loss_l2_regularization_scale,
+            loss_grad1_mse=self.args.loss.loss_grad1_mse,
+            loss_grad1_mse_scale=self.args.loss.loss_grad1_mse_scale,
+            f0_max=self.args.model.f0_max,
+            f0_min=self.args.model.f0_min,
+            confidence=self.args.model.confidence,
+        )
+        model.to(self.device).to(self.dtype)
+        model.load_state_dict(ckpt["model"])
+        model.eval()
+        self.model = model
+        self.wav2mel = Wav2Mel(self.args, dtype=self.dtype, device=self.device)
+
+    @torch.no_grad()
+    def __call__(self, audio, sr, threshold=0.05):
+        self.model.threshold = threshold
+        audio = audio[None, :]
+        mel = self.wav2mel(audio=audio, sample_rate=sr).to(self.dtype)
+        f0 = self.model(mel=mel, infer=True, return_hz_f0=True)
+        return f0
+
+
+class Wav2Mel:
+    def __init__(self, args, device=None, dtype=torch.float32):
+        self.sample_rate = args.mel.sampling_rate
+        self.hop_size = args.mel.hop_size
+        if device is None:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.device = device
+        self.dtype = dtype
+        self.stft = STFT(
+            args.mel.sampling_rate,
+            args.mel.num_mels,
+            args.mel.n_fft,
+            args.mel.win_size,
+            args.mel.hop_size,
+            args.mel.fmin,
+            args.mel.fmax,
+        )
+        self.resample_kernel = {}
+
+    def extract_nvstft(self, audio, keyshift=0, train=False):
+        mel = self.stft.get_mel(audio, keyshift=keyshift, train=train).transpose(1, 2)
+        return mel
+
+    def extract_mel(self, audio, sample_rate, keyshift=0, train=False):
+        audio = audio.to(self.dtype).to(self.device)
+        if sample_rate == self.sample_rate:
+            audio_res = audio
+        else:
+            key_str = str(sample_rate)
+            if key_str not in self.resample_kernel:
+                self.resample_kernel[key_str] = Resample(
+                    sample_rate, self.sample_rate, lowpass_filter_width=128
+                )
+            self.resample_kernel[key_str] = (
+                self.resample_kernel[key_str].to(self.dtype).to(self.device)
+            )
+            audio_res = self.resample_kernel[key_str](audio)
+
+        mel = self.extract_nvstft(
+            audio_res, keyshift=keyshift, train=train
+        )  # B, n_frames, bins
+        n_frames = int(audio.shape[1] // self.hop_size) + 1
+        mel = (
+            torch.cat((mel, mel[:, -1:, :]), 1) if n_frames > int(mel.shape[1]) else mel
+        )
+        mel = mel[:, :n_frames, :] if n_frames < int(mel.shape[1]) else mel
+        return mel
+
+    def __call__(self, audio, sample_rate, keyshift=0, train=False):
+        return self.extract_mel(audio, sample_rate, keyshift=keyshift, train=train)
+
+
+class DotDict(dict):
+    def __getattr__(*args):
+        val = dict.get(*args)
+        return DotDict(val) if type(val) is dict else val
+
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+
+class F0Predictor(object):
+    def compute_f0(self, wav, p_len):
+        pass
+
+    def compute_f0_uv(self, wav, p_len):
+        pass
+
+
+class FCPEF0Predictor(F0Predictor):
+    def __init__(
+        self,
+        model_path,
+        hop_length=512,
+        f0_min=50,
+        f0_max=1100,
+        dtype=torch.float32,
+        device=None,
+        sample_rate=44100,
+        threshold=0.05,
+    ):
+        self.fcpe = FCPEInfer(model_path, device=device, dtype=dtype)
+        self.hop_length = hop_length
+        self.f0_min = f0_min
+        self.f0_max = f0_max
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        self.threshold = threshold
+        self.sample_rate = sample_rate
+        self.dtype = dtype
+        self.name = "fcpe"
+
+    def repeat_expand(
+        self,
+        content: Union[torch.Tensor, np.ndarray],
+        target_len: int,
+        mode: str = "nearest",
+    ):
+        ndim = content.ndim
+        content = (
+            content[None, None]
+            if ndim == 1
+            else content[None] if ndim == 2 else content
+        )
+        assert content.ndim == 3
+        is_np = isinstance(content, np.ndarray)
+        content = torch.from_numpy(content) if is_np else content
+        results = torch.nn.functional.interpolate(content, size=target_len, mode=mode)
+        results = results.numpy() if is_np else results
+        return results[0, 0] if ndim == 1 else results[0] if ndim == 2 else results
+
+    def post_process(self, x, sample_rate, f0, pad_to):
+        f0 = (
+            torch.from_numpy(f0).float().to(x.device)
+            if isinstance(f0, np.ndarray)
+            else f0
+        )
+        f0 = self.repeat_expand(f0, pad_to) if pad_to is not None else f0
+
+        vuv_vector = torch.zeros_like(f0)
+        vuv_vector[f0 > 0.0] = 1.0
+        vuv_vector[f0 <= 0.0] = 0.0
+
+        nzindex = torch.nonzero(f0).squeeze()
+        f0 = torch.index_select(f0, dim=0, index=nzindex).cpu().numpy()
+        time_org = self.hop_length / sample_rate * nzindex.cpu().numpy()
+        time_frame = np.arange(pad_to) * self.hop_length / sample_rate
+
+        vuv_vector = F.interpolate(vuv_vector[None, None, :], size=pad_to)[0][0]
+
+        if f0.shape[0] <= 0:
+            return np.zeros(pad_to), vuv_vector.cpu().numpy()
+        if f0.shape[0] == 1:
+            return np.ones(pad_to) * f0[0], vuv_vector.cpu().numpy()
+
+        f0 = np.interp(time_frame, time_org, f0, left=f0[0], right=f0[-1])
+        return f0, vuv_vector.cpu().numpy()
+
+    def compute_f0(self, wav, p_len=None):
+        x = torch.FloatTensor(wav).to(self.dtype).to(self.device)
+        p_len = x.shape[0] // self.hop_length if p_len is None else p_len
+        f0 = self.fcpe(x, sr=self.sample_rate, threshold=self.threshold)[0, :, 0]
+        if torch.all(f0 == 0):
+            return f0.cpu().numpy() if p_len is None else np.zeros(p_len), (
+                f0.cpu().numpy() if p_len is None else np.zeros(p_len)
+            )
+        return self.post_process(x, self.sample_rate, f0, p_len)[0]
+
+    def compute_f0_uv(self, wav, p_len=None):
+        x = torch.FloatTensor(wav).to(self.dtype).to(self.device)
+        p_len = x.shape[0] // self.hop_length if p_len is None else p_len
+        f0 = self.fcpe(x, sr=self.sample_rate, threshold=self.threshold)[0, :, 0]
+        if torch.all(f0 == 0):
+            return f0.cpu().numpy() if p_len is None else np.zeros(p_len), (
+                f0.cpu().numpy() if p_len is None else np.zeros(p_len)
+            )
+        return self.post_process(x, self.sample_rate, f0, p_len)

RMVPE.py

@@ -0,0 +1,560 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+from librosa.filters import mel
+from typing import List
+
+# Constants for readability
+N_MELS = 128
+N_CLASS = 360
+
+
+# Define a helper function for creating convolutional blocks
+class ConvBlockRes(nn.Module):
+    """
+    A convolutional block with residual connection.
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        momentum (float): Momentum for batch normalization.
+    """
+
+    def __init__(self, in_channels, out_channels, momentum=0.01):
+        super(ConvBlockRes, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv2d(
+                in_channels=in_channels,
+                out_channels=out_channels,
+                kernel_size=(3, 3),
+                stride=(1, 1),
+                padding=(1, 1),
+                bias=False,
+            ),
+            nn.BatchNorm2d(out_channels, momentum=momentum),
+            nn.ReLU(),
+            nn.Conv2d(
+                in_channels=out_channels,
+                out_channels=out_channels,
+                kernel_size=(3, 3),
+                stride=(1, 1),
+                padding=(1, 1),
+                bias=False,
+            ),
+            nn.BatchNorm2d(out_channels, momentum=momentum),
+            nn.ReLU(),
+        )
+        if in_channels != out_channels:
+            self.shortcut = nn.Conv2d(in_channels, out_channels, (1, 1))
+            self.is_shortcut = True
+        else:
+            self.is_shortcut = False
+
+    def forward(self, x):
+        if self.is_shortcut:
+            return self.conv(x) + self.shortcut(x)
+        else:
+            return self.conv(x) + x
+
+
+# Define a class for residual encoder blocks
+class ResEncoderBlock(nn.Module):
+    """
+    A residual encoder block.
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        kernel_size (tuple): Size of the average pooling kernel.
+        n_blocks (int): Number of convolutional blocks in the block.
+        momentum (float): Momentum for batch normalization.
+    """
+
+    def __init__(
+        self, in_channels, out_channels, kernel_size, n_blocks=1, momentum=0.01
+    ):
+        super(ResEncoderBlock, self).__init__()
+        self.n_blocks = n_blocks
+        self.conv = nn.ModuleList()
+        self.conv.append(ConvBlockRes(in_channels, out_channels, momentum))
+        for _ in range(n_blocks - 1):
+            self.conv.append(ConvBlockRes(out_channels, out_channels, momentum))
+        self.kernel_size = kernel_size
+        if self.kernel_size is not None:
+            self.pool = nn.AvgPool2d(kernel_size=kernel_size)
+
+    def forward(self, x):
+        for i in range(self.n_blocks):
+            x = self.conv[i](x)
+        if self.kernel_size is not None:
+            return x, self.pool(x)
+        else:
+            return x
+
+
+# Define a class for the encoder
+class Encoder(nn.Module):
+    """
+    The encoder part of the DeepUnet.
+
+    Args:
+        in_channels (int): Number of input channels.
+        in_size (int): Size of the input tensor.
+        n_encoders (int): Number of encoder blocks.
+        kernel_size (tuple): Size of the average pooling kernel.
+        n_blocks (int): Number of convolutional blocks in each encoder block.
+        out_channels (int): Number of output channels for the first encoder block.
+        momentum (float): Momentum for batch normalization.
+    """
+
+    def __init__(
+        self,
+        in_channels,
+        in_size,
+        n_encoders,
+        kernel_size,
+        n_blocks,
+        out_channels=16,
+        momentum=0.01,
+    ):
+        super(Encoder, self).__init__()
+        self.n_encoders = n_encoders
+        self.bn = nn.BatchNorm2d(in_channels, momentum=momentum)
+        self.layers = nn.ModuleList()
+        self.latent_channels = []
+        for i in range(self.n_encoders):
+            self.layers.append(
+                ResEncoderBlock(
+                    in_channels, out_channels, kernel_size, n_blocks, momentum=momentum
+                )
+            )
+            self.latent_channels.append([out_channels, in_size])
+            in_channels = out_channels
+            out_channels *= 2
+            in_size //= 2
+        self.out_size = in_size
+        self.out_channel = out_channels
+
+    def forward(self, x: torch.Tensor):
+        concat_tensors: List[torch.Tensor] = []
+        x = self.bn(x)
+        for i in range(self.n_encoders):
+            t, x = self.layers[i](x)
+            concat_tensors.append(t)
+        return x, concat_tensors
+
+
+# Define a class for the intermediate layer
+class Intermediate(nn.Module):
+    """
+    The intermediate layer of the DeepUnet.
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        n_inters (int): Number of convolutional blocks in the intermediate layer.
+        n_blocks (int): Number of convolutional blocks in each intermediate block.
+        momentum (float): Momentum for batch normalization.
+    """
+
+    def __init__(self, in_channels, out_channels, n_inters, n_blocks, momentum=0.01):
+        super(Intermediate, self).__init__()
+        self.n_inters = n_inters
+        self.layers = nn.ModuleList()
+        self.layers.append(
+            ResEncoderBlock(in_channels, out_channels, None, n_blocks, momentum)
+        )
+        for _ in range(self.n_inters - 1):
+            self.layers.append(
+                ResEncoderBlock(out_channels, out_channels, None, n_blocks, momentum)
+            )
+
+    def forward(self, x):
+        for i in range(self.n_inters):
+            x = self.layers[i](x)
+        return x
+
+
+# Define a class for residual decoder blocks
+class ResDecoderBlock(nn.Module):
+    """
+    A residual decoder block.
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        stride (tuple): Stride for transposed convolution.
+        n_blocks (int): Number of convolutional blocks in the block.
+        momentum (float): Momentum for batch normalization.
+    """
+
+    def __init__(self, in_channels, out_channels, stride, n_blocks=1, momentum=0.01):
+        super(ResDecoderBlock, self).__init__()
+        out_padding = (0, 1) if stride == (1, 2) else (1, 1)
+        self.n_blocks = n_blocks
+        self.conv1 = nn.Sequential(
+            nn.ConvTranspose2d(
+                in_channels=in_channels,
+                out_channels=out_channels,
+                kernel_size=(3, 3),
+                stride=stride,
+                padding=(1, 1),
+                output_padding=out_padding,
+                bias=False,
+            ),
+            nn.BatchNorm2d(out_channels, momentum=momentum),
+            nn.ReLU(),
+        )
+        self.conv2 = nn.ModuleList()
+        self.conv2.append(ConvBlockRes(out_channels * 2, out_channels, momentum))
+        for _ in range(n_blocks - 1):
+            self.conv2.append(ConvBlockRes(out_channels, out_channels, momentum))
+
+    def forward(self, x, concat_tensor):
+        x = self.conv1(x)
+        x = torch.cat((x, concat_tensor), dim=1)
+        for i in range(self.n_blocks):
+            x = self.conv2[i](x)
+        return x
+
+
+# Define a class for the decoder
+class Decoder(nn.Module):
+    """
+    The decoder part of the DeepUnet.
+
+    Args:
+        in_channels (int): Number of input channels.
+        n_decoders (int): Number of decoder blocks.
+        stride (tuple): Stride for transposed convolution.
+        n_blocks (int): Number of convolutional blocks in each decoder block.
+        momentum (float): Momentum for batch normalization.
+    """
+
+    def __init__(self, in_channels, n_decoders, stride, n_blocks, momentum=0.01):
+        super(Decoder, self).__init__()
+        self.layers = nn.ModuleList()
+        self.n_decoders = n_decoders
+        for _ in range(self.n_decoders):
+            out_channels = in_channels // 2
+            self.layers.append(
+                ResDecoderBlock(in_channels, out_channels, stride, n_blocks, momentum)
+            )
+            in_channels = out_channels
+
+    def forward(self, x, concat_tensors):
+        for i in range(self.n_decoders):
+            x = self.layers[i](x, concat_tensors[-1 - i])
+        return x
+
+
+# Define a class for the DeepUnet architecture
+class DeepUnet(nn.Module):
+    """
+    The DeepUnet architecture.
+
+    Args:
+        kernel_size (tuple): Size of the average pooling kernel.
+        n_blocks (int): Number of convolutional blocks in each encoder/decoder block.
+        en_de_layers (int): Number of encoder/decoder layers.
+        inter_layers (int): Number of convolutional blocks in the intermediate layer.
+        in_channels (int): Number of input channels.
+        en_out_channels (int): Number of output channels for the first encoder block.
+    """
+
+    def __init__(
+        self,
+        kernel_size,
+        n_blocks,
+        en_de_layers=5,
+        inter_layers=4,
+        in_channels=1,
+        en_out_channels=16,
+    ):
+        super(DeepUnet, self).__init__()
+        self.encoder = Encoder(
+            in_channels, 128, en_de_layers, kernel_size, n_blocks, en_out_channels
+        )
+        self.intermediate = Intermediate(
+            self.encoder.out_channel // 2,
+            self.encoder.out_channel,
+            inter_layers,
+            n_blocks,
+        )
+        self.decoder = Decoder(
+            self.encoder.out_channel, en_de_layers, kernel_size, n_blocks
+        )
+
+    def forward(self, x):
+        x, concat_tensors = self.encoder(x)
+        x = self.intermediate(x)
+        x = self.decoder(x, concat_tensors)
+        return x
+
+
+# Define a class for the end-to-end model
+class E2E(nn.Module):
+    """
+    The end-to-end model.
+
+    Args:
+        n_blocks (int): Number of convolutional blocks in each encoder/decoder block.
+        n_gru (int): Number of GRU layers.
+        kernel_size (tuple): Size of the average pooling kernel.
+        en_de_layers (int): Number of encoder/decoder layers.
+        inter_layers (int): Number of convolutional blocks in the intermediate layer.
+        in_channels (int): Number of input channels.
+        en_out_channels (int): Number of output channels for the first encoder block.
+    """
+
+    def __init__(
+        self,
+        n_blocks,
+        n_gru,
+        kernel_size,
+        en_de_layers=5,
+        inter_layers=4,
+        in_channels=1,
+        en_out_channels=16,
+    ):
+        super(E2E, self).__init__()
+        self.unet = DeepUnet(
+            kernel_size,
+            n_blocks,
+            en_de_layers,
+            inter_layers,
+            in_channels,
+            en_out_channels,
+        )
+        self.cnn = nn.Conv2d(en_out_channels, 3, (3, 3), padding=(1, 1))
+        if n_gru:
+            self.fc = nn.Sequential(
+                BiGRU(3 * 128, 256, n_gru),
+                nn.Linear(512, N_CLASS),
+                nn.Dropout(0.25),
+                nn.Sigmoid(),
+            )
+        else:
+            self.fc = nn.Sequential(
+                nn.Linear(3 * N_MELS, N_CLASS), nn.Dropout(0.25), nn.Sigmoid()
+            )
+
+    def forward(self, mel):
+        mel = mel.transpose(-1, -2).unsqueeze(1)
+        x = self.cnn(self.unet(mel)).transpose(1, 2).flatten(-2)
+        x = self.fc(x)
+        return x
+
+
+# Define a class for the MelSpectrogram extractor
+class MelSpectrogram(torch.nn.Module):
+    """
+    Extracts Mel-spectrogram features from audio.
+
+    Args:
+        is_half (bool): Whether to use half-precision floating-point numbers.
+        n_mel_channels (int): Number of Mel-frequency bands.
+        sample_rate (int): Sampling rate of the audio.
+        win_length (int): Length of the window function in samples.
+        hop_length (int): Hop size between frames in samples.
+        n_fft (int, optional): Length of the FFT window. Defaults to None, which uses win_length.
+        mel_fmin (int, optional): Minimum frequency for the Mel filter bank. Defaults to 0.
+        mel_fmax (int, optional): Maximum frequency for the Mel filter bank. Defaults to None.
+        clamp (float, optional): Minimum value for clamping the Mel-spectrogram. Defaults to 1e-5.
+    """
+
+    def __init__(
+        self,
+        is_half,
+        n_mel_channels,
+        sample_rate,
+        win_length,
+        hop_length,
+        n_fft=None,
+        mel_fmin=0,
+        mel_fmax=None,
+        clamp=1e-5,
+    ):
+        super().__init__()
+        n_fft = win_length if n_fft is None else n_fft
+        self.hann_window = {}
+        mel_basis = mel(
+            sr=sample_rate,
+            n_fft=n_fft,
+            n_mels=n_mel_channels,
+            fmin=mel_fmin,
+            fmax=mel_fmax,
+            htk=True,
+        )
+        mel_basis = torch.from_numpy(mel_basis).float()
+        self.register_buffer("mel_basis", mel_basis)
+        self.n_fft = win_length if n_fft is None else n_fft
+        self.hop_length = hop_length
+        self.win_length = win_length
+        self.sample_rate = sample_rate
+        self.n_mel_channels = n_mel_channels
+        self.clamp = clamp
+        self.is_half = is_half
+
+    def forward(self, audio, keyshift=0, speed=1, center=True):
+        factor = 2 ** (keyshift / 12)
+        n_fft_new = int(np.round(self.n_fft * factor))
+        win_length_new = int(np.round(self.win_length * factor))
+        hop_length_new = int(np.round(self.hop_length * speed))
+        keyshift_key = str(keyshift) + "_" + str(audio.device)
+        if keyshift_key not in self.hann_window:
+            self.hann_window[keyshift_key] = torch.hann_window(win_length_new).to(
+                audio.device
+            )
+        fft = torch.stft(
+            audio,
+            n_fft=n_fft_new,
+            hop_length=hop_length_new,
+            win_length=win_length_new,
+            window=self.hann_window[keyshift_key],
+            center=center,
+            return_complex=True,
+        )
+
+        magnitude = torch.sqrt(fft.real.pow(2) + fft.imag.pow(2))
+        if keyshift != 0:
+            size = self.n_fft // 2 + 1
+            resize = magnitude.size(1)
+            if resize < size:
+                magnitude = F.pad(magnitude, (0, 0, 0, size - resize))
+            magnitude = magnitude[:, :size, :] * self.win_length / win_length_new
+        mel_output = torch.matmul(self.mel_basis, magnitude)
+        if self.is_half:
+            mel_output = mel_output.half()
+        log_mel_spec = torch.log(torch.clamp(mel_output, min=self.clamp))
+        return log_mel_spec
+
+
+# Define a class for the RMVPE0 predictor
+class RMVPE0Predictor:
+    """
+    A predictor for fundamental frequency (F0) based on the RMVPE0 model.
+
+    Args:
+        model_path (str): Path to the RMVPE0 model file.
+        is_half (bool): Whether to use half-precision floating-point numbers.
+        device (str, optional): Device to use for computation. Defaults to None, which uses CUDA if available.
+    """
+
+    def __init__(self, model_path, is_half, device=None):
+        self.resample_kernel = {}
+        model = E2E(4, 1, (2, 2))
+        ckpt = torch.load(model_path, map_location="cpu")
+        model.load_state_dict(ckpt)
+        model.eval()
+        if is_half:
+            model = model.half()
+        self.model = model
+        self.resample_kernel = {}
+        self.is_half = is_half
+        self.device = device
+        self.mel_extractor = MelSpectrogram(
+            is_half, N_MELS, 16000, 1024, 160, None, 30, 8000
+        ).to(device)
+        self.model = self.model.to(device)
+        cents_mapping = 20 * np.arange(N_CLASS) + 1997.3794084376191
+        self.cents_mapping = np.pad(cents_mapping, (4, 4))
+
+    def mel2hidden(self, mel):
+        """
+        Converts Mel-spectrogram features to hidden representation.
+
+        Args:
+            mel (torch.Tensor): Mel-spectrogram features.
+        """
+        with torch.no_grad():
+            n_frames = mel.shape[-1]
+            mel = F.pad(
+                mel, (0, 32 * ((n_frames - 1) // 32 + 1) - n_frames), mode="reflect"
+            )
+            hidden = self.model(mel)
+            return hidden[:, :n_frames]
+
+    def decode(self, hidden, thred=0.03):
+        """
+        Decodes hidden representation to F0.
+
+        Args:
+            hidden (np.ndarray): Hidden representation.
+            thred (float, optional): Threshold for salience. Defaults to 0.03.
+        """
+        cents_pred = self.to_local_average_cents(hidden, thred=thred)
+        f0 = 10 * (2 ** (cents_pred / 1200))
+        f0[f0 == 10] = 0
+        return f0
+
+    def infer_from_audio(self, audio, thred=0.03):
+        """
+        Infers F0 from audio.
+
+        Args:
+            audio (np.ndarray): Audio signal.
+            thred (float, optional): Threshold for salience. Defaults to 0.03.
+        """
+        audio = torch.from_numpy(audio).float().to(self.device).unsqueeze(0)
+        mel = self.mel_extractor(audio, center=True)
+        hidden = self.mel2hidden(mel)
+        hidden = hidden.squeeze(0).cpu().numpy()
+        if self.is_half == True:
+            hidden = hidden.astype("float32")
+        f0 = self.decode(hidden, thred=thred)
+        return f0
+
+    def to_local_average_cents(self, salience, thred=0.05):
+        """
+        Converts salience to local average cents.
+
+        Args:
+            salience (np.ndarray): Salience values.
+            thred (float, optional): Threshold for salience. Defaults to 0.05.
+        """
+        center = np.argmax(salience, axis=1)
+        salience = np.pad(salience, ((0, 0), (4, 4)))
+        center += 4
+        todo_salience = []
+        todo_cents_mapping = []
+        starts = center - 4
+        ends = center + 5
+        for idx in range(salience.shape[0]):
+            todo_salience.append(salience[:, starts[idx] : ends[idx]][idx])
+            todo_cents_mapping.append(self.cents_mapping[starts[idx] : ends[idx]])
+        todo_salience = np.array(todo_salience)
+        todo_cents_mapping = np.array(todo_cents_mapping)
+        product_sum = np.sum(todo_salience * todo_cents_mapping, 1)
+        weight_sum = np.sum(todo_salience, 1)
+        devided = product_sum / weight_sum
+        maxx = np.max(salience, axis=1)
+        devided[maxx <= thred] = 0
+        return devided
+
+
+# Define a class for BiGRU (bidirectional GRU)
+class BiGRU(nn.Module):
+    """
+    A bidirectional GRU layer.
+
+    Args:
+        input_features (int): Number of input features.
+        hidden_features (int): Number of hidden features.
+        num_layers (int): Number of GRU layers.
+    """
+
+    def __init__(self, input_features, hidden_features, num_layers):
+        super(BiGRU, self).__init__()
+        self.gru = nn.GRU(
+            input_features,
+            hidden_features,
+            num_layers=num_layers,
+            batch_first=True,
+            bidirectional=True,
+        )
+
+    def forward(self, x):
+        return self.gru(x)[0]

__init__.py

@@ -0,0 +1 @@
+

attentions.py

@@ -0,0 +1,243 @@
+import math
+import torch
+from rvc_cli.rvc.lib.algorithm.commons import convert_pad_shape
+
+
+class MultiHeadAttention(torch.nn.Module):
+    """
+    Multi-head attention module with optional relative positional encoding and proximal bias.
+
+    Args:
+        channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        n_heads (int): Number of attention heads.
+        p_dropout (float, optional): Dropout probability. Defaults to 0.0.
+        window_size (int, optional): Window size for relative positional encoding. Defaults to None.
+        heads_share (bool, optional): Whether to share relative positional embeddings across heads. Defaults to True.
+        block_length (int, optional): Block length for local attention. Defaults to None.
+        proximal_bias (bool, optional): Whether to use proximal bias in self-attention. Defaults to False.
+        proximal_init (bool, optional): Whether to initialize the key projection weights the same as query projection weights. Defaults to False.
+    """
+
+    def __init__(
+        self,
+        channels,
+        out_channels,
+        n_heads,
+        p_dropout=0.0,
+        window_size=None,
+        heads_share=True,
+        block_length=None,
+        proximal_bias=False,
+        proximal_init=False,
+    ):
+        super().__init__()
+        assert (
+            channels % n_heads == 0
+        ), "Channels must be divisible by the number of heads."
+
+        self.channels = channels
+        self.out_channels = out_channels
+        self.n_heads = n_heads
+        self.k_channels = channels // n_heads
+        self.window_size = window_size
+        self.block_length = block_length
+        self.proximal_bias = proximal_bias
+
+        # Define projections
+        self.conv_q = torch.nn.Conv1d(channels, channels, 1)
+        self.conv_k = torch.nn.Conv1d(channels, channels, 1)
+        self.conv_v = torch.nn.Conv1d(channels, channels, 1)
+        self.conv_o = torch.nn.Conv1d(channels, out_channels, 1)
+
+        self.drop = torch.nn.Dropout(p_dropout)
+
+        # Relative positional encodings
+        if window_size:
+            n_heads_rel = 1 if heads_share else n_heads
+            rel_stddev = self.k_channels**-0.5
+            self.emb_rel_k = torch.nn.Parameter(
+                torch.randn(n_heads_rel, 2 * window_size + 1, self.k_channels)
+                * rel_stddev
+            )
+            self.emb_rel_v = torch.nn.Parameter(
+                torch.randn(n_heads_rel, 2 * window_size + 1, self.k_channels)
+                * rel_stddev
+            )
+
+        # Initialize weights
+        torch.nn.init.xavier_uniform_(self.conv_q.weight)
+        torch.nn.init.xavier_uniform_(self.conv_k.weight)
+        torch.nn.init.xavier_uniform_(self.conv_v.weight)
+        torch.nn.init.xavier_uniform_(self.conv_o.weight)
+
+        if proximal_init:
+            with torch.no_grad():
+                self.conv_k.weight.copy_(self.conv_q.weight)
+                self.conv_k.bias.copy_(self.conv_q.bias)
+
+    def forward(self, x, c, attn_mask=None):
+        # Compute query, key, value projections
+        q, k, v = self.conv_q(x), self.conv_k(c), self.conv_v(c)
+
+        # Compute attention
+        x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+        # Final output projection
+        return self.conv_o(x)
+
+    def attention(self, query, key, value, mask=None):
+        # Reshape and compute scaled dot-product attention
+        b, d, t_s, t_t = (*key.size(), query.size(2))
+        query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
+        key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+        value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+
+        scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
+
+        if self.window_size:
+            assert t_s == t_t, "Relative attention only supports self-attention."
+            scores += self._compute_relative_scores(query, t_s)
+
+        if self.proximal_bias:
+            assert t_s == t_t, "Proximal bias only supports self-attention."
+            scores += self._attention_bias_proximal(t_s).to(scores.device, scores.dtype)
+
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e4)
+            if self.block_length:
+                block_mask = (
+                    torch.ones_like(scores)
+                    .triu(-self.block_length)
+                    .tril(self.block_length)
+                )
+                scores = scores.masked_fill(block_mask == 0, -1e4)
+
+        # Apply softmax and dropout
+        p_attn = self.drop(torch.nn.functional.softmax(scores, dim=-1))
+
+        # Compute attention output
+        output = torch.matmul(p_attn, value)
+
+        if self.window_size:
+            output += self._apply_relative_values(p_attn, t_s)
+
+        return output.transpose(2, 3).contiguous().view(b, d, t_t), p_attn
+
+    def _compute_relative_scores(self, query, length):
+        rel_emb = self._get_relative_embeddings(self.emb_rel_k, length)
+        rel_logits = self._matmul_with_relative_keys(
+            query / math.sqrt(self.k_channels), rel_emb
+        )
+        return self._relative_position_to_absolute_position(rel_logits)
+
+    def _apply_relative_values(self, p_attn, length):
+        rel_weights = self._absolute_position_to_relative_position(p_attn)
+        rel_emb = self._get_relative_embeddings(self.emb_rel_v, length)
+        return self._matmul_with_relative_values(rel_weights, rel_emb)
+
+    # Helper methods
+    def _matmul_with_relative_values(self, x, y):
+        return torch.matmul(x, y.unsqueeze(0))
+
+    def _matmul_with_relative_keys(self, x, y):
+        return torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+
+    def _get_relative_embeddings(self, embeddings, length):
+        pad_length = max(length - (self.window_size + 1), 0)
+        start = max((self.window_size + 1) - length, 0)
+        end = start + 2 * length - 1
+
+        if pad_length > 0:
+            embeddings = torch.nn.functional.pad(
+                embeddings,
+                convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
+            )
+        return embeddings[:, start:end]
+
+    def _relative_position_to_absolute_position(self, x):
+        batch, heads, length, _ = x.size()
+        x = torch.nn.functional.pad(
+            x, convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]])
+        )
+        x_flat = x.view(batch, heads, length * 2 * length)
+        x_flat = torch.nn.functional.pad(
+            x_flat, convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
+        )
+        return x_flat.view(batch, heads, length + 1, 2 * length - 1)[
+            :, :, :length, length - 1 :
+        ]
+
+    def _absolute_position_to_relative_position(self, x):
+        batch, heads, length, _ = x.size()
+        x = torch.nn.functional.pad(
+            x, convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
+        )
+        x_flat = x.view(batch, heads, length**2 + length * (length - 1))
+        x_flat = torch.nn.functional.pad(
+            x_flat, convert_pad_shape([[0, 0], [0, 0], [length, 0]])
+        )
+        return x_flat.view(batch, heads, length, 2 * length)[:, :, :, 1:]
+
+    def _attention_bias_proximal(self, length):
+        r = torch.arange(length, dtype=torch.float32)
+        diff = r.unsqueeze(0) - r.unsqueeze(1)
+        return -torch.log1p(torch.abs(diff)).unsqueeze(0).unsqueeze(0)
+
+
+class FFN(torch.nn.Module):
+    """
+    Feed-forward network module.
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        filter_channels (int): Number of filter channels in the convolution layers.
+        kernel_size (int): Kernel size of the convolution layers.
+        p_dropout (float, optional): Dropout probability. Defaults to 0.0.
+        activation (str, optional): Activation function to use. Defaults to None.
+        causal (bool, optional): Whether to use causal padding in the convolution layers. Defaults to False.
+    """
+
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        filter_channels,
+        kernel_size,
+        p_dropout=0.0,
+        activation=None,
+        causal=False,
+    ):
+        super().__init__()
+        self.padding_fn = self._causal_padding if causal else self._same_padding
+
+        self.conv_1 = torch.nn.Conv1d(in_channels, filter_channels, kernel_size)
+        self.conv_2 = torch.nn.Conv1d(filter_channels, out_channels, kernel_size)
+        self.drop = torch.nn.Dropout(p_dropout)
+
+        self.activation = activation
+
+    def forward(self, x, x_mask):
+        x = self.conv_1(self.padding_fn(x * x_mask))
+        x = self._apply_activation(x)
+        x = self.drop(x)
+        x = self.conv_2(self.padding_fn(x * x_mask))
+        return x * x_mask
+
+    def _apply_activation(self, x):
+        if self.activation == "gelu":
+            return x * torch.sigmoid(1.702 * x)
+        return torch.relu(x)
+
+    def _causal_padding(self, x):
+        pad_l, pad_r = self.conv_1.kernel_size[0] - 1, 0
+        return torch.nn.functional.pad(
+            x, convert_pad_shape([[0, 0], [0, 0], [pad_l, pad_r]])
+        )
+
+    def _same_padding(self, x):
+        pad = (self.conv_1.kernel_size[0] - 1) // 2
+        return torch.nn.functional.pad(
+            x, convert_pad_shape([[0, 0], [0, 0], [pad, pad]])
+        )

commons.py

@@ -0,0 +1,207 @@
+import math
+import torch
+from typing import List, Optional
+
+
+def init_weights(m, mean=0.0, std=0.01):
+    """
+    Initialize the weights of a module.
+
+    Args:
+        m: The module to initialize.
+        mean: The mean of the normal distribution.
+        std: The standard deviation of the normal distribution.
+    """
+    classname = m.__class__.__name__
+    if classname.find("Conv") != -1:
+        m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size, dilation=1):
+    """
+    Calculate the padding needed for a convolution.
+
+    Args:
+        kernel_size: The size of the kernel.
+        dilation: The dilation of the convolution.
+    """
+    return int((kernel_size * dilation - dilation) / 2)
+
+
+def convert_pad_shape(pad_shape):
+    """
+    Convert the pad shape to a list of integers.
+
+    Args:
+        pad_shape: The pad shape..
+    """
+    l = pad_shape[::-1]
+    pad_shape = [item for sublist in l for item in sublist]
+    return pad_shape
+
+
+def kl_divergence(m_p, logs_p, m_q, logs_q):
+    """
+    Calculate the KL divergence between two distributions.
+
+    Args:
+        m_p: The mean of the first distribution.
+        logs_p: The log of the standard deviation of the first distribution.
+        m_q: The mean of the second distribution.
+        logs_q: The log of the standard deviation of the second distribution.
+    """
+    kl = (logs_q - logs_p) - 0.5
+    kl += (
+        0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q)
+    )
+    return kl
+
+
+def slice_segments(
+    x: torch.Tensor, ids_str: torch.Tensor, segment_size: int = 4, dim: int = 2
+):
+    """
+    Slice segments from a tensor, handling tensors with different numbers of dimensions.
+
+    Args:
+        x (torch.Tensor): The tensor to slice.
+        ids_str (torch.Tensor): The starting indices of the segments.
+        segment_size (int, optional): The size of each segment. Defaults to 4.
+        dim (int, optional): The dimension to slice across (2D or 3D tensors). Defaults to 2.
+    """
+    if dim == 2:
+        ret = torch.zeros_like(x[:, :segment_size])
+    elif dim == 3:
+        ret = torch.zeros_like(x[:, :, :segment_size])
+
+    for i in range(x.size(0)):
+        idx_str = ids_str[i].item()
+        idx_end = idx_str + segment_size
+        if dim == 2:
+            ret[i] = x[i, idx_str:idx_end]
+        else:
+            ret[i] = x[i, :, idx_str:idx_end]
+
+    return ret
+
+
+def rand_slice_segments(x, x_lengths=None, segment_size=4):
+    """
+    Randomly slice segments from a tensor.
+
+    Args:
+        x: The tensor to slice.
+        x_lengths: The lengths of the sequences.
+        segment_size: The size of each segment.
+    """
+    b, d, t = x.size()
+    if x_lengths is None:
+        x_lengths = t
+    ids_str_max = x_lengths - segment_size + 1
+    ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
+    ret = slice_segments(x, ids_str, segment_size, dim=3)
+    return ret, ids_str
+
+
+def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
+    """
+    Generate a 1D timing signal.
+
+    Args:
+        length: The length of the signal.
+        channels: The number of channels of the signal.
+        min_timescale: The minimum timescale.
+        max_timescale: The maximum timescale.
+    """
+    position = torch.arange(length, dtype=torch.float)
+    num_timescales = channels // 2
+    log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / (
+        num_timescales - 1
+    )
+    inv_timescales = min_timescale * torch.exp(
+        torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment
+    )
+    scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
+    signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
+    signal = torch.nn.functional.pad(signal, [0, 0, 0, channels % 2])
+    signal = signal.view(1, channels, length)
+    return signal
+
+
+def subsequent_mask(length):
+    """
+    Generate a subsequent mask.
+
+    Args:
+        length: The length of the sequence.
+    """
+    mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+    return mask
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+    """
+    Fused add tanh sigmoid multiply operation.
+
+    Args:
+        input_a: The first input tensor.
+        input_b: The second input tensor.
+        n_channels: The number of channels.
+    """
+    n_channels_int = n_channels[0]
+    in_act = input_a + input_b
+    t_act = torch.tanh(in_act[:, :n_channels_int, :])
+    s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+    acts = t_act * s_act
+    return acts
+
+
+def convert_pad_shape(pad_shape: List[List[int]]) -> List[int]:
+    """
+    Convert the pad shape to a list of integers.
+
+    Args:
+        pad_shape: The pad shape.
+    """
+    return torch.tensor(pad_shape).flip(0).reshape(-1).int().tolist()
+
+
+def sequence_mask(length: torch.Tensor, max_length: Optional[int] = None):
+    """
+    Generate a sequence mask.
+
+    Args:
+        length: The lengths of the sequences.
+        max_length: The maximum length of the sequences.
+    """
+    if max_length is None:
+        max_length = length.max()
+    x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+    return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def clip_grad_value(parameters, clip_value, norm_type=2):
+    """
+    Clip the gradients of a list of parameters.
+
+    Args:
+        parameters: The list of parameters to clip.
+        clip_value: The maximum value of the gradients.
+        norm_type: The type of norm to use for clipping.
+    """
+    if isinstance(parameters, torch.Tensor):
+        parameters = [parameters]
+    parameters = list(filter(lambda p: p.grad is not None, parameters))
+    norm_type = float(norm_type)
+    if clip_value is not None:
+        clip_value = float(clip_value)
+
+    total_norm = 0
+    for p in parameters:
+        param_norm = p.grad.data.norm(norm_type)
+        total_norm += param_norm.item() ** norm_type
+        if clip_value is not None:
+            p.grad.data.clamp_(min=-clip_value, max=clip_value)
+    total_norm = total_norm ** (1.0 / norm_type)
+    return total_norm

config.py

@@ -0,0 +1,159 @@
+import torch
+import json
+import os
+
+CONFIG_BASE_PATH = os.path.join("rvc_cli", "rvc", "configs")
+
+version_config_paths = [
+    os.path.join("v1", "32000.json"),
+    os.path.join("v1", "40000.json"),
+    os.path.join("v1", "48000.json"),
+    os.path.join("v2", "48000.json"),
+    os.path.join("v2", "40000.json"),
+    os.path.join("v2", "32000.json"),
+]
+
+
+def singleton(cls):
+    instances = {}
+
+    def get_instance(*args, **kwargs):
+        if cls not in instances:
+            instances[cls] = cls(*args, **kwargs)
+        return instances[cls]
+
+    return get_instance
+
+
+@singleton
+class Config:
+    def __init__(self):
+        self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        self.is_half = self.device.startswith("cuda")
+        self.gpu_name = (
+            torch.cuda.get_device_name(int(self.device.split(":")[-1]))
+            if self.device.startswith("cuda")
+            else None
+        )
+        self.json_config = self.load_config_json()
+        self.gpu_mem = None
+        self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config()
+
+    def load_config_json(self) -> dict:
+        configs = {}
+        for config_file in version_config_paths:
+            config_path = os.path.join(CONFIG_BASE_PATH, config_file)
+
+            if not os.path.exists(config_path):
+                print(f"[WARNING] Config file not found: {config_path}")
+                continue  # Skip missing config files
+
+            try:
+                with open(config_path, "r") as f:
+                    configs[config_file] = json.load(f)
+            except json.JSONDecodeError:
+                print(f"[ERROR] Failed to parse JSON in {config_path}")
+
+        return configs
+
+    def has_mps(self) -> bool:
+        return torch.backends.mps.is_available()
+
+    def has_xpu(self) -> bool:
+        return hasattr(torch, "xpu") and torch.xpu.is_available()
+
+    def set_precision(self, precision):
+        if precision not in ["fp32", "fp16"]:
+            raise ValueError("Invalid precision type. Must be 'fp32' or 'fp16'.")
+
+        fp16_run_value = precision == "fp16"
+        for config_path in version_config_paths:
+            full_config_path = os.path.join(CONFIG_BASE_PATH, config_path)
+            if not os.path.exists(full_config_path):
+                print(f"[WARNING] Config file missing: {full_config_path}")
+                continue
+
+            try:
+                with open(full_config_path, "r") as f:
+                    config = json.load(f)
+                config["train"]["fp16_run"] = fp16_run_value
+                with open(full_config_path, "w") as f:
+                    json.dump(config, f, indent=4)
+            except (FileNotFoundError, json.JSONDecodeError):
+                print(f"[ERROR] Failed to update {full_config_path}")
+
+        return f"Set precision to {precision} in available config files."
+
+    def get_precision(self):
+        if not version_config_paths:
+            raise FileNotFoundError("No configuration paths provided.")
+
+        full_config_path = os.path.join(CONFIG_BASE_PATH, version_config_paths[0])
+        if not os.path.exists(full_config_path):
+            print(f"[ERROR] Config file missing: {full_config_path}")
+            return None
+
+        try:
+            with open(full_config_path, "r") as f:
+                config = json.load(f)
+            return "fp16" if config["train"].get("fp16_run", False) else "fp32"
+        except json.JSONDecodeError:
+            print(f"[ERROR] JSON parsing failed in {full_config_path}")
+            return None
+
+    def device_config(self) -> tuple:
+        if self.device.startswith("cuda"):
+            self.set_cuda_config()
+        elif self.has_mps():
+            self.device = "mps"
+            self.is_half = False
+            self.set_precision("fp32")
+        else:
+            self.device = "cpu"
+            self.is_half = False
+            self.set_precision("fp32")
+
+        x_pad, x_query, x_center, x_max = (
+            (3, 10, 60, 65) if self.is_half else (1, 6, 38, 41)
+        )
+        if self.gpu_mem is not None and self.gpu_mem <= 4:
+            x_pad, x_query, x_center, x_max = (1, 5, 30, 32)
+
+        return x_pad, x_query, x_center, x_max
+
+    def set_cuda_config(self):
+        i_device = int(self.device.split(":")[-1])
+        self.gpu_name = torch.cuda.get_device_name(i_device)
+        low_end_gpus = ["16", "P40", "P10", "1060", "1070", "1080"]
+        if (
+            any(gpu in self.gpu_name for gpu in low_end_gpus)
+            and "V100" not in self.gpu_name.upper()
+        ):
+            self.is_half = False
+            self.set_precision("fp32")
+
+        self.gpu_mem = torch.cuda.get_device_properties(i_device).total_memory // (
+            1024**3
+        )
+
+
+def max_vram_gpu(gpu):
+    if torch.cuda.is_available():
+        gpu_properties = torch.cuda.get_device_properties(gpu)
+        return round(gpu_properties.total_memory / 1024 / 1024 / 1024)
+    return 8
+
+
+def get_gpu_info():
+    ngpu = torch.cuda.device_count()
+    gpu_infos = []
+    if torch.cuda.is_available() or ngpu != 0:
+        for i in range(ngpu):
+            gpu_name = torch.cuda.get_device_name(i)
+            mem = int(torch.cuda.get_device_properties(i).total_memory / 1024**3 + 0.4)
+            gpu_infos.append(f"{i}: {gpu_name} ({mem} GB)")
+    return "\n".join(gpu_infos) if gpu_infos else "No compatible GPU found."
+
+
+def get_number_of_gpus():
+    return "-".join(map(str, range(torch.cuda.device_count()))) if torch.cuda.is_available() else "-"

core.py

@@ -0,0 +1,1519 @@
+import os
+import sys
+import json
+import argparse
+import subprocess
+from functools import lru_cache
+from distutils.util import strtobool
+from rvc_cli.rvc.lib.tools.model_download import model_download_pipeline
+from rvc_cli.rvc.lib.tools.prerequisites import prequisites_download_pipeline
+
+
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+
+current_script_directory = os.path.dirname(os.path.realpath(__file__))
+logs_path = os.path.join(current_script_directory, "logs")
+
+
+python = sys.executable
+
+
+
+@lru_cache(maxsize=None)
+def import_voice_converter():
+    from rvc_cli.rvc.infer.infer import VoiceConverter
+
+    return VoiceConverter()
+
+
+@lru_cache(maxsize=1)
+def get_config():
+    from rvc_cli.rvc.configs.config import Config
+
+    return Config()
+
+
+# Infer
+def run_infer_script(
+    pitch: int,
+    filter_radius: int,
+    index_rate: float,
+    volume_envelope: int,
+    protect: float,
+    hop_length: int,
+    f0_method: str,
+    input_path: str,
+    output_path: str,
+    pth_path: str,
+    index_path: str,
+    split_audio: bool,
+    f0_autotune: bool,
+    f0_autotune_strength: float,
+    clean_audio: bool,
+    clean_strength: float,
+    export_format: str,
+    f0_file: str,
+    embedder_model: str,
+    embedder_model_custom: str = None,
+    formant_shifting: bool = False,
+    formant_qfrency: float = 1.0,
+    formant_timbre: float = 1.0,
+    post_process: bool = False,
+    reverb: bool = False,
+    pitch_shift: bool = False,
+    limiter: bool = False,
+    gain: bool = False,
+    distortion: bool = False,
+    chorus: bool = False,
+    bitcrush: bool = False,
+    clipping: bool = False,
+    compressor: bool = False,
+    delay: bool = False,
+    reverb_room_size: float = 0.5,
+    reverb_damping: float = 0.5,
+    reverb_wet_gain: float = 0.5,
+    reverb_dry_gain: float = 0.5,
+    reverb_width: float = 0.5,
+    reverb_freeze_mode: float = 0.5,
+    pitch_shift_semitones: float = 0.0,
+    limiter_threshold: float = -6,
+    limiter_release_time: float = 0.01,
+    gain_db: float = 0.0,
+    distortion_gain: float = 25,
+    chorus_rate: float = 1.0,
+    chorus_depth: float = 0.25,
+    chorus_center_delay: float = 7,
+    chorus_feedback: float = 0.0,
+    chorus_mix: float = 0.5,
+    bitcrush_bit_depth: int = 8,
+    clipping_threshold: float = -6,
+    compressor_threshold: float = 0,
+    compressor_ratio: float = 1,
+    compressor_attack: float = 1.0,
+    compressor_release: float = 100,
+    delay_seconds: float = 0.5,
+    delay_feedback: float = 0.0,
+    delay_mix: float = 0.5,
+    sid: int = 0,
+):
+    kwargs = {
+        "audio_input_path": input_path,
+        "audio_output_path": output_path,
+        "model_path": pth_path,
+        "index_path": index_path,
+        "pitch": pitch,
+        "filter_radius": filter_radius,
+        "index_rate": index_rate,
+        "volume_envelope": volume_envelope,
+        "protect": protect,
+        "hop_length": hop_length,
+        "f0_method": f0_method,
+        "pth_path": pth_path,
+        "index_path": index_path,
+        "split_audio": split_audio,
+        "f0_autotune": f0_autotune,
+        "f0_autotune_strength": f0_autotune_strength,
+        "clean_audio": clean_audio,
+        "clean_strength": clean_strength,
+        "export_format": export_format,
+        "f0_file": f0_file,
+        "embedder_model": embedder_model,
+        "embedder_model_custom": embedder_model_custom,
+        "post_process": post_process,
+        "formant_shifting": formant_shifting,
+        "formant_qfrency": formant_qfrency,
+        "formant_timbre": formant_timbre,
+        "reverb": reverb,
+        "pitch_shift": pitch_shift,
+        "limiter": limiter,
+        "gain": gain,
+        "distortion": distortion,
+        "chorus": chorus,
+        "bitcrush": bitcrush,
+        "clipping": clipping,
+        "compressor": compressor,
+        "delay": delay,
+        "reverb_room_size": reverb_room_size,
+        "reverb_damping": reverb_damping,
+        "reverb_wet_level": reverb_wet_gain,
+        "reverb_dry_level": reverb_dry_gain,
+        "reverb_width": reverb_width,
+        "reverb_freeze_mode": reverb_freeze_mode,
+        "pitch_shift_semitones": pitch_shift_semitones,
+        "limiter_threshold": limiter_threshold,
+        "limiter_release": limiter_release_time,
+        "gain_db": gain_db,
+        "distortion_gain": distortion_gain,
+        "chorus_rate": chorus_rate,
+        "chorus_depth": chorus_depth,
+        "chorus_delay": chorus_center_delay,
+        "chorus_feedback": chorus_feedback,
+        "chorus_mix": chorus_mix,
+        "bitcrush_bit_depth": bitcrush_bit_depth,
+        "clipping_threshold": clipping_threshold,
+        "compressor_threshold": compressor_threshold,
+        "compressor_ratio": compressor_ratio,
+        "compressor_attack": compressor_attack,
+        "compressor_release": compressor_release,
+        "delay_seconds": delay_seconds,
+        "delay_feedback": delay_feedback,
+        "delay_mix": delay_mix,
+        "sid": sid,
+    }
+    infer_pipeline = import_voice_converter()
+    infer_pipeline.convert_audio(
+        **kwargs,
+    )
+    return f"File {input_path} inferred successfully.", output_path.replace(
+        ".wav", f".{export_format.lower()}"
+    )
+
+
+# Batch infer
+def run_batch_infer_script(
+    pitch: int,
+    filter_radius: int,
+    index_rate: float,
+    volume_envelope: int,
+    protect: float,
+    hop_length: int,
+    f0_method: str,
+    input_folder: str,
+    output_folder: str,
+    pth_path: str,
+    index_path: str,
+    split_audio: bool,
+    f0_autotune: bool,
+    f0_autotune_strength: float,
+    clean_audio: bool,
+    clean_strength: float,
+    export_format: str,
+    f0_file: str,
+    embedder_model: str,
+    embedder_model_custom: str = None,
+    formant_shifting: bool = False,
+    formant_qfrency: float = 1.0,
+    formant_timbre: float = 1.0,
+    post_process: bool = False,
+    reverb: bool = False,
+    pitch_shift: bool = False,
+    limiter: bool = False,
+    gain: bool = False,
+    distortion: bool = False,
+    chorus: bool = False,
+    bitcrush: bool = False,
+    clipping: bool = False,
+    compressor: bool = False,
+    delay: bool = False,
+    reverb_room_size: float = 0.5,
+    reverb_damping: float = 0.5,
+    reverb_wet_gain: float = 0.5,
+    reverb_dry_gain: float = 0.5,
+    reverb_width: float = 0.5,
+    reverb_freeze_mode: float = 0.5,
+    pitch_shift_semitones: float = 0.0,
+    limiter_threshold: float = -6,
+    limiter_release_time: float = 0.01,
+    gain_db: float = 0.0,
+    distortion_gain: float = 25,
+    chorus_rate: float = 1.0,
+    chorus_depth: float = 0.25,
+    chorus_center_delay: float = 7,
+    chorus_feedback: float = 0.0,
+    chorus_mix: float = 0.5,
+    bitcrush_bit_depth: int = 8,
+    clipping_threshold: float = -6,
+    compressor_threshold: float = 0,
+    compressor_ratio: float = 1,
+    compressor_attack: float = 1.0,
+    compressor_release: float = 100,
+    delay_seconds: float = 0.5,
+    delay_feedback: float = 0.0,
+    delay_mix: float = 0.5,
+    sid: int = 0,
+):
+    kwargs = {
+        "audio_input_paths": input_folder,
+        "audio_output_path": output_folder,
+        "model_path": pth_path,
+        "index_path": index_path,
+        "pitch": pitch,
+        "filter_radius": filter_radius,
+        "index_rate": index_rate,
+        "volume_envelope": volume_envelope,
+        "protect": protect,
+        "hop_length": hop_length,
+        "f0_method": f0_method,
+        "pth_path": pth_path,
+        "index_path": index_path,
+        "split_audio": split_audio,
+        "f0_autotune": f0_autotune,
+        "f0_autotune_strength": f0_autotune_strength,
+        "clean_audio": clean_audio,
+        "clean_strength": clean_strength,
+        "export_format": export_format,
+        "f0_file": f0_file,
+        "embedder_model": embedder_model,
+        "embedder_model_custom": embedder_model_custom,
+        "post_process": post_process,
+        "formant_shifting": formant_shifting,
+        "formant_qfrency": formant_qfrency,
+        "formant_timbre": formant_timbre,
+        "reverb": reverb,
+        "pitch_shift": pitch_shift,
+        "limiter": limiter,
+        "gain": gain,
+        "distortion": distortion,
+        "chorus": chorus,
+        "bitcrush": bitcrush,
+        "clipping": clipping,
+        "compressor": compressor,
+        "delay": delay,
+        "reverb_room_size": reverb_room_size,
+        "reverb_damping": reverb_damping,
+        "reverb_wet_level": reverb_wet_gain,
+        "reverb_dry_level": reverb_dry_gain,
+        "reverb_width": reverb_width,
+        "reverb_freeze_mode": reverb_freeze_mode,
+        "pitch_shift_semitones": pitch_shift_semitones,
+        "limiter_threshold": limiter_threshold,
+        "limiter_release": limiter_release_time,
+        "gain_db": gain_db,
+        "distortion_gain": distortion_gain,
+        "chorus_rate": chorus_rate,
+        "chorus_depth": chorus_depth,
+        "chorus_delay": chorus_center_delay,
+        "chorus_feedback": chorus_feedback,
+        "chorus_mix": chorus_mix,
+        "bitcrush_bit_depth": bitcrush_bit_depth,
+        "clipping_threshold": clipping_threshold,
+        "compressor_threshold": compressor_threshold,
+        "compressor_ratio": compressor_ratio,
+        "compressor_attack": compressor_attack,
+        "compressor_release": compressor_release,
+        "delay_seconds": delay_seconds,
+        "delay_feedback": delay_feedback,
+        "delay_mix": delay_mix,
+        "sid": sid,
+    }
+    infer_pipeline = import_voice_converter()
+    infer_pipeline.convert_audio_batch(
+        **kwargs,
+    )
+
+    return f"Files from {input_folder} inferred successfully."
+
+   
+
+# Download
+def run_download_script(model_link: str):
+    model_download_pipeline(model_link)
+    return f"Model downloaded successfully."
+
+        
+
+
+
+# Parse arguments
+def parse_arguments():
+    parser = argparse.ArgumentParser(
+        description="Run the main.py script with specific parameters."
+    )
+    subparsers = parser.add_subparsers(
+        title="subcommands", dest="mode", help="Choose a mode"
+    )
+
+    # Parser for 'infer' mode
+    infer_parser = subparsers.add_parser("infer", help="Run inference")
+    pitch_description = (
+        "Set the pitch of the audio. Higher values result in a higher pitch."
+    )
+    infer_parser.add_argument(
+        "--pitch",
+        type=int,
+        help=pitch_description,
+        choices=range(-24, 25),
+        default=0,
+    )
+    filter_radius_description = "Apply median filtering to the extracted pitch values if this value is greater than or equal to three. This can help reduce breathiness in the output audio."
+    infer_parser.add_argument(
+        "--filter_radius",
+        type=int,
+        help=filter_radius_description,
+        choices=range(11),
+        default=3,
+    )
+    index_rate_description = "Control the influence of the index file on the output. Higher values mean stronger influence. Lower values can help reduce artifacts but may result in less accurate voice cloning."
+    infer_parser.add_argument(
+        "--index_rate",
+        type=float,
+        help=index_rate_description,
+        choices=[i / 100.0 for i in range(0, 101)],
+        default=0.3,
+    )
+    volume_envelope_description = "Control the blending of the output's volume envelope. A value of 1 means the output envelope is fully used."
+    infer_parser.add_argument(
+        "--volume_envelope",
+        type=float,
+        help=volume_envelope_description,
+        choices=[i / 100.0 for i in range(0, 101)],
+        default=1,
+    )
+    protect_description = "Protect consonants and breathing sounds from artifacts. A value of 0.5 offers the strongest protection, while lower values may reduce the protection level but potentially mitigate the indexing effect."
+    infer_parser.add_argument(
+        "--protect",
+        type=float,
+        help=protect_description,
+        choices=[i / 1000.0 for i in range(0, 501)],
+        default=0.33,
+    )
+    hop_length_description = "Only applicable for the Crepe pitch extraction method. Determines the time it takes for the system to react to a significant pitch change. Smaller values require more processing time but can lead to better pitch accuracy."
+    infer_parser.add_argument(
+        "--hop_length",
+        type=int,
+        help=hop_length_description,
+        choices=range(1, 513),
+        default=128,
+    )
+    f0_method_description = "Choose the pitch extraction algorithm for the conversion. 'rmvpe' is the default and generally recommended."
+    infer_parser.add_argument(
+        "--f0_method",
+        type=str,
+        help=f0_method_description,
+        choices=[
+            "crepe",
+            "crepe-tiny",
+            "rmvpe",
+            "fcpe",
+            "hybrid[crepe+rmvpe]",
+            "hybrid[crepe+fcpe]",
+            "hybrid[rmvpe+fcpe]",
+            "hybrid[crepe+rmvpe+fcpe]",
+        ],
+        default="rmvpe",
+    )
+    infer_parser.add_argument(
+        "--input_path",
+        type=str,
+        help="Full path to the input audio file.",
+        required=True,
+    )
+    infer_parser.add_argument(
+        "--output_path",
+        type=str,
+        help="Full path to the output audio file.",
+        required=True,
+    )
+    pth_path_description = "Full path to the RVC model file (.pth)."
+    infer_parser.add_argument(
+        "--pth_path", type=str, help=pth_path_description, required=True
+    )
+    index_path_description = "Full path to the index file (.index)."
+    infer_parser.add_argument(
+        "--index_path", type=str, help=index_path_description, required=True
+    )
+    split_audio_description = "Split the audio into smaller segments before inference. This can improve the quality of the output for longer audio files."
+    infer_parser.add_argument(
+        "--split_audio",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=split_audio_description,
+        default=False,
+    )
+    f0_autotune_description = "Apply a light autotune to the inferred audio. Particularly useful for singing voice conversions."
+    infer_parser.add_argument(
+        "--f0_autotune",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=f0_autotune_description,
+        default=False,
+    )
+    f0_autotune_strength_description = "Set the autotune strength - the more you increase it the more it will snap to the chromatic grid."
+    infer_parser.add_argument(
+        "--f0_autotune_strength",
+        type=float,
+        help=f0_autotune_strength_description,
+        choices=[(i / 10) for i in range(11)],
+        default=1.0,
+    )
+    clean_audio_description = "Clean the output audio using noise reduction algorithms. Recommended for speech conversions."
+    infer_parser.add_argument(
+        "--clean_audio",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=clean_audio_description,
+        default=False,
+    )
+    clean_strength_description = "Adjust the intensity of the audio cleaning process. Higher values result in stronger cleaning, but may lead to a more compressed sound."
+    infer_parser.add_argument(
+        "--clean_strength",
+        type=float,
+        help=clean_strength_description,
+        choices=[(i / 10) for i in range(11)],
+        default=0.7,
+    )
+    export_format_description = "Select the desired output audio format."
+    infer_parser.add_argument(
+        "--export_format",
+        type=str,
+        help=export_format_description,
+        choices=["WAV", "MP3", "FLAC", "OGG", "M4A"],
+        default="WAV",
+    )
+    embedder_model_description = (
+        "Choose the model used for generating speaker embeddings."
+    )
+    infer_parser.add_argument(
+        "--embedder_model",
+        type=str,
+        help=embedder_model_description,
+        choices=[
+            "contentvec",
+            "chinese-hubert-base",
+            "japanese-hubert-base",
+            "korean-hubert-base",
+            "custom",
+        ],
+        default="contentvec",
+    )
+    embedder_model_custom_description = "Specify the path to a custom model for speaker embedding. Only applicable if 'embedder_model' is set to 'custom'."
+    infer_parser.add_argument(
+        "--embedder_model_custom",
+        type=str,
+        help=embedder_model_custom_description,
+        default=None,
+    )
+    f0_file_description = "Full path to an external F0 file (.f0). This allows you to use pre-computed pitch values for the input audio."
+    infer_parser.add_argument(
+        "--f0_file",
+        type=str,
+        help=f0_file_description,
+        default=None,
+    )
+    formant_shifting_description = "Apply formant shifting to the input audio. This can help adjust the timbre of the voice."
+    infer_parser.add_argument(
+        "--formant_shifting",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=formant_shifting_description,
+        default=False,
+        required=False,
+    )
+    formant_qfrency_description = "Control the frequency of the formant shifting effect. Higher values result in a more pronounced effect."
+    infer_parser.add_argument(
+        "--formant_qfrency",
+        type=float,
+        help=formant_qfrency_description,
+        default=1.0,
+        required=False,
+    )
+    formant_timbre_description = "Control the timbre of the formant shifting effect. Higher values result in a more pronounced effect."
+    infer_parser.add_argument(
+        "--formant_timbre",
+        type=float,
+        help=formant_timbre_description,
+        default=1.0,
+        required=False,
+    )
+    sid_description = "Speaker ID for multi-speaker models."
+    infer_parser.add_argument(
+        "--sid",
+        type=int,
+        help=sid_description,
+        default=0,
+        required=False,
+    )
+    post_process_description = "Apply post-processing effects to the output audio."
+    infer_parser.add_argument(
+        "--post_process",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=post_process_description,
+        default=False,
+        required=False,
+    )
+    reverb_description = "Apply reverb effect to the output audio."
+    infer_parser.add_argument(
+        "--reverb",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=reverb_description,
+        default=False,
+        required=False,
+    )
+
+    pitch_shift_description = "Apply pitch shifting effect to the output audio."
+    infer_parser.add_argument(
+        "--pitch_shift",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=pitch_shift_description,
+        default=False,
+        required=False,
+    )
+
+    limiter_description = "Apply limiter effect to the output audio."
+    infer_parser.add_argument(
+        "--limiter",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=limiter_description,
+        default=False,
+        required=False,
+    )
+
+    gain_description = "Apply gain effect to the output audio."
+    infer_parser.add_argument(
+        "--gain",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=gain_description,
+        default=False,
+        required=False,
+    )
+
+    distortion_description = "Apply distortion effect to the output audio."
+    infer_parser.add_argument(
+        "--distortion",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=distortion_description,
+        default=False,
+        required=False,
+    )
+
+    chorus_description = "Apply chorus effect to the output audio."
+    infer_parser.add_argument(
+        "--chorus",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=chorus_description,
+        default=False,
+        required=False,
+    )
+
+    bitcrush_description = "Apply bitcrush effect to the output audio."
+    infer_parser.add_argument(
+        "--bitcrush",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=bitcrush_description,
+        default=False,
+        required=False,
+    )
+
+    clipping_description = "Apply clipping effect to the output audio."
+    infer_parser.add_argument(
+        "--clipping",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=clipping_description,
+        default=False,
+        required=False,
+    )
+
+    compressor_description = "Apply compressor effect to the output audio."
+    infer_parser.add_argument(
+        "--compressor",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=compressor_description,
+        default=False,
+        required=False,
+    )
+
+    delay_description = "Apply delay effect to the output audio."
+    infer_parser.add_argument(
+        "--delay",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=delay_description,
+        default=False,
+        required=False,
+    )
+
+    reverb_room_size_description = "Control the room size of the reverb effect. Higher values result in a larger room size."
+    infer_parser.add_argument(
+        "--reverb_room_size",
+        type=float,
+        help=reverb_room_size_description,
+        default=0.5,
+        required=False,
+    )
+
+    reverb_damping_description = "Control the damping of the reverb effect. Higher values result in a more damped sound."
+    infer_parser.add_argument(
+        "--reverb_damping",
+        type=float,
+        help=reverb_damping_description,
+        default=0.5,
+        required=False,
+    )
+
+    reverb_wet_gain_description = "Control the wet gain of the reverb effect. Higher values result in a stronger reverb effect."
+    infer_parser.add_argument(
+        "--reverb_wet_gain",
+        type=float,
+        help=reverb_wet_gain_description,
+        default=0.5,
+        required=False,
+    )
+
+    reverb_dry_gain_description = "Control the dry gain of the reverb effect. Higher values result in a stronger dry signal."
+    infer_parser.add_argument(
+        "--reverb_dry_gain",
+        type=float,
+        help=reverb_dry_gain_description,
+        default=0.5,
+        required=False,
+    )
+
+    reverb_width_description = "Control the stereo width of the reverb effect. Higher values result in a wider stereo image."
+    infer_parser.add_argument(
+        "--reverb_width",
+        type=float,
+        help=reverb_width_description,
+        default=0.5,
+        required=False,
+    )
+
+    reverb_freeze_mode_description = "Control the freeze mode of the reverb effect. Higher values result in a stronger freeze effect."
+    infer_parser.add_argument(
+        "--reverb_freeze_mode",
+        type=float,
+        help=reverb_freeze_mode_description,
+        default=0.5,
+        required=False,
+    )
+
+    pitch_shift_semitones_description = "Control the pitch shift in semitones. Positive values increase the pitch, while negative values decrease it."
+    infer_parser.add_argument(
+        "--pitch_shift_semitones",
+        type=float,
+        help=pitch_shift_semitones_description,
+        default=0.0,
+        required=False,
+    )
+
+    limiter_threshold_description = "Control the threshold of the limiter effect. Higher values result in a stronger limiting effect."
+    infer_parser.add_argument(
+        "--limiter_threshold",
+        type=float,
+        help=limiter_threshold_description,
+        default=-6,
+        required=False,
+    )
+
+    limiter_release_time_description = "Control the release time of the limiter effect. Higher values result in a longer release time."
+    infer_parser.add_argument(
+        "--limiter_release_time",
+        type=float,
+        help=limiter_release_time_description,
+        default=0.01,
+        required=False,
+    )
+
+    gain_db_description = "Control the gain in decibels. Positive values increase the gain, while negative values decrease it."
+    infer_parser.add_argument(
+        "--gain_db",
+        type=float,
+        help=gain_db_description,
+        default=0.0,
+        required=False,
+    )
+
+    distortion_gain_description = "Control the gain of the distortion effect. Higher values result in a stronger distortion effect."
+    infer_parser.add_argument(
+        "--distortion_gain",
+        type=float,
+        help=distortion_gain_description,
+        default=25,
+        required=False,
+    )
+
+    chorus_rate_description = "Control the rate of the chorus effect. Higher values result in a faster chorus effect."
+    infer_parser.add_argument(
+        "--chorus_rate",
+        type=float,
+        help=chorus_rate_description,
+        default=1.0,
+        required=False,
+    )
+
+    chorus_depth_description = "Control the depth of the chorus effect. Higher values result in a stronger chorus effect."
+    infer_parser.add_argument(
+        "--chorus_depth",
+        type=float,
+        help=chorus_depth_description,
+        default=0.25,
+        required=False,
+    )
+
+    chorus_center_delay_description = "Control the center delay of the chorus effect. Higher values result in a longer center delay."
+    infer_parser.add_argument(
+        "--chorus_center_delay",
+        type=float,
+        help=chorus_center_delay_description,
+        default=7,
+        required=False,
+    )
+
+    chorus_feedback_description = "Control the feedback of the chorus effect. Higher values result in a stronger feedback effect."
+    infer_parser.add_argument(
+        "--chorus_feedback",
+        type=float,
+        help=chorus_feedback_description,
+        default=0.0,
+        required=False,
+    )
+
+    chorus_mix_description = "Control the mix of the chorus effect. Higher values result in a stronger chorus effect."
+    infer_parser.add_argument(
+        "--chorus_mix",
+        type=float,
+        help=chorus_mix_description,
+        default=0.5,
+        required=False,
+    )
+
+    bitcrush_bit_depth_description = "Control the bit depth of the bitcrush effect. Higher values result in a stronger bitcrush effect."
+    infer_parser.add_argument(
+        "--bitcrush_bit_depth",
+        type=int,
+        help=bitcrush_bit_depth_description,
+        default=8,
+        required=False,
+    )
+
+    clipping_threshold_description = "Control the threshold of the clipping effect. Higher values result in a stronger clipping effect."
+    infer_parser.add_argument(
+        "--clipping_threshold",
+        type=float,
+        help=clipping_threshold_description,
+        default=-6,
+        required=False,
+    )
+
+    compressor_threshold_description = "Control the threshold of the compressor effect. Higher values result in a stronger compressor effect."
+    infer_parser.add_argument(
+        "--compressor_threshold",
+        type=float,
+        help=compressor_threshold_description,
+        default=0,
+        required=False,
+    )
+
+    compressor_ratio_description = "Control the ratio of the compressor effect. Higher values result in a stronger compressor effect."
+    infer_parser.add_argument(
+        "--compressor_ratio",
+        type=float,
+        help=compressor_ratio_description,
+        default=1,
+        required=False,
+    )
+
+    compressor_attack_description = "Control the attack of the compressor effect. Higher values result in a stronger compressor effect."
+    infer_parser.add_argument(
+        "--compressor_attack",
+        type=float,
+        help=compressor_attack_description,
+        default=1.0,
+        required=False,
+    )
+
+    compressor_release_description = "Control the release of the compressor effect. Higher values result in a stronger compressor effect."
+    infer_parser.add_argument(
+        "--compressor_release",
+        type=float,
+        help=compressor_release_description,
+        default=100,
+        required=False,
+    )
+
+    delay_seconds_description = "Control the delay time in seconds. Higher values result in a longer delay time."
+    infer_parser.add_argument(
+        "--delay_seconds",
+        type=float,
+        help=delay_seconds_description,
+        default=0.5,
+        required=False,
+    )
+    delay_feedback_description = "Control the feedback of the delay effect. Higher values result in a stronger feedback effect."
+    infer_parser.add_argument(
+        "--delay_feedback",
+        type=float,
+        help=delay_feedback_description,
+        default=0.0,
+        required=False,
+    )
+    delay_mix_description = "Control the mix of the delay effect. Higher values result in a stronger delay effect."
+    infer_parser.add_argument(
+        "--delay_mix",
+        type=float,
+        help=delay_mix_description,
+        default=0.5,
+        required=False,
+    )
+
+    # Parser for 'batch_infer' mode
+    batch_infer_parser = subparsers.add_parser(
+        "batch_infer",
+        help="Run batch inference",
+    )
+    batch_infer_parser.add_argument(
+        "--pitch",
+        type=int,
+        help=pitch_description,
+        choices=range(-24, 25),
+        default=0,
+    )
+    batch_infer_parser.add_argument(
+        "--filter_radius",
+        type=int,
+        help=filter_radius_description,
+        choices=range(11),
+        default=3,
+    )
+    batch_infer_parser.add_argument(
+        "--index_rate",
+        type=float,
+        help=index_rate_description,
+        choices=[i / 100.0 for i in range(0, 101)],
+        default=0.3,
+    )
+    batch_infer_parser.add_argument(
+        "--volume_envelope",
+        type=float,
+        help=volume_envelope_description,
+        choices=[i / 100.0 for i in range(0, 101)],
+        default=1,
+    )
+    batch_infer_parser.add_argument(
+        "--protect",
+        type=float,
+        help=protect_description,
+        choices=[i / 1000.0 for i in range(0, 501)],
+        default=0.33,
+    )
+    batch_infer_parser.add_argument(
+        "--hop_length",
+        type=int,
+        help=hop_length_description,
+        choices=range(1, 513),
+        default=128,
+    )
+    batch_infer_parser.add_argument(
+        "--f0_method",
+        type=str,
+        help=f0_method_description,
+        choices=[
+            "crepe",
+            "crepe-tiny",
+            "rmvpe",
+            "fcpe",
+            "hybrid[crepe+rmvpe]",
+            "hybrid[crepe+fcpe]",
+            "hybrid[rmvpe+fcpe]",
+            "hybrid[crepe+rmvpe+fcpe]",
+        ],
+        default="rmvpe",
+    )
+    batch_infer_parser.add_argument(
+        "--input_folder",
+        type=str,
+        help="Path to the folder containing input audio files.",
+        required=True,
+    )
+    batch_infer_parser.add_argument(
+        "--output_folder",
+        type=str,
+        help="Path to the folder for saving output audio files.",
+        required=True,
+    )
+    batch_infer_parser.add_argument(
+        "--pth_path", type=str, help=pth_path_description, required=True
+    )
+    batch_infer_parser.add_argument(
+        "--index_path", type=str, help=index_path_description, required=True
+    )
+    batch_infer_parser.add_argument(
+        "--split_audio",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=split_audio_description,
+        default=False,
+    )
+    batch_infer_parser.add_argument(
+        "--f0_autotune",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=f0_autotune_description,
+        default=False,
+    )
+    batch_infer_parser.add_argument(
+        "--f0_autotune_strength",
+        type=float,
+        help=clean_strength_description,
+        choices=[(i / 10) for i in range(11)],
+        default=1.0,
+    )
+    batch_infer_parser.add_argument(
+        "--clean_audio",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=clean_audio_description,
+        default=False,
+    )
+    batch_infer_parser.add_argument(
+        "--clean_strength",
+        type=float,
+        help=clean_strength_description,
+        choices=[(i / 10) for i in range(11)],
+        default=0.7,
+    )
+    batch_infer_parser.add_argument(
+        "--export_format",
+        type=str,
+        help=export_format_description,
+        choices=["WAV", "MP3", "FLAC", "OGG", "M4A"],
+        default="WAV",
+    )
+    batch_infer_parser.add_argument(
+        "--embedder_model",
+        type=str,
+        help=embedder_model_description,
+        choices=[
+            "contentvec",
+            "chinese-hubert-base",
+            "japanese-hubert-base",
+            "korean-hubert-base",
+            "custom",
+        ],
+        default="contentvec",
+    )
+    batch_infer_parser.add_argument(
+        "--embedder_model_custom",
+        type=str,
+        help=embedder_model_custom_description,
+        default=None,
+    )
+    batch_infer_parser.add_argument(
+        "--f0_file",
+        type=str,
+        help=f0_file_description,
+        default=None,
+    )
+    batch_infer_parser.add_argument(
+        "--formant_shifting",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=formant_shifting_description,
+        default=False,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--formant_qfrency",
+        type=float,
+        help=formant_qfrency_description,
+        default=1.0,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--formant_timbre",
+        type=float,
+        help=formant_timbre_description,
+        default=1.0,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--sid",
+        type=int,
+        help=sid_description,
+        default=0,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--post_process",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=post_process_description,
+        default=False,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--reverb",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=reverb_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--pitch_shift",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=pitch_shift_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--limiter",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=limiter_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--gain",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=gain_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--distortion",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=distortion_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--chorus",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=chorus_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--bitcrush",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=bitcrush_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--clipping",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=clipping_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--compressor",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=compressor_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--delay",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        help=delay_description,
+        default=False,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--reverb_room_size",
+        type=float,
+        help=reverb_room_size_description,
+        default=0.5,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--reverb_damping",
+        type=float,
+        help=reverb_damping_description,
+        default=0.5,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--reverb_wet_gain",
+        type=float,
+        help=reverb_wet_gain_description,
+        default=0.5,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--reverb_dry_gain",
+        type=float,
+        help=reverb_dry_gain_description,
+        default=0.5,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--reverb_width",
+        type=float,
+        help=reverb_width_description,
+        default=0.5,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--reverb_freeze_mode",
+        type=float,
+        help=reverb_freeze_mode_description,
+        default=0.5,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--pitch_shift_semitones",
+        type=float,
+        help=pitch_shift_semitones_description,
+        default=0.0,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--limiter_threshold",
+        type=float,
+        help=limiter_threshold_description,
+        default=-6,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--limiter_release_time",
+        type=float,
+        help=limiter_release_time_description,
+        default=0.01,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--gain_db",
+        type=float,
+        help=gain_db_description,
+        default=0.0,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--distortion_gain",
+        type=float,
+        help=distortion_gain_description,
+        default=25,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--chorus_rate",
+        type=float,
+        help=chorus_rate_description,
+        default=1.0,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--chorus_depth",
+        type=float,
+        help=chorus_depth_description,
+        default=0.25,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--chorus_center_delay",
+        type=float,
+        help=chorus_center_delay_description,
+        default=7,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--chorus_feedback",
+        type=float,
+        help=chorus_feedback_description,
+        default=0.0,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--chorus_mix",
+        type=float,
+        help=chorus_mix_description,
+        default=0.5,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--bitcrush_bit_depth",
+        type=int,
+        help=bitcrush_bit_depth_description,
+        default=8,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--clipping_threshold",
+        type=float,
+        help=clipping_threshold_description,
+        default=-6,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--compressor_threshold",
+        type=float,
+        help=compressor_threshold_description,
+        default=0,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--compressor_ratio",
+        type=float,
+        help=compressor_ratio_description,
+        default=1,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--compressor_attack",
+        type=float,
+        help=compressor_attack_description,
+        default=1.0,
+        required=False,
+    )
+
+    batch_infer_parser.add_argument(
+        "--compressor_release",
+        type=float,
+        help=compressor_release_description,
+        default=100,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--delay_seconds",
+        type=float,
+        help=delay_seconds_description,
+        default=0.5,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--delay_feedback",
+        type=float,
+        help=delay_feedback_description,
+        default=0.0,
+        required=False,
+    )
+    batch_infer_parser.add_argument(
+        "--delay_mix",
+        type=float,
+        help=delay_mix_description,
+        default=0.5,
+        required=False,
+    )
+
+
+    # Parser for 'download' mode
+    download_parser = subparsers.add_parser(
+        "download", help="Download a model from a provided link."
+    )
+    download_parser.add_argument(
+        "--model_link", type=str, help="Direct link to the model file.", required=True
+    )
+
+    # Parser for 'prerequisites' mode
+    prerequisites_parser = subparsers.add_parser(
+        "prerequisites", help="Install prerequisites for RVC."
+    )
+    prerequisites_parser.add_argument(
+        "--models",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        default=True,
+        help="Download additional models.",
+    )
+    prerequisites_parser.add_argument(
+        "--exe",
+        type=lambda x: bool(strtobool(x)),
+        choices=[True, False],
+        default=True,
+        help="Download required executables.",
+    )
+
+
+    return parser.parse_args()
+
+
+def main():
+    if len(sys.argv) == 1:
+        print("Please run the script with '-h' for more information.")
+        sys.exit(1)
+
+    args = parse_arguments()
+
+    try:
+        if args.mode == "infer":
+            run_infer_script(
+                pitch=args.pitch,
+                filter_radius=args.filter_radius,
+                index_rate=args.index_rate,
+                volume_envelope=args.volume_envelope,
+                protect=args.protect,
+                hop_length=args.hop_length,
+                f0_method=args.f0_method,
+                input_path=args.input_path,
+                output_path=args.output_path,
+                pth_path=args.pth_path,
+                index_path=args.index_path,
+                split_audio=args.split_audio,
+                f0_autotune=args.f0_autotune,
+                f0_autotune_strength=args.f0_autotune_strength,
+                clean_audio=args.clean_audio,
+                clean_strength=args.clean_strength,
+                export_format=args.export_format,
+                embedder_model=args.embedder_model,
+                embedder_model_custom=args.embedder_model_custom,
+                f0_file=args.f0_file,
+                formant_shifting=args.formant_shifting,
+                formant_qfrency=args.formant_qfrency,
+                formant_timbre=args.formant_timbre,
+                sid=args.sid,
+                post_process=args.post_process,
+                reverb=args.reverb,
+                pitch_shift=args.pitch_shift,
+                limiter=args.limiter,
+                gain=args.gain,
+                distortion=args.distortion,
+                chorus=args.chorus,
+                bitcrush=args.bitcrush,
+                clipping=args.clipping,
+                compressor=args.compressor,
+                delay=args.delay,
+                reverb_room_size=args.reverb_room_size,
+                reverb_damping=args.reverb_damping,
+                reverb_wet_gain=args.reverb_wet_gain,
+                reverb_dry_gain=args.reverb_dry_gain,
+                reverb_width=args.reverb_width,
+                reverb_freeze_mode=args.reverb_freeze_mode,
+                pitch_shift_semitones=args.pitch_shift_semitones,
+                limiter_threshold=args.limiter_threshold,
+                limiter_release_time=args.limiter_release_time,
+                gain_db=args.gain_db,
+                distortion_gain=args.distortion_gain,
+                chorus_rate=args.chorus_rate,
+                chorus_depth=args.chorus_depth,
+                chorus_center_delay=args.chorus_center_delay,
+                chorus_feedback=args.chorus_feedback,
+                chorus_mix=args.chorus_mix,
+                bitcrush_bit_depth=args.bitcrush_bit_depth,
+                clipping_threshold=args.clipping_threshold,
+                compressor_threshold=args.compressor_threshold,
+                compressor_ratio=args.compressor_ratio,
+                compressor_attack=args.compressor_attack,
+                compressor_release=args.compressor_release,
+                delay_seconds=args.delay_seconds,
+                delay_feedback=args.delay_feedback,
+                delay_mix=args.delay_mix,
+            )
+        elif args.mode == "batch_infer":
+            run_batch_infer_script(
+                pitch=args.pitch,
+                filter_radius=args.filter_radius,
+                index_rate=args.index_rate,
+                volume_envelope=args.volume_envelope,
+                protect=args.protect,
+                hop_length=args.hop_length,
+                f0_method=args.f0_method,
+                input_folder=args.input_folder,
+                output_folder=args.output_folder,
+                pth_path=args.pth_path,
+                index_path=args.index_path,
+                split_audio=args.split_audio,
+                f0_autotune=args.f0_autotune,
+                f0_autotune_strength=args.f0_autotune_strength,
+                clean_audio=args.clean_audio,
+                clean_strength=args.clean_strength,
+                export_format=args.export_format,
+                embedder_model=args.embedder_model,
+                embedder_model_custom=args.embedder_model_custom,
+                f0_file=args.f0_file,
+                formant_shifting=args.formant_shifting,
+                formant_qfrency=args.formant_qfrency,
+                formant_timbre=args.formant_timbre,
+                sid=args.sid,
+                post_process=args.post_process,
+                reverb=args.reverb,
+                pitch_shift=args.pitch_shift,
+                limiter=args.limiter,
+                gain=args.gain,
+                distortion=args.distortion,
+                chorus=args.chorus,
+                bitcrush=args.bitcrush,
+                clipping=args.clipping,
+                compressor=args.compressor,
+                delay=args.delay,
+                reverb_room_size=args.reverb_room_size,
+                reverb_damping=args.reverb_damping,
+                reverb_wet_gain=args.reverb_wet_gain,
+                reverb_dry_gain=args.reverb_dry_gain,
+                reverb_width=args.reverb_width,
+                reverb_freeze_mode=args.reverb_freeze_mode,
+                pitch_shift_semitones=args.pitch_shift_semitones,
+                limiter_threshold=args.limiter_threshold,
+                limiter_release_time=args.limiter_release_time,
+                gain_db=args.gain_db,
+                distortion_gain=args.distortion_gain,
+                chorus_rate=args.chorus_rate,
+                chorus_depth=args.chorus_depth,
+                chorus_center_delay=args.chorus_center_delay,
+                chorus_feedback=args.chorus_feedback,
+                chorus_mix=args.chorus_mix,
+                bitcrush_bit_depth=args.bitcrush_bit_depth,
+                clipping_threshold=args.clipping_threshold,
+                compressor_threshold=args.compressor_threshold,
+                compressor_ratio=args.compressor_ratio,
+                compressor_attack=args.compressor_attack,
+                compressor_release=args.compressor_release,
+                delay_seconds=args.delay_seconds,
+                delay_feedback=args.delay_feedback,
+                delay_mix=args.delay_mix,
+            )
+
+        elif args.mode == "prerequisites":
+            run_prerequisites_script(
+                pretraineds_v1_f0=args.pretraineds_v1_f0,
+                pretraineds_v1_nof0=args.pretraineds_v1_nof0,
+                pretraineds_v2_f0=args.pretraineds_v2_f0,
+                pretraineds_v2_nof0=args.pretraineds_v2_nof0,
+                models=args.models,
+                exe=args.exe,
+            )
+        elif args.mode == "download":
+            run_download_script(
+                model_link=args.model_link,
+            )
+    except Exception as error:
+        print(f"An error occurred during execution: {error}")
+
+        import traceback
+
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

discriminators.py

@@ -0,0 +1,160 @@
+import torch
+from torch.nn.utils.parametrizations import spectral_norm, weight_norm
+
+from rvc_cli.rvc.lib.algorithm.commons import get_padding
+from rvc_cli.rvc.lib.algorithm.residuals import LRELU_SLOPE
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+    """
+    Multi-period discriminator.
+
+    This class implements a multi-period discriminator, which is used to
+    discriminate between real and fake audio signals. The discriminator
+    is composed of a series of convolutional layers that are applied to
+    the input signal at different periods.
+
+    Args:
+        periods (str): Periods of the discriminator. V1 = [2, 3, 5, 7, 11, 17], V2 = [2, 3, 5, 7, 11, 17, 23, 37].
+        use_spectral_norm (bool): Whether to use spectral normalization.
+            Defaults to False.
+    """
+
+    def __init__(self, version, use_spectral_norm=False):
+        super(MultiPeriodDiscriminator, self).__init__()
+        periods = (
+            [2, 3, 5, 7, 11, 17] if version == "v1" else [2, 3, 5, 7, 11, 17, 23, 37]
+        )
+        self.discriminators = torch.nn.ModuleList(
+            [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
+            + [DiscriminatorP(p, use_spectral_norm=use_spectral_norm) for p in periods]
+        )
+
+    def forward(self, y, y_hat):
+        """
+        Forward pass of the multi-period discriminator.
+
+        Args:
+            y (torch.Tensor): Real audio signal.
+            y_hat (torch.Tensor): Fake audio signal.
+        """
+        y_d_rs, y_d_gs, fmap_rs, fmap_gs = [], [], [], []
+        for d in self.discriminators:
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            y_d_rs.append(y_d_r)
+            y_d_gs.append(y_d_g)
+            fmap_rs.append(fmap_r)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+
+class DiscriminatorS(torch.nn.Module):
+    """
+    Discriminator for the short-term component.
+
+    This class implements a discriminator for the short-term component
+    of the audio signal. The discriminator is composed of a series of
+    convolutional layers that are applied to the input signal.
+    """
+
+    def __init__(self, use_spectral_norm=False):
+        super(DiscriminatorS, self).__init__()
+        norm_f = spectral_norm if use_spectral_norm else weight_norm
+        self.convs = torch.nn.ModuleList(
+            [
+                norm_f(torch.nn.Conv1d(1, 16, 15, 1, padding=7)),
+                norm_f(torch.nn.Conv1d(16, 64, 41, 4, groups=4, padding=20)),
+                norm_f(torch.nn.Conv1d(64, 256, 41, 4, groups=16, padding=20)),
+                norm_f(torch.nn.Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
+                norm_f(torch.nn.Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
+                norm_f(torch.nn.Conv1d(1024, 1024, 5, 1, padding=2)),
+            ]
+        )
+        self.conv_post = norm_f(torch.nn.Conv1d(1024, 1, 3, 1, padding=1))
+        self.lrelu = torch.nn.LeakyReLU(LRELU_SLOPE)
+
+    def forward(self, x):
+        """
+        Forward pass of the discriminator.
+
+        Args:
+            x (torch.Tensor): Input audio signal.
+        """
+        fmap = []
+        for conv in self.convs:
+            x = self.lrelu(conv(x))
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+        return x, fmap
+
+
+class DiscriminatorP(torch.nn.Module):
+    """
+    Discriminator for the long-term component.
+
+    This class implements a discriminator for the long-term component
+    of the audio signal. The discriminator is composed of a series of
+    convolutional layers that are applied to the input signal at a given
+    period.
+
+    Args:
+        period (int): Period of the discriminator.
+        kernel_size (int): Kernel size of the convolutional layers.
+            Defaults to 5.
+        stride (int): Stride of the convolutional layers. Defaults to 3.
+        use_spectral_norm (bool): Whether to use spectral normalization.
+            Defaults to False.
+    """
+
+    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+        super(DiscriminatorP, self).__init__()
+        self.period = period
+        norm_f = spectral_norm if use_spectral_norm else weight_norm
+
+        in_channels = [1, 32, 128, 512, 1024]
+        out_channels = [32, 128, 512, 1024, 1024]
+
+        self.convs = torch.nn.ModuleList(
+            [
+                norm_f(
+                    torch.nn.Conv2d(
+                        in_ch,
+                        out_ch,
+                        (kernel_size, 1),
+                        (stride, 1),
+                        padding=(get_padding(kernel_size, 1), 0),
+                    )
+                )
+                for in_ch, out_ch in zip(in_channels, out_channels)
+            ]
+        )
+
+        self.conv_post = norm_f(torch.nn.Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+        self.lrelu = torch.nn.LeakyReLU(LRELU_SLOPE)
+
+    def forward(self, x):
+        """
+        Forward pass of the discriminator.
+
+        Args:
+            x (torch.Tensor): Input audio signal.
+        """
+        fmap = []
+        b, c, t = x.shape
+        if t % self.period != 0:
+            n_pad = self.period - (t % self.period)
+            x = torch.nn.functional.pad(x, (0, n_pad), "reflect")
+        x = x.view(b, c, -1, self.period)
+
+        for conv in self.convs:
+            x = self.lrelu(conv(x))
+            fmap.append(x)
+
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+        return x, fmap

encoders.py

@@ -0,0 +1,218 @@
+import math
+import torch
+from typing import Optional
+
+from rvc_cli.rvc.lib.algorithm.commons import sequence_mask
+from rvc_cli.rvc.lib.algorithm.modules import WaveNet
+from rvc_cli.rvc.lib.algorithm.normalization import LayerNorm
+from rvc_cli.rvc.lib.algorithm.attentions import FFN, MultiHeadAttention
+
+
+class Encoder(torch.nn.Module):
+    """
+    Encoder module for the Transformer model.
+
+    Args:
+        hidden_channels (int): Number of hidden channels in the encoder.
+        filter_channels (int): Number of filter channels in the feed-forward network.
+        n_heads (int): Number of attention heads.
+        n_layers (int): Number of encoder layers.
+        kernel_size (int, optional): Kernel size of the convolution layers in the feed-forward network. Defaults to 1.
+        p_dropout (float, optional): Dropout probability. Defaults to 0.0.
+        window_size (int, optional): Window size for relative positional encoding. Defaults to 10.
+    """
+
+    def __init__(
+        self,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size=1,
+        p_dropout=0.0,
+        window_size=10,
+    ):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+
+        self.drop = torch.nn.Dropout(p_dropout)
+        self.attn_layers = torch.nn.ModuleList()
+        self.norm_layers_1 = torch.nn.ModuleList()
+        self.ffn_layers = torch.nn.ModuleList()
+        self.norm_layers_2 = torch.nn.ModuleList()
+        for i in range(self.n_layers):
+            self.attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels,
+                    hidden_channels,
+                    n_heads,
+                    p_dropout=p_dropout,
+                    window_size=window_size,
+                )
+            )
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(
+                    hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                )
+            )
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask):
+        attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.attn_layers[i](x, x, attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class TextEncoder(torch.nn.Module):
+    """Text Encoder with configurable embedding dimension.
+
+    Args:
+        out_channels (int): Output channels of the encoder.
+        hidden_channels (int): Hidden channels of the encoder.
+        filter_channels (int): Filter channels of the encoder.
+        n_heads (int): Number of attention heads.
+        n_layers (int): Number of encoder layers.
+        kernel_size (int): Kernel size of the convolutional layers.
+        p_dropout (float): Dropout probability.
+        embedding_dim (int): Embedding dimension for phone embeddings (v1 = 256, v2 = 768).
+        f0 (bool, optional): Whether to use F0 embedding. Defaults to True.
+    """
+
+    def __init__(
+        self,
+        out_channels,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size,
+        p_dropout,
+        embedding_dim,
+        f0=True,
+    ):
+        super(TextEncoder, self).__init__()
+        self.out_channels = out_channels
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = float(p_dropout)
+        self.emb_phone = torch.nn.Linear(embedding_dim, hidden_channels)
+        self.lrelu = torch.nn.LeakyReLU(0.1, inplace=True)
+        if f0:
+            self.emb_pitch = torch.nn.Embedding(256, hidden_channels)
+        self.encoder = Encoder(
+            hidden_channels,
+            filter_channels,
+            n_heads,
+            n_layers,
+            kernel_size,
+            float(p_dropout),
+        )
+        self.proj = torch.nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+    def forward(
+        self, phone: torch.Tensor, pitch: Optional[torch.Tensor], lengths: torch.Tensor
+    ):
+        if pitch is None:
+            x = self.emb_phone(phone)
+        else:
+            x = self.emb_phone(phone) + self.emb_pitch(pitch)
+        x = x * math.sqrt(self.hidden_channels)  # [b, t, h]
+        x = self.lrelu(x)
+        x = torch.transpose(x, 1, -1)  # [b, h, t]
+        x_mask = torch.unsqueeze(sequence_mask(lengths, x.size(2)), 1).to(x.dtype)
+        x = self.encoder(x * x_mask, x_mask)
+        stats = self.proj(x) * x_mask
+
+        m, logs = torch.split(stats, self.out_channels, dim=1)
+        return m, logs, x_mask
+
+
+class PosteriorEncoder(torch.nn.Module):
+    """Posterior Encoder for inferring latent representation.
+
+    Args:
+        in_channels (int): Number of channels in the input.
+        out_channels (int): Number of channels in the output.
+        hidden_channels (int): Number of hidden channels in the encoder.
+        kernel_size (int): Kernel size of the convolutional layers.
+        dilation_rate (int): Dilation rate of the convolutional layers.
+        n_layers (int): Number of layers in the encoder.
+        gin_channels (int, optional): Number of channels for the global conditioning input. Defaults to 0.
+    """
+
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        gin_channels=0,
+    ):
+        super(PosteriorEncoder, self).__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+
+        self.pre = torch.nn.Conv1d(in_channels, hidden_channels, 1)
+        self.enc = WaveNet(
+            hidden_channels,
+            kernel_size,
+            dilation_rate,
+            n_layers,
+            gin_channels=gin_channels,
+        )
+        self.proj = torch.nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+    def forward(
+        self, x: torch.Tensor, x_lengths: torch.Tensor, g: Optional[torch.Tensor] = None
+    ):
+        x_mask = torch.unsqueeze(sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
+        x = self.pre(x) * x_mask
+        x = self.enc(x, x_mask, g=g)
+        stats = self.proj(x) * x_mask
+        m, logs = torch.split(stats, self.out_channels, dim=1)
+        z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
+        return z, m, logs, x_mask
+
+    def remove_weight_norm(self):
+        """Removes weight normalization from the encoder."""
+        self.enc.remove_weight_norm()
+
+    def __prepare_scriptable__(self):
+        """Prepares the module for scripting."""
+        for hook in self.enc._forward_pre_hooks.values():
+            if (
+                hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                and hook.__class__.__name__ == "WeightNorm"
+            ):
+                torch.nn.utils.remove_weight_norm(self.enc)
+        return self

generators.py

@@ -0,0 +1,231 @@
+import torch
+import numpy as np
+from torch.nn.utils import remove_weight_norm
+from torch.nn.utils.parametrizations import weight_norm
+from typing import Optional
+
+from rvc_cli.rvc.lib.algorithm.residuals import LRELU_SLOPE, ResBlock1, ResBlock2
+from rvc_cli.rvc.lib.algorithm.commons import init_weights
+
+
+class Generator(torch.nn.Module):
+    """Generator for synthesizing audio.
+
+    Args:
+        initial_channel (int): Number of channels in the initial convolutional layer.
+        resblock (str): Type of residual block to use (1 or 2).
+        resblock_kernel_sizes (list): Kernel sizes of the residual blocks.
+        resblock_dilation_sizes (list): Dilation rates of the residual blocks.
+        upsample_rates (list): Upsampling rates.
+        upsample_initial_channel (int): Number of channels in the initial upsampling layer.
+        upsample_kernel_sizes (list): Kernel sizes of the upsampling layers.
+        gin_channels (int, optional): Number of channels for the global conditioning input. Defaults to 0.
+    """
+
+    def __init__(
+        self,
+        initial_channel,
+        resblock,
+        resblock_kernel_sizes,
+        resblock_dilation_sizes,
+        upsample_rates,
+        upsample_initial_channel,
+        upsample_kernel_sizes,
+        gin_channels=0,
+    ):
+        super(Generator, self).__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+        self.conv_pre = torch.nn.Conv1d(
+            initial_channel, upsample_initial_channel, 7, 1, padding=3
+        )
+        resblock = ResBlock1 if resblock == "1" else ResBlock2
+
+        self.ups = torch.nn.ModuleList()
+        self.resblocks = torch.nn.ModuleList()
+
+        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+            self.ups.append(
+                weight_norm(
+                    torch.nn.ConvTranspose1d(
+                        upsample_initial_channel // (2**i),
+                        upsample_initial_channel // (2 ** (i + 1)),
+                        k,
+                        u,
+                        padding=(k - u) // 2,
+                    )
+                )
+            )
+            ch = upsample_initial_channel // (2 ** (i + 1))
+            for j, (k, d) in enumerate(
+                zip(resblock_kernel_sizes, resblock_dilation_sizes)
+            ):
+                self.resblocks.append(resblock(ch, k, d))
+
+        self.conv_post = torch.nn.Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+        self.ups.apply(init_weights)
+
+        if gin_channels != 0:
+            self.cond = torch.nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+    def forward(self, x: torch.Tensor, g: Optional[torch.Tensor] = None):
+        x = self.conv_pre(x)
+        if g is not None:
+            x = x + self.cond(g)
+
+        for i in range(self.num_upsamples):
+            x = torch.nn.functional.leaky_relu(x, LRELU_SLOPE)
+            x = self.ups[i](x)
+            xs = None
+            for j in range(self.num_kernels):
+                if xs == None:
+                    xs = self.resblocks[i * self.num_kernels + j](x)
+                else:
+                    xs += self.resblocks[i * self.num_kernels + j](x)
+            x = xs / self.num_kernels
+
+        x = torch.nn.functional.leaky_relu(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+
+        return x
+
+    def __prepare_scriptable__(self):
+        """Prepares the module for scripting."""
+        for l in self.ups_and_resblocks:
+            for hook in l._forward_pre_hooks.values():
+                if (
+                    hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                    and hook.__class__.__name__ == "WeightNorm"
+                ):
+                    torch.nn.utils.remove_weight_norm(l)
+        return self
+
+    def remove_weight_norm(self):
+        """Removes weight normalization from the upsampling and residual blocks."""
+        for l in self.ups:
+            remove_weight_norm(l)
+        for l in self.resblocks:
+            l.remove_weight_norm()
+
+
+class SineGenerator(torch.nn.Module):
+    """
+    A sine wave generator that synthesizes waveforms with optional harmonic overtones and noise.
+
+    Args:
+        sampling_rate (int): The sampling rate in Hz.
+        num_harmonics (int, optional): The number of harmonic overtones to include. Defaults to 0.
+        sine_amplitude (float, optional): The amplitude of the sine waveform. Defaults to 0.1.
+        noise_stddev (float, optional): The standard deviation of Gaussian noise. Defaults to 0.003.
+        voiced_threshold (float, optional): F0 threshold for distinguishing voiced/unvoiced frames. Defaults to 0.
+    """
+
+    def __init__(
+        self,
+        sampling_rate: int,
+        num_harmonics: int = 0,
+        sine_amplitude: float = 0.1,
+        noise_stddev: float = 0.003,
+        voiced_threshold: float = 0.0,
+    ):
+        super(SineGenerator, self).__init__()
+        self.sampling_rate = sampling_rate
+        self.num_harmonics = num_harmonics
+        self.sine_amplitude = sine_amplitude
+        self.noise_stddev = noise_stddev
+        self.voiced_threshold = voiced_threshold
+        self.waveform_dim = self.num_harmonics + 1  # fundamental + harmonics
+
+    def _compute_voiced_unvoiced(self, f0: torch.Tensor) -> torch.Tensor:
+        """
+        Generate a binary mask to indicate voiced/unvoiced frames.
+
+        Args:
+            f0 (torch.Tensor): Fundamental frequency tensor (batch_size, length).
+        """
+        uv_mask = (f0 > self.voiced_threshold).float()
+        return uv_mask
+
+    def _generate_sine_wave(
+        self, f0: torch.Tensor, upsampling_factor: int
+    ) -> torch.Tensor:
+        """
+        Generate sine waves for the fundamental frequency and its harmonics.
+
+        Args:
+            f0 (torch.Tensor): Fundamental frequency tensor (batch_size, length, 1).
+            upsampling_factor (int): Upsampling factor.
+        """
+        batch_size, length, _ = f0.shape
+
+        # Create an upsampling grid
+        upsampling_grid = torch.arange(
+            1, upsampling_factor + 1, dtype=f0.dtype, device=f0.device
+        )
+
+        # Calculate phase increments
+        phase_increments = (f0 / self.sampling_rate) * upsampling_grid
+        phase_remainder = torch.fmod(phase_increments[:, :-1, -1:] + 0.5, 1.0) - 0.5
+        cumulative_phase = phase_remainder.cumsum(dim=1).fmod(1.0).to(f0.dtype)
+        phase_increments += torch.nn.functional.pad(
+            cumulative_phase, (0, 0, 1, 0), mode="constant"
+        )
+
+        # Reshape to match the sine wave shape
+        phase_increments = phase_increments.reshape(batch_size, -1, 1)
+
+        # Scale for harmonics
+        harmonic_scale = torch.arange(
+            1, self.waveform_dim + 1, dtype=f0.dtype, device=f0.device
+        ).reshape(1, 1, -1)
+        phase_increments *= harmonic_scale
+
+        # Add random phase offset (except for the fundamental)
+        random_phase = torch.rand(1, 1, self.waveform_dim, device=f0.device)
+        random_phase[..., 0] = 0  # Fundamental frequency has no random offset
+        phase_increments += random_phase
+
+        # Generate sine waves
+        sine_waves = torch.sin(2 * np.pi * phase_increments)
+        return sine_waves
+
+    def forward(self, f0: torch.Tensor, upsampling_factor: int):
+        """
+        Forward pass to generate sine waveforms with noise and voiced/unvoiced masking.
+
+        Args:
+            f0 (torch.Tensor): Fundamental frequency tensor (batch_size, length, 1).
+            upsampling_factor (int): Upsampling factor.
+        """
+        with torch.no_grad():
+            # Expand `f0` to include waveform dimensions
+            f0 = f0.unsqueeze(-1)
+
+            # Generate sine waves
+            sine_waves = (
+                self._generate_sine_wave(f0, upsampling_factor) * self.sine_amplitude
+            )
+
+            # Compute voiced/unvoiced mask
+            voiced_mask = self._compute_voiced_unvoiced(f0)
+
+            # Upsample voiced/unvoiced mask
+            voiced_mask = torch.nn.functional.interpolate(
+                voiced_mask.transpose(2, 1),
+                scale_factor=float(upsampling_factor),
+                mode="nearest",
+            ).transpose(2, 1)
+
+            # Compute noise amplitude
+            noise_amplitude = voiced_mask * self.noise_stddev + (1 - voiced_mask) * (
+                self.sine_amplitude / 3
+            )
+
+            # Add Gaussian noise
+            noise = noise_amplitude * torch.randn_like(sine_waves)
+
+            # Combine sine waves and noise
+            sine_waveforms = sine_waves * voiced_mask + noise
+
+        return sine_waveforms, voiced_mask, noise

infer.py

@@ -0,0 +1,495 @@
+import os
+import sys
+import time
+import torch
+import librosa
+import logging
+import traceback
+import numpy as np
+import soundfile as sf
+import noisereduce as nr
+from pedalboard import (
+    Pedalboard,
+    Chorus,
+    Distortion,
+    Reverb,
+    PitchShift,
+    Limiter,
+    Gain,
+    Bitcrush,
+    Clipping,
+    Compressor,
+    Delay,
+)
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+
+from rvc_cli.rvc.infer.pipeline import Pipeline as VC
+from rvc_cli.rvc.lib.utils import load_audio_infer, load_embedding
+from rvc_cli.rvc.lib.tools.split_audio import process_audio, merge_audio
+from rvc_cli.rvc.lib.algorithm.synthesizers import Synthesizer
+from rvc_cli.rvc.configs.config import Config
+
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+logging.getLogger("faiss").setLevel(logging.WARNING)
+logging.getLogger("faiss.loader").setLevel(logging.WARNING)
+
+
+class VoiceConverter:
+    """
+    A class for performing voice conversion using the Retrieval-Based Voice Conversion (RVC) method.
+    """
+
+    def __init__(self):
+        """
+        Initializes the VoiceConverter with default configuration, and sets up models and parameters.
+        """
+        self.config = Config()  # Load RVC configuration
+        self.hubert_model = (
+            None  # Initialize the Hubert model (for embedding extraction)
+        )
+        self.last_embedder_model = None  # Last used embedder model
+        self.tgt_sr = None  # Target sampling rate for the output audio
+        self.net_g = None  # Generator network for voice conversion
+        self.vc = None  # Voice conversion pipeline instance
+        self.cpt = None  # Checkpoint for loading model weights
+        self.version = None  # Model version
+        self.n_spk = None  # Number of speakers in the model
+        self.use_f0 = None  # Whether the model uses F0
+        self.loaded_model = None
+
+    def load_hubert(self, embedder_model: str, embedder_model_custom: str = None):
+        """
+        Loads the HuBERT model for speaker embedding extraction.
+
+        Args:
+            embedder_model (str): Path to the pre-trained HuBERT model.
+            embedder_model_custom (str): Path to the custom HuBERT model.
+        """
+        self.hubert_model = load_embedding(embedder_model, embedder_model_custom)
+        self.hubert_model.to(self.config.device)
+        self.hubert_model = (
+            self.hubert_model.half()
+            if self.config.is_half
+            else self.hubert_model.float()
+        )
+        self.hubert_model.eval()
+
+    @staticmethod
+    def remove_audio_noise(data, sr, reduction_strength=0.7):
+        """
+        Removes noise from an audio file using the NoiseReduce library.
+
+        Args:
+            data (numpy.ndarray): The audio data as a NumPy array.
+            sr (int): The sample rate of the audio data.
+            reduction_strength (float): Strength of the noise reduction. Default is 0.7.
+        """
+        try:
+            reduced_noise = nr.reduce_noise(
+                y=data, sr=sr, prop_decrease=reduction_strength
+            )
+            return reduced_noise
+        except Exception as error:
+            print(f"An error occurred removing audio noise: {error}")
+            return None
+
+    @staticmethod
+    def convert_audio_format(input_path, output_path, output_format):
+        """
+        Converts an audio file to a specified output format.
+
+        Args:
+            input_path (str): Path to the input audio file.
+            output_path (str): Path to the output audio file.
+            output_format (str): Desired audio format (e.g., "WAV", "MP3").
+        """
+        try:
+            if output_format != "WAV":
+                print(f"Saving audio as {output_format}...")
+                audio, sample_rate = librosa.load(input_path, sr=None)
+                common_sample_rates = [
+                    8000,
+                    11025,
+                    12000,
+                    16000,
+                    22050,
+                    24000,
+                    32000,
+                    44100,
+                    48000,
+                ]
+                target_sr = min(common_sample_rates, key=lambda x: abs(x - sample_rate))
+                audio = librosa.resample(
+                    audio, orig_sr=sample_rate, target_sr=target_sr
+                )
+                sf.write(output_path, audio, target_sr, format=output_format.lower())
+            return output_path
+        except Exception as error:
+            print(f"An error occurred converting the audio format: {error}")
+
+    @staticmethod
+    def post_process_audio(
+        audio_input,
+        sample_rate,
+        **kwargs,
+    ):
+        board = Pedalboard()
+        if kwargs.get("reverb", False):
+            reverb = Reverb(
+                room_size=kwargs.get("reverb_room_size", 0.5),
+                damping=kwargs.get("reverb_damping", 0.5),
+                wet_level=kwargs.get("reverb_wet_level", 0.33),
+                dry_level=kwargs.get("reverb_dry_level", 0.4),
+                width=kwargs.get("reverb_width", 1.0),
+                freeze_mode=kwargs.get("reverb_freeze_mode", 0),
+            )
+            board.append(reverb)
+        if kwargs.get("pitch_shift", False):
+            pitch_shift = PitchShift(semitones=kwargs.get("pitch_shift_semitones", 0))
+            board.append(pitch_shift)
+        if kwargs.get("limiter", False):
+            limiter = Limiter(
+                threshold_db=kwargs.get("limiter_threshold", -6),
+                release_ms=kwargs.get("limiter_release", 0.05),
+            )
+            board.append(limiter)
+        if kwargs.get("gain", False):
+            gain = Gain(gain_db=kwargs.get("gain_db", 0))
+            board.append(gain)
+        if kwargs.get("distortion", False):
+            distortion = Distortion(drive_db=kwargs.get("distortion_gain", 25))
+            board.append(distortion)
+        if kwargs.get("chorus", False):
+            chorus = Chorus(
+                rate_hz=kwargs.get("chorus_rate", 1.0),
+                depth=kwargs.get("chorus_depth", 0.25),
+                centre_delay_ms=kwargs.get("chorus_delay", 7),
+                feedback=kwargs.get("chorus_feedback", 0.0),
+                mix=kwargs.get("chorus_mix", 0.5),
+            )
+            board.append(chorus)
+        if kwargs.get("bitcrush", False):
+            bitcrush = Bitcrush(bit_depth=kwargs.get("bitcrush_bit_depth", 8))
+            board.append(bitcrush)
+        if kwargs.get("clipping", False):
+            clipping = Clipping(threshold_db=kwargs.get("clipping_threshold", 0))
+            board.append(clipping)
+        if kwargs.get("compressor", False):
+            compressor = Compressor(
+                threshold_db=kwargs.get("compressor_threshold", 0),
+                ratio=kwargs.get("compressor_ratio", 1),
+                attack_ms=kwargs.get("compressor_attack", 1.0),
+                release_ms=kwargs.get("compressor_release", 100),
+            )
+            board.append(compressor)
+        if kwargs.get("delay", False):
+            delay = Delay(
+                delay_seconds=kwargs.get("delay_seconds", 0.5),
+                feedback=kwargs.get("delay_feedback", 0.0),
+                mix=kwargs.get("delay_mix", 0.5),
+            )
+            board.append(delay)
+        return board(audio_input, sample_rate)
+
+    def convert_audio(
+        self,
+        audio_input_path: str,
+        audio_output_path: str,
+        model_path: str,
+        index_path: str,
+        pitch: int = 0,
+        f0_file: str = None,
+        f0_method: str = "rmvpe",
+        index_rate: float = 0.75,
+        volume_envelope: float = 1,
+        protect: float = 0.5,
+        hop_length: int = 128,
+        split_audio: bool = False,
+        f0_autotune: bool = False,
+        f0_autotune_strength: float = 1,
+        filter_radius: int = 3,
+        embedder_model: str = "contentvec",
+        embedder_model_custom: str = None,
+        clean_audio: bool = False,
+        clean_strength: float = 0.5,
+        export_format: str = "WAV",
+        upscale_audio: bool = False,
+        post_process: bool = False,
+        resample_sr: int = 0,
+        sid: int = 0,
+        **kwargs,
+    ):
+        """
+        Performs voice conversion on the input audio.
+
+        Args:
+            pitch (int): Key for F0 up-sampling.
+            filter_radius (int): Radius for filtering.
+            index_rate (float): Rate for index matching.
+            volume_envelope (int): RMS mix rate.
+            protect (float): Protection rate for certain audio segments.
+            hop_length (int): Hop length for audio processing.
+            f0_method (str): Method for F0 extraction.
+            audio_input_path (str): Path to the input audio file.
+            audio_output_path (str): Path to the output audio file.
+            model_path (str): Path to the voice conversion model.
+            index_path (str): Path to the index file.
+            split_audio (bool): Whether to split the audio for processing.
+            f0_autotune (bool): Whether to use F0 autotune.
+            clean_audio (bool): Whether to clean the audio.
+            clean_strength (float): Strength of the audio cleaning.
+            export_format (str): Format for exporting the audio.
+            upscale_audio (bool): Whether to upscale the audio.
+            f0_file (str): Path to the F0 file.
+            embedder_model (str): Path to the embedder model.
+            embedder_model_custom (str): Path to the custom embedder model.
+            resample_sr (int, optional): Resample sampling rate. Default is 0.
+            sid (int, optional): Speaker ID. Default is 0.
+            **kwargs: Additional keyword arguments.
+        """
+        self.get_vc(model_path, sid)
+        try:
+            start_time = time.time()
+            print(f"Converting audio '{audio_input_path}'...")
+
+            audio = load_audio_infer(
+                audio_input_path,
+                16000,
+                **kwargs,
+            )
+            audio_max = np.abs(audio).max() / 0.95
+
+            if audio_max > 1:
+                audio /= audio_max
+
+            if not self.hubert_model or embedder_model != self.last_embedder_model:
+                self.load_hubert(embedder_model, embedder_model_custom)
+                self.last_embedder_model = embedder_model
+
+            file_index = (
+                index_path.strip()
+                .strip('"')
+                .strip("\n")
+                .strip('"')
+                .strip()
+                .replace("trained", "added")
+            )
+
+            if self.tgt_sr != resample_sr >= 16000:
+                self.tgt_sr = resample_sr
+
+            if split_audio:
+                chunks, intervals = process_audio(audio, 16000)
+                print(f"Audio split into {len(chunks)} chunks for processing.")
+            else:
+                chunks = []
+                chunks.append(audio)
+
+            converted_chunks = []
+            for c in chunks:
+                audio_opt = self.vc.pipeline(
+                    model=self.hubert_model,
+                    net_g=self.net_g,
+                    sid=sid,
+                    audio=c,
+                    pitch=pitch,
+                    f0_method=f0_method,
+                    file_index=file_index,
+                    index_rate=index_rate,
+                    pitch_guidance=self.use_f0,
+                    filter_radius=filter_radius,
+                    volume_envelope=volume_envelope,
+                    version=self.version,
+                    protect=protect,
+                    hop_length=hop_length,
+                    f0_autotune=f0_autotune,
+                    f0_autotune_strength=f0_autotune_strength,
+                    f0_file=f0_file,
+                )
+                converted_chunks.append(audio_opt)
+                if split_audio:
+                    print(f"Converted audio chunk {len(converted_chunks)}")
+
+            if split_audio:
+                audio_opt = merge_audio(converted_chunks, intervals, 16000, self.tgt_sr)
+            else:
+                audio_opt = converted_chunks[0]
+
+            if clean_audio:
+                cleaned_audio = self.remove_audio_noise(
+                    audio_opt, self.tgt_sr, clean_strength
+                )
+                if cleaned_audio is not None:
+                    audio_opt = cleaned_audio
+
+            if post_process:
+                audio_opt = self.post_process_audio(
+                    audio_input=audio_opt,
+                    sample_rate=self.tgt_sr,
+                    **kwargs,
+                )
+
+            sf.write(audio_output_path, audio_opt, self.tgt_sr, format="WAV")
+            output_path_format = audio_output_path.replace(
+                ".wav", f".{export_format.lower()}"
+            )
+            audio_output_path = self.convert_audio_format(
+                audio_output_path, output_path_format, export_format
+            )
+
+            elapsed_time = time.time() - start_time
+            print(
+                f"Conversion completed at '{audio_output_path}' in {elapsed_time:.2f} seconds."
+            )
+        except Exception as error:
+            print(f"An error occurred during audio conversion: {error}")
+            print(traceback.format_exc())
+
+    def convert_audio_batch(
+        self,
+        audio_input_paths: str,
+        audio_output_path: str,
+        **kwargs,
+    ):
+        """
+        Performs voice conversion on a batch of input audio files.
+
+        Args:
+            audio_input_paths (str): List of paths to the input audio files.
+            audio_output_path (str): Path to the output audio file.
+            resample_sr (int, optional): Resample sampling rate. Default is 0.
+            sid (int, optional): Speaker ID. Default is 0.
+            **kwargs: Additional keyword arguments.
+        """
+        pid = os.getpid()
+        try:
+            with open(
+                os.path.join(now_dir, "assets", "infer_pid.txt"), "w"
+            ) as pid_file:
+                pid_file.write(str(pid))
+            start_time = time.time()
+            print(f"Converting audio batch '{audio_input_paths}'...")
+            audio_files = [
+                f
+                for f in os.listdir(audio_input_paths)
+                if f.endswith(
+                    (
+                        "wav",
+                        "mp3",
+                        "flac",
+                        "ogg",
+                        "opus",
+                        "m4a",
+                        "mp4",
+                        "aac",
+                        "alac",
+                        "wma",
+                        "aiff",
+                        "webm",
+                        "ac3",
+                    )
+                )
+            ]
+            print(f"Detected {len(audio_files)} audio files for inference.")
+            for a in audio_files:
+                new_input = os.path.join(audio_input_paths, a)
+                new_output = os.path.splitext(a)[0] + "_output.wav"
+                new_output = os.path.join(audio_output_path, new_output)
+                if os.path.exists(new_output):
+                    continue
+                self.convert_audio(
+                    audio_input_path=new_input,
+                    audio_output_path=new_output,
+                    **kwargs,
+                )
+            print(f"Conversion completed at '{audio_input_paths}'.")
+            elapsed_time = time.time() - start_time
+            print(f"Batch conversion completed in {elapsed_time:.2f} seconds.")
+        except Exception as error:
+            print(f"An error occurred during audio batch conversion: {error}")
+            print(traceback.format_exc())
+        finally:
+            os.remove(os.path.join(now_dir, "assets", "infer_pid.txt"))
+
+    def get_vc(self, weight_root, sid):
+        """
+        Loads the voice conversion model and sets up the pipeline.
+
+        Args:
+            weight_root (str): Path to the model weights.
+            sid (int): Speaker ID.
+        """
+        if sid == "" or sid == []:
+            self.cleanup_model()
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+
+        if not self.loaded_model or self.loaded_model != weight_root:
+            self.load_model(weight_root)
+            if self.cpt is not None:
+                self.setup_network()
+                self.setup_vc_instance()
+            self.loaded_model = weight_root
+
+    def cleanup_model(self):
+        """
+        Cleans up the model and releases resources.
+        """
+        if self.hubert_model is not None:
+            del self.net_g, self.n_spk, self.vc, self.hubert_model, self.tgt_sr
+            self.hubert_model = self.net_g = self.n_spk = self.vc = self.tgt_sr = None
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+
+        del self.net_g, self.cpt
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        self.cpt = None
+
+    def load_model(self, weight_root):
+        """
+        Loads the model weights from the specified path.
+
+        Args:
+            weight_root (str): Path to the model weights.
+        """
+        self.cpt = (
+            torch.load(weight_root, map_location="cpu")
+            if os.path.isfile(weight_root)
+            else None
+        )
+
+    def setup_network(self):
+        """
+        Sets up the network configuration based on the loaded checkpoint.
+        """
+        if self.cpt is not None:
+            self.tgt_sr = self.cpt["config"][-1]
+            self.cpt["config"][-3] = self.cpt["weight"]["emb_g.weight"].shape[0]
+            self.use_f0 = self.cpt.get("f0", 1)
+
+            self.version = self.cpt.get("version", "v1")
+            self.text_enc_hidden_dim = 768 if self.version == "v2" else 256
+            self.net_g = Synthesizer(
+                *self.cpt["config"],
+                use_f0=self.use_f0,
+                text_enc_hidden_dim=self.text_enc_hidden_dim,
+                is_half=self.config.is_half,
+            )
+            del self.net_g.enc_q
+            self.net_g.load_state_dict(self.cpt["weight"], strict=False)
+            self.net_g.eval().to(self.config.device)
+            self.net_g = (
+                self.net_g.half() if self.config.is_half else self.net_g.float()
+            )
+
+    def setup_vc_instance(self):
+        """
+        Sets up the voice conversion pipeline instance based on the target sampling rate and configuration.
+        """
+        if self.cpt is not None:
+            self.vc = VC(self.tgt_sr, self.config)
+            self.n_spk = self.cpt["config"][-3]

modules.py

@@ -0,0 +1,124 @@
+import torch
+from rvc_cli.rvc.lib.algorithm.commons import fused_add_tanh_sigmoid_multiply
+
+
+class WaveNet(torch.nn.Module):
+    """WaveNet residual blocks as used in WaveGlow.
+
+    Args:
+        hidden_channels (int): Number of hidden channels.
+        kernel_size (int): Size of the convolutional kernel.
+        dilation_rate (int): Dilation rate of the convolution.
+        n_layers (int): Number of convolutional layers.
+        gin_channels (int, optional): Number of conditioning channels. Defaults to 0.
+        p_dropout (float, optional): Dropout probability. Defaults to 0.
+    """
+
+    def __init__(
+        self,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        gin_channels=0,
+        p_dropout=0,
+    ):
+        super().__init__()
+        assert kernel_size % 2 == 1, "Kernel size must be odd for proper padding."
+
+        self.hidden_channels = hidden_channels
+        self.kernel_size = (kernel_size,)
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+        self.p_dropout = p_dropout
+        self.n_channels_tensor = torch.IntTensor([hidden_channels])  # Static tensor
+
+        self.in_layers = torch.nn.ModuleList()
+        self.res_skip_layers = torch.nn.ModuleList()
+        self.drop = torch.nn.Dropout(p_dropout)
+
+        # Conditional layer for global conditioning
+        if gin_channels:
+            self.cond_layer = torch.nn.utils.parametrizations.weight_norm(
+                torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1),
+                name="weight",
+            )
+
+        # Precompute dilations and paddings
+        dilations = [dilation_rate**i for i in range(n_layers)]
+        paddings = [(kernel_size * d - d) // 2 for d in dilations]
+
+        # Initialize layers
+        for i in range(n_layers):
+            self.in_layers.append(
+                torch.nn.utils.parametrizations.weight_norm(
+                    torch.nn.Conv1d(
+                        hidden_channels,
+                        2 * hidden_channels,
+                        kernel_size,
+                        dilation=dilations[i],
+                        padding=paddings[i],
+                    ),
+                    name="weight",
+                )
+            )
+
+            res_skip_channels = (
+                hidden_channels if i == n_layers - 1 else 2 * hidden_channels
+            )
+            self.res_skip_layers.append(
+                torch.nn.utils.parametrizations.weight_norm(
+                    torch.nn.Conv1d(hidden_channels, res_skip_channels, 1),
+                    name="weight",
+                )
+            )
+
+    def forward(self, x, x_mask, g=None):
+        """Forward pass.
+
+        Args:
+            x (torch.Tensor): Input tensor (batch_size, hidden_channels, time_steps).
+            x_mask (torch.Tensor): Mask tensor (batch_size, 1, time_steps).
+            g (torch.Tensor, optional): Conditioning tensor (batch_size, gin_channels, time_steps).
+        """
+        output = x.clone().zero_()
+
+        # Apply conditional layer if global conditioning is provided
+        g = self.cond_layer(g) if g is not None else None
+
+        for i in range(self.n_layers):
+            x_in = self.in_layers[i](x)
+            g_l = (
+                g[
+                    :,
+                    i * 2 * self.hidden_channels : (i + 1) * 2 * self.hidden_channels,
+                    :,
+                ]
+                if g is not None
+                else 0
+            )
+
+            # Activation with fused Tanh-Sigmoid
+            acts = fused_add_tanh_sigmoid_multiply(x_in, g_l, self.n_channels_tensor)
+            acts = self.drop(acts)
+
+            # Residual and skip connections
+            res_skip_acts = self.res_skip_layers[i](acts)
+            if i < self.n_layers - 1:
+                res_acts = res_skip_acts[:, : self.hidden_channels, :]
+                x = (x + res_acts) * x_mask
+                output = output + res_skip_acts[:, self.hidden_channels :, :]
+            else:
+                output = output + res_skip_acts
+
+        return output * x_mask
+
+    def remove_weight_norm(self):
+        """Remove weight normalization from the module."""
+        if self.gin_channels:
+            torch.nn.utils.remove_weight_norm(self.cond_layer)
+        for layer in self.in_layers:
+            torch.nn.utils.remove_weight_norm(layer)
+        for layer in self.res_skip_layers:
+            torch.nn.utils.remove_weight_norm(layer)

normalization.py

@@ -0,0 +1,31 @@
+import torch
+
+
+class LayerNorm(torch.nn.Module):
+    """Layer normalization module.
+
+    Args:
+        channels (int): Number of channels.
+        eps (float, optional): Epsilon value for numerical stability. Defaults to 1e-5.
+    """
+
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.eps = eps
+        self.gamma = torch.nn.Parameter(torch.ones(channels))
+        self.beta = torch.nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        """Forward pass.
+
+        Args:
+            x (torch.Tensor): Input tensor of shape (batch_size, channels, time_steps).
+
+        """
+        # Transpose to (batch_size, time_steps, channels) for layer_norm
+        x = x.transpose(1, -1)
+        x = torch.nn.functional.layer_norm(
+            x, (x.size(-1),), self.gamma, self.beta, self.eps
+        )
+        # Transpose back to (batch_size, channels, time_steps)
+        return x.transpose(1, -1)

nsf.py

@@ -0,0 +1,196 @@
+import math
+import torch
+from torch.nn.utils import remove_weight_norm
+from torch.nn.utils.parametrizations import weight_norm
+from typing import Optional
+
+from rvc_cli.rvc.lib.algorithm.generators import SineGenerator
+from rvc_cli.rvc.lib.algorithm.residuals import LRELU_SLOPE, ResBlock1, ResBlock2
+from rvc_cli.rvc.lib.algorithm.commons import init_weights
+
+
+class SourceModuleHnNSF(torch.nn.Module):
+    """
+    Source Module for harmonic-plus-noise excitation.
+
+    Args:
+        sample_rate (int): Sampling rate in Hz.
+        harmonic_num (int, optional): Number of harmonics above F0. Defaults to 0.
+        sine_amp (float, optional): Amplitude of sine source signal. Defaults to 0.1.
+        add_noise_std (float, optional): Standard deviation of additive Gaussian noise. Defaults to 0.003.
+        voiced_threshod (float, optional): Threshold to set voiced/unvoiced given F0. Defaults to 0.
+        is_half (bool, optional): Whether to use half precision. Defaults to True.
+    """
+
+    def __init__(
+        self,
+        sample_rate,
+        harmonic_num=0,
+        sine_amp=0.1,
+        add_noise_std=0.003,
+        voiced_threshod=0,
+        is_half=True,
+    ):
+        super(SourceModuleHnNSF, self).__init__()
+
+        self.sine_amp = sine_amp
+        self.noise_std = add_noise_std
+        self.is_half = is_half
+
+        self.l_sin_gen = SineGenerator(
+            sample_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod
+        )
+        self.l_linear = torch.nn.Linear(harmonic_num + 1, 1)
+        self.l_tanh = torch.nn.Tanh()
+
+    def forward(self, x: torch.Tensor, upsample_factor: int = 1):
+        sine_wavs, uv, _ = self.l_sin_gen(x, upsample_factor)
+        sine_wavs = sine_wavs.to(dtype=self.l_linear.weight.dtype)
+        sine_merge = self.l_tanh(self.l_linear(sine_wavs))
+        return sine_merge, None, None
+
+
+class GeneratorNSF(torch.nn.Module):
+    """
+    Generator for synthesizing audio using the NSF (Neural Source Filter) approach.
+
+    Args:
+        initial_channel (int): Number of channels in the initial convolutional layer.
+        resblock (str): Type of residual block to use (1 or 2).
+        resblock_kernel_sizes (list): Kernel sizes of the residual blocks.
+        resblock_dilation_sizes (list): Dilation rates of the residual blocks.
+        upsample_rates (list): Upsampling rates.
+        upsample_initial_channel (int): Number of channels in the initial upsampling layer.
+        upsample_kernel_sizes (list): Kernel sizes of the upsampling layers.
+        gin_channels (int): Number of channels for the global conditioning input.
+        sr (int): Sampling rate.
+        is_half (bool, optional): Whether to use half precision. Defaults to False.
+    """
+
+    def __init__(
+        self,
+        initial_channel,
+        resblock,
+        resblock_kernel_sizes,
+        resblock_dilation_sizes,
+        upsample_rates,
+        upsample_initial_channel,
+        upsample_kernel_sizes,
+        gin_channels,
+        sr,
+        is_half=False,
+    ):
+        super(GeneratorNSF, self).__init__()
+
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+        self.f0_upsamp = torch.nn.Upsample(scale_factor=math.prod(upsample_rates))
+        self.m_source = SourceModuleHnNSF(
+            sample_rate=sr, harmonic_num=0, is_half=is_half
+        )
+
+        self.conv_pre = torch.nn.Conv1d(
+            initial_channel, upsample_initial_channel, 7, 1, padding=3
+        )
+        resblock_cls = ResBlock1 if resblock == "1" else ResBlock2
+
+        self.ups = torch.nn.ModuleList()
+        self.noise_convs = torch.nn.ModuleList()
+
+        channels = [
+            upsample_initial_channel // (2 ** (i + 1))
+            for i in range(len(upsample_rates))
+        ]
+        stride_f0s = [
+            math.prod(upsample_rates[i + 1 :]) if i + 1 < len(upsample_rates) else 1
+            for i in range(len(upsample_rates))
+        ]
+
+        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+            self.ups.append(
+                weight_norm(
+                    torch.nn.ConvTranspose1d(
+                        upsample_initial_channel // (2**i),
+                        channels[i],
+                        k,
+                        u,
+                        padding=(k - u) // 2,
+                    )
+                )
+            )
+
+            self.noise_convs.append(
+                torch.nn.Conv1d(
+                    1,
+                    channels[i],
+                    kernel_size=(stride_f0s[i] * 2 if stride_f0s[i] > 1 else 1),
+                    stride=stride_f0s[i],
+                    padding=(stride_f0s[i] // 2 if stride_f0s[i] > 1 else 0),
+                )
+            )
+
+        self.resblocks = torch.nn.ModuleList(
+            [
+                resblock_cls(channels[i], k, d)
+                for i in range(len(self.ups))
+                for k, d in zip(resblock_kernel_sizes, resblock_dilation_sizes)
+            ]
+        )
+
+        self.conv_post = torch.nn.Conv1d(channels[-1], 1, 7, 1, padding=3, bias=False)
+        self.ups.apply(init_weights)
+
+        if gin_channels != 0:
+            self.cond = torch.nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+        self.upp = math.prod(upsample_rates)
+        self.lrelu_slope = LRELU_SLOPE
+
+    def forward(self, x, f0, g: Optional[torch.Tensor] = None):
+        har_source, _, _ = self.m_source(f0, self.upp)
+        har_source = har_source.transpose(1, 2)
+        x = self.conv_pre(x)
+
+        if g is not None:
+            x = x + self.cond(g)
+
+        for i, (ups, noise_convs) in enumerate(zip(self.ups, self.noise_convs)):
+            x = torch.nn.functional.leaky_relu(x, self.lrelu_slope)
+            x = ups(x)
+            x = x + noise_convs(har_source)
+
+            xs = sum(
+                [
+                    resblock(x)
+                    for j, resblock in enumerate(self.resblocks)
+                    if j in range(i * self.num_kernels, (i + 1) * self.num_kernels)
+                ]
+            )
+            x = xs / self.num_kernels
+
+        x = torch.nn.functional.leaky_relu(x)
+        x = torch.tanh(self.conv_post(x))
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.ups:
+            remove_weight_norm(l)
+        for l in self.resblocks:
+            l.remove_weight_norm()
+
+    def __prepare_scriptable__(self):
+        for l in self.ups:
+            for hook in l._forward_pre_hooks.values():
+                if (
+                    hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                    and hook.__class__.__name__ == "WeightNorm"
+                ):
+                    remove_weight_norm(l)
+        for l in self.resblocks:
+            for hook in l._forward_pre_hooks.values():
+                if (
+                    hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                    and hook.__class__.__name__ == "WeightNorm"
+                ):
+                    remove_weight_norm(l)
+        return self

pipeline.py

@@ -0,0 +1,708 @@
+import os
+import gc
+import re
+import sys
+import torch
+import torch.nn.functional as F
+import torchcrepe
+import faiss
+import librosa
+import numpy as np
+from scipy import signal
+from torch import Tensor
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+
+from rvc_cli.rvc.lib.predictors.RMVPE import RMVPE0Predictor
+from rvc_cli.rvc.lib.predictors.FCPE import FCPEF0Predictor
+
+import logging
+
+logging.getLogger("faiss").setLevel(logging.WARNING)
+
+# Constants for high-pass filter
+FILTER_ORDER = 5
+CUTOFF_FREQUENCY = 48  # Hz
+SAMPLE_RATE = 16000  # Hz
+bh, ah = signal.butter(
+    N=FILTER_ORDER, Wn=CUTOFF_FREQUENCY, btype="high", fs=SAMPLE_RATE
+)
+
+input_audio_path2wav = {}
+
+
+class AudioProcessor:
+    """
+    A class for processing audio signals, specifically for adjusting RMS levels.
+    """
+
+    def change_rms(
+        source_audio: np.ndarray,
+        source_rate: int,
+        target_audio: np.ndarray,
+        target_rate: int,
+        rate: float,
+    ) -> np.ndarray:
+        """
+        Adjust the RMS level of target_audio to match the RMS of source_audio, with a given blending rate.
+
+        Args:
+            source_audio: The source audio signal as a NumPy array.
+            source_rate: The sampling rate of the source audio.
+            target_audio: The target audio signal to adjust.
+            target_rate: The sampling rate of the target audio.
+            rate: The blending rate between the source and target RMS levels.
+        """
+        # Calculate RMS of both audio data
+        rms1 = librosa.feature.rms(
+            y=source_audio,
+            frame_length=source_rate // 2 * 2,
+            hop_length=source_rate // 2,
+        )
+        rms2 = librosa.feature.rms(
+            y=target_audio,
+            frame_length=target_rate // 2 * 2,
+            hop_length=target_rate // 2,
+        )
+
+        # Interpolate RMS to match target audio length
+        rms1 = F.interpolate(
+            torch.from_numpy(rms1).float().unsqueeze(0),
+            size=target_audio.shape[0],
+            mode="linear",
+        ).squeeze()
+        rms2 = F.interpolate(
+            torch.from_numpy(rms2).float().unsqueeze(0),
+            size=target_audio.shape[0],
+            mode="linear",
+        ).squeeze()
+        rms2 = torch.maximum(rms2, torch.zeros_like(rms2) + 1e-6)
+
+        # Adjust target audio RMS based on the source audio RMS
+        adjusted_audio = (
+            target_audio
+            * (torch.pow(rms1, 1 - rate) * torch.pow(rms2, rate - 1)).numpy()
+        )
+        return adjusted_audio
+
+
+class Autotune:
+    """
+    A class for applying autotune to a given fundamental frequency (F0) contour.
+    """
+
+    def __init__(self, ref_freqs):
+        """
+        Initializes the Autotune class with a set of reference frequencies.
+
+        Args:
+            ref_freqs: A list of reference frequencies representing musical notes.
+        """
+        self.ref_freqs = ref_freqs
+        self.note_dict = self.ref_freqs  # No interpolation needed
+
+    def autotune_f0(self, f0, f0_autotune_strength):
+        """
+        Autotunes a given F0 contour by snapping each frequency to the closest reference frequency.
+
+        Args:
+            f0: The input F0 contour as a NumPy array.
+        """
+        autotuned_f0 = np.zeros_like(f0)
+        for i, freq in enumerate(f0):
+            closest_note = min(self.note_dict, key=lambda x: abs(x - freq))
+            autotuned_f0[i] = freq + (closest_note - freq) * f0_autotune_strength
+        return autotuned_f0
+
+
+class Pipeline:
+    """
+    The main pipeline class for performing voice conversion, including preprocessing, F0 estimation,
+    voice conversion using a model, and post-processing.
+    """
+
+    def __init__(self, tgt_sr, config):
+        """
+        Initializes the Pipeline class with target sampling rate and configuration parameters.
+
+        Args:
+            tgt_sr: The target sampling rate for the output audio.
+            config: A configuration object containing various parameters for the pipeline.
+        """
+        self.x_pad = config.x_pad
+        self.x_query = config.x_query
+        self.x_center = config.x_center
+        self.x_max = config.x_max
+        self.is_half = config.is_half
+        self.sample_rate = 16000
+        self.window = 160
+        self.t_pad = self.sample_rate * self.x_pad
+        self.t_pad_tgt = tgt_sr * self.x_pad
+        self.t_pad2 = self.t_pad * 2
+        self.t_query = self.sample_rate * self.x_query
+        self.t_center = self.sample_rate * self.x_center
+        self.t_max = self.sample_rate * self.x_max
+        self.time_step = self.window / self.sample_rate * 1000
+        self.f0_min = 50
+        self.f0_max = 1100
+        self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)
+        self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700)
+        self.device = config.device
+        self.ref_freqs = [
+            49.00,  # G1
+            51.91,  # G#1 / Ab1
+            55.00,  # A1
+            58.27,  # A#1 / Bb1
+            61.74,  # B1
+            65.41,  # C2
+            69.30,  # C#2 / Db2
+            73.42,  # D2
+            77.78,  # D#2 / Eb2
+            82.41,  # E2
+            87.31,  # F2
+            92.50,  # F#2 / Gb2
+            98.00,  # G2
+            103.83,  # G#2 / Ab2
+            110.00,  # A2
+            116.54,  # A#2 / Bb2
+            123.47,  # B2
+            130.81,  # C3
+            138.59,  # C#3 / Db3
+            146.83,  # D3
+            155.56,  # D#3 / Eb3
+            164.81,  # E3
+            174.61,  # F3
+            185.00,  # F#3 / Gb3
+            196.00,  # G3
+            207.65,  # G#3 / Ab3
+            220.00,  # A3
+            233.08,  # A#3 / Bb3
+            246.94,  # B3
+            261.63,  # C4
+            277.18,  # C#4 / Db4
+            293.66,  # D4
+            311.13,  # D#4 / Eb4
+            329.63,  # E4
+            349.23,  # F4
+            369.99,  # F#4 / Gb4
+            392.00,  # G4
+            415.30,  # G#4 / Ab4
+            440.00,  # A4
+            466.16,  # A#4 / Bb4
+            493.88,  # B4
+            523.25,  # C5
+            554.37,  # C#5 / Db5
+            587.33,  # D5
+            622.25,  # D#5 / Eb5
+            659.25,  # E5
+            698.46,  # F5
+            739.99,  # F#5 / Gb5
+            783.99,  # G5
+            830.61,  # G#5 / Ab5
+            880.00,  # A5
+            932.33,  # A#5 / Bb5
+            987.77,  # B5
+            1046.50,  # C6
+        ]
+        self.autotune = Autotune(self.ref_freqs)
+        self.note_dict = self.autotune.note_dict
+        self.model_rmvpe = RMVPE0Predictor(
+            os.path.join("rvc_cli", "rvc", "models", "predictors", "rmvpe.pt"),
+            is_half=self.is_half,
+            device=self.device,
+        )
+
+    def get_f0_crepe(
+        self,
+        x,
+        f0_min,
+        f0_max,
+        p_len,
+        hop_length,
+        model="full",
+    ):
+        """
+        Estimates the fundamental frequency (F0) of a given audio signal using the Crepe model.
+
+        Args:
+            x: The input audio signal as a NumPy array.
+            f0_min: Minimum F0 value to consider.
+            f0_max: Maximum F0 value to consider.
+            p_len: Desired length of the F0 output.
+            hop_length: Hop length for the Crepe model.
+            model: Crepe model size to use ("full" or "tiny").
+        """
+        x = x.astype(np.float32)
+        x /= np.quantile(np.abs(x), 0.999)
+        audio = torch.from_numpy(x).to(self.device, copy=True)
+        audio = torch.unsqueeze(audio, dim=0)
+        if audio.ndim == 2 and audio.shape[0] > 1:
+            audio = torch.mean(audio, dim=0, keepdim=True).detach()
+        audio = audio.detach()
+        pitch: Tensor = torchcrepe.predict(
+            audio,
+            self.sample_rate,
+            hop_length,
+            f0_min,
+            f0_max,
+            model,
+            batch_size=hop_length * 2,
+            device=self.device,
+            pad=True,
+        )
+        p_len = p_len or x.shape[0] // hop_length
+        source = np.array(pitch.squeeze(0).cpu().float().numpy())
+        source[source < 0.001] = np.nan
+        target = np.interp(
+            np.arange(0, len(source) * p_len, len(source)) / p_len,
+            np.arange(0, len(source)),
+            source,
+        )
+        f0 = np.nan_to_num(target)
+        return f0
+
+    def get_f0_hybrid(
+        self,
+        methods_str,
+        x,
+        f0_min,
+        f0_max,
+        p_len,
+        hop_length,
+    ):
+        """
+        Estimates the fundamental frequency (F0) using a hybrid approach combining multiple methods.
+
+        Args:
+            methods_str: A string specifying the methods to combine (e.g., "hybrid[crepe+rmvpe]").
+            x: The input audio signal as a NumPy array.
+            f0_min: Minimum F0 value to consider.
+            f0_max: Maximum F0 value to consider.
+            p_len: Desired length of the F0 output.
+            hop_length: Hop length for F0 estimation methods.
+        """
+        methods_str = re.search("hybrid\[(.+)\]", methods_str)
+        if methods_str:
+            methods = [method.strip() for method in methods_str.group(1).split("+")]
+        f0_computation_stack = []
+        print(f"Calculating f0 pitch estimations for methods: {', '.join(methods)}")
+        x = x.astype(np.float32)
+        x /= np.quantile(np.abs(x), 0.999)
+        for method in methods:
+            f0 = None
+            if method == "crepe":
+                f0 = self.get_f0_crepe_computation(
+                    x, f0_min, f0_max, p_len, int(hop_length)
+                )
+            elif method == "rmvpe":
+                f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
+                f0 = f0[1:]
+            elif method == "fcpe":
+                self.model_fcpe = FCPEF0Predictor(
+                    os.path.join("rvc_cli", "rvc", "models", "predictors", "fcpe.pt"),
+                    f0_min=int(f0_min),
+                    f0_max=int(f0_max),
+                    dtype=torch.float32,
+                    device=self.device,
+                    sample_rate=self.sample_rate,
+                    threshold=0.03,
+                )
+                f0 = self.model_fcpe.compute_f0(x, p_len=p_len)
+                del self.model_fcpe
+                gc.collect()
+            f0_computation_stack.append(f0)
+
+        f0_computation_stack = [fc for fc in f0_computation_stack if fc is not None]
+        f0_median_hybrid = None
+        if len(f0_computation_stack) == 1:
+            f0_median_hybrid = f0_computation_stack[0]
+        else:
+            f0_median_hybrid = np.nanmedian(f0_computation_stack, axis=0)
+        return f0_median_hybrid
+
+    def get_f0(
+        self,
+        input_audio_path,
+        x,
+        p_len,
+        pitch,
+        f0_method,
+        filter_radius,
+        hop_length,
+        f0_autotune,
+        f0_autotune_strength,
+        inp_f0=None,
+    ):
+        """
+        Estimates the fundamental frequency (F0) of a given audio signal using various methods.
+
+        Args:
+            input_audio_path: Path to the input audio file.
+            x: The input audio signal as a NumPy array.
+            p_len: Desired length of the F0 output.
+            pitch: Key to adjust the pitch of the F0 contour.
+            f0_method: Method to use for F0 estimation (e.g., "crepe").
+            filter_radius: Radius for median filtering the F0 contour.
+            hop_length: Hop length for F0 estimation methods.
+            f0_autotune: Whether to apply autotune to the F0 contour.
+            inp_f0: Optional input F0 contour to use instead of estimating.
+        """
+        global input_audio_path2wav
+        if f0_method == "crepe":
+            f0 = self.get_f0_crepe(x, self.f0_min, self.f0_max, p_len, int(hop_length))
+        elif f0_method == "crepe-tiny":
+            f0 = self.get_f0_crepe(
+                x, self.f0_min, self.f0_max, p_len, int(hop_length), "tiny"
+            )
+        elif f0_method == "rmvpe":
+            f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
+        elif f0_method == "fcpe":
+            self.model_fcpe = FCPEF0Predictor(
+                os.path.join("rvc_cli", "rvc", "models", "predictors", "fcpe.pt"),
+                f0_min=int(self.f0_min),
+                f0_max=int(self.f0_max),
+                dtype=torch.float32,
+                device=self.device,
+                sample_rate=self.sample_rate,
+                threshold=0.03,
+            )
+            f0 = self.model_fcpe.compute_f0(x, p_len=p_len)
+            del self.model_fcpe
+            gc.collect()
+        elif "hybrid" in f0_method:
+            input_audio_path2wav[input_audio_path] = x.astype(np.double)
+            f0 = self.get_f0_hybrid(
+                f0_method,
+                x,
+                self.f0_min,
+                self.f0_max,
+                p_len,
+                hop_length,
+            )
+
+        if f0_autotune is True:
+            f0 = Autotune.autotune_f0(self, f0, f0_autotune_strength)
+
+        f0 *= pow(2, pitch / 12)
+        tf0 = self.sample_rate // self.window
+        if inp_f0 is not None:
+            delta_t = np.round(
+                (inp_f0[:, 0].max() - inp_f0[:, 0].min()) * tf0 + 1
+            ).astype("int16")
+            replace_f0 = np.interp(
+                list(range(delta_t)), inp_f0[:, 0] * 100, inp_f0[:, 1]
+            )
+            shape = f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)].shape[0]
+            f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)] = replace_f0[
+                :shape
+            ]
+        f0bak = f0.copy()
+        f0_mel = 1127 * np.log(1 + f0 / 700)
+        f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * 254 / (
+            self.f0_mel_max - self.f0_mel_min
+        ) + 1
+        f0_mel[f0_mel <= 1] = 1
+        f0_mel[f0_mel > 255] = 255
+        f0_coarse = np.rint(f0_mel).astype(int)
+
+        return f0_coarse, f0bak
+
+    def voice_conversion(
+        self,
+        model,
+        net_g,
+        sid,
+        audio0,
+        pitch,
+        pitchf,
+        index,
+        big_npy,
+        index_rate,
+        version,
+        protect,
+    ):
+        """
+        Performs voice conversion on a given audio segment.
+
+        Args:
+            model: The feature extractor model.
+            net_g: The generative model for synthesizing speech.
+            sid: Speaker ID for the target voice.
+            audio0: The input audio segment.
+            pitch: Quantized F0 contour for pitch guidance.
+            pitchf: Original F0 contour for pitch guidance.
+            index: FAISS index for speaker embedding retrieval.
+            big_npy: Speaker embeddings stored in a NumPy array.
+            index_rate: Blending rate for speaker embedding retrieval.
+            version: Model version ("v1" or "v2").
+            protect: Protection level for preserving the original pitch.
+        """
+        with torch.no_grad():
+            pitch_guidance = pitch != None and pitchf != None
+            # prepare source audio
+            feats = (
+                torch.from_numpy(audio0).half()
+                if self.is_half
+                else torch.from_numpy(audio0).float()
+            )
+            feats = feats.mean(-1) if feats.dim() == 2 else feats
+            assert feats.dim() == 1, feats.dim()
+            feats = feats.view(1, -1).to(self.device)
+            # extract features
+            feats = model(feats)["last_hidden_state"]
+            feats = (
+                model.final_proj(feats[0]).unsqueeze(0) if version == "v1" else feats
+            )
+            # make a copy for pitch guidance and protection
+            feats0 = feats.clone() if pitch_guidance else None
+            if (
+                index
+            ):  # set by parent function, only true if index is available, loaded, and index rate > 0
+                feats = self._retrieve_speaker_embeddings(
+                    feats, index, big_npy, index_rate
+                )
+            # feature upsampling
+            feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(
+                0, 2, 1
+            )
+            # adjust the length if the audio is short
+            p_len = min(audio0.shape[0] // self.window, feats.shape[1])
+            if pitch_guidance:
+                feats0 = F.interpolate(feats0.permute(0, 2, 1), scale_factor=2).permute(
+                    0, 2, 1
+                )
+                pitch, pitchf = pitch[:, :p_len], pitchf[:, :p_len]
+                # Pitch protection blending
+                if protect < 0.5:
+                    pitchff = pitchf.clone()
+                    pitchff[pitchf > 0] = 1
+                    pitchff[pitchf < 1] = protect
+                    feats = feats * pitchff.unsqueeze(-1) + feats0 * (
+                        1 - pitchff.unsqueeze(-1)
+                    )
+                    feats = feats.to(feats0.dtype)
+            else:
+                pitch, pitchf = None, None
+            p_len = torch.tensor([p_len], device=self.device).long()
+            audio1 = (
+                (net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0])
+                .data.cpu()
+                .float()
+                .numpy()
+            )
+            # clean up
+            del feats, feats0, p_len
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+        return audio1
+
+    def _retrieve_speaker_embeddings(self, feats, index, big_npy, index_rate):
+        npy = feats[0].cpu().numpy()
+        npy = npy.astype("float32") if self.is_half else npy
+        score, ix = index.search(npy, k=8)
+        weight = np.square(1 / score)
+        weight /= weight.sum(axis=1, keepdims=True)
+        npy = np.sum(big_npy[ix] * np.expand_dims(weight, axis=2), axis=1)
+        npy = npy.astype("float16") if self.is_half else npy
+        feats = (
+            torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate
+            + (1 - index_rate) * feats
+        )
+        return feats
+
+    def pipeline(
+        self,
+        model,
+        net_g,
+        sid,
+        audio,
+        pitch,
+        f0_method,
+        file_index,
+        index_rate,
+        pitch_guidance,
+        filter_radius,
+        volume_envelope,
+        version,
+        protect,
+        hop_length,
+        f0_autotune,
+        f0_autotune_strength,
+        f0_file,
+    ):
+        """
+        The main pipeline function for performing voice conversion.
+
+        Args:
+            model: The feature extractor model.
+            net_g: The generative model for synthesizing speech.
+            sid: Speaker ID for the target voice.
+            audio: The input audio signal.
+            input_audio_path: Path to the input audio file.
+            pitch: Key to adjust the pitch of the F0 contour.
+            f0_method: Method to use for F0 estimation.
+            file_index: Path to the FAISS index file for speaker embedding retrieval.
+            index_rate: Blending rate for speaker embedding retrieval.
+            pitch_guidance: Whether to use pitch guidance during voice conversion.
+            filter_radius: Radius for median filtering the F0 contour.
+            tgt_sr: Target sampling rate for the output audio.
+            resample_sr: Resampling rate for the output audio.
+            volume_envelope: Blending rate for adjusting the RMS level of the output audio.
+            version: Model version.
+            protect: Protection level for preserving the original pitch.
+            hop_length: Hop length for F0 estimation methods.
+            f0_autotune: Whether to apply autotune to the F0 contour.
+            f0_file: Path to a file containing an F0 contour to use.
+        """
+        if file_index != "" and os.path.exists(file_index) and index_rate > 0:
+            try:
+                index = faiss.read_index(file_index)
+                big_npy = index.reconstruct_n(0, index.ntotal)
+            except Exception as error:
+                print(f"An error occurred reading the FAISS index: {error}")
+                index = big_npy = None
+        else:
+            index = big_npy = None
+        audio = signal.filtfilt(bh, ah, audio)
+        audio_pad = np.pad(audio, (self.window // 2, self.window // 2), mode="reflect")
+        opt_ts = []
+        if audio_pad.shape[0] > self.t_max:
+            audio_sum = np.zeros_like(audio)
+            for i in range(self.window):
+                audio_sum += audio_pad[i : i - self.window]
+            for t in range(self.t_center, audio.shape[0], self.t_center):
+                opt_ts.append(
+                    t
+                    - self.t_query
+                    + np.where(
+                        np.abs(audio_sum[t - self.t_query : t + self.t_query])
+                        == np.abs(audio_sum[t - self.t_query : t + self.t_query]).min()
+                    )[0][0]
+                )
+        s = 0
+        audio_opt = []
+        t = None
+        audio_pad = np.pad(audio, (self.t_pad, self.t_pad), mode="reflect")
+        p_len = audio_pad.shape[0] // self.window
+        inp_f0 = None
+        if hasattr(f0_file, "name"):
+            try:
+                with open(f0_file.name, "r") as f:
+                    lines = f.read().strip("\n").split("\n")
+                inp_f0 = []
+                for line in lines:
+                    inp_f0.append([float(i) for i in line.split(",")])
+                inp_f0 = np.array(inp_f0, dtype="float32")
+            except Exception as error:
+                print(f"An error occurred reading the F0 file: {error}")
+        sid = torch.tensor(sid, device=self.device).unsqueeze(0).long()
+        if pitch_guidance:
+            pitch, pitchf = self.get_f0(
+                "input_audio_path",  # questionable purpose of making a key for an array
+                audio_pad,
+                p_len,
+                pitch,
+                f0_method,
+                filter_radius,
+                hop_length,
+                f0_autotune,
+                f0_autotune_strength,
+                inp_f0,
+            )
+            pitch = pitch[:p_len]
+            pitchf = pitchf[:p_len]
+            if self.device == "mps":
+                pitchf = pitchf.astype(np.float32)
+            pitch = torch.tensor(pitch, device=self.device).unsqueeze(0).long()
+            pitchf = torch.tensor(pitchf, device=self.device).unsqueeze(0).float()
+        for t in opt_ts:
+            t = t // self.window * self.window
+            if pitch_guidance:
+                audio_opt.append(
+                    self.voice_conversion(
+                        model,
+                        net_g,
+                        sid,
+                        audio_pad[s : t + self.t_pad2 + self.window],
+                        pitch[:, s // self.window : (t + self.t_pad2) // self.window],
+                        pitchf[:, s // self.window : (t + self.t_pad2) // self.window],
+                        index,
+                        big_npy,
+                        index_rate,
+                        version,
+                        protect,
+                    )[self.t_pad_tgt : -self.t_pad_tgt]
+                )
+            else:
+                audio_opt.append(
+                    self.voice_conversion(
+                        model,
+                        net_g,
+                        sid,
+                        audio_pad[s : t + self.t_pad2 + self.window],
+                        None,
+                        None,
+                        index,
+                        big_npy,
+                        index_rate,
+                        version,
+                        protect,
+                    )[self.t_pad_tgt : -self.t_pad_tgt]
+                )
+            s = t
+        if pitch_guidance:
+            audio_opt.append(
+                self.voice_conversion(
+                    model,
+                    net_g,
+                    sid,
+                    audio_pad[t:],
+                    pitch[:, t // self.window :] if t is not None else pitch,
+                    pitchf[:, t // self.window :] if t is not None else pitchf,
+                    index,
+                    big_npy,
+                    index_rate,
+                    version,
+                    protect,
+                )[self.t_pad_tgt : -self.t_pad_tgt]
+            )
+        else:
+            audio_opt.append(
+                self.voice_conversion(
+                    model,
+                    net_g,
+                    sid,
+                    audio_pad[t:],
+                    None,
+                    None,
+                    index,
+                    big_npy,
+                    index_rate,
+                    version,
+                    protect,
+                )[self.t_pad_tgt : -self.t_pad_tgt]
+            )
+        audio_opt = np.concatenate(audio_opt)
+        if volume_envelope != 1:
+            audio_opt = AudioProcessor.change_rms(
+                audio, self.sample_rate, audio_opt, self.sample_rate, volume_envelope
+            )
+        # if resample_sr >= self.sample_rate and tgt_sr != resample_sr:
+        #    audio_opt = librosa.resample(
+        #        audio_opt, orig_sr=tgt_sr, target_sr=resample_sr
+        #    )
+        # audio_max = np.abs(audio_opt).max() / 0.99
+        # max_int16 = 32768
+        # if audio_max > 1:
+        #    max_int16 /= audio_max
+        # audio_opt = (audio_opt * 32768).astype(np.int16)
+        audio_max = np.abs(audio_opt).max() / 0.99
+        if audio_max > 1:
+            audio_opt /= audio_max
+        if pitch_guidance:
+            del pitch, pitchf
+        del sid
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio_opt

residuals.py

@@ -0,0 +1,250 @@
+from typing import Optional
+import torch
+from torch.nn.utils import remove_weight_norm
+from torch.nn.utils.parametrizations import weight_norm
+
+from rvc_cli.rvc.lib.algorithm.modules import WaveNet
+from rvc_cli.rvc.lib.algorithm.commons import get_padding, init_weights
+
+LRELU_SLOPE = 0.1
+
+
+def create_conv1d_layer(channels, kernel_size, dilation):
+    return weight_norm(
+        torch.nn.Conv1d(
+            channels,
+            channels,
+            kernel_size,
+            1,
+            dilation=dilation,
+            padding=get_padding(kernel_size, dilation),
+        )
+    )
+
+
+def apply_mask(tensor, mask):
+    return tensor * mask if mask is not None else tensor
+
+
+class ResBlockBase(torch.nn.Module):
+    def __init__(self, channels, kernel_size, dilations):
+        super(ResBlockBase, self).__init__()
+        self.convs1 = torch.nn.ModuleList(
+            [create_conv1d_layer(channels, kernel_size, d) for d in dilations]
+        )
+        self.convs1.apply(init_weights)
+
+        self.convs2 = torch.nn.ModuleList(
+            [create_conv1d_layer(channels, kernel_size, 1) for _ in dilations]
+        )
+        self.convs2.apply(init_weights)
+
+    def forward(self, x, x_mask=None):
+        for c1, c2 in zip(self.convs1, self.convs2):
+            xt = torch.nn.functional.leaky_relu(x, LRELU_SLOPE)
+            xt = apply_mask(xt, x_mask)
+            xt = torch.nn.functional.leaky_relu(c1(xt), LRELU_SLOPE)
+            xt = apply_mask(xt, x_mask)
+            xt = c2(xt)
+            x = xt + x
+        return apply_mask(x, x_mask)
+
+    def remove_weight_norm(self):
+        for conv in self.convs1 + self.convs2:
+            remove_weight_norm(conv)
+
+
+class ResBlock1(ResBlockBase):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
+        super(ResBlock1, self).__init__(channels, kernel_size, dilation)
+
+
+class ResBlock2(ResBlockBase):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
+        super(ResBlock2, self).__init__(channels, kernel_size, dilation)
+
+
+class Flip(torch.nn.Module):
+    """Flip module for flow-based models.
+
+    This module flips the input along the time dimension.
+    """
+
+    def forward(self, x, *args, reverse=False, **kwargs):
+        """Forward pass.
+
+        Args:
+            x (torch.Tensor): Input tensor.
+            reverse (bool, optional): Whether to reverse the operation. Defaults to False.
+        """
+        x = torch.flip(x, [1])
+        if not reverse:
+            logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
+            return x, logdet
+        else:
+            return x
+
+
+class ResidualCouplingBlock(torch.nn.Module):
+    """Residual Coupling Block for normalizing flow.
+
+    Args:
+        channels (int): Number of channels in the input.
+        hidden_channels (int): Number of hidden channels in the coupling layer.
+        kernel_size (int): Kernel size of the convolutional layers.
+        dilation_rate (int): Dilation rate of the convolutional layers.
+        n_layers (int): Number of layers in the coupling layer.
+        n_flows (int, optional): Number of coupling layers in the block. Defaults to 4.
+        gin_channels (int, optional): Number of channels for the global conditioning input. Defaults to 0.
+    """
+
+    def __init__(
+        self,
+        channels,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        n_flows=4,
+        gin_channels=0,
+    ):
+        super(ResidualCouplingBlock, self).__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.n_flows = n_flows
+        self.gin_channels = gin_channels
+
+        self.flows = torch.nn.ModuleList()
+        for i in range(n_flows):
+            self.flows.append(
+                ResidualCouplingLayer(
+                    channels,
+                    hidden_channels,
+                    kernel_size,
+                    dilation_rate,
+                    n_layers,
+                    gin_channels=gin_channels,
+                    mean_only=True,
+                )
+            )
+            self.flows.append(Flip())
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        x_mask: torch.Tensor,
+        g: Optional[torch.Tensor] = None,
+        reverse: bool = False,
+    ):
+        if not reverse:
+            for flow in self.flows:
+                x, _ = flow(x, x_mask, g=g, reverse=reverse)
+        else:
+            for flow in reversed(self.flows):
+                x = flow.forward(x, x_mask, g=g, reverse=reverse)
+        return x
+
+    def remove_weight_norm(self):
+        """Removes weight normalization from the coupling layers."""
+        for i in range(self.n_flows):
+            self.flows[i * 2].remove_weight_norm()
+
+    def __prepare_scriptable__(self):
+        """Prepares the module for scripting."""
+        for i in range(self.n_flows):
+            for hook in self.flows[i * 2]._forward_pre_hooks.values():
+                if (
+                    hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                    and hook.__class__.__name__ == "WeightNorm"
+                ):
+                    torch.nn.utils.remove_weight_norm(self.flows[i * 2])
+
+        return self
+
+
+class ResidualCouplingLayer(torch.nn.Module):
+    """Residual coupling layer for flow-based models.
+
+    Args:
+        channels (int): Number of channels.
+        hidden_channels (int): Number of hidden channels.
+        kernel_size (int): Size of the convolutional kernel.
+        dilation_rate (int): Dilation rate of the convolution.
+        n_layers (int): Number of convolutional layers.
+        p_dropout (float, optional): Dropout probability. Defaults to 0.
+        gin_channels (int, optional): Number of conditioning channels. Defaults to 0.
+        mean_only (bool, optional): Whether to use mean-only coupling. Defaults to False.
+    """
+
+    def __init__(
+        self,
+        channels,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        p_dropout=0,
+        gin_channels=0,
+        mean_only=False,
+    ):
+        assert channels % 2 == 0, "channels should be divisible by 2"
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.half_channels = channels // 2
+        self.mean_only = mean_only
+
+        self.pre = torch.nn.Conv1d(self.half_channels, hidden_channels, 1)
+        self.enc = WaveNet(
+            hidden_channels,
+            kernel_size,
+            dilation_rate,
+            n_layers,
+            p_dropout=p_dropout,
+            gin_channels=gin_channels,
+        )
+        self.post = torch.nn.Conv1d(
+            hidden_channels, self.half_channels * (2 - mean_only), 1
+        )
+        self.post.weight.data.zero_()
+        self.post.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        """Forward pass.
+
+        Args:
+            x (torch.Tensor): Input tensor of shape (batch_size, channels, time_steps).
+            x_mask (torch.Tensor): Mask tensor of shape (batch_size, 1, time_steps).
+            g (torch.Tensor, optional): Conditioning tensor of shape (batch_size, gin_channels, time_steps).
+                Defaults to None.
+            reverse (bool, optional): Whether to reverse the operation. Defaults to False.
+        """
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0) * x_mask
+        h = self.enc(h, x_mask, g=g)
+        stats = self.post(h) * x_mask
+        if not self.mean_only:
+            m, logs = torch.split(stats, [self.half_channels] * 2, 1)
+        else:
+            m = stats
+            logs = torch.zeros_like(m)
+
+        if not reverse:
+            x1 = m + x1 * torch.exp(logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            logdet = torch.sum(logs, [1, 2])
+            return x, logdet
+        else:
+            x1 = (x1 - m) * torch.exp(-logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            return x
+
+    def remove_weight_norm(self):
+        """Remove weight normalization from the module."""
+        self.enc.remove_weight_norm()

synthesizers.py

@@ -0,0 +1,237 @@
+import torch
+from typing import Optional
+
+from rvc_cli.rvc.lib.algorithm.nsf import GeneratorNSF
+from rvc_cli.rvc.lib.algorithm.generators import Generator
+from rvc_cli.rvc.lib.algorithm.commons import slice_segments, rand_slice_segments
+from rvc_cli.rvc.lib.algorithm.residuals import ResidualCouplingBlock
+from rvc_cli.rvc.lib.algorithm.encoders import TextEncoder, PosteriorEncoder
+
+
+class Synthesizer(torch.nn.Module):
+    """
+    Base Synthesizer model.
+
+    Args:
+        spec_channels (int): Number of channels in the spectrogram.
+        segment_size (int): Size of the audio segment.
+        inter_channels (int): Number of channels in the intermediate layers.
+        hidden_channels (int): Number of channels in the hidden layers.
+        filter_channels (int): Number of channels in the filter layers.
+        n_heads (int): Number of attention heads.
+        n_layers (int): Number of layers in the encoder.
+        kernel_size (int): Size of the convolution kernel.
+        p_dropout (float): Dropout probability.
+        resblock (str): Type of residual block.
+        resblock_kernel_sizes (list): Kernel sizes for the residual blocks.
+        resblock_dilation_sizes (list): Dilation sizes for the residual blocks.
+        upsample_rates (list): Upsampling rates for the decoder.
+        upsample_initial_channel (int): Number of channels in the initial upsampling layer.
+        upsample_kernel_sizes (list): Kernel sizes for the upsampling layers.
+        spk_embed_dim (int): Dimension of the speaker embedding.
+        gin_channels (int): Number of channels in the global conditioning vector.
+        sr (int): Sampling rate of the audio.
+        use_f0 (bool): Whether to use F0 information.
+        text_enc_hidden_dim (int): Hidden dimension for the text encoder.
+        kwargs: Additional keyword arguments.
+    """
+
+    def __init__(
+        self,
+        spec_channels,
+        segment_size,
+        inter_channels,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size,
+        p_dropout,
+        resblock,
+        resblock_kernel_sizes,
+        resblock_dilation_sizes,
+        upsample_rates,
+        upsample_initial_channel,
+        upsample_kernel_sizes,
+        spk_embed_dim,
+        gin_channels,
+        sr,
+        use_f0,
+        text_enc_hidden_dim=768,
+        **kwargs
+    ):
+        super(Synthesizer, self).__init__()
+        self.spec_channels = spec_channels
+        self.inter_channels = inter_channels
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = float(p_dropout)
+        self.resblock = resblock
+        self.resblock_kernel_sizes = resblock_kernel_sizes
+        self.resblock_dilation_sizes = resblock_dilation_sizes
+        self.upsample_rates = upsample_rates
+        self.upsample_initial_channel = upsample_initial_channel
+        self.upsample_kernel_sizes = upsample_kernel_sizes
+        self.segment_size = segment_size
+        self.gin_channels = gin_channels
+        self.spk_embed_dim = spk_embed_dim
+        self.use_f0 = use_f0
+
+        self.enc_p = TextEncoder(
+            inter_channels,
+            hidden_channels,
+            filter_channels,
+            n_heads,
+            n_layers,
+            kernel_size,
+            float(p_dropout),
+            text_enc_hidden_dim,
+            f0=use_f0,
+        )
+
+        if use_f0:
+            self.dec = GeneratorNSF(
+                inter_channels,
+                resblock,
+                resblock_kernel_sizes,
+                resblock_dilation_sizes,
+                upsample_rates,
+                upsample_initial_channel,
+                upsample_kernel_sizes,
+                gin_channels=gin_channels,
+                sr=sr,
+                is_half=kwargs["is_half"],
+            )
+        else:
+            self.dec = Generator(
+                inter_channels,
+                resblock,
+                resblock_kernel_sizes,
+                resblock_dilation_sizes,
+                upsample_rates,
+                upsample_initial_channel,
+                upsample_kernel_sizes,
+                gin_channels=gin_channels,
+            )
+
+        self.enc_q = PosteriorEncoder(
+            spec_channels,
+            inter_channels,
+            hidden_channels,
+            5,
+            1,
+            16,
+            gin_channels=gin_channels,
+        )
+        self.flow = ResidualCouplingBlock(
+            inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels
+        )
+        self.emb_g = torch.nn.Embedding(self.spk_embed_dim, gin_channels)
+
+    def remove_weight_norm(self):
+        """Removes weight normalization from the model."""
+        self.dec.remove_weight_norm()
+        self.flow.remove_weight_norm()
+        self.enc_q.remove_weight_norm()
+
+    def __prepare_scriptable__(self):
+        for hook in self.dec._forward_pre_hooks.values():
+            if (
+                hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                and hook.__class__.__name__ == "WeightNorm"
+            ):
+                torch.nn.utils.remove_weight_norm(self.dec)
+        for hook in self.flow._forward_pre_hooks.values():
+            if (
+                hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                and hook.__class__.__name__ == "WeightNorm"
+            ):
+                torch.nn.utils.remove_weight_norm(self.flow)
+        if hasattr(self, "enc_q"):
+            for hook in self.enc_q._forward_pre_hooks.values():
+                if (
+                    hook.__module__ == "torch.nn.utils.parametrizations.weight_norm"
+                    and hook.__class__.__name__ == "WeightNorm"
+                ):
+                    torch.nn.utils.remove_weight_norm(self.enc_q)
+        return self
+
+    @torch.jit.ignore
+    def forward(
+        self,
+        phone: torch.Tensor,
+        phone_lengths: torch.Tensor,
+        pitch: Optional[torch.Tensor] = None,
+        pitchf: Optional[torch.Tensor] = None,
+        y: torch.Tensor = None,
+        y_lengths: torch.Tensor = None,
+        ds: Optional[torch.Tensor] = None,
+    ):
+        """
+        Forward pass of the model.
+
+        Args:
+            phone (torch.Tensor): Phoneme sequence.
+            phone_lengths (torch.Tensor): Lengths of the phoneme sequences.
+            pitch (torch.Tensor, optional): Pitch sequence.
+            pitchf (torch.Tensor, optional): Fine-grained pitch sequence.
+            y (torch.Tensor, optional): Target spectrogram.
+            y_lengths (torch.Tensor, optional): Lengths of the target spectrograms.
+            ds (torch.Tensor, optional): Speaker embedding. Defaults to None.
+        """
+        g = self.emb_g(ds).unsqueeze(-1)
+        m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
+        if y is not None:
+            z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
+            z_p = self.flow(z, y_mask, g=g)
+            z_slice, ids_slice = rand_slice_segments(z, y_lengths, self.segment_size)
+            if self.use_f0:
+                pitchf = slice_segments(pitchf, ids_slice, self.segment_size, 2)
+                o = self.dec(z_slice, pitchf, g=g)
+            else:
+                o = self.dec(z_slice, g=g)
+            return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
+        else:
+            return None, None, x_mask, None, (None, None, m_p, logs_p, None, None)
+
+    @torch.jit.export
+    def infer(
+        self,
+        phone: torch.Tensor,
+        phone_lengths: torch.Tensor,
+        pitch: Optional[torch.Tensor] = None,
+        nsff0: Optional[torch.Tensor] = None,
+        sid: torch.Tensor = None,
+        rate: Optional[torch.Tensor] = None,
+    ):
+        """
+        Inference of the model.
+
+        Args:
+            phone (torch.Tensor): Phoneme sequence.
+            phone_lengths (torch.Tensor): Lengths of the phoneme sequences.
+            pitch (torch.Tensor, optional): Pitch sequence.
+            nsff0 (torch.Tensor, optional): Fine-grained pitch sequence.
+            sid (torch.Tensor): Speaker embedding.
+            rate (torch.Tensor, optional): Rate for time-stretching. Defaults to None.
+        """
+        g = self.emb_g(sid).unsqueeze(-1)
+        m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
+        z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
+        if rate is not None:
+            assert isinstance(rate, torch.Tensor)
+            head = int(z_p.shape[2] * (1.0 - rate.item()))
+            z_p = z_p[:, :, head:]
+            x_mask = x_mask[:, :, head:]
+            if self.use_f0:
+                nsff0 = nsff0[:, head:]
+        if self.use_f0:
+            z = self.flow(z_p, x_mask, g=g, reverse=True)
+            o = self.dec(z * x_mask, nsff0, g=g)
+        else:
+            z = self.flow(z_p, x_mask, g=g, reverse=True)
+            o = self.dec(z * x_mask, g=g)
+        return o, x_mask, (z, z_p, m_p, logs_p)

utils.py

@@ -0,0 +1,137 @@
+import os, sys
+import librosa
+import soundfile as sf
+import numpy as np
+import re
+import unicodedata
+import wget
+from pydub import AudioSegment
+from torch import nn
+
+import logging
+from transformers import HubertModel
+import warnings
+
+# Remove this to see warnings about transformers models
+warnings.filterwarnings("ignore")
+
+logging.getLogger("fairseq").setLevel(logging.ERROR)
+logging.getLogger("faiss.loader").setLevel(logging.ERROR)
+logging.getLogger("transformers").setLevel(logging.ERROR)
+logging.getLogger("torch").setLevel(logging.ERROR)
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+
+base_path = os.path.join(now_dir, "rvc_cli", "rvc", "models", "formant", "stftpitchshift")
+stft = base_path + ".exe" if sys.platform == "win32" else base_path
+
+
+class HubertModelWithFinalProj(HubertModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.final_proj = nn.Linear(config.hidden_size, config.classifier_proj_size)
+
+
+def load_audio(file, sample_rate):
+    try:
+        file = file.strip(" ").strip('"').strip("\n").strip('"').strip(" ")
+        audio, sr = sf.read(file)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio.T)
+        if sr != sample_rate:
+            audio = librosa.resample(audio, orig_sr=sr, target_sr=sample_rate)
+    except Exception as error:
+        raise RuntimeError(f"An error occurred loading the audio: {error}")
+
+    return audio.flatten()
+
+
+def load_audio_infer(
+    file,
+    sample_rate,
+    **kwargs,
+):
+    formant_shifting = kwargs.get("formant_shifting", False)
+    try:
+        file = file.strip(" ").strip('"').strip("\n").strip('"').strip(" ")
+        if not os.path.isfile(file):
+            raise FileNotFoundError(f"File not found: {file}")
+        audio, sr = sf.read(file)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio.T)
+        if sr != sample_rate:
+            audio = librosa.resample(audio, orig_sr=sr, target_sr=sample_rate)
+        if formant_shifting:
+            formant_qfrency = kwargs.get("formant_qfrency", 0.8)
+            formant_timbre = kwargs.get("formant_timbre", 0.8)
+
+            from stftpitchshift import StftPitchShift
+
+            pitchshifter = StftPitchShift(1024, 32, sample_rate)
+            audio = pitchshifter.shiftpitch(
+                audio,
+                factors=1,
+                quefrency=formant_qfrency * 1e-3,
+                distortion=formant_timbre,
+            )
+    except Exception as error:
+        raise RuntimeError(f"An error occurred loading the audio: {error}")
+    return np.array(audio).flatten()
+
+
+def format_title(title):
+    formatted_title = (
+        unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode("utf-8")
+    )
+    formatted_title = re.sub(r"[\u2500-\u257F]+", "", formatted_title)
+    formatted_title = re.sub(r"[^\w\s.-]", "", formatted_title)
+    formatted_title = re.sub(r"\s+", "_", formatted_title)
+    return formatted_title
+
+
+def load_embedding(embedder_model, custom_embedder=None):
+    embedder_root = os.path.join(now_dir, "rvc", "models", "embedders")
+    embedding_list = {
+        "contentvec": os.path.join(embedder_root, "contentvec"),
+        "chinese-hubert-base": os.path.join(embedder_root, "chinese_hubert_base"),
+        "japanese-hubert-base": os.path.join(embedder_root, "japanese_hubert_base"),
+        "korean-hubert-base": os.path.join(embedder_root, "korean_hubert_base"),
+    }
+
+    online_embedders = {
+        "contentvec": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/contentvec/pytorch_model.bin",
+        "chinese-hubert-base": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/chinese_hubert_base/pytorch_model.bin",
+        "japanese-hubert-base": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/japanese_hubert_base/pytorch_model.bin",
+        "korean-hubert-base": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/korean_hubert_base/pytorch_model.bin",
+    }
+
+    config_files = {
+        "contentvec": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/contentvec/config.json",
+        "chinese-hubert-base": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/chinese_hubert_base/config.json",
+        "japanese-hubert-base": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/japanese_hubert_base/config.json",
+        "korean-hubert-base": "https://huggingface.co/IAHispano/Applio/resolve/main/Resources/embedders/korean_hubert_base/config.json",
+    }
+
+    if embedder_model == "custom":
+        if os.path.exists(custom_embedder):
+            model_path = custom_embedder
+        else:
+            print(f"Custom embedder not found: {custom_embedder}, using contentvec")
+            model_path = embedding_list["contentvec"]
+    else:
+        model_path = embedding_list[embedder_model]
+        bin_file = os.path.join(model_path, "pytorch_model.bin")
+        json_file = os.path.join(model_path, "config.json")
+        os.makedirs(model_path, exist_ok=True)
+        if not os.path.exists(bin_file):
+            url = online_embedders[embedder_model]
+            print(f"Downloading {url} to {model_path}...")
+            wget.download(url, out=bin_file)
+        if not os.path.exists(json_file):
+            url = config_files[embedder_model]
+            print(f"Downloading {url} to {model_path}...")
+            wget.download(url, out=json_file)
+
+    models = HubertModelWithFinalProj.from_pretrained(model_path)
+    return models

zluda.py

@@ -0,0 +1,43 @@
+import torch
+
+if torch.cuda.is_available() and torch.cuda.get_device_name().endswith("[ZLUDA]"):
+    _torch_stft = torch.stft
+
+    def z_stft(
+        audio: torch.Tensor,
+        n_fft: int,
+        hop_length: int = None,
+        win_length: int = None,
+        window: torch.Tensor = None,
+        center: bool = True,
+        pad_mode: str = "reflect",
+        normalized: bool = False,
+        onesided: bool = None,
+        return_complex: bool = None,
+    ):
+        sd = audio.device
+        return _torch_stft(
+            audio.to("cpu"),
+            n_fft=n_fft,
+            hop_length=hop_length,
+            win_length=win_length,
+            window=window.to("cpu"),
+            center=center,
+            pad_mode=pad_mode,
+            normalized=normalized,
+            onesided=onesided,
+            return_complex=return_complex,
+        ).to(sd)
+
+    def z_jit(f, *_, **__):
+        f.graph = torch._C.Graph()
+        return f
+
+    # hijacks
+    torch.stft = z_stft
+    torch.jit.script = z_jit
+    # disabling unsupported cudnn
+    torch.backends.cudnn.enabled = False
+    torch.backends.cuda.enable_flash_sdp(False)
+    torch.backends.cuda.enable_math_sdp(True)
+    torch.backends.cuda.enable_mem_efficient_sdp(False)