app/infer.py

import os
import shutil
import hashlib
from pathlib import Path
from typing import Tuple
from demucs.separate import main as demucs
import gradio as gr
import numpy as np
import soundfile as sf
from zerorvc import RVC
from .zero import zero
from .model import device
import yt_dlp


def download_audio(url):
    ydl_opts = {
        "format": "bestaudio/best",
        "outtmpl": "ytdl/%(title)s.%(ext)s",
        "postprocessors": [
            {
                "key": "FFmpegExtractAudio",
                "preferredcodec": "wav",
                "preferredquality": "192",
            }
        ],
    }

    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
        info_dict = ydl.extract_info(url, download=True)
        file_path = ydl.prepare_filename(info_dict).rsplit(".", 1)[0] + ".wav"
        sample_rate, audio_data = read(file_path)
        audio_array = np.asarray(audio_data, dtype=np.int16)

        return sample_rate, audio_array


@zero(duration=120)
def infer(
    exp_dir: str, original_audio: str, pitch_mod: int, protect: float
) -> Tuple[int, np.ndarray]:
    checkpoint_dir = os.path.join(exp_dir, "checkpoints")
    if not os.path.exists(checkpoint_dir):
        raise gr.Error("Model not found")

    # rename the original audio to the hash
    with open(original_audio, "rb") as f:
        original_audio_hash = hashlib.md5(f.read()).hexdigest()
    ext = Path(original_audio).suffix
    original_audio_hashed = os.path.join(exp_dir, f"{original_audio_hash}{ext}")
    shutil.copy(original_audio, original_audio_hashed)

    out = os.path.join("separated", "htdemucs", original_audio_hash, "vocals.wav")
    if not os.path.exists(out):
        demucs(
            [
                "--two-stems",
                "vocals",
                "-d",
                str(device),
                "-n",
                "htdemucs",
                original_audio_hashed,
            ]
        )

    rvc = RVC.from_pretrained(checkpoint_dir)
    samples = rvc.convert(out, pitch_modification=pitch_mod, protect=protect)
    file = os.path.join(exp_dir, "infer.wav")
    sf.write(file, samples, rvc.sr)

    return file


def merge(exp_dir: str, original_audio: str, vocal: Tuple[int, np.ndarray]) -> str:
    with open(original_audio, "rb") as f:
        original_audio_hash = hashlib.md5(f.read()).hexdigest()
    music = os.path.join("separated", "htdemucs", original_audio_hash, "no_vocals.wav")

    tmp = os.path.join(exp_dir, "tmp.wav")
    sf.write(tmp, vocal[1], vocal[0])

    os.system(
        f"ffmpeg -i {music} -i {tmp} -filter_complex '[1]volume=2[a];[0][a]amix=inputs=2:duration=first:dropout_transition=2' -ac 2 -y {tmp}.merged.mp3"
    )

    return f"{tmp}.merged.mp3"


class InferenceTab:
    def __init__(self):
        pass

    def ui(self):
        gr.Markdown("# Inference")
        gr.Markdown(
            "After trained model is pruned, you can use it to infer on new music. \n"
            "Upload the original audio and adjust the F0 add value to generate the inferred audio."
        )

        with gr.Row():
            self.original_audio = gr.Audio(
                label="Upload original audio",
                type="filepath",
                show_download_button=True,
            )
            with gr.Accordion("inference by Link", open=False):
                with gr.Row():
                    youtube_link = gr.Textbox(
                        label="Link",
                        placeholder="Paste the link here",
                        interactive=True,
                    )
                with gr.Row():
                    gr.Markdown(
                        "You can paste the link to the video/audio from many sites, check the complete list [here](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)"
                    )
                with gr.Row():
                    download_button = gr.Button("Download!", variant="primary")
                    download_button.click(
                        download_audio, [youtube_link], [self.original_audio]
                    )

            with gr.Column():
                self.pitch_mod = gr.Slider(
                    label="Pitch Modification +/-",
                    minimum=-16,
                    maximum=16,
                    step=1,
                    value=0,
                )
                self.protect = gr.Slider(
                    label="Protect",
                    minimum=0,
                    maximum=0.5,
                    step=0.01,
                    value=0.33,
                )

            self.infer_btn = gr.Button(value="Infer", variant="primary")
        with gr.Row():
            self.infer_output = gr.Audio(
                label="Inferred audio", show_download_button=True, format="mp3"
            )
        with gr.Row():
            self.merge_output = gr.Audio(
                label="Merged audio", show_download_button=True, format="mp3"
            )

    def build(self, exp_dir: gr.Textbox):
        self.infer_btn.click(
            fn=infer,
            inputs=[
                exp_dir,
                self.original_audio,
                self.pitch_mod,
                self.protect,
            ],
            outputs=[self.infer_output],
        ).success(
            fn=merge,
            inputs=[exp_dir, self.original_audio, self.infer_output],
            outputs=[self.merge_output],
        )