inference.py

import argparse
import json
import os
from functools import partial
from typing import Union

import gradio as gr
import librosa
import numpy as np
import soundfile as sf
import torch
from fish_audio_preprocess.utils import loudness_norm, separate_audio
from loguru import logger
from mmengine import Config

from fish_diffusion.feature_extractors import FEATURE_EXTRACTORS, PITCH_EXTRACTORS
from fish_diffusion.utils.audio import get_mel_from_audio, slice_audio
from fish_diffusion.utils.inference import load_checkpoint
from fish_diffusion.utils.tensor import repeat_expand


@torch.no_grad()
def inference(
    config,
    checkpoint,
    input_path,
    output_path,
    speaker_id=0,
    pitch_adjust=0,
    silence_threshold=30,
    max_slice_duration=5,
    extract_vocals=True,
    merge_non_vocals=True,
    vocals_loudness_gain=0.0,
    sampler_interval=None,
    sampler_progress=False,
    device="cuda",
    gradio_progress=None,
):
    """Inference

    Args:
        config: config
        checkpoint: checkpoint path
        input_path: input path
        output_path: output path
        speaker_id: speaker id
        pitch_adjust: pitch adjust
        silence_threshold: silence threshold of librosa.effects.split
        max_slice_duration: maximum duration of each slice
        extract_vocals: extract vocals
        merge_non_vocals: merge non-vocals, only works when extract_vocals is True
        vocals_loudness_gain: loudness gain of vocals (dB)
        sampler_interval: sampler interval, lower value means higher quality
        sampler_progress: show sampler progress
        device: device
        gradio_progress: gradio progress callback
    """

    if sampler_interval is not None:
        config.model.diffusion.sampler_interval = sampler_interval

    if os.path.isdir(checkpoint):
        # Find the latest checkpoint
        checkpoints = sorted(os.listdir(checkpoint))
        logger.info(f"Found {len(checkpoints)} checkpoints, using {checkpoints[-1]}")
        checkpoint = os.path.join(checkpoint, checkpoints[-1])

    audio, sr = librosa.load(input_path, sr=config.sampling_rate, mono=True)

    # Extract vocals

    if extract_vocals:
        logger.info("Extracting vocals...")

        if gradio_progress is not None:
            gradio_progress(0, "Extracting vocals...")

        model = separate_audio.init_model("htdemucs", device=device)
        audio = librosa.resample(audio, orig_sr=sr, target_sr=model.samplerate)[None]

        # To two channels
        audio = np.concatenate([audio, audio], axis=0)
        audio = torch.from_numpy(audio).to(device)
        tracks = separate_audio.separate_audio(
            model, audio, shifts=1, num_workers=0, progress=True
        )
        audio = separate_audio.merge_tracks(tracks, filter=["vocals"]).cpu().numpy()
        non_vocals = (
            separate_audio.merge_tracks(tracks, filter=["drums", "bass", "other"])
            .cpu()
            .numpy()
        )

        audio = librosa.resample(audio[0], orig_sr=model.samplerate, target_sr=sr)
        non_vocals = librosa.resample(
            non_vocals[0], orig_sr=model.samplerate, target_sr=sr
        )

        # Normalize loudness
        non_vocals = loudness_norm.loudness_norm(non_vocals, sr)

    # Normalize loudness
    audio = loudness_norm.loudness_norm(audio, sr)

    # Slice into segments
    segments = list(
        slice_audio(
            audio, sr, max_duration=max_slice_duration, top_db=silence_threshold
        )
    )
    logger.info(f"Sliced into {len(segments)} segments")

    # Load models
    text_features_extractor = FEATURE_EXTRACTORS.build(
        config.preprocessing.text_features_extractor
    ).to(device)
    text_features_extractor.eval()

    model = load_checkpoint(config, checkpoint, device=device)

    pitch_extractor = PITCH_EXTRACTORS.build(config.preprocessing.pitch_extractor)
    assert pitch_extractor is not None, "Pitch extractor not found"

    generated_audio = np.zeros_like(audio)
    audio_torch = torch.from_numpy(audio).to(device)[None]

    for idx, (start, end) in enumerate(segments):
        if gradio_progress is not None:
            gradio_progress(idx / len(segments), "Generating audio...")

        segment = audio_torch[:, start:end]
        logger.info(
            f"Processing segment {idx + 1}/{len(segments)}, duration: {segment.shape[-1] / sr:.2f}s"
        )

        # Extract mel
        mel = get_mel_from_audio(segment, sr)

        # Extract pitch (f0)
        pitch = pitch_extractor(segment, sr, pad_to=mel.shape[-1]).float()
        pitch *= 2 ** (pitch_adjust / 12)

        # Extract text features
        text_features = text_features_extractor(segment, sr)[0]
        text_features = repeat_expand(text_features, mel.shape[-1]).T

        # Predict
        src_lens = torch.tensor([mel.shape[-1]]).to(device)

        features = model.model.forward_features(
            speakers=torch.tensor([speaker_id]).long().to(device),
            contents=text_features[None].to(device),
            src_lens=src_lens,
            max_src_len=max(src_lens),
            mel_lens=src_lens,
            max_mel_len=max(src_lens),
            pitches=pitch[None].to(device),
        )

        result = model.model.diffusion(features["features"], progress=sampler_progress)
        wav = model.vocoder.spec2wav(result[0].T, f0=pitch).cpu().numpy()
        max_wav_len = generated_audio.shape[-1] - start
        generated_audio[start : start + wav.shape[-1]] = wav[:max_wav_len]

    # Loudness normalization
    generated_audio = loudness_norm.loudness_norm(generated_audio, sr)

    # Loudness gain
    loudness_float = 10 ** (vocals_loudness_gain / 20)
    generated_audio = generated_audio * loudness_float

    # Merge non-vocals
    if extract_vocals and merge_non_vocals:
        generated_audio = (generated_audio + non_vocals) / 2

    logger.info("Done")

    if output_path is not None:
        sf.write(output_path, generated_audio, sr)

    return generated_audio, sr


def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--config",
        type=str,
        required=True,
        help="Path to the config file",
    )

    parser.add_argument(
        "--checkpoint",
        type=str,
        required=True,
        help="Path to the checkpoint file",
    )

    parser.add_argument(
        "--gradio",
        action="store_true",
        help="Run in gradio mode",
    )

    parser.add_argument(
        "--gradio_share",
        action="store_true",
        help="Share gradio app",
    )

    parser.add_argument(
        "--input",
        type=str,
        required=False,
        help="Path to the input audio file",
    )

    parser.add_argument(
        "--output",
        type=str,
        required=False,
        help="Path to the output audio file",
    )

    parser.add_argument(
        "--speaker_id",
        type=int,
        default=0,
        help="Speaker id",
    )

    parser.add_argument(
        "--speaker_mapping",
        type=str,
        default=None,
        help="Speaker mapping file (gradio mode only)",
    )

    parser.add_argument(
        "--pitch_adjust",
        type=int,
        default=0,
        help="Pitch adjustment in semitones",
    )

    parser.add_argument(
        "--extract_vocals",
        action="store_true",
        help="Extract vocals",
    )

    parser.add_argument(
        "--merge_non_vocals",
        action="store_true",
        help="Merge non-vocals",
    )

    parser.add_argument(
        "--vocals_loudness_gain",
        type=float,
        default=0,
        help="Loudness gain for vocals",
    )

    parser.add_argument(
        "--sampler_interval",
        type=int,
        default=None,
        required=False,
        help="Sampler interval, if not specified, will be taken from config",
    )

    parser.add_argument(
        "--sampler_progress",
        action="store_true",
        help="Show sampler progress",
    )

    parser.add_argument(
        "--device",
        type=str,
        default=None,
        required=False,
        help="Device to use",
    )

    return parser.parse_args()


def run_inference(
    config_path: str,
    model_path: str,
    input_path: str,
    speaker: Union[int, str],
    pitch_adjust: int,
    sampler_interval: int,
    extract_vocals: bool,
    device: str,
    progress=gr.Progress(),
    speaker_mapping: dict = None,
):
    if speaker_mapping is not None and isinstance(speaker, str):
        speaker = speaker_mapping[speaker]

    audio, sr = inference(
        Config.fromfile(config_path),
        model_path,
        input_path=input_path,
        output_path=None,
        speaker_id=speaker,
        pitch_adjust=pitch_adjust,
        sampler_interval=round(sampler_interval),
        extract_vocals=extract_vocals,
        merge_non_vocals=False,
        device=device,
        gradio_progress=progress,
    )

    return (sr, audio)


def launch_gradio(args):
    with gr.Blocks(title="Fish Diffusion") as app:
        gr.Markdown("# Fish Diffusion SVC Inference")

        with gr.Row():
            with gr.Column():
                input_audio = gr.Audio(
                    label="Input Audio",
                    type="filepath",
                    value=args.input,
                )
                output_audio = gr.Audio(label="Output Audio")

            with gr.Column():
                if args.speaker_mapping is not None:
                    speaker_mapping = json.load(open(args.speaker_mapping))

                    speaker = gr.Dropdown(
                        label="Speaker Name (Used for Multi-Speaker Models)",
                        choices=list(speaker_mapping.keys()),
                        value=list(speaker_mapping.keys())[0],
                    )
                else:
                    speaker_mapping = None
                    speaker = gr.Number(
                        label="Speaker ID (Used for Multi-Speaker Models)",
                        value=args.speaker_id,
                    )

                pitch_adjust = gr.Number(
                    label="Pitch Adjust (Semitones)", value=args.pitch_adjust
                )
                sampler_interval = gr.Slider(
                    label="Sampler Interval (⬆️ Faster Generation, ⬇️ Better Quality)",
                    value=args.sampler_interval or 10,
                    minimum=1,
                    maximum=100,
                )
                extract_vocals = gr.Checkbox(
                    label="Extract Vocals (For low quality audio)",
                    value=args.extract_vocals,
                )
                device = gr.Radio(
                    label="Device", choices=["cuda", "cpu"], value=args.device or "cuda"
                )

                run_btn = gr.Button(label="Run")

            run_btn.click(
                partial(
                    run_inference,
                    args.config,
                    args.checkpoint,
                    speaker_mapping=speaker_mapping,
                ),
                [
                    input_audio,
                    speaker,
                    pitch_adjust,
                    sampler_interval,
                    extract_vocals,
                    device,
                ],
                output_audio,
            )

    app.queue(concurrency_count=2).launch(share=args.gradio_share)


if __name__ == "__main__":
    args = parse_args()

    assert args.gradio or (
        args.input is not None and args.output is not None
    ), "Either --gradio or --input and --output should be specified"

    if args.device is None:
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    else:
        device = torch.device(args.device)

    if args.gradio:
        args.device = device
        launch_gradio(args)

    else:

        inference(
            Config.fromfile(args.config),
            args.checkpoint,
            args.input,
            args.output,
            speaker_id=args.speaker_id,
            pitch_adjust=args.pitch_adjust,
            extract_vocals=args.extract_vocals,
            merge_non_vocals=args.merge_non_vocals,
            vocals_loudness_gain=args.vocals_loudness_gain,
            sampler_interval=args.sampler_interval,
            sampler_progress=args.sampler_progress,
            device=device,
        )