app.py

import gradio as gr
from model.CLAPSep import CLAPSep
import torch
import librosa
import numpy as np


model_config = {"lan_embed_dim": 1024,
    "depths": [1, 1, 1, 1],
    "embed_dim": 128,
    "encoder_embed_dim": 128,
    "phase": False,
    "spec_factor": 8,
    "d_attn": 640,
    "n_masker_layer": 3,
    "conv": False}
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
CLAP_path = "model/music_audioset_epoch_15_esc_90.14.pt"


model = CLAPSep(model_config, CLAP_path).to(DEVICE)
ckpt = torch.load('model/best_model.ckpt', map_location=DEVICE)
model.load_state_dict(ckpt, strict=False)
model.eval()



def inference(audio_file_path: str, text_p: str, audio_file_path_p: str, text_n: str, audio_file_path_n: str):
    # handling queries
    with torch.no_grad():
        embed_pos, embed_neg = torch.chunk(model.clap_model.get_text_embedding([text_p, text_n],
                                                                              use_tensor=True), dim=0, chunks=2)
        embed_pos = torch.zeros_like(embed_pos) if text_p == '' else embed_pos
        embed_neg = torch.zeros_like(embed_neg) if text_n == '' else embed_neg
        embed_pos += (model.clap_model.get_audio_embedding_from_filelist(
            [audio_file_path_p]) if audio_file_path_p is not None else torch.zeros_like(embed_pos))
        embed_neg += (model.clap_model.get_audio_embedding_from_filelist(
            [audio_file_path_n]) if audio_file_path_n is not None else torch.zeros_like(embed_neg))



    print(f"Separate audio from [{audio_file_path}] with textual query p: [{text_p}] and n: [{text_n}]")

    mixture, _ = librosa.load(audio_file_path, sr=32000)

    pad = (320000 - (len(mixture) % 320000))if len(mixture) % 320000 != 0 else 0

    mixture =torch.tensor(np.pad(mixture,(0,pad)))
    
    max_value = torch.max(torch.abs(mixture))
    if max_value > 1:
        mixture *= 0.9 / max_value
    
    mixture_chunks = torch.chunk(mixture, dim=0, chunks=len(mixture)//320000)
    sep_segments = []
    for chunk in mixture_chunks:
        with torch.no_grad():
            sep_segments.append(model.inference_from_data(chunk.unsqueeze(0), embed_pos, embed_neg))

    sep_segment = torch.concat(sep_segments, dim=1)

    return 32000, sep_segment.squeeze().numpy()


with gr.Blocks(title="CLAPSep") as demo:
    with gr.Row():
        with gr.Column():
            input_audio = gr.Audio(label="Mixture", type="filepath")
            text_p = gr.Textbox(label="Positive Query Text")
            text_n = gr.Textbox(label="Negative Query Text")
            query_audio_p = gr.Audio(label="Positive Query Audio (optional)", type="filepath")
            query_audio_n = gr.Audio(label="Negative Query Audio (optional)", type="filepath")
        with gr.Column():
            with gr.Column():
                output_audio = gr.Audio(label="Separation Result", scale=10)
                button = gr.Button(
                    "Separate",
                    variant="primary",
                    scale=2,
                    size="lg",
                    interactive=True,
                )
                button.click(
                    fn=inference, inputs=[input_audio, text_p, query_audio_p, text_n, query_audio_n], outputs=[output_audio]
                )


demo.queue().launch(share=True)