app.py

from __future__ import annotations

import gradio as gr
try:
    import spaces
except ImportError:  # 'spaces' is only provided by Hugging Face Spaces
    import types as _types

    def _gpu(*args, **kwargs):
        if len(args) == 1 and callable(args[0]) and not kwargs:
            return args[0]

        def _decorator(func):
            return func

        return _decorator

    spaces = _types.SimpleNamespace(GPU=_gpu)


import tensorflow as tf
import gin
import ddsp.training.inference
import ddsp.core
import librosa
import soundfile
import numpy as np
import requests
import zipfile
import io
import tempfile
import os
import crepe

# Global variables for model and parameters
MODEL = None
SAMPLE_RATE = None
N_SAMPLES = None
N_FRAMES = None

def _download_and_extract_model(model_name):
    model_url = f'https://storage.googleapis.com/ddsp/models/{model_name}/{model_name}.zip'
    print(f"Downloading model from: {model_url}")
    response = requests.get(model_url)
    response.raise_for_status() # Raise an exception for HTTP errors

    # Create a temporary directory for the model
    temp_dir = tempfile.mkdtemp()
    zip_path = os.path.join(temp_dir, f'{model_name}.zip')

    with open(zip_path, 'wb') as f:
        f.write(response.content)

    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
        zip_ref.extractall(temp_dir)

    # The actual model checkpoint is usually in a subdirectory
    # Find the directory that contains the 'operative_config.gin'
    model_ckpt_dir = None
    for root, dirs, files in os.walk(temp_dir):
        if 'operative_config.gin' in files:
            model_ckpt_dir = root
            break
    
    if not model_ckpt_dir:
        raise FileNotFoundError("Could not find operative_config.gin in the downloaded model.")

    print(f"Model extracted to: {model_ckpt_dir}")
    return model_ckpt_dir

# --- Model Loading ---
MODEL_NAME = 'violin_ddsp_2020_03_03' # Example model from DDSP demos
CKPT_DIR = _download_and_extract_model(MODEL_NAME)

# Initialize the model with fixed parameters for this deployment
LENGTH_SECONDS = 4 # Default length for output audio
REMOVE_REVERB = True # Whether to remove reverb from the output

# Need to unlock gin config before parsing
with gin.unlock_config():
    MODEL = ddsp.training.inference.AutoencoderInference(
        ckpt=CKPT_DIR,
        length_seconds=LENGTH_SECONDS,
        remove_reverb=REMOVE_REVERB
    )

# Extract model parameters after initialization
SAMPLE_RATE = MODEL.sample_rate
N_SAMPLES = MODEL.n_samples
N_FRAMES = MODEL.n_frames

# Build the network by running a fake batch to initialize weights
MODEL.build_network()

# --- Helper functions for audio processing ---
def _load_audio_for_inference(audio_path, sr, n_samples):
    audio, _ = librosa.load(audio_path, sr=sr, mono=True)
    # Pad or trim audio to n_samples
    if len(audio) < n_samples:
        audio = np.pad(audio, (0, n_samples - len(audio)), mode='constant')
    else:
        audio = audio[:n_samples]
    return audio

def _save_audio_for_inference(audio_tensor, sr):
    output_path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
    audio_np = audio_tensor.numpy() if tf.is_tensor(audio_tensor) else audio_tensor
    soundfile.write(output_path, audio_np, sr)
    return output_path


@spaces.GPU
def predict(input_audio):
    # Load audio
    audio_np = _load_audio_for_inference(input_audio, SAMPLE_RATE, N_SAMPLES)

    # Extract pitch using crepe
    # crepe.predict expects a 1D numpy array
    # Calculate step_size in milliseconds for crepe based on model's frame rate
    step_size_ms = int(1000 * N_SAMPLES / (SAMPLE_RATE * N_FRAMES))
    _, f0_hz, _, _ = crepe.predict(
        audio_np, SAMPLE_RATE, viterbi=True, step_size=step_size_ms
    )

    # Resample f0_hz to N_FRAMES
    f0_hz_resampled = ddsp.core.resample(
        tf.convert_to_tensor(f0_hz, dtype=tf.float32), N_FRAMES
    )

    # Compute loudness
    loudness_db = ddsp.core.compute_loudness(
        tf.expand_dims(tf.convert_to_tensor(audio_np, dtype=tf.float32), axis=0),
        SAMPLE_RATE
    )

    # Prepare features dictionary
    audio_features = {
        'audio': tf.expand_dims(tf.convert_to_tensor(audio_np, dtype=tf.float32), axis=0),
        'f0_hz': tf.expand_dims(f0_hz_resampled, axis=0),
        'loudness_db': loudness_db,
    }

    # Preprocess features using the model's preprocessor
    processed_features = MODEL.preprocessor(audio_features)

    # Run inference
    outputs = MODEL(processed_features, training=False)

    # Get synthesized audio from outputs
    synthesized_audio = MODEL.get_audio_from_outputs(outputs)

    # Save and return the output audio file path
    output_audio_path = _save_audio_for_inference(synthesized_audio[0], SAMPLE_RATE)
    return output_audio_path


demo = gr.Interface(
    fn=predict,
    inputs=[
        gr.Audio(type="filepath", label="Input Audio"),
    ],
    outputs=[
        gr.Audio(type="filepath", label="Output Audio"),
    ],
    title="DDSP Autoencoder Inference",
    description="Resynthesize input audio using a pre-trained DDSP autoencoder model. This model takes an audio input, extracts its fundamental frequency (f0) and loudness, and then uses a DDSP autoencoder to synthesize a new audio output, effectively performing a timbre transfer based on the loaded model.",
)

if __name__ == "__main__":
    demo.queue().launch(server_name="0.0.0.0", server_port=7860, show_error=True)