inference.py

import argparse
import os

import soundfile as sf
import torch
from diffusers import StableDiffusionPipeline
from huggingface_hub import snapshot_download

from auffusion.auffusion_pipeline import Generator, denormalize_spectrogram


MODEL_ID = "dinhhung1508/SwiftAudio"
SAMPLE_RATE = 16000


def load_model(model_id=MODEL_ID, device=None):
    device = device or ("cuda" if torch.cuda.is_available() else "cpu")
    dtype = torch.float16 if device == "cuda" else torch.float32
    model_path = model_id if os.path.isdir(model_id) else snapshot_download(model_id)

    pipeline = StableDiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
    pipeline = pipeline.to(device)
    pipeline.set_progress_bar_config(disable=True)

    vocoder = Generator.from_pretrained(model_path, subfolder="vocoder")
    vocoder = vocoder.to(device=device, dtype=dtype)
    return pipeline, vocoder, device, dtype


@torch.inference_mode()
def generate_audio(pipeline, vocoder, prompt, seed=42, device=None, dtype=None):
    device = device or pipeline.device.type
    dtype = dtype or pipeline.unet.dtype

    text_inputs = pipeline.tokenizer(
        prompt,
        padding="max_length",
        max_length=pipeline.tokenizer.model_max_length,
        truncation=True,
        return_tensors="pt",
    )
    prompt_embeds = pipeline.text_encoder(text_inputs.input_ids.to(device))[0]
    prompt_embeds = prompt_embeds.to(device=device, dtype=dtype)

    generator = torch.Generator(device=device).manual_seed(seed)
    noise = torch.randn(
        (1, pipeline.unet.config.in_channels, 256 // 8, 1024 // 8),
        generator=generator,
        device=device,
        dtype=dtype,
    )

    timestep = torch.full(
        (1,),
        pipeline.scheduler.config.num_train_timesteps - 1,
        device=device,
        dtype=torch.int64,
    )
    model_pred = pipeline.unet(
        noise, timestep, prompt_embeds, return_dict=False
    )[0]

    alpha_t = 0.0047**0.5
    sigma_t = (1 - 0.0047) ** 0.5
    latents = (noise - sigma_t * model_pred) / alpha_t
    latents = latents / pipeline.vae.config.scaling_factor

    image = pipeline.vae.decode(latents, return_dict=False)[0]
    image = pipeline.image_processor.postprocess(
        image, output_type="pt", do_denormalize=[True]
    )[0]
    spectrogram = denormalize_spectrogram(image)
    return vocoder.inference(spectrogram, lengths=10 * SAMPLE_RATE)[0]


def parse_args():
    parser = argparse.ArgumentParser(description="Generate audio with SwiftAudio.")
    parser.add_argument("--prompt", required=True, help="Text description of the audio.")
    parser.add_argument("--output", default="output.wav", help="Output WAV path.")
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--model", default=MODEL_ID, help="Hub model ID or local path.")
    parser.add_argument("--device", choices=["cuda", "cpu"], default=None)
    return parser.parse_args()


def main():
    args = parse_args()
    pipeline, vocoder, device, dtype = load_model(args.model, args.device)
    audio = generate_audio(
        pipeline, vocoder, args.prompt, args.seed, device=device, dtype=dtype
    )
    sf.write(args.output, audio, SAMPLE_RATE)
    print(f"Saved audio to {args.output}")


if __name__ == "__main__":
    main()