app.py

from __future__ import annotations

import tempfile
import uuid
from pathlib import Path

import spaces
import gradio as gr
import torch
import torchaudio
from huggingface_hub import hf_hub_download
from pyharp import ModelCard, build_endpoint
from pyharp.tags import Subcategory

from data.tokenizer import AudioTokenizer, TextTokenizer
from inference_speech_editing_scale import inference_one_sample
from models.voicecraft import VoiceCraft


DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
CODEC_RATE = 50
OUTPUT_DIR = Path(tempfile.gettempdir()) / "voicecraft_outputs"
OUTPUT_DIR.mkdir(exist_ok=True)

model = VoiceCraft.from_pretrained("pyp1/VoiceCraft_830M_TTSEnhanced").to(DEVICE).eval()
codec_path = hf_hub_download("pyp1/VoiceCraft", "encodec_4cb2048_giga.th")
audio_tokenizer = AudioTokenizer(device=DEVICE, signature=codec_path)
text_tokenizer = TextTokenizer(backend="espeak")


@spaces.GPU(duration=60)
def edit_speech(audio_path: str, target_text: str, start_time: float, end_time: float) -> str:
    if not audio_path:
        raise gr.Error("Upload a speech recording.")
    target_text = target_text.strip()
    if not target_text:
        raise gr.Error("Enter the full transcript with the replacement text.")

    info = torchaudio.info(audio_path)
    duration = info.num_frames / info.sample_rate
    if not 0 <= start_time < end_time <= duration:
        raise gr.Error(f"Select an edit interval within the {duration:.2f} second recording.")

    start = max(start_time - 0.08, 1 / CODEC_RATE)
    end = min(end_time + 0.08, duration)
    mask = torch.tensor([[round(start * CODEC_RATE), round(end * CODEC_RATE)]])
    decode_config = {
        "top_k": 0,
        "top_p": 0.9,
        "temperature": 1.0,
        "stop_repetition": 3,
        "kvcache": 1,
        "codec_sr": CODEC_RATE,
        "silence_tokens": [1388, 1898, 131],
    }
    generated = inference_one_sample(
        model,
        model.args,
        model.args.phn2num,
        text_tokenizer,
        audio_tokenizer,
        audio_path,
        target_text,
        mask,
        DEVICE,
        decode_config,
    )
    output_path = OUTPUT_DIR / f"{uuid.uuid4().hex}.wav"
    torchaudio.save(str(output_path), generated[0].cpu(), 16000)
    return str(output_path)


if __name__ == "__main__":
    with gr.Blocks(title="VoiceCraft") as demo:
        build_endpoint(
            model_card=ModelCard(
                name="VoiceCraft",
                description=(
                    "Edit English speech while retaining the surrounding recording. "
                    "Repository: https://github.com/jasonppy/VoiceCraft Paper: "
                    "https://arxiv.org/abs/2403.16973"
                ),
                author="Puyuan Peng, Po-Yao Huang, Abdelrahman Mohamed, David Harwath",
                tags=[Subcategory.EDITING, 'speech-editing'],
            ),
            input_components=[
                gr.Audio(type="filepath", label="Speech Recording")
                .harp_required(True)
                .set_info("English speech to edit."),
                gr.Textbox(label="Target Transcript", lines=4)
                .harp_required(True)
                .set_info("Full transcript of the recording, with the replacement words in place."),
                gr.Number(label="Edit Start (seconds)", value=0.5, precision=2)
                .set_info("Start of the words to replace."),
                gr.Number(label="Edit End (seconds)", value=1.5, precision=2)
                .set_info("End of the words to replace."),
            ],
            output_components=[
                gr.Audio(type="filepath", label="Edited Speech")
                .set_info("16 kHz WAV recording."),
            ],
            process_fn=edit_speech,
        )


    demo.queue(default_concurrency_limit=1).launch(show_error=True, pwa=True)