app.py

#!/usr/bin/env python3

import gradio as gr
import numpy as np
import torch
import json
import io
import soundfile as sf
from PIL import Image
import matplotlib
import joblib
from sklearn.decomposition import PCA
from collections import OrderedDict
import nltk

matplotlib.use("Agg")  # Use non-interactive backend
import matplotlib.pyplot as plt

# -------------------------------------------------------------------
# IMPORT OR DEFINE YOUR TEXT-TO-SPEECH FUNCTIONS
# (Adjust these imports to match your local TTS code)
# -------------------------------------------------------------------
from text2speech import tts_randomized, parse_speed, tts_with_style_vector

# Constants and Paths
VOICES_JSON_PATH = "voices.json"
PCA_MODEL_PATH = "pca_model.pkl"
ANNOTATED_FEATURES_PATH = "annotated_features.npy"
VECTOR_DIMENSION = 256
ANNOTATED_FEATURES_NAMES = ["Gender", "Tone", "Quality", "Enunciation", "Pace", "Style"]
ANNOTATED_FEATURES_INFO = [
    "Male | Female",
    "High | Low",
    "Noisy | Clean",
    "Clear | Unclear",
    "Rapid | Slow",
    "Colloquial | Formal",
]

# Download necessary NLTK data
nltk.download("punkt_tab")

##############################################################################
# DEVICE CONFIGURATION
##############################################################################
# Detect if CUDA is available and set the device accordingly
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")

##############################################################################
# LOAD PCA MODEL AND ANNOTATED FEATURES
##############################################################################

try:
    pca = joblib.load(PCA_MODEL_PATH)
    print("PCA model loaded successfully.")
except FileNotFoundError:
    print(f"Error: PCA model file '{PCA_MODEL_PATH}' not found.")
    pca = None

try:
    annotated_features = np.load(ANNOTATED_FEATURES_PATH)
    print("Annotated features loaded successfully.")
except FileNotFoundError:
    print(f"Error: Annotated features file '{ANNOTATED_FEATURES_PATH}' not found.")
    annotated_features = None

##############################################################################
# UTILITY FUNCTIONS
##############################################################################


def load_voices_json():
    """Load the voices.json file."""
    try:
        with open(VOICES_JSON_PATH, "r") as f:
            return json.load(f, object_pairs_hook=OrderedDict)
    except FileNotFoundError:
        print(f"Warning: {VOICES_JSON_PATH} not found. Creating a new one.")
        return OrderedDict()
    except json.JSONDecodeError:
        print(f"Warning: {VOICES_JSON_PATH} is not valid JSON.")
        return OrderedDict()


def save_voices_json(data, path=VOICES_JSON_PATH):
    """Save to voices.json."""
    with open(path, "w") as f:
        json.dump(data, f, indent=2)
    print(f"Voices saved to '{path}'.")


def update_sliders(voice_name):
    """
    Update slider values based on the selected predefined voice using reverse PCA.
    Returns a list of PCA component values to set the sliders.
    """
    if not voice_name:
        # Return default slider values (e.g., zeros) if no voice is selected
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)

    voices_data = load_voices_json()
    if voice_name not in voices_data:
        print(f"Voice '{voice_name}' not found in {VOICES_JSON_PATH}.")
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)

    style_vector = np.array(voices_data[voice_name], dtype=np.float32).reshape(1, -1)

    if pca is None:
        print("PCA model is not loaded.")
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)

    try:
        # Transform the style vector into PCA component values
        pca_components = pca.transform(style_vector)[0]
        return pca_components.tolist()
    except Exception as e:
        print(f"Error transforming style vector to PCA components: {e}")
        return [0.0] * len(ANNOTATED_FEATURES_NAMES)


def generate_audio_with_voice(text, voice_key, speed_val):
    """
    Generate audio using the style vector of the selected predefined voice.
    Returns (audio_tuple, style_vector) or (None, error_message).
    """
    try:
        # Load voices data
        voices_data = load_voices_json()
        if voice_key not in voices_data:
            msg = f"Voice '{voice_key}' not found in {VOICES_JSON_PATH}."
            print(msg)
            return None, msg

        style_vector = np.array(voices_data[voice_key], dtype=np.float32).reshape(1, -1)
        print(f"Selected Voice: {voice_key}")
        print(f"Style Vector (First 6): {style_vector[0][:6]}")

        # Convert to torch tensor and move to device
        style_vec_torch = torch.from_numpy(style_vector).float().to(device)

        # Generate audio
        audio_np = tts_with_style_vector(
            text,
            style_vec=style_vec_torch,
            speed=speed_val,
            alpha=0.3,
            beta=0.7,
            diffusion_steps=7,
            embedding_scale=1.0,
        )

        if audio_np is None:
            msg = "Audio generation failed."
            print(msg)
            return None, msg

        sr = 24000
        audio_tuple = (sr, audio_np)
        return audio_tuple, style_vector.tolist()

    except Exception as e:
        print(f"Error in generate_audio_with_voice: {e}")
        return None, "An error occurred during audio generation."


def build_modified_vector(voice_key, top6_values):
    """Reconstruct a style vector by applying inverse PCA on the given 6 slider values."""
    voices_data = load_voices_json()
    if voice_key not in voices_data:
        print(f"Voice '{voice_key}' not found in {VOICES_JSON_PATH}.")
        return None

    arr = np.array(voices_data[voice_key], dtype=np.float32).squeeze()
    if arr.ndim != 1 or arr.shape[0] != VECTOR_DIMENSION:
        print(f"Voice '{voice_key}' has invalid shape {arr.shape}. Expected (256,).")
        return None

    try:
        pca_components = np.array(top6_values).reshape(1, -1)
        reconstructed_vec = pca.inverse_transform(pca_components)[0]
        return reconstructed_vec
    except Exception as e:
        print(f"Error reconstructing style vector: {e}")
        return None


def generate_custom_audio(text, voice_key, randomize, speed_val, *slider_values):
    """
    Generate audio with either a random style vector or a reconstructed vector
    from the 6 PCA sliders. Returns (audio_tuple, style_vector) or (None, None).
    """
    try:
        if randomize:
            # Generate randomized style vector
            audio_np, random_style_vec = tts_randomized(text, speed=speed_val)
            if random_style_vec is None:
                print("Failed to generate randomized style vector.")
                return None, None
            final_vec = (
                random_style_vec.cpu().numpy().flatten()
                if isinstance(random_style_vec, torch.Tensor)
                else np.array(random_style_vec).flatten()
            )
            print("Randomized Style Vector (First 6):", final_vec[:6])
        else:
            # Reconstruct vector from PCA sliders
            reconstructed_vec = build_modified_vector(voice_key, slider_values)
            if reconstructed_vec is None:
                print("No reconstructed vector. Skipping audio generation.")
                return None, None

            style_vec_torch = (
                torch.from_numpy(reconstructed_vec).float().unsqueeze(0).to(device)
            )
            audio_np = tts_with_style_vector(
                text,
                style_vec=style_vec_torch,
                speed=speed_val,
                alpha=0.3,
                beta=0.7,
                diffusion_steps=7,
                embedding_scale=1.0,
            )
            final_vec = reconstructed_vec
            print("Reconstructed Style Vector (First 6):", final_vec[:6])

        if audio_np is None:
            print("Audio generation failed.")
            return None, None

        sr = 24000
        audio_tuple = (sr, audio_np)
        return audio_tuple, final_vec.tolist()

    except Exception as e:
        print(f"Error generating audio and style: {e}")
        return None, None


def save_style_to_json(style_data, style_name):
    """
    Saves the provided style_data (list of floats) into voices.json under style_name.
    Returns a status message.
    """
    if not style_name.strip():
        return "Please enter a new style name before saving."

    voices_data = load_voices_json()
    if style_name in voices_data:
        return (
            f"Style name '{style_name}' already exists. Please choose a different name."
        )

    if len(style_data) != VECTOR_DIMENSION:
        return f"Style vector length mismatch. Expected {VECTOR_DIMENSION}, got {len(style_data)}."

    voices_data[style_name] = style_data
    save_voices_json(voices_data)
    return f"Saved style as '{style_name}' in {VOICES_JSON_PATH}."


def rearrange_voices(new_order):
    """
    Rearrange the voices in voices.json based on the comma-separated `new_order`.
    Returns (status_msg, updated_list_of_voices).
    """
    voices_data = load_voices_json()
    new_order_list = [name.strip() for name in new_order.split(",")]
    if not all(name in voices_data for name in new_order_list):
        return "Error: New order contains invalid voice names.", list(
            voices_data.keys()
        )

    ordered_data = OrderedDict()
    for name in new_order_list:
        ordered_data[name] = voices_data[name]

    save_voices_json(ordered_data)
    print(f"Voices rearranged: {list(ordered_data.keys())}")
    return "Voices rearranged successfully.", list(ordered_data.keys())


def delete_voice(selected):
    """Delete voices from the voices.json. Returns (status_msg, updated_list_of_voices)."""
    if not selected:
        return "No voices selected for deletion.", list(load_voices_json().keys())
    voices_data = load_voices_json()
    for voice_name in selected:
        if voice_name in voices_data:
            del voices_data[voice_name]
            print(f"Voice '{voice_name}' deleted.")
    save_voices_json(voices_data)
    return "Deleted selected voices successfully.", list(voices_data.keys())


def upload_new_voices(uploaded_file):
    """Upload new voices from a JSON file. Returns (status_msg, updated_list_of_voices)."""
    if uploaded_file is None:
        return "No file uploaded.", list(load_voices_json().keys())
    try:
        uploaded_data = json.load(uploaded_file)
        if not isinstance(uploaded_data, dict):
            return (
                "Invalid JSON format. Expected a dictionary of voices.",
                list(load_voices_json().keys()),
            )
        voices_data = load_voices_json()
        voices_data.update(uploaded_data)
        save_voices_json(voices_data)
        print(f"Voices uploaded: {list(uploaded_data.keys())}")
        return "Voices uploaded successfully.", list(voices_data.keys())
    except json.JSONDecodeError:
        return "Uploaded file is not valid JSON.", list(load_voices_json().keys())


# -------------------------------------------------------------------
# GRADIO INTERFACE
# -------------------------------------------------------------------


def create_combined_interface():
    # We'll initially load the voices to get a default set for the dropdown
    voices_data = load_voices_json()
    voice_choices = list(voices_data.keys())
    default_voice = voice_choices[0] if voice_choices else None

    css = """
    h4 {
        text-align: center;
        display:block;
    }
    """

    with gr.Blocks(theme=gr.themes.Ocean(), css=css) as demo:
        gr.Markdown("# StyleTTS2 Studio - Build custom voices")

        # -------------------------------------------------------
        # 1) Text-to-Speech Tab
        # -------------------------------------------------------
        with gr.Tab("Text-to-Speech"):
            gr.Markdown("### Generate Speech with Predefined Voices")

            with gr.Column():
                text_input = gr.Textbox(
                    label="Text to Synthesize",
                    value="How much wood could a woodchuck chuck if a woodchuck could chuck wood?",
                    lines=3,
                )
                voice_dropdown = gr.Dropdown(
                    choices=voice_choices,
                    label="Select Base Voice",
                    value=default_voice,
                    interactive=True,
                )
                speed_slider = gr.Slider(
                    minimum=50,
                    maximum=200,
                    step=1,
                    label="Speed (%)",
                    value=120,
                )
                generate_btn = gr.Button("Generate Audio")
                status_tts = gr.Textbox(label="Status", visible=False)
            audio_output = gr.Audio(label="Synthesized Audio")

            # Generate TTS callback
            def on_generate_tts(text, voice, speed):
                if not voice:
                    return None, "No voice selected."
                speed_val = speed / 100  # Convert percentage to multiplier
                audio_result, msg = generate_audio_with_voice(text, voice, speed_val)
                if audio_result is None:
                    return None, msg
                return audio_result, "Audio generated successfully."

            generate_btn.click(
                fn=on_generate_tts,
                inputs=[text_input, voice_dropdown, speed_slider],
                outputs=[audio_output, status_tts],
            )

        # -------------------------------------------------------
        # 2) Voice Studio Tab
        # -------------------------------------------------------
        with gr.Tab("Voice Studio"):
            gr.Markdown("### Customize and Create New Voices")

            with gr.Column():
                text_input_studio = gr.Textbox(
                    label="Text to Synthesize",
                    value="Use the sliders to customize a voice!",
                    lines=3,
                )
                voice_dropdown_studio = gr.Dropdown(
                    choices=voice_choices,
                    label="Select Base Voice",
                    value=default_voice,
                )
                speed_slider_studio = gr.Slider(
                    minimum=50,
                    maximum=200,
                    step=1,
                    label="Speed (%)",
                    value=120,
                )
                # Sliders for PCA components (6 sliders)
                pca_sliders = [
                    gr.Slider(
                        minimum=-2.0,
                        maximum=2.0,
                        value=0.0,
                        step=0.1,
                        label=feature,
                    )
                    for feature in ANNOTATED_FEATURES_NAMES
                ]

            generate_btn_studio = gr.Button("Generate Customized Audio")
            audio_output_studio = gr.Audio(label="Customized Synthesized Audio")
            new_style_name = gr.Textbox(label="New Style Name", value="")
            save_btn_studio = gr.Button("Save Customized Voice")
            status_text = gr.Textbox(label="Status", visible=True)

            # State to hold the last style vector
            style_vector_state_studio = gr.State()

            # Generate customized audio callback
            def on_generate_studio(text, voice, speed, *pca_values):
                if not voice:
                    return None, "No voice selected.", None
                speed_val = speed / 100
                audio_tuple, style_vector = generate_custom_audio(
                    text, voice, False, speed_val, *pca_values
                )
                if audio_tuple is None:
                    return None, "Failed to generate audio.", None
                return audio_tuple, "Audio generated successfully.", style_vector

            generate_btn_studio.click(
                fn=on_generate_studio,
                inputs=[text_input_studio, voice_dropdown_studio, speed_slider_studio]
                + pca_sliders,
                outputs=[audio_output_studio, status_text, style_vector_state_studio],
            )

            # Save customized voice callback
            def on_save_style_studio(style_vector, style_name):
                """Save the new style, then update the dropdown choices."""
                if not style_vector or not style_name:
                    return (
                        gr.update(value="Please enter a name for the new voice!"),
                        gr.update(),
                        gr.update(),
                    )
                # Save the style
                result = save_style_to_json(style_vector, style_name)
                # Reload the voices to get the new list
                new_choices = list(load_voices_json().keys())

                # Return dictionary updates to existing components
                return (
                    gr.update(value=result),
                    gr.update(choices=new_choices),
                    gr.update(choices=new_choices),
                )

            save_btn_studio.click(
                fn=on_save_style_studio,
                inputs=[style_vector_state_studio, new_style_name],
                # We update: status_text, voice_dropdown, voice_dropdown_studio
                outputs=[status_text, voice_dropdown, voice_dropdown_studio],
            )

            # Update sliders callback
            voice_dropdown_studio.change(
                fn=update_sliders,
                inputs=voice_dropdown_studio,
                outputs=pca_sliders,
            )

        # -------------------------------------------------------
        # Optionally: Reload voices on page load
        # -------------------------------------------------------
        def on_page_load():
            new_choices = list(load_voices_json().keys())
            return {
                voice_dropdown: gr.update(choices=new_choices),
                voice_dropdown_studio: gr.update(choices=new_choices),
            }

        # This automatically refreshes dropdowns every time the user loads/refreshes the page
        demo.load(
            on_page_load, inputs=None, outputs=[voice_dropdown, voice_dropdown_studio]
        )

        gr.Markdown(
            "#### Based on [StyleTTS2](https://github.com/yl4579/StyleTTS2) and [artificial StyleTTS2](https://huggingface.co/dkounadis/artificial-styletts2/tree/main)"
        )

    return demo


if __name__ == "__main__":
    try:
        interface = create_combined_interface()
        interface.launch(share=False)  # or share=True if you want a public share link
    except Exception as e:
        print(f"An error occurred while launching the interface: {e}")