app.py

# app.py
# Malayalam TTS (Free) – Multi-style, Prosody (rate & pitch), Batch paragraphs, WAV+MP3
# Model: AI4Bharat VITS (supports Malayalam among 13 Indian languages)

import gradio as gr
import soundfile as sf
import tempfile
import torch
from transformers import AutoModel, AutoTokenizer
import numpy as np
import os

# Optional MP3 conversion
try:
    from pydub import AudioSegment
    _HAS_PYDUB = True
except Exception:
    _HAS_PYDUB = False

MODEL_ID = "ai4bharat/vits_rasa_13"

device = "cuda" if torch.cuda.is_available() else "cpu"
model = AutoModel.from_pretrained(MODEL_ID, trust_remote_code=True).to(device)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)

DEFAULT_SPEAKER = 11  # MAL_F
DEFAULT_TEXT = (
    "മലയാളം ടെക്സ്റ്റ് ശബ്ദമായി മാറ്റാൻ ഇതുപയോഗിക്കുക. താഴെ ഒരു ഉദാഹരണം നൽകുന്നു.\n\n"
    "ഇത് ഒരു രണ്ടാം പാരագրാഫ് ആണ്."
)

STYLE_LABELS = {
    0: "ALEXA",
    1: "ANGER",
    2: "BB",
    3: "BOOK",
    4: "CONV",
    5: "DIGI",
    6: "DISGUST",
    7: "FEAR",
    8: "HAPPY",
    10: "NEWS",
    12: "SAD",
    14: "SURPRISE",
    15: "UMANG",
    16: "WIKI",
}


def split_paragraphs(text: str):
    # Split on blank lines; ignore empty chunks
    parts = [p.strip() for p in text.replace('\r','').split('\n\n')]
    parts = [p for p in parts if p]
    return parts if parts else ([text.strip()] if text.strip() else [])


def time_scale(wav: np.ndarray, rate: float) -> np.ndarray:
    """Naive time scaling by linear interpolation. rate>1 -> faster (shorter)."""
    if rate <= 0:
        rate = 1.0
    if abs(rate - 1.0) < 1e-6:
        return wav
    n = len(wav)
    new_len = max(1, int(n / rate))
    x_old = np.linspace(0.0, 1.0, n, endpoint=False)
    x_new = np.linspace(0.0, 1.0, new_len, endpoint=False)
    return np.interp(x_new, x_old, wav).astype(wav.dtype)


def apply_prosody(wav: np.ndarray, sr: int, rate: float, pitch_semitones: float):
    """
    Approximate prosody control without heavy DSP:
    - We implement pitch by changing the output *sample rate* by factor pf = 2**(semitones/12).
    - Changing sample rate also changes playback speed by pf, so we pre-scale time by rate/pf
      to keep the final perceived speaking rate close to the requested rate.
    """
    pf = 2.0 ** (pitch_semitones / 12.0)
    pre_rate = max(0.25, min(4.0, rate / max(pf, 1e-6)))
    y = time_scale(wav, pre_rate)
    out_sr = int(sr * pf)
    return y, out_sr


def synthesize_once(text: str, speaker_id: int, style_id: int):
    inputs = tokenizer(text=text, return_tensors="pt").to(device)
    outputs = model(inputs['input_ids'], speaker_id=int(speaker_id), emotion_id=int(style_id))
    wav = outputs.waveform.squeeze().detach().cpu().numpy()
    sr = model.config.sampling_rate
    return wav, sr


def save_audio_pair(wav: np.ndarray, sr: int, base_name: str, make_mp3: bool):
    # Save WAV
    wav_path = base_name + ".wav"
    sf.write(wav_path, wav, sr)
    out_files = [wav_path]
    # Optionally save MP3 via pydub/ffmpeg
    if make_mp3 and _HAS_PYDUB:
        try:
            mp3_path = base_name + ".mp3"
            seg = AudioSegment.from_wav(wav_path)
            seg.export(mp3_path, format="mp3")
            out_files.append(mp3_path)
        except Exception:
            pass
    return out_files


def parse_style(choice: str) -> int:
    try:
        return int(choice.split(":", 1)[0])
    except Exception:
        return 0


with gr.Blocks(theme=gr.themes.Soft()) as demo:
    gr.Markdown(
        """
        # മലയാളം Text → AI Voice (Free)
        Open‑source Malayalam TTS powered by **AI4Bharat VITS**.
        Now supports **multiple voice styles**, **prosody (rate & pitch)**, **batch paragraphs**, and **WAV + MP3** output.
        """
    )

    with gr.Row():
        txt = gr.Textbox(label="Malayalam Text (single or multiple paragraphs)", value=DEFAULT_TEXT, lines=8, placeholder="ഒരു അല്ലെങ്കിൽ നിരവധി പാരഗ്രാഫുകൾ ഇവിടെ പേസ്റ്റ് ചെയ്യുക… രണ്ട് newline ഉപയോഗിച്ച് വേർതിരിക്കുക.")
    with gr.Row():
        speaker = gr.Slider(0, 19, value=DEFAULT_SPEAKER, step=1, label="Speaker ID (MAL_F = 11)")
        styles = gr.CheckboxGroup(
            choices=[f"{k}:{v}" for k, v in STYLE_LABELS.items()],
            value=["0:ALEXA", "10:NEWS", "3:BOOK"],
            label="Voice styles (select one or more)"
        )
    with gr.Row():
        rate = gr.Slider(minimum=0.5, maximum=1.5, value=1.0, step=0.05, label="Speaking rate (0.5–1.5)")
        pitch = gr.Slider(minimum=-4, maximum=+4, value=0, step=1, label="Pitch (semitones, -4 to +4)")
        batch = gr.Checkbox(value=True, label="Batch: split by blank lines (paragraphs)")
        make_mp3 = gr.Checkbox(value=True, label="Also export MP3 (needs ffmpeg)")
    with gr.Row():
        btn = gr.Button("Generate", variant="primary")
    audio = gr.Audio(label="Preview (first file)", type="filepath")
    files_out = gr.Files(label="All generated files")
    note = gr.Markdown()

    def run(text, speaker_id, style_choices, rate, pitch, batch, make_mp3):
        text = (text or "").strip()
        if not text:
            raise gr.Error("ദയവായി മലയാളത്തിൽ ഒരു വാചകം/പാരഗ്രാഫ് നൽകുക.")
        paras = split_paragraphs(text) if batch else [text]
        if not style_choices:
            style_choices = ["0:ALEXA"]
        total = len(paras) * len(style_choices)
        if total > 30:
            raise gr.Error(f"താങ്കൾ വളരെ കൂടുതൽ ഔട്ട്‌പുട്ടുകൾ ആവശ്യപ്പെടുന്നു ({total}). ദയവായി കുറച്ച് പാരഗ്രാഫുകൾ/സ്റ്റൈലുകൾ തിരഞ്ഞെടുക്കുക (<= 30 files).")

        all_files = []
        preview = None
        details = []
        idx = 1
        for pi, para in enumerate(paras, start=1):
            wav_raw, sr_raw = synthesize_once(para, int(speaker_id), parse_style(style_choices[0]))  # synthesize once per paragraph using first style to get base prosody; style will be applied per file below anyway
            for sc in style_choices:
                stid = parse_style(sc)
                # Re-synthesize for each style to reflect emotion_id
                wav, sr = synthesize_once(para, int(speaker_id), stid)
                # Apply prosody approximation
                wav2, sr2 = apply_prosody(wav, sr, float(rate), float(pitch))
                base = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name[:-4]
                base_named = f"{base}_p{pi:02d}_style-{stid}_{STYLE_LABELS.get(stid, 'STYLE')}"
                outs = save_audio_pair(wav2, sr2, base_named, bool(make_mp3))
                all_files.extend(outs)
                if preview is None:
                    preview = outs[0]
                details.append(f"• P{pi} – {STYLE_LABELS.get(stid, sc)} → {os.path.basename(outs[0])}{' (+MP3)' if len(outs)>1 else ''}")
                idx += 1

        summary = (
            f"Generated **{len(all_files)}** files for {len(paras)} paragraph(s) × {len(style_choices)} style(s).\n\n"
            + "\n".join(details)
            + ("\n\n**Note:** Rate & pitch are approximations using resampling; for studio-grade SSML prosody use a managed TTS like Azure." if True else "")
        )
        return preview, all_files, summary

    btn.click(run, inputs=[txt, speaker, styles, rate, pitch, batch, make_mp3], outputs=[audio, files_out, note])

    gr.Markdown(
        """
        **Prosody controls**  
        *Speaking rate* slows/speeds audio; *Pitch* raises/lowers tone (in semitones). These are **approximate** controls based on resampling. For high‑fidelity prosody, consider SSML in Azure TTS.
        
        **Batch mode**  
        Split input into paragraphs using a blank line. The app creates one file per **paragraph × style**.
        
        **MP3 output**  
        Requires `ffmpeg` (available on Hugging Face Spaces). If unavailable, only WAV will be produced.
        """
    )

if __name__ == "__main__":
    demo.launch()