no

app.py

@@ -5,86 +5,176 @@ from src.chatterbox.mtl_tts import ChatterboxMultilingualTTS, SUPPORTED_LANGUAGE
 import gradio as gr
 import spaces
 
-# ✅ Detect CUDA
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-print(f"✅ Model ready on {DEVICE}")
+print(f"🚀 Running on device: {DEVICE}")
 
-# --- Global Model Cache ---
+# --- Global Model Initialization ---
 MODEL = None
 
-# --- Language Defaults ---
 LANGUAGE_CONFIG = {
+    "ar": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/ar_f/ar_prompts2.flac",
+        "text": "في الشهر الماضي، وصلنا إلى معلم جديد بمليارين من المشاهدات على قناتنا على يوتيوب."
+    },
+    "da": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/da_m1.flac",
+        "text": "Sidste måned nåede vi en ny milepæl med to milliarder visninger på vores YouTube-kanal."
+    },
+    "de": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/de_f1.flac",
+        "text": "Letzten Monat haben wir einen neuen Meilenstein erreicht: zwei Milliarden Aufrufe auf unserem YouTube-Kanal."
+    },
+    "el": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/el_m.flac",
+        "text": "Τον περασμένο μήνα, φτάσαμε σε ένα νέο ορόσημο με δύο δισεκατομμύρια προβολές στο κανάλι μας στο YouTube."
+    },
     "en": {
         "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/en_f1.flac",
         "text": "Last month, we reached a new milestone with two billion views on our YouTube channel."
     },
+    "es": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/es_f1.flac",
+        "text": "El mes pasado alcanzamos un nuevo hito: dos mil millones de visualizaciones en nuestro canal de YouTube."
+    },
+    "fi": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/fi_m.flac",
+        "text": "Viime kuussa saavutimme uuden virstanpylvään kahden miljardin katselukerran kanssa YouTube-kanavallamme."
+    },
     "fr": {
         "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/fr_f1.flac",
         "text": "Le mois dernier, nous avons atteint un nouveau jalon avec deux milliards de vues sur notre chaîne YouTube."
     },
+    "he": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/he_m1.flac",
+        "text": "בחודש שעבר הגענו לאבן דרך חדשה עם שני מיליארד צפיות בערוץ היוטיוב שלנו."
+    },
     "hi": {
         "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/hi_f1.flac",
         "text": "पिछले महीने हमने एक नया मील का पत्थर छुआ: हमारे YouTube चैनल पर दो अरब व्यूज़।"
     },
-    "es": {
-        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/es_f1.flac",
-        "text": "El mes pasado alcanzamos un nuevo hito: dos mil millones de visualizaciones en nuestro canal de YouTube."
+    "it": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/it_m1.flac",
+        "text": "Il mese scorso abbiamo raggiunto un nuovo traguardo: due miliardi di visualizzazioni sul nostro canale YouTube."
     },
     "ja": {
         "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/ja/ja_prompts1.flac",
         "text": "先月、私たちのYouTubeチャンネルで二十億回の再生回数という新たなマイルストーンに到達しました。"
     },
+    "ko": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/ko_f.flac",
+        "text": "지난달 우리는 유튜브 채널에서 이십억 조회수라는 새로운 이정표에 도달했습니다."
+    },
+    "ms": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/ms_f.flac",
+        "text": "Bulan lepas, kami mencapai pencapaian baru dengan dua bilion tontonan di saluran YouTube kami."
+    },
+    "nl": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/nl_m.flac",
+        "text": "Vorige maand bereikten we een nieuwe mijlpaal met twee miljard weergaven op ons YouTube-kanaal."
+    },
+    "no": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/no_f1.flac",
+        "text": "Forrige måned nådde vi en ny milepæl med to milliarder visninger på YouTube-kanalen vår."
+    },
+    "pl": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/pl_m.flac",
+        "text": "W zeszłym miesiącu osiągnęliśmy nowy kamień milowy z dwoma miliardami wyświetleń na naszym kanale YouTube."
+    },
+    "pt": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/pt_m1.flac",
+        "text": "No mês passado, alcançámos um novo marco: dois mil milhões de visualizações no nosso canal do YouTube."
+    },
+    "ru": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/ru_m.flac",
+        "text": "В прошлом месяце мы достигли нового рубежа: два миллиарда просмотров на нашем YouTube-канале."
+    },
+    "sv": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/sv_f.flac",
+        "text": "Förra månaden nådde vi en ny milstolpe med två miljarder visningar på vår YouTube-kanal."
+    },
+    "sw": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/sw_m.flac",
+        "text": "Mwezi uliopita, tulifika hatua mpya ya maoni ya bilioni mbili kweny kituo chetu cha YouTube."
+    },
+    "tr": {
+        "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/tr_m.flac",
+        "text": "Geçen ay YouTube kanalımızda iki milyar görüntüleme ile yeni bir dönüm noktasına ulaştık."
+    },
     "zh": {
         "audio": "https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/zh_f2.flac",
         "text": "上个月，我们达到了一个新的里程碑。 我们的YouTube频道观看次数达到了二十亿次，这绝对令人难以置信。"
     },
 }
 
-# --- Helpers ---
+# --- UI Helpers ---
 def default_audio_for_ui(lang: str) -> str | None:
     return LANGUAGE_CONFIG.get(lang, {}).get("audio")
 
+
 def default_text_for_ui(lang: str) -> str:
     return LANGUAGE_CONFIG.get(lang, {}).get("text", "")
 
+
 def get_supported_languages_display() -> str:
-    items = [f"**{name}** (`{code}`)" for code, name in sorted(SUPPORTED_LANGUAGES.items())]
-    mid = len(items) // 2
+    """Generate a formatted display of all supported languages."""
+    language_items = []
+    for code, name in sorted(SUPPORTED_LANGUAGES.items()):
+        language_items.append(f"**{name}** (`{code}`)")
+    
+    # Split into 2 lines
+    mid = len(language_items) // 2
+    line1 = " • ".join(language_items[:mid])
+    line2 = " • ".join(language_items[mid:])
+    
     return f"""
-### 🌍 Supported Languages ({len(SUPPORTED_LANGUAGES)})
-{' • '.join(items[:mid])}
+### 🌍 Supported Languages ({len(SUPPORTED_LANGUAGES)} total)
+{line1}
 
-{' • '.join(items[mid:])}
+{line2}
 """
 
-# --- Model Loader ---
+
 def get_or_load_model():
+    """Loads the ChatterboxMultilingualTTS model if it hasn't been loaded already,
+    and ensures it's on the correct device."""
     global MODEL
     if MODEL is None:
-        print("🔄 Loading Chatterbox Multilingual TTS model...")
+        print("Model not loaded, initializing...")
         try:
             MODEL = ChatterboxMultilingualTTS.from_pretrained(DEVICE)
-            if hasattr(MODEL, "to") and str(MODEL.device) != DEVICE:
+            if hasattr(MODEL, 'to') and str(MODEL.device) != DEVICE:
                 MODEL.to(DEVICE)
-            print(f"✅ Model ready on {getattr(MODEL, 'device', DEVICE)}")
+            print(f"Model loaded successfully. Internal device: {getattr(MODEL, 'device', 'N/A')}")
         except Exception as e:
-            print(f"❌ Error loading model: {e}")
+            print(f"Error loading model: {e}")
             raise
     return MODEL
 
-# Preload model silently (non-blocking for Spaces)
+# Attempt to load the model at startup.
 try:
     get_or_load_model()
 except Exception as e:
-    print(f"⚠️ Model preload failed: {e}")
+    print(f"CRITICAL: Failed to load model on startup. Application may not function. Error: {e}")
 
 def set_seed(seed: int):
+    """Sets the random seed for reproducibility across torch, numpy, and random."""
     torch.manual_seed(seed)
     if DEVICE == "cuda":
         torch.cuda.manual_seed(seed)
         torch.cuda.manual_seed_all(seed)
     random.seed(seed)
     np.random.seed(seed)
+    
+def resolve_audio_prompt(language_id: str, provided_path: str | None) -> str | None:
+    """
+    Decide which audio prompt to use:
+    - If user provided a path (upload/mic/url), use it.
+    - Else, fall back to language-specific default (if any).
+    """
+    if provided_path and str(provided_path).strip():
+        return provided_path
+    return LANGUAGE_CONFIG.get(language_id, {}).get("audio")
+
 
 @spaces.GPU
 def generate_tts_audio(
@@ -96,15 +186,39 @@ def generate_tts_audio(
     seed_num_input: int = 0,
     cfgw_input: float = 0.5
 ) -> tuple[int, np.ndarray]:
-    """Generate multilingual TTS output."""
-    model = get_or_load_model()
-    if model is None:
-        raise RuntimeError("Model not loaded")
+    """
+    Generate high-quality speech audio from text using Chatterbox Multilingual model with optional reference audio styling.
+    Supported languages: English, French, German, Spanish, Italian, Portuguese, and Hindi.
+    
+    This tool synthesizes natural-sounding speech from input text. When a reference audio file 
+    is provided, it captures the speaker's voice characteristics and speaking style. The generated audio 
+    maintains the prosody, tone, and vocal qualities of the reference speaker, or uses default voice if no reference is provided.
+
+    Args:
+        text_input (str): The text to synthesize into speech (maximum 300 characters)
+        language_id (str): The language code for synthesis (eg. en, fr, de, es, it, pt, hi)
+        audio_prompt_path_input (str, optional): File path or URL to the reference audio file that defines the target voice style. Defaults to None.
+        exaggeration_input (float, optional): Controls speech expressiveness (0.25-2.0, neutral=0.5, extreme values may be unstable). Defaults to 0.5.
+        temperature_input (float, optional): Controls randomness in generation (0.05-5.0, higher=more varied). Defaults to 0.8.
+        seed_num_input (int, optional): Random seed for reproducible results (0 for random generation). Defaults to 0.
+        cfgw_input (float, optional): CFG/Pace weight controlling generation guidance (0.2-1.0). Defaults to 0.5, 0 for language transfer. 
+
+    Returns:
+        tuple[int, np.ndarray]: A tuple containing the sample rate (int) and the generated audio waveform (numpy.ndarray)
+    """
+    current_model = get_or_load_model()
+
+    if current_model is None:
+        raise RuntimeError("TTS model is not loaded.")
 
     if seed_num_input != 0:
         set_seed(int(seed_num_input))
 
+    print(f"Generating audio for text: '{text_input[:50]}...'")
+    
+    # Handle optional audio prompt
     chosen_prompt = audio_prompt_path_input or default_audio_for_ui(language_id)
+
     generate_kwargs = {
         "exaggeration": exaggeration_input,
         "temperature": temperature_input,
@@ -112,58 +226,96 @@ def generate_tts_audio(
     }
     if chosen_prompt:
         generate_kwargs["audio_prompt_path"] = chosen_prompt
-        print(f"🎧 Using reference: {chosen_prompt}")
+        print(f"Using audio prompt: {chosen_prompt}")
     else:
-        print("🎙️ No reference audio — using default voice")
-
-    wav = model.generate(text_input[:300], language_id=language_id, **generate_kwargs)
-    print("✅ Audio generated.")
-    return (model.sr, wav.squeeze(0).numpy())
+        print("No audio prompt provided; using default voice.")
+        
+    wav = current_model.generate(
+        text_input[:300],  # Truncate text to max chars
+        language_id=language_id,
+        **generate_kwargs
+    )
+    print("Audio generation complete.")
+    return (current_model.sr, wav.squeeze(0).numpy())
 
-# --- Gradio UI ---
-with gr.Blocks(title="Chatterbox Multilingual TTS") as demo:
-    gr.Markdown("# 🌍 Chatterbox Multilingual TTS Demo\nGenerate high-quality multilingual speech with optional voice cloning.")
+with gr.Blocks() as demo:
+    gr.Markdown(
+        """
+        # Chatterbox Multilingual Demo
+        Generate high-quality multilingual speech from text with reference audio styling, supporting 23 languages.
+        
+        For a hosted version of Chatterbox Multilingual and for finetuning, please visit [resemble.ai](https://app.resemble.ai)
+        """
+    )
+    
+    # Display supported languages
     gr.Markdown(get_supported_languages_display())
-
     with gr.Row():
         with gr.Column():
-            initial_lang = "en"
-            text = gr.Textbox(value=default_text_for_ui(initial_lang), label="Text", max_lines=5)
+            initial_lang = "fr"
+            text = gr.Textbox(
+                value=default_text_for_ui(initial_lang),
+                label="Text to synthesize (max chars 300)",
+                max_lines=5
+            )
+            
             language_id = gr.Dropdown(
                 choices=list(ChatterboxMultilingualTTS.get_supported_languages().keys()),
-                value=initial_lang, label="Language"
+                value=initial_lang,
+                label="Language",
+                info="Select the language for text-to-speech synthesis"
             )
+            
             ref_wav = gr.Audio(
                 sources=["upload", "microphone"],
                 type="filepath",
-                label="Reference Audio (Optional)",
+                label="Reference Audio File (Optional)",
                 value=default_audio_for_ui(initial_lang)
             )
-
-            exaggeration = gr.Slider(0.25, 2, 0.05, label="Exaggeration (emotion)", value=0.5)
-            cfg_weight = gr.Slider(0.2, 1, 0.05, label="CFG / Pace", value=0.5)
+            
+            gr.Markdown(
+                "💡 **Note**: Ensure that the reference clip matches the specified language tag. Otherwise, language transfer outputs may inherit the accent of the reference clip's language. To mitigate this, set the CFG weight to 0.",
+                elem_classes=["audio-note"]
+            )
+            
+            exaggeration = gr.Slider(
+                0.25, 2, step=.05, label="Exaggeration (Neutral = 0.5, extreme values can be unstable)", value=.5
+            )
+            cfg_weight = gr.Slider(
+                0.2, 1, step=.05, label="CFG/Pace", value=0.5
+            )
 
             with gr.Accordion("More options", open=False):
-                seed_num = gr.Number(value=0, label="Seed (0=random)")
-                temp = gr.Slider(0.05, 5, 0.05, label="Temperature", value=0.8)
+                seed_num = gr.Number(value=0, label="Random seed (0 for random)")
+                temp = gr.Slider(0.05, 5, step=.05, label="Temperature", value=.8)
 
-            run_btn = gr.Button("🎤 Generate", variant="primary")
+            run_btn = gr.Button("Generate", variant="primary")
 
         with gr.Column():
-            audio_output = gr.Audio(label="Generated Audio")
+            audio_output = gr.Audio(label="Output Audio")
 
-        def on_lang_change(lang, _, __):
+        def on_language_change(lang, current_ref, current_text):
             return default_audio_for_ui(lang), default_text_for_ui(lang)
 
-        language_id.change(fn=on_lang_change, inputs=[language_id, ref_wav, text],
-                           outputs=[ref_wav, text], show_progress=False)
+        language_id.change(
+            fn=on_language_change,
+            inputs=[language_id, ref_wav, text],
+            outputs=[ref_wav, text],
+            show_progress=False
+        )
 
     run_btn.click(
         fn=generate_tts_audio,
-        inputs=[text, language_id, ref_wav, exaggeration, temp, seed_num, cfg_weight],
-        outputs=[audio_output]
+        inputs=[
+            text,
+            language_id,
+            ref_wav,
+            exaggeration,
+            temp,
+            seed_num,
+            cfg_weight,
+        ],
+        outputs=[audio_output],
     )
 
-# --- Launch ---
-if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860, show_api=False)
+demo.launch(mcp_server=True)

requirements.txt

@@ -1,26 +1,19 @@
-# Core dependencies
-torch==2.4.1
-torchaudio==2.4.1
-transformers==4.46.3
-diffusers==0.29.0
-safetensors
-omegaconf==2.3.0
+gradio
 numpy==1.26.0
 resampy==0.4.3
 librosa==0.10.0
-soundfile
-gradio==4.20.1   # ✅ Node-free stable build for T4 GPU (no "--import" issue)
-spaces==0.26.1
-tqdm
-
-# Speech/voice-related packages
+s3tokenizer
+transformers==4.46.3
+diffusers==0.29.0
+omegaconf==2.3.0
+resemble-perth==1.0.1
 silero-vad==5.1.2
 conformer==0.3.2
-resemble-perth==1.0.1
-s3tokenizer
+safetensors
 
-# Language-specific utilities (optional, uncomment as needed)
-# spacy_pkuseg           # Chinese text segmentation
-# pykakasi>=2.2.0        # Japanese text processing (Kanji → Hiragana)
-# russian-text-stresser @ git+https://github.com/Vuizur/add-stress-to-epub
-# dicta-onnx>=0.1.0      # Hebrew diacritization
+# Optional language-specific dependencies
+# Uncomment the ones you need for specific languages:
+ spacy_pkuseg          # For Chinese text segmentation
+ pykakasi>=2.2.0       # For Japanese text processing (Kanji to Hiragana)
+ russian-text-stresser @ git+https://github.com/Vuizur/add-stress-to-epub
+# dicta-onnx>=0.1.0     # For Hebrew diacritization