Upload app.py

app.py

@@ -144,15 +144,30 @@ def resolve_text(text_input, file_input):
 # SINGLE-SPEAKER MODES
 # ==========================================
 @spaces.GPU
-def generate_custom_voice(text, file_input, language, speaker_label, instruction):
+def generate_custom_voice(text, file_input, language, speaker_label, instruction, auto_emotions):
     resolved_text = resolve_text(text, file_input)
-    model = get_model("custom")
     speaker = speaker_label.split("--")[0].strip()
     lang = language if language != "Auto" else "Auto"
+
+    if auto_emotions:
+        client = get_llm_client()
+        if client:
+            tmp_dir = os.path.join(OUTPUT_DIR, f"cv_{int(time.time())}")
+            os.makedirs(tmp_dir, exist_ok=True)
+            segments = analyze_emotions_for_segments(client, resolved_text)
+            audio_files = generate_segments_with_emotions(
+                segments, language, speaker, "custom", tmp_dir,
+            )
+            if audio_files:
+                final = os.path.join(OUTPUT_DIR, f"custom_{int(time.time())}.wav")
+                concatenate_wavs(audio_files, final)
+                return final
+
+    # Fallback: single generation
+    model = get_model("custom")
     kwargs = {"text": resolved_text, "language": lang, "speaker": speaker}
     if instruction and instruction.strip():
         kwargs["instruct"] = instruction.strip()
-    print(f"[TTS] Custom: speaker={speaker}, lang={lang}")
     wavs, sr = model.generate_custom_voice(**kwargs)
     path = os.path.join(OUTPUT_DIR, f"custom_{int(time.time())}.wav")
     sf.write(path, wavs[0], sr)
@@ -160,23 +175,39 @@ def generate_custom_voice(text, file_input, language, speaker_label, instruction
 
 
 @spaces.GPU
-def generate_voice_design(text, file_input, language, voice_description):
+def generate_voice_design(text, file_input, language, voice_description, auto_emotions):
     resolved_text = resolve_text(text, file_input)
     if not voice_description.strip():
         raise gr.Error("Please describe the voice you want.")
-    model = get_model("design")
     lang = language if language != "Auto" else "Auto"
-    print(f"[TTS] Design: lang={lang}, desc={voice_description[:60]}")
+
+    if auto_emotions:
+        client = get_llm_client()
+        if client:
+            tmp_dir = os.path.join(OUTPUT_DIR, f"vd_{int(time.time())}")
+            os.makedirs(tmp_dir, exist_ok=True)
+            segments = analyze_emotions_for_segments(client, resolved_text)
+            audio_files = generate_segments_with_emotions(
+                segments, language, None, "design", tmp_dir,
+                voice_desc=voice_description.strip(),
+            )
+            if audio_files:
+                final = os.path.join(OUTPUT_DIR, f"design_{int(time.time())}.wav")
+                concatenate_wavs(audio_files, final)
+                return final
+
+    # Fallback: single generation
+    model = get_model("design")
     wavs, sr = model.generate_voice_design(text=resolved_text, language=lang, instruct=voice_description)
     path = os.path.join(OUTPUT_DIR, f"design_{int(time.time())}.wav")
     sf.write(path, wavs[0], sr)
     return path
 
 
-def enhance_text_with_emotions(client, text):
-    """Use AI to rewrite text with emotional delivery cues that TTS picks up naturally."""
+def analyze_emotions_for_segments(client, text):
+    """Split text into segments with emotion instructions for single-speaker mode."""
     if not client:
-        return text
+        return [{"text": text, "emotion": ""}]
     try:
         response = client.chat.completions.create(
             model=OMNI_MODEL, modalities=["text"],
@@ -184,32 +215,93 @@ def enhance_text_with_emotions(client, text):
                 {
                     "role": "system",
                     "content": (
-                        "You are an audiobook performance coach. Rewrite the text to add natural "
-                        "emotional delivery cues that a TTS voice will pick up from context.\n\n"
-                        "Techniques:\n"
-                        "- Add pause markers: '...' for dramatic pauses\n"
-                        "- Use punctuation for emphasis: '!' for energy, '?' for curiosity\n"
-                        "- Add breath/pacing cues with commas and dashes\n"
-                        "- Extend words for emphasis: 'sooo beautiful'\n"
-                        "- Add interjections: 'Oh!', 'Hmm...', 'Ah,'\n"
-                        "- Use ellipsis for trailing off: 'I never thought...'\n\n"
+                        "You are an audiobook director. Split this text into segments where "
+                        "the emotional tone changes. For each segment, provide an emotion/delivery "
+                        "instruction for the voice actor.\n\n"
+                        "Output ONLY valid JSON:\n"
+                        '{"segments": [\n'
+                        '  {"text": "The lighthouse stood tall against the storm.", "emotion": "Atmospheric, steady, painting the scene"},\n'
+                        '  {"text": "She ran to the door, heart pounding.", "emotion": "Urgent, breathless, rising tension"},\n'
+                        '  {"text": "And then... silence.", "emotion": "Quiet, dramatic pause, barely above a whisper"}\n'
+                        "]}\n\n"
                         "Rules:\n"
-                        "1. Keep ALL the original meaning and content\n"
-                        "2. Do NOT add stage directions or [brackets]\n"
-                        "3. The output must be speakable text only\n"
-                        "4. Be subtle — enhance, don't overdo\n"
-                        "5. Output ONLY the enhanced text"
+                        "- Include ALL text, do not skip anything\n"
+                        "- Keep segments in original order\n"
+                        "- Each segment should be 1-4 sentences with a consistent emotion\n"
+                        "- Be specific with emotions: not just 'sad' but 'quietly heartbroken, voice trailing off'\n"
+                        "- Merge text with the same emotion\n"
+                        "- Dialogue should have the emotion of the speaker\n"
+                        "- Output ONLY JSON, no markdown, no backticks"
                     ),
                 },
-                {"role": "user", "content": f"Enhance this for emotional TTS delivery:\n\n{text}"},
+                {"role": "user", "content": f"Direct this text with emotions:\n\n{text[:6000]}"},
             ],
         )
-        enhanced = response.choices[0].message.content.strip()
-        print(f"[Emotions] Original: {len(text)} chars -> Enhanced: {len(enhanced)} chars")
-        return enhanced
+        raw = response.choices[0].message.content.strip()
+        raw = re.sub(r'^```json\s*', '', raw)
+        raw = re.sub(r'\s*```$', '', raw)
+        data = json.loads(raw)
+        segments = data.get("segments", [])
+        if segments:
+            print(f"[Emotions] Split into {len(segments)} emotional segments")
+            return segments
     except Exception as e:
-        print(f"[Emotions] Enhancement failed: {e}")
-        return text
+        print(f"[Emotions] Analysis failed: {e}")
+    return [{"text": text, "emotion": ""}]
+
+
+@spaces.GPU
+def generate_segments_with_emotions(segments, language, speaker, model_type, tmp_dir,
+                                     voice_desc=None, ref_audio=None, ref_text=None):
+    """Generate audio for multiple emotion-tagged segments using one voice."""
+    model = get_model(model_type)
+    lang = language if language != "Auto" else "Auto"
+    audio_files = []
+
+    # Create pause between segments
+    pause_path = os.path.join(tmp_dir, "seg_pause.wav")
+    generate_silence(0.6, pause_path)
+
+    for i, seg in enumerate(segments):
+        seg_text = seg.get("text", "").strip()
+        emotion = seg.get("emotion", "")
+        if not seg_text:
+            continue
+
+        path = os.path.join(tmp_dir, f"emoseg_{i:04d}.wav")
+        try:
+            if model_type == "custom":
+                kwargs = {"text": seg_text, "language": lang, "speaker": speaker}
+                if emotion:
+                    kwargs["instruct"] = emotion
+                wavs, sr = model.generate_custom_voice(**kwargs)
+            elif model_type == "design":
+                instruct = voice_desc or ""
+                if emotion:
+                    instruct = f"{instruct}. {emotion}" if instruct else emotion
+                wavs, sr = model.generate_voice_design(text=seg_text, language=lang, instruct=instruct)
+            elif model_type == "clone":
+                kwargs = {"text": seg_text, "language": lang, "ref_audio": ref_audio}
+                if ref_text:
+                    kwargs["ref_text"] = ref_text
+                else:
+                    kwargs["x_vector_only_mode"] = True
+                wavs, sr = model.generate_voice_clone(**kwargs)
+
+            sf.write(path, wavs[0], sr)
+            audio_files.append(path)
+            print(f"[Emotions] Seg {i}: '{emotion[:40]}' -> OK")
+        except Exception as e:
+            print(f"[Emotions] Seg {i} failed: {e}")
+            fail = os.path.join(tmp_dir, f"fail_{i:04d}.wav")
+            generate_silence(1.0, fail)
+            audio_files.append(fail)
+
+        # Add pause between segments
+        if i < len(segments) - 1:
+            audio_files.append(pause_path)
+
+    return audio_files
 
 
 def extract_pdf_sections(filepath):
@@ -277,23 +369,29 @@ def generate_voice_clone(text, file_input, language, ref_audio, ref_text, add_em
     if ref_audio is None:
         raise gr.Error("Please upload a reference audio sample.")
 
-    # Enhance text with emotions if requested
-    final_text = resolved_text
     if add_emotions:
         client = get_llm_client()
         if client:
-            final_text = enhance_text_with_emotions(client, text)
-        else:
-            print("[Clone] No LLM client for emotion enhancement, using raw text")
+            tmp_dir = os.path.join(OUTPUT_DIR, f"vc_{int(time.time())}")
+            os.makedirs(tmp_dir, exist_ok=True)
+            segments = analyze_emotions_for_segments(client, resolved_text)
+            audio_files = generate_segments_with_emotions(
+                segments, language, None, "clone", tmp_dir,
+                ref_audio=ref_audio, ref_text=ref_text if ref_text and ref_text.strip() else None,
+            )
+            if audio_files:
+                final = os.path.join(OUTPUT_DIR, f"clone_{int(time.time())}.wav")
+                concatenate_wavs(audio_files, final)
+                return final
 
+    # Fallback: single generation
     model = get_model("clone")
     lang = language if language != "Auto" else "Auto"
-    kwargs = {"text": final_text, "language": lang, "ref_audio": ref_audio}
+    kwargs = {"text": resolved_text, "language": lang, "ref_audio": ref_audio}
     if ref_text and ref_text.strip():
         kwargs["ref_text"] = ref_text.strip()
     else:
         kwargs["x_vector_only_mode"] = True
-    print(f"[TTS] Clone: lang={lang}, emotions={'yes' if add_emotions else 'no'}")
     wavs, sr = model.generate_voice_clone(**kwargs)
     path = os.path.join(OUTPUT_DIR, f"clone_{int(time.time())}.wav")
     sf.write(path, wavs[0], sr)
@@ -608,13 +706,15 @@ with gr.Blocks(title="Qwen3-TTS Demo") as demo:
                 cv_speaker = gr.Dropdown(choices=SPEAKER_CHOICES,
                                          value="Ryan -- Dynamic male, strong rhythmic drive (English)",
                                          label="Speaker")
-                cv_instruct = gr.Textbox(label="Emotion / Style (optional)",
+                cv_instruct = gr.Textbox(label="Emotion / Style (optional, used when auto-emotions is off)",
                                          placeholder="e.g. Very happy, Whisper softly, Speak with authority...")
+                cv_emotions = gr.Checkbox(value=True, label="Auto-detect emotions",
+                                          info="AI analyzes text and varies tone/emotion per segment. Requires DASHSCOPE_API_KEY.")
                 cv_btn = gr.Button("Generate", variant="primary")
             with gr.Column():
                 cv_audio = gr.Audio(label="Generated Speech", type="filepath")
         cv_btn.click(fn=generate_custom_voice,
-                     inputs=[cv_text, cv_file, cv_lang, cv_speaker, cv_instruct], outputs=cv_audio)
+                     inputs=[cv_text, cv_file, cv_lang, cv_speaker, cv_instruct, cv_emotions], outputs=cv_audio)
 
     # ── Tab 2: Voice Design ──
     with gr.Tab("Voice Design"):
@@ -637,11 +737,13 @@ with gr.Blocks(title="Qwen3-TTS Demo") as demo:
                     ],
                     inputs=[vd_desc], label="Examples",
                 )
+                vd_emotions = gr.Checkbox(value=True, label="Auto-detect emotions",
+                                          info="AI varies tone/emotion per segment. Requires DASHSCOPE_API_KEY.")
                 vd_btn = gr.Button("Generate", variant="primary")
             with gr.Column():
                 vd_audio = gr.Audio(label="Generated Speech", type="filepath")
         vd_btn.click(fn=generate_voice_design,
-                     inputs=[vd_text, vd_file, vd_lang, vd_desc], outputs=vd_audio)
+                     inputs=[vd_text, vd_file, vd_lang, vd_desc, vd_emotions], outputs=vd_audio)
 
     # ── Tab 3: Voice Clone ──
     with gr.Tab("Voice Clone"):