import os import html import math import re import struct import wave import gradio as gr import time from pathlib import Path from tts import split_into_chunks, generate_audio_stream from inference import transcribe_audio, answer_story_question from voice_clone import load_default_profile, list_saved_profiles # Create directories for sample audio files os.makedirs("sample_sounds", exist_ok=True) # Generate relaxing audio chime waveforms procedurally # This runs fully client/server-side with standard Python, eliminating mock limits or Torch wait times def generate_chimes_wav(filename, duration=120, melody_type="lullaby"): sample_rate = 16000 n_samples = int(duration * sample_rate) with wave.open(filename, 'wb') as wav_file: wav_file.setnchannels(1) wav_file.setsampwidth(2) wav_file.setframerate(sample_rate) # Pentatonic cozy scales if melody_type == "lullaby": notes = [261.63, 293.66, 329.63, 392.00, 440.00] # C4, D4, E4, G4, A4 elif melody_type == "adventure": notes = [196.00, 246.94, 293.63, 329.63, 392.00] # G3, B3, D4, E4, G4 else: notes = [220.00, 261.63, 293.66, 329.63, 392.00] # A3, C4, D4, E4, G4 for i in range(n_samples): # cycle notes based on melody speed note_speed = sample_rate * 1.5 if melody_type == "lullaby" else sample_rate * 1.2 note_idx = int((i / note_speed) % len(notes)) freq = notes[note_idx] # envelope to prevent crackles sample_in_note = i % int(note_speed) envelope = 1.0 if sample_in_note < 1200: # attack envelope = sample_in_note / 1200 else: # decay decay_length = note_speed - 1200 envelope = max(0.0, 1.0 - (sample_in_note - 1200) / decay_length) # Synthesize voice-harmonic chime val = math.sin(2 * math.pi * freq * i / sample_rate) val += 0.45 * math.sin(2 * math.pi * (freq * 1.5) * i / sample_rate) val += 0.25 * math.sin(2 * math.pi * (freq * 2.0) * i / sample_rate) val = val / 1.7 * envelope packed_val = struct.pack(' str: p = COVERS_DIR / f"{slug}.png" return f"/gradio_api/file={p.resolve().as_posix()}" def load_books_from_stories(stories_dir="stories"): books = [] story_files = sorted( f for f in os.listdir(stories_dir) if f.endswith(".txt") ) if os.path.isdir(stories_dir) else [] for i, filename in enumerate(story_files): title = filename.replace(".txt", "").replace("_", " ") slug = filename.replace(".txt", "") synopsis = "" try: with open(os.path.join(stories_dir, filename), encoding="utf-8") as f: lines = [l.strip() for l in f.readlines() if l.strip()] synopsis = " ".join(lines[1:4]) if len(lines) > 1 else lines[0] if lines else "" except Exception: pass books.append({ "id": str(i + 1), "title": title, "author": "Public Domain", "cover_url": cover_path(slug), "voice_name": "Mom's Voice", "duration": 0, "elapsed_time": 0, "synopsis": synopsis, "category": "Children's Story", "is_cloned": False, "percentage": 0, "audio_path": None, "story_path": os.path.join(stories_dir, filename), }) return books mock_books = load_books_from_stories() mock_voices = [ { "id": "v1", "name": "Mom's Voice", "avatar_url": "👩‍đŸĻŗ", "description": "Warm, soft, with a natural soothing cadence. Best for bedtime stories.", "gender": "Female", "created_date": "2026-06-01", "status": "ready" }, { "id": "v2", "name": "Grandpa Joseph", "avatar_url": "👴", "description": "Deep, resonant, with an occasional comforting chuckle. Perfect for classic adventure stories.", "gender": "Male", "created_date": "2026-06-03", "status": "ready" }, { "id": "v3", "name": "Aunt Sarah", "avatar_url": "👩", "description": "Bubbly, expressive, brilliant at acting out funny character voices.", "gender": "Female", "created_date": "2026-06-06", "status": "processing" } ] with open(os.path.join(os.path.dirname(__file__), "static", "style.css"), encoding="utf-8") as _f: css_code = _f.read() def generate_dashboard_html(books_list): featured = books_list[0] others = books_list[1:] featured_js_title = featured['title'].replace("'", "\\'").replace('"', '\\"') html = f"""
Welcome Back, Sarah

Explore Your Warm Audiobook Shelves

Featured Clone

{featured['title']}

by {featured['author']}

{featured['synopsis']}

⏱ī¸ 20 Min Remaining 🎙ī¸ Narrator: {featured['voice_name']} ⭐ Editorial Spotlight

Continue Reading with Channeled Voices

""" for bk in books_list: js_safe_title = bk['title'].replace("'", "\\'").replace('"', '\\"') html += f"""
{bk['category']} {bk['percentage']}%
{bk['title']}

by {bk['author']}

🎙ī¸ {bk['voice_name']}
""" html += """
""" return html def generate_library_html(books_list, query="", category_filt="All", empty_mode=False): if empty_mode: return """
📚 🎙ī¸

Your Bookshelf is Waiting to be Voiced

Choose a comforting companion, clone their warm voice, and hear any classical or modern tale come to life immediately.

Use 'Clone Voice' tab to craft a companion
""" filtered = [] for bk in books_list: match_query = (query.lower() in bk["title"].lower()) or (query.lower() in bk["author"].lower()) or (query.lower() in bk["voice_name"].lower()) match_cat = True if category_filt == "In Progress": match_cat = 0 < bk["percentage"] < 100 elif category_filt == "Completed": match_cat = bk["percentage"] == 100 if match_query and match_cat: filtered.append(bk) if len(filtered) == 0: return """
🔍

No audiobooks match your search criteria.

""" html_out = """
""" for bk in filtered: safe_title = html.escape(bk['title']) js_safe_title = bk['title'].replace("'", "\\'").replace('"', '\\"') html_out += f"""
{bk['percentage']}%
{bk['category']}
{safe_title}
Narrator: 🌟 {bk['voice_name']}
â–ļī¸ Tap to Play
""" html_out += "
" return html_out def load_paragraphs(story_path: str) -> list: try: with open(story_path, encoding="utf-8") as f: raw = f.read() paras = [p.strip() for p in raw.split("\n\n") if p.strip()] return paras except Exception: return [] def render_story_text(paragraphs: list, current_idx: int) -> str: if not paragraphs: return "" result = """
""" for i, para in enumerate(paragraphs): safe_para = html.escape(para) if i == current_idx: result += f"""

{safe_para}

""" else: result += f"""

{safe_para}

""" result += "
" return result def render_cloned_voices_html(voices_list): markup = """
""" for v in voices_list: status_color = "#16a34a" if v["status"] == "ready" else "#d97706" status_bg = "#f0fdf4" if v["status"] == "ready" else "#fef3c7" safe_name = html.escape(v['name']) safe_desc = html.escape(v['description']) voice_id = html.escape(v.get('profile_id', v['id'])) onclick = f"""onclick=" const sel = document.querySelector('#voice_selector textarea, #voice_selector input'); if (sel) {{ sel.value = '{voice_id}'; sel.dispatchEvent(new Event('input', {{bubbles: true}})); sel.dispatchEvent(new Event('change', {{bubbles: true}})); }} document.querySelectorAll('.voice-card-selected').forEach(c => c.classList.remove('voice-card-selected')); this.classList.add('voice-card-selected'); " """ markup += f"""
{v['avatar_url']}
{safe_name}
{v['status']}

{safe_desc}

Gender: {v['gender']} Created: {v['created_date']}
""" markup += "
" return markup # Load default voice profile from Voice_Profile/ (if any saved from previous sessions) _default_profile_id = load_default_profile() _saved_profiles = list_saved_profiles() if _default_profile_id: _default_voice_name = next( (p["voice_name"] for p in _saved_profiles if p["profile_id"] == _default_profile_id), "Cloned Voice" ) print(f"[MomsVoice] Default voice profile loaded: {_default_profile_id} ({_default_voice_name})") else: _default_voice_name = None print("[MomsVoice] No saved voice profile — using stock voice as default.") # Preload models at startup for fast response print("[MomsVoice] Preloading Qwen3-TTS 0.6B CustomVoice model...") from voice_clone import get_custom_voice_model get_custom_voice_model() print("[MomsVoice] TTS model ready.") print("[MomsVoice] Preloading Whisper-small ASR model...") from inference import get_asr_pipeline, get_qa_model get_asr_pipeline() print("[MomsVoice] ASR model ready.") print("[MomsVoice] Preloading Qwen2.5-3B-Instruct Q&A model...") get_qa_model() print("[MomsVoice] Q&A model ready. All models preloaded.") # Gradio Application Core setup with gr.Blocks(title="MomsVoice", css=css_code) as demo: books_state = gr.State(mock_books) voices_state = gr.State(mock_voices) paragraphs_state = gr.State([]) tts_chunks_state = gr.State([]) voice_profile_state = gr.State(_default_profile_id) gr.HTML("""
đŸ’ŋ

VoiceBook

AI Bedtime Story Narrator
Hugging Face Space Node: ● Active
""") with gr.Tabs() as main_tabs: # TAB 1: Explore Dashboard with gr.TabItem("🏛ī¸ Explore Space") as explore_tab: dashboard_container = gr.HTML(value=generate_dashboard_html(mock_books)) gr.HTML("""
🎧 Head over to My Library and tap any book to start listening!
""") # TAB 2: Library + Player (í†ĩ합) with gr.TabItem("📚 My Library") as library_tab: with gr.Row(): # Left: book grid with gr.Column(scale=3): with gr.Row(): with gr.Column(scale=3): search_input = gr.Textbox(placeholder="Search by title, author, or voice...", label="Filter", show_label=False) with gr.Column(scale=1): category_filter = gr.Radio(["All", "In Progress", "Completed"], value="All", label="Progress", show_label=False) shelf_grid = gr.HTML(value=generate_library_html(mock_books)) # Hidden radio — book cards click this via JS onclick book_titles_list = [b["title"] for b in mock_books] book_selector = gr.Radio( choices=book_titles_list, value=None, label="", show_label=False, elem_classes="library-book-selector", visible=False, ) # Right: player panel with gr.Column(scale=2, elem_classes="dark-card-container"): gr.HTML("""
Now Playing SYNCHRONIZED PITCH
""") player_title_info = gr.HTML("""
📖

Select a story

Tap any book on the left to begin

""") player_audio_control = gr.Audio( visible=False, interactive=False, label="Audio Stream", type="numpy", streaming=False, autoplay=True, ) player_status_bar = gr.HTML("""
SELECT A STORY
""") with gr.Row(): play_btn = gr.Button("â–ļī¸ Play", variant="primary", scale=1, visible=False) pause_btn = gr.Button("⏸ Pause", variant="secondary", scale=1, visible=False) resume_btn = gr.Button("↩ī¸ Resume Story", variant="secondary", scale=2, visible=False) with gr.Row(): ask_btn = gr.Button("❓ Ask a Question", variant="primary", scale=2, visible=False) chunk_status = gr.HTML("""
Chunk 0 / 0
""") story_finished_panel = gr.HTML("""
🎉

Story Complete!

Tap another book to start a new story.

""", visible=False) # Story text display + slider story_text_display = gr.HTML(visible=False) timeline_slider = gr.Slider(minimum=0, maximum=100, step=1, value=0, label="📍 Story position (paragraph)", visible=False) # Q&A Panel qa_panel = gr.HTML("""

❓ Ask About the Story

""", visible=False) with gr.Group(visible=False) as qa_input_group: question_audio = gr.Audio( sources=["microphone"], type="filepath", label="🎤 Ask your question — answer generates when you stop recording", show_label=True, streaming=False, elem_id="qa_audio_input", ) question_text = gr.Textbox(visible=False) submit_question_btn = gr.Button("🔍 Get Answer in Narrator's Voice", variant="primary", elem_id="qa_submit_btn") answer_display = gr.HTML(visible=False) answer_audio = gr.Audio(label="Answer in Narrator's Voice", interactive=False, visible=False, autoplay=True) # TAB 3: Clone Voice Studio with gr.TabItem("🎙ī¸ Clone Voice Studio") as clone_tab: with gr.Row(): with gr.Column(scale=1, elem_classes="card-container"): gr.HTML("""

🧙 Voice Details Form

Provide a comforting nickname and details below to prepare vocal weights matching.

""") new_voice_name = gr.Textbox(placeholder="Enter a descriptive nickname (e.g., Grandma Judy, Dad)...", label="Voice Nickname") new_voice_gender = gr.Radio(["Female", "Male"], value="Female", label="Voice Gender Avatar") gr.HTML("""
Story Script to read aloud:

"Deep within the ancient forest, the giant oak tree stood. Its roots whispered stories of forgotten eras. 'Close your eyes,' the wind hummed softly, 'for our beautiful journey begins tonight.'"

""") mic_recorder = gr.Audio(sources=["microphone"], type="filepath", label="Record speaking story prompt (Min 10 seconds sample)") extract_btn = gr.Button("đŸĒ„ Analyze & Synthesize Customized Vocal Profile", variant="primary") with gr.Column(scale=1, elem_classes="card-container"): gr.HTML("""

Status Pipeline Visualizer

Track extraction, de-noising, and pitch tuning.

""") cloning_progress_msg = gr.HTML("""
🎙ī¸

Fill parameters on the left and read the story script aloud to begin extraction.

""") voice_cloning_success_panel = gr.HTML(visible=False) voice_sample_preview_widget = gr.Audio(value="sample_sounds/cloned_preview.wav", visible=False, label="Pre-listening synthesized vocal pitch weights match preview", interactive=False) add_to_library_btn = gr.Button("✨ Synced successfully! Bind Companion Voice to Inventory", visible=False) gr.HTML("""

Currently Available Voices

Click a voice card to select it as the narrator.

""") cloned_voices_list_grid = gr.HTML(value=render_cloned_voices_html(mock_voices)) voice_selector = gr.Textbox(visible=False, elem_id="voice_selector") # TAB 4: Profile with gr.TabItem("⚙ī¸ Profile") as profile_tab: with gr.Row(): with gr.Column(scale=1, elem_classes="card-container"): gr.HTML("""
👩‍đŸ’ģ

Your Profile

ReadBookMom — Hugging Face Hackathon 2026

""") gr.HTML("""
📚 5 Stories
🧠 3 Models
🎙ī¸ Qwen3 TTS Engine
""") with gr.Column(scale=1, elem_classes="card-container"): gr.HTML("""

Model Pipeline

Active AI models powering the app.

""") gr.HTML("""
🎙ī¸ Voice Cloning: Qwen3-TTS-1.7B-Base (bfloat16)
đŸ—Ŗī¸ Stock TTS: Qwen3-TTS-0.6B-CustomVoice
👂 ASR: Whisper-small
🧠 Q&A: Qwen2.5-3B-Instruct
""") # REACTIVE LOGIC # 1. Search / filter shelf def filter_books_shelf(query, category, current_books): return generate_library_html(current_books, query=query, category_filt=category) search_input.change(filter_books_shelf, inputs=[search_input, category_filter, books_state], outputs=shelf_grid) category_filter.change(filter_books_shelf, inputs=[search_input, category_filter, books_state], outputs=shelf_grid) # 2. Book card click → load into player def handle_book_select(title_chosen, current_inventory, profile_id): if not title_chosen: return [gr.update()] * 12 selected = next((b for b in current_inventory if b["title"] == title_chosen), current_inventory[0]) # Show cloned voice name if profile exists from voice_clone import has_profile, list_saved_profiles as _list_profiles if profile_id and has_profile(profile_id): saved = _list_profiles() voice_label = next( (p["voice_name"] for p in saved if p["profile_id"] == profile_id), "Cloned Voice" ) narrator_label = f"🎙ī¸ {voice_label}" narrator_color = "#4ade80" else: narrator_label = f"🔊 {selected['voice_name']} (stock)" narrator_color = "#ffd3a6" safe_title = html.escape(selected['title']) safe_author = html.escape(selected['author']) player_html = f"""

{safe_title}

by {safe_author}

Active Narrator: {narrator_label}
""" paras = load_paragraphs(selected["story_path"]) total = max(len(paras) - 1, 0) tts_chunks = split_into_chunks("\n\n".join(paras)) chunk_html = f"""
{len(tts_chunks)} chunks ready — tap Play
""" status_html = """
READY
""" story_html = render_story_text(paras, 0) return ( player_html, gr.Audio(visible=True), status_html, chunk_html, gr.Button(visible=True), # play_btn gr.Button(visible=False), # pause_btn gr.Button(visible=False), # ask_btn gr.HTML(visible=False), # story_finished_panel paras, # paragraphs_state tts_chunks, # tts_chunks_state gr.HTML(value=story_html, visible=True), # story_text_display gr.Slider(minimum=0, maximum=total, step=1, value=0, visible=True), # timeline_slider ) book_selector.change( handle_book_select, inputs=[book_selector, books_state, voice_profile_state], outputs=[player_title_info, player_audio_control, player_status_bar, chunk_status, play_btn, pause_btn, ask_btn, story_finished_panel, paragraphs_state, tts_chunks_state, story_text_display, timeline_slider] ) # 3. Extract vocal weights trigger def animate_cloning_pipeline(v_name, v_gender, recorder_data, progress=gr.Progress()): if not v_name.strip(): return ( """
Please provide a comforting nickname for your cloning candidate.
""", gr.HTML(visible=False), gr.Audio(visible=False), gr.Button(visible=False), None, ) if recorder_data is None: return ( """
Microphone recording sample missing. Speak some script lines!
""", gr.HTML(visible=False), gr.Audio(visible=False), gr.Button(visible=False), None, ) from voice_clone import create_voice_profile, synthesize_cloned_preview import soundfile as sf progress(0.1, desc="Extracting speaker embedding from recording...") try: profile_id = create_voice_profile(recorder_data, voice_name=v_name.strip()) except Exception as e: import logging, traceback logging.getLogger(__name__).exception("Voice cloning failed") return ( f"""
Voice cloning failed: {html.escape(str(e))}
""", gr.HTML(visible=False), gr.Audio(visible=False), gr.Button(visible=False), None, ) progress(0.7, desc="Generating voice preview...") try: preview_wav, preview_sr = synthesize_cloned_preview(profile_id) preview_path = f"sample_sounds/clone_preview_{profile_id}.wav" sf.write(preview_path, preview_wav, preview_sr) except Exception as e: import logging logging.getLogger(__name__).exception("Preview synthesis failed") preview_path = None progress(1.0, desc="Voice cloned successfully!") safe_name = html.escape(v_name) avatar = "👩" if v_gender == "Female" else "👨" cloned_card_html = f"""
{avatar}

{safe_name} Cloned successfully

Synthesized today using Qwen3-TTS voice cloning

Voice profile ready. Select a story and tap Play to hear narration in this voice.
""" return ( gr.HTML(visible=False), gr.HTML(value=cloned_card_html, visible=True), gr.Audio(value=preview_path, visible=preview_path is not None), gr.Button(visible=True), profile_id, ) extract_btn.click( animate_cloning_pipeline, inputs=[new_voice_name, new_voice_gender, mic_recorder], outputs=[cloning_progress_msg, voice_cloning_success_panel, voice_sample_preview_widget, add_to_library_btn, voice_profile_state] ) # 4. Add cloned voice to inventory def save_new_cloned_voice(v_name, v_gender, list_voices, current_profile_id): avatar = "👩" if v_gender == "Female" else "👨" new_voice_item = { "id": f"v-{time.time()}", "name": v_name, "avatar_url": avatar, "description": f"Custom cloned {v_gender.lower()} companion voice. Highly optimized.", "gender": v_gender, "created_date": "2026-06-06", "status": "ready", "profile_id": current_profile_id, } updated = [new_voice_item] + list_voices return ( updated, render_cloned_voices_html(updated), gr.HTML(visible=True, value="""
🎙ī¸

Vocal profile saved successfully!

"""), gr.HTML(visible=False), gr.Audio(visible=False), gr.Button(visible=False), gr.Textbox(value="") ) add_to_library_btn.click( save_new_cloned_voice, inputs=[new_voice_name, new_voice_gender, voices_state, voice_profile_state], outputs=[voices_state, cloned_voices_list_grid, cloning_progress_msg, voice_cloning_success_panel, voice_sample_preview_widget, add_to_library_btn, new_voice_name] ) # Voice card selection — update voice_profile_state when a card is clicked def handle_voice_select(selected_id, voices): """Map selected voice ID to profile_id for TTS.""" if not selected_id: return gr.update() # Check if it matches a saved profile ID directly from voice_clone import has_profile if has_profile(selected_id): return selected_id # Check if it matches a voice in the list with a profile_id for v in voices: if v.get('profile_id') == selected_id or v.get('id') == selected_id: pid = v.get('profile_id') if pid and has_profile(pid): return pid # No valid profile — use stock voice (None) return None voice_selector.change( handle_voice_select, inputs=[voice_selector, voices_state], outputs=[voice_profile_state] ) # 5. Play button — streams TTS audio chunk by chunk def stream_tts(tts_chunks, paras, profile_id): import logging as _log _log.getLogger(__name__).info("stream_tts called with profile_id=%s", profile_id) voice_mode = "🎙ī¸ Cloned Voice" if profile_id else "🔊 Stock Voice" _status_playing = f"""
PLAYING — {voice_mode}
""" _status_done = """
DONE
""" n_paras = len(paras) if paras else 1 if not tts_chunks: yield (gr.update(), _status_done, "
No content loaded
", gr.Button(visible=True), gr.Button(visible=False), gr.Button(visible=False), gr.Slider(value=0), render_story_text(paras, 0) if paras else "", gr.HTML(visible=False), gr.HTML(visible=False), gr.Group(visible=False), gr.Button(visible=False)) return n = len(tts_chunks) yield (gr.update(), _status_playing, f"
Generating chunk 1 / {n}â€Ļ
", gr.Button(visible=False), gr.Button(visible=True), gr.Button(visible=False), gr.Slider(value=0), render_story_text(paras, 0) if paras else "", gr.HTML(visible=False), gr.HTML(visible=False), gr.Group(visible=False), gr.Button(visible=False)) for sample_rate, wav, i, total, err in generate_audio_stream(tts_chunks, voice_profile_id=profile_id): # Map chunk index to paragraph index para_idx = min(int(i * n_paras / total), n_paras - 1) if total > 0 else 0 if err: yield (gr.update(), f"
Error on chunk {i+1}: {err}
", "", gr.Button(visible=True), gr.Button(visible=False), gr.Button(visible=False), gr.Slider(value=para_idx), gr.update(), gr.HTML(visible=False), gr.HTML(visible=False), gr.Group(visible=False), gr.Button(visible=False)) return para_idx = min(int(i * n_paras / total), n_paras - 1) if total > 0 else 0 next_label = f"chunk {i+2} / {total}" if i + 1 < total else "last chunk" chunk_html = f"
â–ļ Playing chunk {i+1} / {total} — generating {next_label}
" is_last = (i + 1 >= total) yield ((sample_rate, wav), _status_playing, chunk_html, gr.Button(visible=False), gr.Button(visible=True), gr.Button(visible=True), gr.Slider(value=para_idx), render_story_text(paras, para_idx), gr.HTML(visible=is_last), gr.HTML(visible=False), gr.Group(visible=False), gr.Button(visible=False)) done_html = f"
✅ {n} chunks complete
" yield (gr.update(), _status_done, done_html, gr.Button(visible=True), gr.Button(visible=False), gr.Button(visible=False), gr.Slider(value=max(n_paras - 1, 0)), render_story_text(paras, max(n_paras - 1, 0)), gr.HTML(visible=True), gr.HTML(visible=False), gr.Group(visible=False), gr.Button(visible=False)) play_event = play_btn.click( stream_tts, inputs=[tts_chunks_state, paragraphs_state, voice_profile_state], outputs=[player_audio_control, player_status_bar, chunk_status, play_btn, pause_btn, ask_btn, timeline_slider, story_text_display, story_finished_panel, qa_panel, qa_input_group, resume_btn] ) # 6. Pause button — PAUSED state def enter_paused_state(slider_val, paras): total = len(paras) if paras else 1 chunk_html = f"""
Paragraph {int(slider_val) + 1} / {total} — paused
""" status_html = """
PAUSED
""" return ( status_html, chunk_html, gr.Button(visible=True), gr.Button(visible=False), ) pause_btn.click( enter_paused_state, inputs=[timeline_slider, paragraphs_state], outputs=[player_status_bar, chunk_status, play_btn, pause_btn], cancels=[play_event] ) # 7. Timeline slider → highlight current paragraph def update_story_highlight(slider_val, paras): idx = int(slider_val) total = len(paras) chunk_html = f"""
Paragraph {idx + 1} / {total}
""" return render_story_text(paras, idx), chunk_html timeline_slider.change(update_story_highlight, inputs=[timeline_slider, paragraphs_state], outputs=[story_text_display, chunk_status]) # 8. Ask button — PAUSED-ASKING state def enter_asking_state(): status_html = """
PAUSED — ASKING
""" return ( status_html, gr.Button(visible=False), gr.Button(visible=False), gr.Button(visible=False), gr.Button(visible=True), gr.HTML(visible=True), gr.Group(visible=True), gr.HTML(visible=False), gr.Audio(visible=False), ) ask_btn.click( enter_asking_state, inputs=[], outputs=[player_status_bar, play_btn, pause_btn, ask_btn, resume_btn, qa_panel, qa_input_group, answer_display, answer_audio], cancels=[play_event], js="""() => { // Auto-start microphone recording after a brief delay for UI to render setTimeout(() => { const recordBtn = document.querySelector('#qa_audio_input button[aria-label="Record"], #qa_audio_input button.record-button, .qa-audio-group button[aria-label="Record"]'); if (recordBtn) { recordBtn.click(); } else { // Fallback: find any record button inside the audio component const btns = document.querySelectorAll('button'); for (const b of btns) { if (b.getAttribute('aria-label') === 'Record' || b.textContent.includes('Record')) { b.click(); break; } } } }, 500); }""" ) # 9. Submit question — always auto-submits when recording stops def handle_question_submit(question_txt, question_audio_path, paragraphs, slider_val, profile_id): q_txt = (question_txt or "").strip() has_audio = question_audio_path is not None and question_audio_path != "" if not q_txt and not has_audio: answer_html = """
Please type a question or record one with the microphone.
""" return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False) # Step 9: Transcribe audio if text is empty (on-demand ASR) if q_txt: q_text = q_txt else: try: q_text = transcribe_audio(question_audio_path) if not q_text: q_text = "(could not understand audio)" except Exception as e: q_text = f"(transcription failed: {e})" # Step 10: Generate grounded answer from story context using Qwen current_idx = int(slider_val) if slider_val else 0 try: answer_text = answer_story_question(q_text, paragraphs, current_idx) if not answer_text: answer_text = "Hmm, I'm not sure about that! Let's keep listening to find out." except Exception as e: answer_text = f"Oops, I couldn't think of an answer right now. Let's keep reading! ({type(e).__name__})" import logging logging.getLogger(__name__).exception("Q&A failed: %s", e) # Synthesize answer in cloned voice via TTS answer_audio_path = None try: from tts import split_into_chunks as _split, generate_audio_stream as _gen_stream import soundfile as sf import numpy as np import uuid chunks = _split(answer_text) audio_segments = [] sample_rate = 16000 for sr, wav, idx, total, err in _gen_stream(chunks, voice_profile_id=profile_id): if wav is not None: audio_segments.append(wav) sample_rate = sr if audio_segments: full_audio = np.concatenate(audio_segments) answer_audio_path = f"sample_sounds/qa_answer_{uuid.uuid4().hex[:8]}.wav" sf.write(answer_audio_path, full_audio, sample_rate) except Exception: # Fall back to no audio if TTS fails pass safe_answer = html.escape(answer_text) safe_question = html.escape(q_text) answer_html = f"""
Answer in Narrator's Voice
“{safe_answer}”
Q: {safe_question}
""" if answer_audio_path: return gr.HTML(value=answer_html, visible=True), gr.Audio(value=answer_audio_path, visible=True) return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False) submit_question_btn.click( handle_question_submit, inputs=[question_text, question_audio, paragraphs_state, timeline_slider, voice_profile_state], outputs=[answer_display, answer_audio] ) # Auto-submit when voice recording completes # In Gradio 5.35, .stop_recording may not pass audio value reliably. # Use .stop_recording with a small delay via JS to click the submit button. question_audio.stop_recording( fn=None, inputs=None, outputs=None, js="""() => { // Wait for audio value to be committed, then click submit setTimeout(() => { const btn = document.querySelector('#qa_submit_btn button, #qa_submit_btn'); if (btn) btn.click(); }, 500); }""" ) # .change() as primary Python handler — fires after audio value is committed def auto_submit_on_audio(audio_path, question_txt, paragraphs, slider_val, profile_id): """Auto-submit when a recording is available.""" if not audio_path: return gr.update(), gr.update() return handle_question_submit(question_txt, audio_path, paragraphs, slider_val, profile_id) question_audio.change( auto_submit_on_audio, inputs=[question_audio, question_text, paragraphs_state, timeline_slider, voice_profile_state], outputs=[answer_display, answer_audio] ) # 10. Resume button — dismiss Q&A panel, show Play to continue def enter_resume_state(): status_html = """
READY — Tap Play to continue
""" return ( status_html, gr.Button(visible=True), # play_btn gr.Button(visible=False), # pause_btn gr.Button(visible=True), # ask_btn — keep visible so user can ask again gr.Button(visible=False), # resume_btn gr.HTML(visible=False), # qa_panel gr.Group(visible=False), # qa_input_group gr.HTML(visible=False), # answer_display gr.Audio(visible=False), # answer_audio gr.Textbox(value=""), # question_text — clear previous question None, # question_audio — clear previous recording ) resume_btn.click( enter_resume_state, inputs=[], outputs=[player_status_bar, play_btn, pause_btn, ask_btn, resume_btn, qa_panel, qa_input_group, answer_display, answer_audio, question_text, question_audio] ) # 11. Pause playback when switching to other tabs def pause_on_tab_switch(): status_html = """
PAUSED
""" return status_html, gr.Button(visible=True), gr.Button(visible=False) explore_tab.select( pause_on_tab_switch, inputs=[], outputs=[player_status_bar, play_btn, pause_btn], cancels=[play_event] ) clone_tab.select( pause_on_tab_switch, inputs=[], outputs=[player_status_bar, play_btn, pause_btn], cancels=[play_event] ) if __name__ == "__main__": demo.queue().launch(allowed_paths=[str(Path(__file__).parent / "assets")])