Smooth live audio streaming

README.md

@@ -41,6 +41,8 @@ session. Context is held only in volatile worker memory and disappears when the
 This is near-full-duplex rather than mathematically simultaneous model inference: LFM is turn-based, so visitor
 speech cancels the active reply before the new utterance is processed. Gradio streaming audio is used instead of
 FastRTC/WebRTC because the earlier ZeroGPU WebRTC transport never passed physical media flow.
+Lyra's native 80 ms decoder frames are grouped into 320 ms playback packets to reduce HLS segment churn, while
+the microphone uploads every 500 ms with background progress indicators hidden.
 
 ## Privacy and safety
 

live_app.py

@@ -24,6 +24,9 @@ MODEL_REVISION = "c362a0625dfe45aa588dce5f0ada28a7e5707628"
 DEVICE = "cuda"
 OUTPUT_SAMPLE_RATE = 24_000
 STREAM_PRIMER_SAMPLES = OUTPUT_SAMPLE_RATE // 4
+OUTPUT_PACKET_SAMPLES = 7_680
+MICROPHONE_STREAM_SECONDS = 0.5
+STATUS_HEARTBEAT_SECONDS = 2.0
 SESSION_SECONDS = 90
 MAX_NEW_TOKENS = 320
 MAX_TURNS = 8
@@ -272,7 +275,7 @@ def drain_queue(input_queue):
             return chunks
 
 
-def session_metrics(phase, started, turns, counters, first_reply, interruptions, model_seconds):
+def session_metrics(phase, started, turns, counters, first_reply, interruptions, model_seconds, playback):
     return json.dumps(
         {
             "phase": phase,
@@ -283,6 +286,10 @@ def session_metrics(phase, started, turns, counters, first_reply, interruptions,
             "microphone_chunks_received": int(counters.get("received_chunks", 0)),
             "microphone_chunks_dropped": int(counters.get("dropped_chunks", 0)),
             "model_generation_seconds": round(model_seconds, 3),
+            "reply_packets_emitted": playback["packets"],
+            "reply_audio_emitted_seconds": round(playback["samples"] / OUTPUT_SAMPLE_RATE, 3),
+            "reply_packet_target_seconds": round(OUTPUT_PACKET_SAMPLES / OUTPUT_SAMPLE_RATE, 3),
+            "microphone_stream_interval_seconds": MICROPHONE_STREAM_SECONDS,
             "gpu_peak_gib": round(torch.cuda.max_memory_allocated() / (1024**3), 3),
             "transport": "Gradio streaming audio; no WebRTC/ICE/TURN",
             "mode": "interruptible near-full-duplex LFM2.5-Audio",
@@ -335,6 +342,7 @@ def run_live_session(scene, request: gr.Request):
     interruptions = 0
     first_reply = None
     model_seconds = 0.0
+    playback = {"packets": 0, "samples": 0}
     ready_event.set()
     live_started = time.perf_counter()
     last_heartbeat = 0.0
@@ -345,12 +353,12 @@ def run_live_session(scene, request: gr.Request):
             sample_rate, audio = input_queue.get(timeout=0.1)
         except queue.Empty:
             now = time.perf_counter()
-            if now - last_heartbeat >= 0.75:
+            if now - last_heartbeat >= STATUS_HEARTBEAT_SECONDS:
                 yield (
                     gr.skip(),
                     "**Listening.** Speak naturally; Lyra replies after a short pause.",
                     gr.skip(),
-                    session_metrics("listening", started, turns, counters, first_reply, interruptions, model_seconds),
+                    session_metrics("listening", started, turns, counters, first_reply, interruptions, model_seconds, playback),
                 )
                 last_heartbeat = now
             continue
@@ -388,6 +396,8 @@ def run_live_session(scene, request: gr.Request):
         text_tokens = []
         audio_tokens = []
         modalities = []
+        pcm_frames = []
+        pcm_samples = 0
         response_started = time.perf_counter()
         interrupted = False
         with torch.inference_mode(), mimi.streaming(1):
@@ -430,7 +440,16 @@ def run_live_session(scene, request: gr.Request):
                     waveform = mimi.decode(token[None, :, None])[0]
                     pcm = np.clip(waveform.detach().float().cpu().numpy(), -1.0, 1.0)
                     pcm = (pcm * 32767.0).astype(np.int16).reshape(-1)
-                    if first_reply is None and np.sqrt(np.mean((pcm.astype(np.float32) / 32768.0) ** 2)) > 0.003:
+                    pcm_frames.append(pcm)
+                    pcm_samples += pcm.size
+                    if pcm_samples < OUTPUT_PACKET_SAMPLES:
+                        continue
+                    packet = np.concatenate(pcm_frames)
+                    pcm_frames = []
+                    pcm_samples = 0
+                    playback["packets"] += 1
+                    playback["samples"] += packet.size
+                    if first_reply is None and np.sqrt(np.mean((packet.astype(np.float32) / 32768.0) ** 2)) > 0.003:
                         first_reply = time.perf_counter() - response_started
                     current_text = (
                         processor.text.decode(torch.cat(text_tokens)).removesuffix("<|text_end|>").strip()
@@ -438,12 +457,30 @@ def run_live_session(scene, request: gr.Request):
                         else ""
                     )
                     yield (
-                        (OUTPUT_SAMPLE_RATE, pcm),
+                        (OUTPUT_SAMPLE_RATE, packet),
                         "**Lyra is speaking.** You can interrupt by speaking clearly.",
                         render_transcript(entries) + (f"\n\n**Current Lyra reply:** {safe_text(current_text)}" if current_text else ""),
-                        session_metrics("speaking", started, turns, counters, first_reply, interruptions, model_seconds),
+                        session_metrics("speaking", started, turns, counters, first_reply, interruptions, model_seconds, playback),
                     )
 
+        if pcm_frames and not interrupted:
+            packet = np.concatenate(pcm_frames)
+            playback["packets"] += 1
+            playback["samples"] += packet.size
+            if first_reply is None and np.sqrt(np.mean((packet.astype(np.float32) / 32768.0) ** 2)) > 0.003:
+                first_reply = time.perf_counter() - response_started
+            current_text = (
+                processor.text.decode(torch.cat(text_tokens)).removesuffix("<|text_end|>").strip()
+                if text_tokens
+                else ""
+            )
+            yield (
+                (OUTPUT_SAMPLE_RATE, packet),
+                "**Lyra is speaking.** You can interrupt by speaking clearly.",
+                render_transcript(entries) + (f"\n\n**Current Lyra reply:** {safe_text(current_text)}" if current_text else ""),
+                session_metrics("speaking", started, turns, counters, first_reply, interruptions, model_seconds, playback),
+            )
+
         model_seconds += time.perf_counter() - response_started
         response_text = (
             processor.text.decode(torch.cat(text_tokens)).removesuffix("<|text_end|>").strip()
@@ -473,7 +510,7 @@ def run_live_session(scene, request: gr.Request):
             gr.skip(),
             "**Listening.** Speak naturally; Lyra replies after a short pause.",
             render_transcript(entries),
-            session_metrics("listening", started, turns, counters, first_reply, interruptions, model_seconds),
+            session_metrics("listening", started, turns, counters, first_reply, interruptions, model_seconds, playback),
         )
 
     ready_event.clear()
@@ -483,7 +520,7 @@ def run_live_session(scene, request: gr.Request):
         gr.skip(),
         final_status,
         render_transcript(entries),
-        session_metrics("complete", started, turns, counters, first_reply, interruptions, model_seconds),
+        session_metrics("complete", started, turns, counters, first_reply, interruptions, model_seconds, playback),
     )
 
 
@@ -570,6 +607,7 @@ with gr.Blocks(title="RoleForge Live Voice NPC", theme=gr.themes.Soft(), css=CSS
         new_live_session,
         outputs=[status, metrics, transcript],
         queue=False,
+        show_progress="hidden",
         api_name="new_live_session",
     )
     run_event = start_event.then(
@@ -577,18 +615,21 @@ with gr.Blocks(title="RoleForge Live Voice NPC", theme=gr.themes.Soft(), css=CSS
         inputs=[scene],
         outputs=[reply_audio, status, transcript, metrics],
         concurrency_limit=1,
+        show_progress="hidden",
         api_name="run_live_session",
     )
     run_event.then(
         finish_live_microphone,
         outputs=[microphone],
         queue=False,
+        show_progress="hidden",
         api_name="finish_live_microphone",
     )
     microphone.stop_recording(
         request_live_stop,
         outputs=[status],
         queue=False,
+        show_progress="hidden",
         api_name="request_live_stop",
     )
     microphone.stream(
@@ -596,14 +637,16 @@ with gr.Blocks(title="RoleForge Live Voice NPC", theme=gr.themes.Soft(), css=CSS
         inputs=[microphone],
         outputs=[],
         queue=False,
-        stream_every=0.25,
-        concurrency_limit=8,
+        stream_every=MICROPHONE_STREAM_SECONDS,
+        concurrency_limit=4,
+        show_progress="hidden",
         api_name="ingest_audio_chunk",
     )
     stop_btn.click(
         stop_live_session_button,
         outputs=[status, microphone],
         queue=False,
+        show_progress="hidden",
         api_name="stop_live_session",
     )