Fix ZeroGPU audio quality and stuck-processing UI

Quality: stop pinning seed=7. That seed reproduced the verified lab take
on torch 2.7.1, but ZeroGPU runs torch 2.8-2.11, where the same seed is
just an arbitrary (bad) draw -- the 'sporadic loud random synth noise'
take. engine.continue_audio now draws best-of-N candidates and keeps the
cleanest by a blind artifact score that rejects both loud random bursts
and silence collapse. Early-accept + an 85s wall-clock budget keep it
inside the 120s GPU window. Stay at the distilled 8-step pingpong regime
(bumping steps adds artifacts, not quality).

UI: collapse the finish_btn .then chain into a single event so one
completion signal clears the spinner -- the chained second event could
hang the 'processing' state if the SSE stream blipped (ClientDisconnect).
Build the summary markdown visible (empty) instead of toggling it. Queue
one job at a time and launch with show_error so a real failure surfaces
in the UI instead of a silently stuck spinner.

app.py

@@ -137,13 +137,15 @@ def _continue_on_gpu(listen_path, total_seconds, vibe):
     starts and finishes fast instead of sitting through a slow analysis preamble
     until the browser aborts the stream.
 
-    Pin the seed to the lab's known-good draw. SA3 is generative: a random seed
-    gives a different (often weaker) continuation every run. seed=7 produced the
-    verified lab_out/sa3 takes, so the app reproduces that instead of re-rolling.
+    Do NOT pin a magic seed. The lab's verified seed=7 take was made on torch
+    2.7.1; ZeroGPU runs a different torch (2.8–2.11), so the same seed reproduces
+    a *different* draw — on the deployed build, the bad "loud random synth noise"
+    one. Instead, leave the seed unset (engine draws several candidates and keeps
+    the cleanest by an artifact score), which is robust across torch builds.
     """
     return engine.continue_audio(
         listen_path, total_seconds=int(total_seconds),
-        prompt=(vibe or "").strip(), seed=7)
+        prompt=(vibe or "").strip())
 
 
 def finish_song(audio_path, total_seconds, vibe, remaster,
@@ -295,7 +297,11 @@ with gr.Blocks(title="CODA") as app:
                 gr.Markdown("### Your finished song")
                 output_audio = gr.Audio(label="", type="filepath",
                                         interactive=False)
-                summary_md = gr.Markdown(visible=False)
+                # built visible with an empty value (an empty Markdown renders
+                # nothing). finish_song fills it in the SAME event that sets the
+                # audio, so there's no second `.then` to toggle visibility — one
+                # event = one completion signal, so the spinner always clears.
+                summary_md = gr.Markdown()
 
     gr.Markdown(PUSHBACK_CREDIT, elem_id="coda-foot")
 
@@ -303,17 +309,21 @@ with gr.Blocks(title="CODA") as app:
                        outputs=[info_md, finish_btn])
     demo_btn.click(fn=load_demo, inputs=[], outputs=[audio_input])
 
-    def _show_summary():
-        return gr.update(visible=True)
-
+    # Single event (no chained `.then`): finish_song returns BOTH the audio path
+    # and the summary markdown, so one completion message clears the spinner.
+    # A chained second event was a place the "stuck processing" state could hang
+    # if the SSE stream blipped (ClientDisconnect) between the two events.
     finish_btn.click(
         fn=finish_song,
         inputs=[audio_input, total_slider, vibe, remaster],
-        outputs=[output_audio, summary_md]).then(
-        fn=_show_summary, inputs=[], outputs=[summary_md])
+        outputs=[output_audio, summary_md])
 
 
 if __name__ == "__main__":
     # Gradio 6 moved theme/css to launch(); pass them here (and they remain on
     # Blocks above) so the dark DAW theme applies however the Space serves it.
-    app.queue().launch(theme=THEME, css=CSS)
+    # queue: one heavy job at a time (single GPU), others wait rather than
+    # contend. show_error surfaces a real error in the UI instead of a silently
+    # stuck spinner — the failure mode we just chased on the deployed Space.
+    app.queue(default_concurrency_limit=1, max_size=10).launch(
+        theme=THEME, css=CSS, show_error=True)

engine.py

@@ -5,8 +5,17 @@ into a finished-sounding track in the same key, tempo and feel. SA3 does that in
 a SINGLE call. Its `generate_diffusion_cond_inpaint` is a native audio-inpainting
 diffusion sampler: place the user's clip at the front of the buffer, mask the
 region after it, and the model fills the masked region conditioned on the kept
-audio — true long-form continuation, 44.1 kHz stereo, no multi-pass chaining,
-no energy guards, no re-roll logic.
+audio — true long-form continuation, 44.1 kHz stereo, no multi-pass chaining
+and no energy guards.
+
+Candidate selection (the deployed quality fix): the lab verified a take with a
+pinned seed on torch 2.7.1, but ZeroGPU runs a different torch (2.8–2.11), so the
+same seed no longer reproduces it — it just freezes one arbitrary draw, which on
+the deployed build was the bad "sporadic loud synth noise" take. Instead of
+trusting a magic seed, CODA draws a few candidates and keeps the cleanest by a
+cheap artifact score (it rejects both failure modes: loud random bursts AND
+silence collapse). A wall-clock budget + early-accept keep this inside the
+ZeroGPU window, so it costs at most a couple of extra fast 8-step draws.
 
 This module is the whole generation core. It returns ONLY the newly generated
 tail plus the source length in seconds; `stitch.py` joins that tail onto the
@@ -28,16 +37,30 @@ Mask convention (verified against the installed library source):
   at the front and mask [lead, lead+new], so SA3 keeps the lead and generates a
   fresh `new`-second tail that continues from the clip's end.
 """
+import time
+
 import numpy as np
 import torch
 
 MODEL_ID = "stabilityai/stable-audio-3-small-music"
 
 SR = 44100               # SA3 native sample rate (model_config: sample_rate)
-STEPS = 8                # SA3 Small is an 8-step adversarially-distilled model
+STEPS = 8                # SA3 Small is an 8-step adversarially-distilled model.
+                         # It was tuned for 8-step pingpong; pushing it to 16/25
+                         # steps is OFF its distilled regime and tends to ADD
+                         # artifacts, not remove them. We stay at 8 and fix
+                         # quality by picking the best of a few draws instead.
 SAMPLER = "pingpong"     # the sampler the distilled model was tuned for
 DEFAULT_CFG = 1.0        # distilled-model guidance; the prompt still conditions
                          # at 1.0 (CFG amplification off, conditional path on)
+
+# Best-of-N: with a random seed each draw differs, so we generate a few and keep
+# the cleanest. Bounded so it never blows the ZeroGPU window.
+DEFAULT_CANDIDATES = 3       # how many draws to consider when no seed is pinned
+GPU_BUDGET_SECONDS = 85.0    # stop drawing once this much wall-clock is spent
+                             # (the @spaces.GPU window is 120s; leave slack)
+EARLY_ACCEPT_SCORE = 4.0     # a draw this clean is taken immediately, no re-draw
+                             # (heuristic — see _tail_artifact_score; tune live)
 MAX_TOTAL_SECONDS = 120  # SA3 Small duration cap (sample_size / sample_rate)
 MIN_NEW_SECONDS = 5      # below this a "continuation" isn't worth a GPU call
 MAX_LEAD_SECONDS = 30    # how much of the clip's TAIL to feed SA3 as run-up.
@@ -144,9 +167,47 @@ def plan_continuation(source_seconds, total_seconds):
     return lead, new_seconds, buffer_seconds
 
 
+def _tail_artifact_score(tail, sr=SR):
+    """Lower is better. A blind, ear-free quality score for a generated tail,
+    used to pick the cleanest of several candidate draws.
+
+    It targets the two ways an SA3 draw goes bad:
+      * "sporadic loud random synth noises" — even a FEW short windows far louder
+        than the body push the loudest window way above the median. (After the
+        whole-buffer peak-normalize, a burst that set the peak crushes the body,
+        making the gap larger still.) Sustained dynamics rarely make any single
+        50 ms window many times the median, so musical loudness doesn't trip it.
+      * silence collapse — a near-silent tail (the other known failure) is caught
+        by the explicit loudness floor below.
+
+    Score = max(window RMS) / median(window RMS)  +  silence penalty.
+    Computed on a mono mix over short (~50 ms) windows. A flat, steady signal
+    scores ~1; isolated loud bursts or a crushed body score high.
+    """
+    mono = tail.mean(axis=0) if tail.ndim == 2 else np.asarray(tail)
+    mono = np.asarray(mono, dtype=np.float64)
+    win = max(1, int(0.05 * sr))
+    if mono.size < win * 4:
+        return float("inf")                 # too short to judge — avoid it
+    n = mono.size // win
+    energies = np.sqrt(
+        np.mean(mono[:n * win].reshape(n, win) ** 2, axis=1) + 1e-12)
+    median = float(np.median(energies)) + 1e-9
+    loudest = float(np.max(energies))
+    spikiness = loudest / median
+    overall = float(np.sqrt(np.mean(mono ** 2)) + 1e-12)
+    silence_penalty = 0.0 if overall > 0.02 else (0.02 - overall) * 200.0
+    return spikiness + silence_penalty
+
+
 def continue_audio(clip_path, total_seconds, prompt="", cfg_scale=DEFAULT_CFG,
-                   seed=-1, progress=None):
-    """Continue `clip_path` up to `total_seconds` in one SA3 inpaint call.
+                   seed=-1, candidates=None, progress=None):
+    """Continue `clip_path` up to `total_seconds` with SA3 inpainting.
+
+    With the default `seed` (< 0) this draws up to `candidates` SA3 inpaint
+    takes and returns the cleanest by `_tail_artifact_score` (early-accepting a
+    clean draw and respecting a GPU wall-clock budget). Pin `seed` >= 0 for a
+    single deterministic draw (debug/repro).
 
     Returns (new_tail, source_seconds, SR) where:
       new_tail       : (2, M) float32 @44.1k — ONLY the generated region
@@ -155,7 +216,7 @@ def continue_audio(clip_path, total_seconds, prompt="", cfg_scale=DEFAULT_CFG,
       SR             : 44100
 
     `progress(stage_name)` is called (best-effort) at each stage so the UI can
-    paint a live status. SA3 is one diffusion call, so progress is stage-based.
+    paint a live status. Progress is stage-based (read / compose / finalize).
     """
     from einops import rearrange
     from stable_audio_tools.inference.generation import (
@@ -172,11 +233,21 @@ def continue_audio(clip_path, total_seconds, prompt="", cfg_scale=DEFAULT_CFG,
     model = _ensure_on_device()
     dev = _device()
 
-    # The library does `np.random.randint(0, 2**32-1)` when seed == -1, which
-    # overflows int32 on Windows/numpy<2. Draw a safe seed ourselves so the
-    # default path works everywhere, not just on the Linux Space.
-    if seed is None or seed < 0:
-        seed = int(np.random.randint(0, 2 ** 31 - 1))
+    # Seed policy. The library does `np.random.randint(0, 2**32-1)` when
+    # seed == -1, which overflows int32 on Windows/numpy<2, so we always draw
+    # our own safe seeds. A caller that pins a seed (>= 0) gets exactly one
+    # deterministic draw (reproducibility/debug paths). The default path
+    # (seed < 0) draws several candidates and keeps the cleanest — that's what
+    # the app uses, because a single pinned seed doesn't survive a torch change.
+    pinned = seed is not None and seed >= 0
+    n_candidates = 1 if pinned else max(1, int(candidates or DEFAULT_CANDIDATES))
+    if pinned:
+        seeds = [int(seed)]
+    else:
+        base = int(np.random.randint(0, 2 ** 31 - 1))
+        # spread the seeds far apart so the draws are genuinely different
+        seeds = [(base + i * 0x9E3779B1) % (2 ** 31 - 1)
+                 for i in range(n_candidates)]
 
     _notify("reading")
     source = _load_source(clip_path)
@@ -202,43 +273,66 @@ def continue_audio(clip_path, total_seconds, prompt="", cfg_scale=DEFAULT_CFG,
           f"(+{new_seconds:.1f}s new), mask=[{mask_start:.1f}s, {mask_end:.1f}s], "
           f"steps={STEPS}, cfg={cfg_scale}, prompt={prompt!r}", flush=True)
 
+    def _draw(draw_seed):
+        """One full SA3 inpaint draw -> normalized generated tail (2, M)."""
+        with torch.no_grad():
+            output = generate_diffusion_cond_inpaint(
+                model,
+                steps=STEPS,
+                cfg_scale=cfg_scale,
+                conditioning=[{"prompt": prompt, "seconds_total": buffer_seconds}],
+                sample_size=_sample_size,
+                sampler_type=SAMPLER,
+                inpaint_audio=(SR, lead_audio),
+                inpaint_mask_start_seconds=mask_start,
+                inpaint_mask_end_seconds=mask_end,
+                seed=int(draw_seed),
+                device=dev,
+            )
+        # (b, d, n) -> (d, b*n); peak-normalize like Stability's reference Space
+        # (unchanged from the verified-good local path).
+        output = rearrange(output, "b d n -> d (b n)")
+        audio = output.to(torch.float32).cpu().numpy()
+        peak = float(np.abs(audio).max())
+        if peak > 1e-9:
+            audio = audio / peak
+        if audio.shape[0] == 1:            # safety: ensure stereo
+            audio = np.repeat(audio, 2, axis=0)
+        # the generated region is [lead, buffer]; lead ends at the clip's true
+        # end, so this slice is the continuation that follows the source.
+        start = int(round(lead * SR))
+        end = min(int(round(buffer_seconds * SR)), audio.shape[-1])
+        return np.ascontiguousarray(audio[:, start:end].astype(np.float32))
+
     _notify("composing")
-    with torch.no_grad():
-        output = generate_diffusion_cond_inpaint(
-            model,
-            steps=STEPS,
-            cfg_scale=cfg_scale,
-            conditioning=[{"prompt": prompt, "seconds_total": buffer_seconds}],
-            sample_size=_sample_size,
-            sampler_type=SAMPLER,
-            inpaint_audio=(SR, lead_audio),
-            inpaint_mask_start_seconds=mask_start,
-            inpaint_mask_end_seconds=mask_end,
-            seed=seed,
-            device=dev,
-        )
+    # Best-of-N: draw, score, keep the cleanest. Early-accept a clean draw and
+    # stop if the GPU wall-clock budget runs low, so this never blows the window.
+    best_tail, best_score, best_seed = None, float("inf"), None
+    t0 = time.time()
+    for i, draw_seed in enumerate(seeds):
+        tail = _draw(draw_seed)
+        score = _tail_artifact_score(tail, SR)
+        elapsed = time.time() - t0
+        print(f"[coda] candidate {i + 1}/{len(seeds)} seed={draw_seed} "
+              f"shape={tail.shape} ({tail.shape[-1] / SR:.1f}s) "
+              f"artifact_score={score:.2f} "
+              f"rms={float(np.sqrt(np.mean(tail ** 2))):.3f} "
+              f"elapsed={elapsed:.1f}s", flush=True)
+        if score < best_score:
+            best_tail, best_score, best_seed = tail, score, draw_seed
+        if best_score <= EARLY_ACCEPT_SCORE:
+            print(f"[coda] candidate {i + 1} clean enough "
+                  f"(score {best_score:.2f} <= {EARLY_ACCEPT_SCORE}); accepting",
+                  flush=True)
+            break
+        if elapsed > GPU_BUDGET_SECONDS and i + 1 < len(seeds):
+            print(f"[coda] GPU budget {GPU_BUDGET_SECONDS:.0f}s reached after "
+                  f"{i + 1} draw(s); keeping best so far", flush=True)
+            break
 
     _notify("finalizing")
-    # (b, d, n) -> (d, b*n); peak-normalize like Stability's reference Space
-    # (unchanged from the verified-good local path).
-    output = rearrange(output, "b d n -> d (b n)")
-    audio = output.to(torch.float32).cpu().numpy()
-    peak = float(np.abs(audio).max())
-    if peak > 1e-9:
-        audio = audio / peak
-
-    if audio.shape[0] == 1:            # safety: ensure stereo
-        audio = np.repeat(audio, 2, axis=0)
-
-    # the generated region is [lead, buffer]; lead ends at the clip's true end,
-    # so this slice is the continuation that follows the source.
-    start = int(round(lead * SR))
-    end = min(int(round(buffer_seconds * SR)), audio.shape[-1])
-    new_tail = audio[:, start:end]
-    new_tail = np.ascontiguousarray(new_tail.astype(np.float32))
-
-    print(f"[coda] generated tail: shape={new_tail.shape} "
-          f"({new_tail.shape[-1] / SR:.1f}s), peak after norm "
-          f"{float(np.abs(new_tail).max()):.3f}, "
-          f"rms {float(np.sqrt(np.mean(new_tail ** 2))):.3f}", flush=True)
-    return new_tail, source_seconds, SR
+    print(f"[coda] selected seed={best_seed} artifact_score={best_score:.2f} "
+          f"tail={best_tail.shape[-1] / SR:.1f}s peak after norm "
+          f"{float(np.abs(best_tail).max()):.3f} "
+          f"rms {float(np.sqrt(np.mean(best_tail ** 2))):.3f}", flush=True)
+    return best_tail, source_seconds, SR