Spaces:
Running on Zero
Running on Zero
File size: 10,081 Bytes
dc98a34 216c878 dc98a34 216c878 dc98a34 216c878 dc98a34 3314beb dc98a34 3314beb dc98a34 3314beb 216c878 dc98a34 3314beb 216c878 3314beb dc98a34 216c878 db7a515 216c878 dc98a34 216c878 db7a515 216c878 db7a515 216c878 db7a515 216c878 db7a515 216c878 dc98a34 216c878 dc98a34 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | from __future__ import annotations
import base64
import json
import re
import time
from pathlib import Path
import gradio as gr
import soundfile as sf
import config
from backend.presets import list_presets
from frontend import live
LOG_HEADERS = ['t_s', 'model', 'decision', 'probability', 'latency_ms']
def _load_sample_manifest() -> list[dict]:
manifest_path = config.SAMPLE_CLIPS_DIR / 'manifest.json'
if not manifest_path.exists():
return []
return json.loads(manifest_path.read_text())
BUCKET_LABEL = {'english': 'English', 'hindi': 'Hindi', 'hinglish': 'Hinglish'}
BUCKET_ORDER = ['english', 'hindi', 'hinglish']
def _sample_clip_choices() -> list[tuple[str, str]]:
manifest = _load_sample_manifest()
by_bucket: dict[str, list[dict]] = {b: [] for b in BUCKET_ORDER}
for entry in manifest:
by_bucket.setdefault(entry.get('bucket', 'other'), []).append(entry)
choices = []
for bucket in BUCKET_ORDER:
for i, entry in enumerate(by_bucket.get(bucket, []), start=1):
label = f'{BUCKET_LABEL.get(bucket, bucket.title())} {i} - {entry['duration_seconds']}s'
choices.append((label, entry['filename']))
for bucket, entries in by_bucket.items():
if bucket in BUCKET_ORDER:
continue
for i, entry in enumerate(entries, start=1):
choices.append((f'{bucket.title()} {i} - {entry['duration_seconds']}s', entry['filename']))
return choices
def _all_public_presets() -> list[dict]:
return live.all_public_presets()
FUTURE_PRESET_LABELS = ['Easy Turn (unavailable)']
def _display_presets() -> list[dict]:
presets = live.all_public_presets()
seen = {p['label'] for p in presets}
for label in FUTURE_PRESET_LABELS:
if label not in seen:
match = next((p for p in list_presets() if p['label'] == label), None)
if match:
presets.append(match)
return presets
def _preset_display_order() -> list[str]:
return [live.DISPLAY_NAME.get(p['label'], p['label']).replace(' (unavailable)', '') for p in _display_presets()]
def _default_active_display_names() -> list[str]:
target = 'Whisper-Tiny + Mean-Pool + Linear (trained)'
name = live.DISPLAY_NAME.get(target, target)
names = _preset_display_order()
return [name] if name in names else [names[0]] if names else []
def _on_preset_change(*values: bool) -> tuple[list[str], dict, dict]:
names = _preset_display_order()
active = [n for n, v in zip(names, values) if v]
ac, se = _param_relevance(active)
return (active, gr.update(interactive=ac), gr.update(interactive=se))
def _param_relevance(active_display_names: list[str]) -> tuple[bool, bool]:
display_to_internal = {v: k for k, v in live.DISPLAY_NAME.items()}
labels = [display_to_internal[n] for n in active_display_names or [] if n in display_to_internal]
acoustic_relevant = any((live.uses_acoustic_weight(label) for label in labels))
semantic_relevant = any((live.uses_semantic_temperature(label) for label in labels))
return (acoustic_relevant, semantic_relevant)
def _load_doc_markdown() -> str:
root = Path(__file__).resolve().parent.parent
doc_path = root / 'docs' / 'Turn Detection - Aman.md'
if not doc_path.exists():
return '_Documentation is not included in this deployment._'
text = doc_path.read_text(encoding='utf-8')
for a, b in [('\\"', '"'), ('\\&', '&'), ('\\'', "'"), ('\\<', '<'), ('\\>', '>')]:
text = text.replace(a, b)
def repl(m: re.Match) -> str:
alt, src = (m.group(1), m.group(2).strip())
if src.startswith(('http', 'data:')):
return m.group(0)
cand = (doc_path.parent / src).resolve()
if cand.exists():
b64 = base64.b64encode(cand.read_bytes()).decode('ascii')
return f''
return m.group(0)
return re.sub('!\\[([^\\]]*)\\]\\(([^)]+)\\)', repl, text)
CUSTOM_CSS = '\n.wrap-row { flex-wrap: wrap; gap: 10px; }\n.wrap-row > * { flex: 1 1 220px; min-width: 170px; }\n'
def build_app() -> gr.Blocks:
with gr.Blocks(title='Turn Detection - Live Dashboard') as demo:
gr.Markdown('# Turn Detection - Live Dashboard\nSpeak, or replay a clip, and watch how different models judge whether the speaker is **done talking** vs. **still going** - plotted directly against the waveform as audio arrives. Open the **Documentation** tab for the full write-up, methodology and results.')
with gr.Tabs():
with gr.Tab('Live Dashboard', id='live'):
session_state = gr.State(live.new_session_state)
active_state = gr.State(_default_active_display_names())
gr.Markdown('### Models to compare *(greyed = not built yet / future work)*')
with gr.Row(elem_classes='wrap-row'):
default_active = _default_active_display_names()
preset_checkboxes = []
for preset in _display_presets():
label = preset['label']
disp = live.DISPLAY_NAME.get(label, label).replace(' (unavailable)', '')
available = bool(preset.get('available'))
cb_label = disp if available else f'{disp} (future)'
preset_checkboxes.append(gr.Checkbox(label=cb_label, value=disp in default_active, interactive=available))
with gr.Row():
with gr.Column(scale=1):
gr.Markdown('### Audio')
mic = gr.Audio(sources=['microphone'], streaming=True, type='numpy', label='Record')
gr.Markdown('*Recordings are saved privately to improve the model - never played back or shown to others.*')
clear_btn = gr.Button('Clear / reset')
gr.Markdown('**...or replay a clip in real time** (paced to its real duration, not dumped in at once) - 10 real English, 10 Hindi, 10 Hinglish')
replay_clip_dropdown = gr.Dropdown(choices=_sample_clip_choices(), value=None, label='Sample clip')
replay_upload = gr.Audio(sources=['upload'], type='filepath', label='...or upload a recording')
replay_btn = gr.Button('Replay in real time')
with gr.Column(scale=2):
gr.Markdown('### Waveform + live probability, on one timeline')
chart = gr.Plot(value=live.render_chart(live.new_session_state(), [], live.DEFAULT_DECISION_THRESHOLD), label=None)
with gr.Row(elem_classes='wrap-row'):
threshold_slider = gr.Slider(0.0, 1.0, value=live.DEFAULT_DECISION_THRESHOLD, step=0.01, label='Decision threshold - probability above this = "complete"')
cadence_slider = gr.Slider(0, 5000, value=live.DEFAULT_CADENCE_MS, step=100, label='Update cadence (ms) - how often each model re-checks (slower models may still lag behind this)')
smoothing_slider = gr.Slider(0.0, 0.9, value=0.5, step=0.05, label="Smoothing - damps short swings on the chart (raw points still shown faintly; doesn't change what's logged)")
acoustic_weight_slider = gr.Slider(0.0, 1.0, value=0.6, label='Acoustic weight - audio tone vs. sentence grammar (fusion models only)', interactive=False)
temperature_slider = gr.Slider(0.0, 1.0, value=0.2, label='Semantic temperature - how deterministic the language judgment is (LLM-based models only)', interactive=False)
with gr.Accordion('History (this session)', open=False):
log_table = gr.Dataframe(headers=LOG_HEADERS, value=[], label=None)
for cb in preset_checkboxes:
cb.change(_on_preset_change, inputs=preset_checkboxes, outputs=[active_state, acoustic_weight_slider, temperature_slider])
def replay_clip(clip_filename, uploaded_path, active_display_names, acoustic_weight, temperature, threshold, cadence_ms, smoothing):
path = uploaded_path or (str(config.SAMPLE_CLIPS_DIR / clip_filename) if clip_filename else None)
if not path:
yield (live.new_session_state(), live.render_chart(live.new_session_state(), [], threshold, smoothing), [])
return
audio, sr = sf.read(path, dtype='float32')
if audio.ndim > 1:
audio = audio.mean(axis=1)
state = live.new_session_state()
for chunk in live.chunk_audio(audio, sr, chunk_seconds=1.0):
chunk_duration_s = len(chunk) / sr
state, fig, log_rows = live.process_chunk(state, (sr, chunk), active_display_names, acoustic_weight, temperature, threshold, cadence_ms, smoothing)
yield (state, fig, log_rows)
time.sleep(chunk_duration_s)
replay_btn.click(replay_clip, inputs=[replay_clip_dropdown, replay_upload, active_state, acoustic_weight_slider, temperature_slider, threshold_slider, cadence_slider, smoothing_slider], outputs=[session_state, chart, log_table])
mic.stream(fn=live.process_chunk, inputs=[session_state, mic, active_state, acoustic_weight_slider, temperature_slider, threshold_slider, cadence_slider, smoothing_slider], outputs=[session_state, chart, log_table], stream_every=1.0, time_limit=None)
clear_btn.click(live.clear_session, inputs=session_state, outputs=[session_state, chart, log_table])
with gr.Tab('Documentation', id='docs'):
gr.HTML('<style>.doc-md { max-height: 78vh; overflow: auto; padding-right: 16px; }.doc-md img { max-width: 100%; height: auto; border: 1px solid #ddd; border-radius: 6px; margin: 8px 0; }</style>')
gr.Markdown(_load_doc_markdown(), elem_classes=['doc-md'])
return demo |