import os import time import gradio as gr import pandas as pd from constants import ( ABOUT_TEXT, APP_TITLE, BANNER, CITATION_TEXT, DEFAULT_CUSTOM_EVAL_EXAMPLE, INTRODUCTION_TEXT, LEADERBOARD_CSS, tab_label, TREBLE_TEAL, treble_gradio_theme, ) import examples_renderer import analytics from family_registry import default_family_id import job_queue from metrics_config import ( HEATMAP_SCENARIO_KEYS, LIVE_SCENARIO_KEYS, SCENARIO_METRICS, resolve_scenario_metric_keys, ) from recipes.registry import RECIPE_CHOICES, resolve_recipe_id from init import ( LATEST_VERSION, is_model_on_hub, list_leaderboard_versions, load_raw_results, load_results, normalize_legacy_csv_row, raw_rows_for_version, ) from utils_display import ( AutoEvalColumn, SCENARIO_DISPLAY_COLS, SCENARIO_DISPLAY_TO_KEY, column_widths_for, fields, format_wer_percent, model_id_from_leaderboard_cell, styled_error, styled_message, styled_warning, ) # --------------------------------------------------------------------------- # Column setup # --------------------------------------------------------------------------- COLS = [c.name for c in fields(AutoEvalColumn)] TYPES = [c.type for c in fields(AutoEvalColumn)] AVG_WER_COL = AutoEvalColumn.avg_wer_core.name # "Avg WER (%)" MODEL_COL = AutoEvalColumn.model.name # "Model" ALWAYS_VISIBLE_COLS = (MODEL_COL, AVG_WER_COL) # Empty table at import; real data loaded on demo.load (avoids blocking Space "preparing"). original_df = pd.DataFrame(columns=COLS) # Raw CSV rows backing the currently displayed leaderboard version. ``None`` means "live" # (filter_main_table falls back to load_raw_results()). Set whenever a version is selected # so the Avg-WER recompute in filter_main_table matches the displayed snapshot. _active_raw_rows: list[dict] | None = None _NEXT_UP_PLACEHOLDER = ( "
" ) # --------------------------------------------------------------------------- # Filter (matches open_asr_leaderboard pattern) # --------------------------------------------------------------------------- def filter_main_table(search_query, selected_columns): filtered_df = original_df.copy() selected_columns = list(selected_columns or []) if MODEL_COL not in filtered_df.columns: return pd.DataFrame(columns=COLS) # Filter by model name search (case-insensitive, supports comma-separated terms). if search_query: terms = [t.strip() for t in search_query.split(",") if t.strip()] if terms: mask = filtered_df[MODEL_COL].str.lower().apply( lambda cell: any(term.lower() in cell for term in terms) ) filtered_df = filtered_df[mask] # Hide toggleable columns that the user unchecked. Model + Avg WER stay visible. all_toggleable = [c for c in COLS if c not in ALWAYS_VISIBLE_COLS] columns_to_hide = set(all_toggleable) - set(selected_columns) filtered_df = filtered_df[[c for c in filtered_df.columns if c not in columns_to_hide]] raw_rows = _active_raw_rows if _active_raw_rows is not None else load_raw_results() for r in raw_rows: normalize_legacy_csv_row(r) raw_by_mid = {r["model_id"]: r for r in raw_rows} visible_metric_keys = [ SCENARIO_DISPLAY_TO_KEY[c] for c in SCENARIO_DISPLAY_COLS if c in selected_columns ] def _row_avg_wer(rdict: dict) -> float | None: if not visible_metric_keys: return None vals = [] for k in visible_metric_keys: try: v = rdict.get(k, "") if v is None or str(v).strip() == "": continue vals.append(float(v)) except (TypeError, ValueError): continue if not vals: return None return sum(vals) / len(vals) def _avg_wer_key(row) -> tuple[int, float]: mid = model_id_from_leaderboard_cell(row[MODEL_COL]) rdict = raw_by_mid.get(mid) if not rdict: return (1, float("inf")) avg = _row_avg_wer(rdict) if avg is None: return (1, float("inf")) return (0, avg) if AVG_WER_COL in filtered_df.columns: def _display_avg(row): mid = model_id_from_leaderboard_cell(row[MODEL_COL]) rdict = raw_by_mid.get(mid) if not rdict: return "NA" avg = _row_avg_wer(rdict) if avg is None: return "NA" return format_wer_percent(avg) filtered_df[AVG_WER_COL] = filtered_df.apply(_display_avg, axis=1) filtered_df = filtered_df.assign(_sort=filtered_df.apply(_avg_wer_key, axis=1)) filtered_df = filtered_df.sort_values(by="_sort").drop(columns="_sort") return filtered_df def _leaderboard_table_update(search_query, selected_columns): """Return a Gradio update with table data and matching column widths.""" df = filter_main_table(search_query, selected_columns) return gr.update(value=df, column_widths=column_widths_for(list(df.columns))) def _format_version_recorded_at(ts: str) -> str: from datetime import datetime try: dt = datetime.fromisoformat(ts.replace("Z", "+00:00")) return dt.strftime("%Y-%m-%d %H:%M UTC") except Exception: return str(ts) def _version_choices() -> list[tuple[str, str]]: """(label, value) pairs for the version dropdown; first entry is always 'Latest'.""" choices: list[tuple[str, str]] = [("Latest", LATEST_VERSION)] try: for v in list_leaderboard_versions(): label = f"{_format_version_recorded_at(v['recorded_at'])} — {v['label']}" choices.append((label, v["version"])) except Exception: pass return choices # --------------------------------------------------------------------------- # Submission handler # --------------------------------------------------------------------------- def _wer_cell(row: dict, key: str) -> str: from utils_display import format_wer_percent normalize_legacy_csv_row(row) v = row.get(key, "") if v is None or (isinstance(v, str) and str(v).strip() == ""): return "N/A" pct = format_wer_percent(v) return f"{pct}%" if pct != "NA" else "N/A" def _model_hub_page_link(model_id: str) -> str: mid = (model_id or "").strip().replace(" ", "") if not mid or "/" not in mid: return ( ""
"Enter a model id (author/name) to open its Hub page.
" f"" f"Open model page on the Hub — request access if the repo is gated.
" ) def submit_model( model_id: str, submission_notes: str, contact_email: str, extra_requirements: str, setup_script: str, custom_script: str, recipe_id: str, is_gated: bool, ) -> str: """Validate and enqueue evaluation; a background worker runs approved jobs (up to 4 parallel Hub Jobs when remote mode is on). The backend always uses the "Auto" family (pipeline → Granite → universal → CTC), which covers every checkpoint we currently support, so there is no per-submission family choice. """ model_id = model_id.strip() if not model_id: return styled_error("Please enter a model ID.") family_id = default_family_id() existing = load_raw_results() for row in existing: if row["model_id"] == model_id: return styled_message( f"Model '{model_id}' has already been evaluated. " f"Near Field Speech: {_wer_cell(row, 'wer_anechoic_speech')} | " f"Lab Measured: {_wer_cell(row, 'wer_lab_measured')} | " f"Lab Simulated: {_wer_cell(row, 'wer_lab_simulated')} | " f"High SNR: {_wer_cell(row, 'wer_realistic_high_snr')} | " f"Mid SNR: {_wer_cell(row, 'wer_realistic_mid_snr')} | " f"Low SNR: {_wer_cell(row, 'wer_realistic_low_snr')} | " f"Moving Sources (Low SNR): {_wer_cell(row, 'wer_moving_sources')}" ) # API / recipe / custom-script models (e.g. zoom/scribe_v1) are not Hub repos, so the # Hub-existence check would wrongly reject them. Skip it whenever a recipe resolves or a # custom evaluator is supplied; otherwise validate the model id against the Hub. resolved_recipe = resolve_recipe_id(model_id, (recipe_id or "").strip() or None) has_custom_script = bool((custom_script or "").strip()) if not resolved_recipe and not has_custom_script: on_hub, err_msg = is_model_on_hub(model_id) if not on_hub: return styled_error(f"Model '{model_id}' {err_msg}") if not (contact_email or "").strip(): return styled_error("Please enter a contact email address.") try: job_queue.sanitize_contact_email(contact_email) except ValueError as exc: msg = str(exc) if msg == "Contact email is required.": return styled_error("Please enter a contact email address.") return styled_error("Please enter a valid email address.") script_hint = "" if (custom_script or "").strip(): dep_warn = job_queue.custom_script_deprecated_api_warning(custom_script) if dep_warn: script_hint += f"Note: {dep_warn}
" argparse_warn = job_queue.custom_script_argparse_warning(custom_script) if argparse_warn: script_hint += f"Note: {argparse_warn}
" if not job_queue.custom_script_defines_evaluate(custom_script): script_hint += ( "Note: Your custom script should define "
"evaluate(file: pathlib.Path) -> str at the top level. "
"It will be called once per sample during evaluation.
FFASR_REMOTE_JOBS=1, token_for_ffasr_jobs, "
"HF_TOKEN, and FFASR_REMOTE_EVAL_REPO_URL."
)
if err == "moderation_misconfigured":
return styled_error(
"This Space has moderation enabled but FFASR_MODERATOR_SECRET is not set. "
"Ask the owner to add it under Space Settings → Secrets."
)
if err == "pending_moderation_full":
return styled_warning(
"Too many models are awaiting moderator approval. Please try again later."
)
if err == "already_in_csv":
existing = load_raw_results()
for row in existing:
if row["model_id"] == model_id:
return styled_message(
f"Model '{model_id}' has already been evaluated. "
f"Near Field Speech: {_wer_cell(row, 'wer_anechoic_speech')} | "
f"Lab Measured: {_wer_cell(row, 'wer_lab_measured')} | "
f"Lab Simulated: {_wer_cell(row, 'wer_lab_simulated')} | "
f"High SNR: {_wer_cell(row, 'wer_realistic_high_snr')} | "
f"Mid SNR: {_wer_cell(row, 'wer_realistic_mid_snr')} | "
f"Low SNR: {_wer_cell(row, 'wer_realistic_low_snr')} | "
f"Moving Sources (Low SNR): {_wer_cell(row, 'wer_moving_sources')}"
)
return styled_error("Could not enqueue; please try again.")
if err == "queue_full":
return styled_warning(
"The evaluation queue is full. Please try again later."
)
if err:
return styled_warning(err)
if awaiting_mod:
return (
script_hint
+ styled_message(
f"Request {job_id} recorded for {model_id}. "
f"A moderator must approve it before evaluation starts "
f"(see the Moderate tab). "
f"Approx. backlog awaiting approval: {position}."
)
)
return (
script_hint
+ styled_message(
f"Queued job {job_id} for {model_id}. "
f"Approx. position in queue: {position}. "
f"Evaluation runs in the background; refresh the Leaderboard tab "
f"after a few minutes to see WER when the job finishes."
)
)
# ---------------------------------------------------------------------------
# Analysis tab: radar + line + bar charts
# ---------------------------------------------------------------------------
_METRIC_BY_KEY = {m.key: m for m in SCENARIO_METRICS}
_METRIC_CHOICES = [
(
f"{_METRIC_BY_KEY[k].short}: {_METRIC_BY_KEY[k].label}"
+ (" (planned)" if _METRIC_BY_KEY[k].status == "planned" else ""),
k,
)
for k in HEATMAP_SCENARIO_KEYS
if k in _METRIC_BY_KEY
]
def _analytics_initial():
raw = load_raw_results()
df = analytics._raw_to_analytics_df(raw)
ids = df["model_id"].tolist() if not df.empty else []
avail = analytics.available_metric_keys(df)
line_default = [k for k in HEATMAP_SCENARIO_KEYS if k in avail] or list(avail)
tn = 10
fig_intel = analytics.plot_avg_wer_bars(df, top_n=tn)
fig_speed = analytics.plot_speed_bars(df, top_n=tn)
fig_hm = analytics.plot_scenario_heatmap(df, line_default, top_n=tn)
fig_pareto = analytics.plot_pareto_frontier(df)
# fig_radar = analytics.plot_robustness_radar(df, sel)
fig_b = analytics.plot_scenario_bar_summary(df, 8)
return (
gr.update(value=line_default),
fig_intel,
fig_speed,
fig_hm,
fig_pareto,
fig_b,
)
def _analytics_apply(line_keys, top_n):
raw = load_raw_results()
df = analytics._raw_to_analytics_df(raw)
avail = analytics.available_metric_keys(df)
lk = resolve_scenario_metric_keys(line_keys, _METRIC_CHOICES)
if not lk:
lk = [k for k in HEATMAP_SCENARIO_KEYS if k in avail] or list(avail)
tn = int(top_n) if top_n else 10
fig_intel = analytics.plot_avg_wer_bars(df, top_n=tn)
fig_speed = analytics.plot_speed_bars(df, top_n=tn)
fig_hm = analytics.plot_scenario_heatmap(df, lk, top_n=tn)
fig_pareto = analytics.plot_pareto_frontier(df)
# fig_radar = analytics.plot_robustness_radar(df, valid_models)
fig_b = analytics.plot_scenario_bar_summary(df, tn)
return fig_intel, fig_speed, fig_hm, fig_pareto, fig_b
# ---------------------------------------------------------------------------
# Gradio App
# ---------------------------------------------------------------------------
_theme = treble_gradio_theme()
with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
gr.HTML(BANNER, elem_id="banner")
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
with gr.Tabs(elem_classes="tab-buttons") as tabs:
with gr.TabItem(tab_label("leaderboard", "Leaderboard"), elem_id="od-benchmark-tab-table", id=0):
with gr.Row():
search_box = gr.Textbox(
label="Search models",
placeholder="Filter by model name -- separate multiple terms with commas, e.g. whisper, nvidia",
scale=4,
)
version_dropdown = gr.Dropdown(
choices=_version_choices(),
value=LATEST_VERSION,
label="Version",
scale=2,
min_width=200,
)
leaderboard_update_btn = gr.Button(
"Update", variant="primary", scale=1, min_width=120
)
toggleable_columns = [c for c in COLS if c not in ALWAYS_VISIBLE_COLS]
_default_visible_cols = [
c
for c in toggleable_columns
if c != AutoEvalColumn.wer_lab_measured.name
and c != AutoEvalColumn.wer_lab_simulated.name
and c != AutoEvalColumn.wer_moving_low.name
and c != AutoEvalColumn.wer_moving_mid.name
and c != AutoEvalColumn.wer_moving_high.name
]
column_checkboxes = gr.CheckboxGroup(
choices=toggleable_columns,
value=_default_visible_cols,
label="Select columns to display",
)
leaderboard_table = gr.components.Dataframe(
value=original_df,
datatype=TYPES,
elem_id="leaderboard-table",
interactive=False,
visible=True,
wrap=False,
column_widths=column_widths_for(COLS),
)
gr.Markdown(
"\* Moving-source evaluations are a beta feature.",
elem_classes="markdown-text",
)
def _set_active_version(version):
"""Load the chosen version into the module globals used by filter_main_table."""
global original_df, _active_raw_rows
if version:
_active_raw_rows = raw_rows_for_version(version)
original_df = load_results(version)
else:
_active_raw_rows = None
original_df = load_results()
def _leaderboard_refresh(search, cols, version):
"""Reload results from storage for the selected version, refresh version list."""
from init import invalidate_results_cache
try:
invalidate_results_cache()
choices = _version_choices()
valid = {v for _, v in choices}
if version not in valid:
version = LATEST_VERSION
_set_active_version(version)
return (
_leaderboard_table_update(search, cols),
gr.update(choices=choices, value=version),
)
except Exception:
return gr.update(
value=original_df,
column_widths=column_widths_for(list(original_df.columns)),
), gr.update()
def _on_version_change(version, search, cols):
"""Swap the displayed leaderboard to the selected version, keep filters."""
try:
_set_active_version(version)
return _leaderboard_table_update(search, cols)
except Exception:
return gr.update(
value=original_df,
column_widths=column_widths_for(list(original_df.columns)),
)
def _on_startup():
"""Load leaderboard + next-up list after UI is up (keeps Space prepare step fast)."""
global original_df, _active_raw_rows
from init import invalidate_results_cache
version_update = gr.update()
try:
invalidate_results_cache()
_active_raw_rows = None
original_df = load_results()
table = _leaderboard_table_update("", _default_visible_cols)
version_update = gr.update(choices=_version_choices(), value=LATEST_VERSION)
except Exception as exc:
table = gr.update(
value=original_df,
column_widths=column_widths_for(list(original_df.columns)),
)
next_html = (
""
)
return table, next_html, version_update
try:
job_queue.ensure_worker_started()
next_html = job_queue.next_up_html()
except Exception as exc:
next_html = (
""
)
return table, next_html, version_update
filter_inputs = [search_box, column_checkboxes]
search_box.change(_leaderboard_table_update, inputs=filter_inputs, outputs=leaderboard_table)
column_checkboxes.change(_leaderboard_table_update, inputs=filter_inputs, outputs=leaderboard_table)
version_dropdown.change(
fn=_on_version_change,
inputs=[version_dropdown, search_box, column_checkboxes],
outputs=leaderboard_table,
)
leaderboard_update_btn.click(
fn=_leaderboard_refresh,
inputs=[search_box, column_checkboxes, version_dropdown],
outputs=[leaderboard_table, version_dropdown],
)
with gr.TabItem(tab_label("submit", "Submit"), elem_id="submit-tab", id=1):
gr.Markdown("## Submit a model for evaluation")
gr.Markdown(
"Paste a Hugging Face model id. The server loads every checkpoint with an automatic backend, so "
"**Whisper, IBM Granite speech, Cohere Transcribe, efficient‑speech / custom Whisper variants, "
"Wav2Vec2 / HuBERT CTC heads, and SpeechBrain ASR**, plus most other ASR stacks on the Hub are "
"supported without a per‑submission choice.\n\n"
"Submissions are **queued**; when Hub Jobs are enabled, up to **four** models evaluate in parallel. "
"Otherwise jobs run **one at a time** on this Space. "
"When moderation is enabled, new requests wait for approval before they run."
)
with gr.Row():
model_input = gr.Textbox(
label="Model ID",
placeholder="e.g. openai/whisper-tiny",
scale=3,
)
submit_btn = gr.Button("Evaluate", variant="primary", scale=1)
status_output = gr.HTML()
recipe_input = gr.Dropdown(
label="Complex model recipe (optional)",
choices=RECIPE_CHOICES,
value="",
info="Pre-fills setup script, evaluate(), and deps for install-heavy stacks (e.g. Mega-ASR).",
)
requirements_input = gr.Textbox(
label="Optional extra Python requirements (one per line, requirements.txt format)",
placeholder="transformers @ git+https://github.com/huggingface/transformers.git\nsentencepiece",
lines=4,
max_length=8000,
)
setup_script_input = gr.Textbox(
label="Optional one-time setup script (shell or Python, runs once per Hub Job)",
placeholder="git clone …\npip install -r requirements.txt\npython scripts/download.py",
lines=6,
max_length=8000,
)
script_input = gr.Code(
label="Optional custom evaluator (Python)",
language="python",
lines=16,
value=DEFAULT_CUSTOM_EVAL_EXAMPLE,
)
gr.Markdown(
"Edit the example below: define **`evaluate(file: Path) -> str`** to transcribe one WAV. "
"Load your model **once** at module level (as in the Cohere example). "
"Use **soundfile** (or numpy) to read each WAV (see the example). "
"Hub Jobs disable torchcodec and fall back to librosa if you use "
"``transformers.audio_utils.load_audio`` on file paths. "
"The function is called **once per audio sample** inside the eval loop. "
"**Model size:** expose your loaded model as a module-level `model` "
"(or set `NUM_PARAMS = Could not load queue: {exc}
" return status, nxt recipe_input.change( fn=_apply_recipe_fields, inputs=[ model_input, recipe_input, requirements_input, setup_script_input, script_input, ], outputs=[requirements_input, setup_script_input, script_input], ) submit_btn.click( fn=_submit_and_refresh, inputs=[ model_input, notes_input, email_input, requirements_input, setup_script_input, script_input, recipe_input, is_gated_input, ], outputs=[status_output, next_up_panel], ) with gr.TabItem(tab_label("moderate", "Moderate"), elem_id="moderate-tab", id=2): from benchmark.dataset import CONDITION_UI_CHOICES _MOD_COND_CHOICES = [(lbl, key) for lbl, key in CONDITION_UI_CHOICES] _MOD_COND_DEFAULT = [key for _, key in CONDITION_UI_CHOICES] gr.Markdown("## Moderator access") mod_unlock_feedback = gr.HTML() mod_secret = gr.Textbox( label="Moderator secret", type="password", placeholder="FFASR_MODERATOR_SECRET", ) mod_unlock_btn = gr.Button("Unlock moderator tools", variant="primary") mod_unlocked = gr.State(False) with gr.Column(visible=False) as mod_panel: gr.Markdown( "WhenFFASR_MODERATION=1 is set, new submissions **wait here** until you approve them. "
"Use per-row **Check**, **Approve**, **Retry**, and **Remove** buttons — popups open for "
"submission review, dataset selection, and approval.\n\n"
"When FFASR_REMOTE_JOBS=1, use **Open Hub Job logs** while a job runs. "
"The bucket results/remote_artifacts/ folder only gets a JSON file **after** a successful run.\n\n"
"**Retry** re-queues failed, done, or queued jobs (not while running). "
"Uncheck datasets in the retry popup to run only selected packed splits; "
"other leaderboard WER columns are left unchanged on success. "
"A full re-run (all datasets selected) **replaces** the row when the model is already listed."
)
gr.Markdown("### Current job progress")
mod_progress = gr.HTML(
value=job_queue.moderation_locked_placeholder_html(),
elem_classes="ffasr-card",
)
mod_refresh_tick = gr.Number(value=0, visible=False)
mod_refresh = gr.Button("Refresh lists")
mod_feedback = gr.HTML()
mod_check_job_id = gr.State("")
mod_retry_job_id = gr.State("")
mod_approve_job_id = gr.State("")
with gr.Column(visible=False) as check_panel:
check_header = gr.Markdown("### Check submission")
check_notes = gr.Textbox(
label="Submitter notes",
lines=2,
interactive=False,
)
check_recipe = gr.Dropdown(
label="Recipe",
choices=RECIPE_CHOICES,
value="",
)
check_reqs = gr.Textbox(
label="Extra Python requirements (one per line)",
lines=4,
max_length=8000,
)
check_setup = gr.Textbox(
label="One-time setup script (shell or Python)",
lines=6,
max_length=8000,
)
check_script = gr.Code(
label="Custom evaluator (Python)",
language="python",
lines=16,
)
with gr.Row():
check_save_btn = gr.Button("Save", variant="primary")
check_cancel_btn = gr.Button("Cancel")
with gr.Column(visible=False) as retry_panel:
retry_header = gr.Markdown("### Retry job")
retry_eval_conditions = gr.CheckboxGroup(
choices=_MOD_COND_CHOICES,
value=_MOD_COND_DEFAULT,
label="Datasets to evaluate (all selected = full benchmark)",
)
with gr.Row():
retry_confirm_btn = gr.Button(
"Confirm retry", variant="primary"
)
retry_cancel_btn = gr.Button("Cancel")
with gr.Column(visible=False) as approve_panel:
approve_header = gr.Markdown("### Approve job")
approve_run_custom = gr.Checkbox(
label="Use submitter's custom evaluate() function (if provided)",
value=False,
)
approve_eval_conditions = gr.CheckboxGroup(
choices=_MOD_COND_CHOICES,
value=_MOD_COND_DEFAULT,
label="Datasets to evaluate (all selected = full benchmark)",
)
with gr.Row():
approve_confirm_btn = gr.Button("Approve", variant="primary")
approve_cancel_btn = gr.Button("Cancel")
_MOD_POPUP_HIDE = (
gr.update(visible=False),
gr.update(visible=False),
gr.update(visible=False),
)
def _mod_bump_tick(tick):
return int(tick or 0) + 1
def _mod_feedback_html(ok: bool, msg: str) -> str:
color = "green" if ok else "red"
return f"{msg}
" def _mod_open_check(jid: str, secret: str, unlocked: bool): hidden = gr.update(visible=False) if not unlocked: return ( gr.update(visible=False), hidden, hidden, "", "### Check submission", "", "", "", "", "", ) job = job_queue.peek_job(str(jid).strip()) if not job: return ( gr.update(visible=False), hidden, hidden, "", "### Check submission", "", "", "", "", "", ) return ( gr.update(visible=True), hidden, hidden, job.id, f"### Check submission: `{job.model_id}`", job.submission_notes or "", job.recipe_id or "", job.extra_requirements or "", job.setup_script or "", job.custom_script or "", ) def _mod_open_retry(jid: str, secret: str, unlocked: bool): hidden = gr.update(visible=False) if not unlocked: return ( hidden, gr.update(visible=False), hidden, "", "### Retry job", ) job = job_queue.peek_job(str(jid).strip()) if not job: return ( hidden, gr.update(visible=False), hidden, "", "### Retry job", ) return ( hidden, gr.update(visible=True), hidden, job.id, f"### Retry: `{job.model_id}` (`{job.id}`)", ) def _mod_open_approve(jid: str, secret: str, unlocked: bool): hidden = gr.update(visible=False) if not unlocked: return ( hidden, hidden, gr.update(visible=False), "", "### Approve job", False, ) job = job_queue.peek_job(str(jid).strip()) if not job: return ( hidden, hidden, gr.update(visible=False), "", "### Approve job", False, ) has_script = bool( (job.custom_script or "").strip() or (job.setup_script or "").strip() or (job.recipe_id or "").strip() ) return ( hidden, hidden, gr.update(visible=True), job.id, f"### Approve: `{job.model_id}` (`{job.id}`)", has_script, ) def _mod_reject_row(jid, secret, unlocked, tick): if not unlocked: return ( "Unlock moderator tools first.
", tick, job_queue.progress_html(), ) ok, msg = job_queue.reject_job(str(jid).strip(), secret) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), job_queue.progress_html(), ) def _mod_remove_row(jid, secret, unlocked, tick): if not unlocked: return ( "Unlock moderator tools first.
", tick, job_queue.progress_html(), ) ok, msg = job_queue.remove_job_entry(str(jid).strip(), secret) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), job_queue.progress_html(), ) def _mod_save_check(jid, secret, unlocked, tick, recipe, reqs, setup, script): if not unlocked: return ( "Unlock moderator tools first.
", tick, *_MOD_POPUP_HIDE, ) ok, msg = job_queue.update_job_script_and_requirements( str(jid).strip(), secret, extra_requirements=reqs or "", setup_script=setup or "", custom_script=script or "", recipe_id=(recipe or "").strip(), ) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), *_MOD_POPUP_HIDE, ) def _mod_confirm_retry(jid, secret, unlocked, tick, eval_conds): if not unlocked: return ( "Unlock moderator tools first.
", tick, *_MOD_POPUP_HIDE, job_queue.progress_html(), ) ok, msg = job_queue.retry_failed_job( str(jid).strip(), secret, eval_conditions=list(eval_conds or []), ) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), *_MOD_POPUP_HIDE, job_queue.progress_html(), ) def _mod_confirm_approve( jid, secret, unlocked, tick, run_custom, eval_conds ): if not unlocked: return ( "Unlock moderator tools first.
", tick, *_MOD_POPUP_HIDE, job_queue.progress_html(), ) ok, msg = job_queue.approve_job( str(jid).strip(), secret, run_custom_script=bool(run_custom), eval_conditions=list(eval_conds or []), ) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), *_MOD_POPUP_HIDE, job_queue.progress_html(), ) def _mod_cancel_popup(tick): return tick, *_MOD_POPUP_HIDE def _mod_retry_all(secret, unlocked, tick): if not unlocked: return ( "Unlock moderator tools first.
", tick, job_queue.progress_html(), ) ok, msg = job_queue.retry_all_eligible_jobs(secret) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), job_queue.progress_html(), ) def _mod_import_artifact( secret, artifact_ref, replace_existing, import_notes, unlocked, tick ): if not unlocked: return ( "Unlock moderator tools first.
", tick, ) ok, msg = job_queue.import_artifact_to_leaderboard( artifact_ref, secret, replace_existing=bool(replace_existing), submission_notes=import_notes or "", ) return (_mod_feedback_html(ok, msg), _mod_bump_tick(tick)) def _mod_refresh_lists(secret, unlocked, tick): if not unlocked: return tick, job_queue.moderation_locked_placeholder_html() ok, msg = job_queue.verify_moderator_secret(secret) if not ok: return tick, job_queue.moderation_locked_placeholder_html() return _mod_bump_tick(tick), job_queue.progress_html() gr.Markdown("### Pending moderation") @gr.render(inputs=[mod_refresh_tick]) def _render_pending_jobs(_tick): try: jobs = job_queue.pending_jobs_for_render() except Exception as exc: gr.Markdown(f"*Queue unavailable: {exc}*") return if not jobs: gr.Markdown("*No jobs awaiting approval.*") return for job in jobs: jid = job["id"] row_cls = job_queue.job_row_elem_classes(job["status"]) gated = " · **gated**" if job["is_gated"] else "" script_bit = ( " · custom/setup/recipe" if job["has_custom_script"] else "" ) extras = ( f"{job['req_count']} extra req(s)" if job["req_count"] else "no extra reqs" ) email_bit = ( f" · 📧 `{job['contact_email']}`" if job.get("contact_email") else " · 📧 *(none)*" ) with gr.Row(elem_classes=row_cls): gr.Markdown( f"**`{jid}`** · `{job['model_id']}` · " f"`{job['family_id']}`{gated}{script_bit} · {extras}{email_bit} · " f"{job['notes_preview'] or '—'} · {job['created_at']}", elem_classes="ffasr-job-info", ) check_btn = gr.Button("Check", size="sm") approve_btn = gr.Button( "Approve", size="sm", variant="primary" ) reject_btn = gr.Button("Reject", size="sm", variant="stop") check_btn.click( fn=_mod_open_check, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_check_job_id, check_header, check_notes, check_recipe, check_reqs, check_setup, check_script, ], ) approve_btn.click( fn=_mod_open_approve, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_approve_job_id, approve_header, approve_run_custom, ], ) reject_btn.click( fn=_mod_reject_row, inputs=[gr.State(jid), mod_secret, mod_unlocked, mod_refresh_tick], outputs=[mod_feedback, mod_refresh_tick, mod_progress], ) gr.Markdown("### Recent job activity") retry_all_btn = gr.Button( "Retry ALL eligible jobs (full benchmark)", variant="secondary", ) gr.Markdown( "Retry ALL re-queues every failed / done / queued " "job against the full dataset set (all packed conditions). Running, dispatching, and " "pending-moderation jobs are skipped. Successful re-runs replace the existing " "leaderboard row for each model." ) @gr.render(inputs=[mod_refresh_tick]) def _render_recent_jobs(_tick): try: jobs = job_queue.recent_jobs_for_render(30) except Exception as exc: gr.Markdown(f"*Queue unavailable: {exc}*") return if not jobs: gr.Markdown("*No job history loaded yet.*") return for job in jobs: jid = job["id"] err = job["error"] or "N/A" hub = job["hub_link_html"] or "" row_cls = job_queue.job_row_elem_classes(job["status"]) email_bit = ( f" · 📧 `{job['contact_email']}`" if job.get("contact_email") else " · 📧 *(none)*" ) info_md = ( f"**`{jid}`** · `{job['model_id']}` · " f"**{job['status']}**{email_bit} · {err} · {job['updated_at']}" ) if hub: info_md += f" · {hub}" with gr.Row(elem_classes=row_cls): gr.Markdown( info_md, elem_classes="ffasr-job-info", ) check_btn = gr.Button("Check", size="sm") if job["can_retry"]: retry_btn = gr.Button( "Retry", size="sm", variant="secondary" ) if job["can_remove"]: remove_btn = gr.Button("Remove", size="sm", variant="stop") check_btn.click( fn=_mod_open_check, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_check_job_id, check_header, check_notes, check_recipe, check_reqs, check_setup, check_script, ], ) if job["can_retry"]: retry_btn.click( fn=_mod_open_retry, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_retry_job_id, retry_header, ], ) if job["can_remove"]: remove_btn.click( fn=_mod_remove_row, inputs=[ gr.State(jid), mod_secret, mod_unlocked, mod_refresh_tick, ], outputs=[mod_feedback, mod_refresh_tick, mod_progress], ) gr.Markdown("### Import result from bucket artifact") gr.Markdown( "If a Hub Job finished but the leaderboard CSV was not updated, paste the artifact " "file name fromresults/remote_artifacts/ (e.g. a1b2c3d4.json) "
"to merge WER/RTFx into the CSV."
)
mod_artifact_ref = gr.Textbox(
label="Artifact JSON file name or bucket path",
placeholder="e.g. a1b2c3d4.json or results/remote_artifacts/a1b2c3d4.json",
)
mod_replace_existing = gr.Checkbox(
label="Replace existing leaderboard row for this model",
value=False,
)
mod_import_notes = gr.Textbox(
label="Optional submission notes override",
placeholder="Leave empty to use notes from the matched queue job, if any",
lines=1,
max_length=4000,
)
import_artifact_btn = gr.Button("Import artifact to CSV", variant="secondary")
def _mod_unlock(secret: str):
ok, msg = job_queue.verify_moderator_secret(secret)
if not ok:
return (
False,
gr.update(visible=False),
f"{msg}
", 0, ) return ( True, gr.update(visible=True), "Moderator tools unlocked for this session.
", 1, ) def _mod_progress_gated(unlocked: bool): if not unlocked: return job_queue.moderation_locked_placeholder_html() return job_queue.progress_html() mod_unlock_btn.click( fn=_mod_unlock, inputs=[mod_secret], outputs=[mod_unlocked, mod_panel, mod_unlock_feedback, mod_refresh_tick], ) mod_refresh.click( fn=_mod_refresh_lists, inputs=[mod_secret, mod_unlocked, mod_refresh_tick], outputs=[mod_refresh_tick, mod_progress], ) check_save_btn.click( fn=_mod_save_check, inputs=[ mod_check_job_id, mod_secret, mod_unlocked, mod_refresh_tick, check_recipe, check_reqs, check_setup, check_script, ], outputs=[ mod_feedback, mod_refresh_tick, check_panel, retry_panel, approve_panel, ], ) check_cancel_btn.click( fn=_mod_cancel_popup, inputs=[mod_refresh_tick], outputs=[mod_refresh_tick, check_panel, retry_panel, approve_panel], ) retry_confirm_btn.click( fn=_mod_confirm_retry, inputs=[ mod_retry_job_id, mod_secret, mod_unlocked, mod_refresh_tick, retry_eval_conditions, ], outputs=[ mod_feedback, mod_refresh_tick, check_panel, retry_panel, approve_panel, mod_progress, ], ) retry_cancel_btn.click( fn=_mod_cancel_popup, inputs=[mod_refresh_tick], outputs=[mod_refresh_tick, check_panel, retry_panel, approve_panel], ) approve_confirm_btn.click( fn=_mod_confirm_approve, inputs=[ mod_approve_job_id, mod_secret, mod_unlocked, mod_refresh_tick, approve_run_custom, approve_eval_conditions, ], outputs=[ mod_feedback, mod_refresh_tick, check_panel, retry_panel, approve_panel, mod_progress, ], ) approve_cancel_btn.click( fn=_mod_cancel_popup, inputs=[mod_refresh_tick], outputs=[mod_refresh_tick, check_panel, retry_panel, approve_panel], ) retry_all_btn.click( fn=_mod_retry_all, inputs=[mod_secret, mod_unlocked, mod_refresh_tick], outputs=[mod_feedback, mod_refresh_tick, mod_progress], ) import_artifact_btn.click( fn=_mod_import_artifact, inputs=[ mod_secret, mod_artifact_ref, mod_replace_existing, mod_import_notes, mod_unlocked, mod_refresh_tick, ], outputs=[mod_feedback, mod_refresh_tick], ) with gr.TabItem(tab_label("analysis", "Analysis"), elem_id="analysis-tab", id=3) as analysis_tab: gr.Markdown("## Scenario analysis") gr.Markdown( "Charts support standard Plotly interaction (legend, zoom, pan, hover).\n\n" "**Leaderboard order** is by **Average WER** (mean across checked scenario columns; lower is better).\n\n" "**Average WER** ranks the top N models by mean WER across live scenarios (lower is better). " "**Speed** ranks the top N models by RTFx (audio sec / inference sec, higher is better). " "Bars are colored per company (e.g. all NVIDIA models share one color).\n\n" "**WER heatmap** shows WER (%) by model and scenario for the top N models (sorted by Average WER); " "lower WER corresponds to greener cells.\n\n" "**Pareto Front** plots Average WER on X and RTFx on Y (log scale). Models on the frontier are " "labeled and connected by the dashed blue line; other models render as faint dots.\n\n" "**WER by scenario** compares raw WER across conditions for the selected top models." ) # with gr.Row(): # an_models = gr.Dropdown( # label="Models on radar (up to 8)", # choices=[], # value=None, # multiselect=True, # max_choices=8, # scale=2, # ) an_topn = gr.Slider( 5, 80, value=10, step=1, label="Top‑N models (WER bars, heatmap & grouped WER)", scale=1, ) an_line_metrics = gr.CheckboxGroup( label="Scenarios shown in the heatmap", choices=_METRIC_CHOICES, value=[k for k in HEATMAP_SCENARIO_KEYS if k in LIVE_SCENARIO_KEYS], ) an_apply = gr.Button("Apply / refresh charts", variant="primary") gr.Markdown( "### Pareto Front: Average WER vs RTFx\n\n" "Models on the Pareto frontier achieve the best trade-off between WER " "and speed (RTFx). Names are shown for frontier models; hover over " "other points to see their names." ) an_plot_pareto = gr.Plot( label="Pareto Front: Average WER vs RTFx", elem_id="analysis-pareto-plot", ) with gr.Row(): an_plot_intelligence = gr.Plot(label="Average WER") an_plot_speed = gr.Plot(label="Speed") an_plot_compare = gr.Plot(label="WER heatmap") # with gr.Row(): # an_plot_radar = gr.Plot(label="Robustness radar") an_plot_bar = gr.Plot(label="WER by scenario") an_apply.click( fn=_analytics_apply, inputs=[ an_line_metrics, an_topn, ], outputs=[ an_plot_intelligence, an_plot_speed, an_plot_compare, an_plot_pareto, an_plot_bar, ], ) _analytics_outputs = [ an_line_metrics, an_plot_intelligence, an_plot_speed, an_plot_compare, an_plot_pareto, an_plot_bar, ] _analytics_loaded = gr.State(False) def _analytics_on_tab_select(loaded: bool): """Load charts once the Analysis tab is visible so Plotly gets the real width.""" if loaded: return (loaded,) + (gr.skip(),) * 6 ( metrics_upd, fig_intel, fig_speed, fig_hm, fig_pareto, fig_b, ) = _analytics_initial() return ( True, metrics_upd, fig_intel, fig_speed, fig_hm, fig_pareto, fig_b, ) analysis_tab.select( fn=_analytics_on_tab_select, inputs=[_analytics_loaded], outputs=[_analytics_loaded, *_analytics_outputs], ) with gr.TabItem(tab_label("examples", "Examples"), elem_id="examples-tab", id=4): examples_renderer.ensure_assets() _dry = examples_renderer.sample_paths("dry") _hi = examples_renderer.sample_paths("high_snr") _mid = examples_renderer.sample_paths("mid_snr") _lo = examples_renderer.sample_paths("low_snr") _scene = examples_renderer.scene_image_path() def _example_audio(path: str | None, label: str): return gr.Audio( value=path, type="filepath", label=label, interactive=False, editable=False, show_download_button=False, show_share_button=False, waveform_options=gr.WaveformOptions( waveform_color="#5c6670", waveform_progress_color=TREBLE_TEAL, show_recording_waveform=True, ), elem_classes="examples-audio", ) with gr.Column(elem_classes="examples-scene-wrap"): gr.Image( value=_scene, type="filepath", label="Treble scene", interactive=False, show_label=False, show_download_button=False, show_fullscreen_button=False, height=300, container=False, ) gr.Markdown("### Near Field Speech") with gr.Row(): _example_audio(_dry[0], "Example") gr.Markdown("### High SNR") with gr.Row(): _example_audio(_hi[0], "Example") gr.Markdown("### Mid SNR") with gr.Row(): _example_audio(_mid[0], "Example") gr.Markdown("### Low SNR") with gr.Row(): _example_audio(_lo[0], "Example") with gr.TabItem(tab_label("about", "About"), elem_id="about-tab", id=5): gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text") _footer_html = ( "