import os import time import gradio as gr import pandas as pd from constants import ( ABOUT_TEXT, APP_TITLE, BANNER, CITATION_TEXT, DEFAULT_CUSTOM_EVAL_EXAMPLE, INTRODUCTION_TEXT, LEADERBOARD_CSS, tab_label, TREBLE_TEAL, treble_gradio_theme, ) import examples_renderer import analytics from family_registry import default_family_id import job_queue from metrics_config import ( HEATMAP_SCENARIO_KEYS, LIVE_SCENARIO_KEYS, SCENARIO_METRICS, resolve_scenario_metric_keys, ) from recipes.registry import RECIPE_CHOICES, resolve_recipe_id from init import ( LATEST_VERSION, is_model_on_hub, list_leaderboard_versions, load_raw_results, load_results, normalize_legacy_csv_row, raw_rows_for_version, ) from utils_display import ( AutoEvalColumn, SCENARIO_DISPLAY_COLS, SCENARIO_DISPLAY_TO_KEY, column_widths_for, fields, format_wer_percent, model_id_from_leaderboard_cell, styled_error, styled_message, styled_warning, ) # --------------------------------------------------------------------------- # Column setup # --------------------------------------------------------------------------- COLS = [c.name for c in fields(AutoEvalColumn)] TYPES = [c.type for c in fields(AutoEvalColumn)] AVG_WER_COL = AutoEvalColumn.avg_wer_core.name # "Avg WER (%)" MODEL_COL = AutoEvalColumn.model.name # "Model" ALWAYS_VISIBLE_COLS = (MODEL_COL, AVG_WER_COL) # Empty table at import; real data loaded on demo.load (avoids blocking Space "preparing"). original_df = pd.DataFrame(columns=COLS) # Raw CSV rows backing the currently displayed leaderboard version. ``None`` means "live" # (filter_main_table falls back to load_raw_results()). Set whenever a version is selected # so the Avg-WER recompute in filter_main_table matches the displayed snapshot. _active_raw_rows: list[dict] | None = None _NEXT_UP_PLACEHOLDER = ( "
" "Loading upcoming evaluations…
" ) # --------------------------------------------------------------------------- # Filter (matches open_asr_leaderboard pattern) # --------------------------------------------------------------------------- def filter_main_table(search_query, selected_columns): filtered_df = original_df.copy() selected_columns = list(selected_columns or []) if MODEL_COL not in filtered_df.columns: return pd.DataFrame(columns=COLS) # Filter by model name search (case-insensitive, supports comma-separated terms). if search_query: terms = [t.strip() for t in search_query.split(",") if t.strip()] if terms: mask = filtered_df[MODEL_COL].str.lower().apply( lambda cell: any(term.lower() in cell for term in terms) ) filtered_df = filtered_df[mask] # Hide toggleable columns that the user unchecked. Model + Avg WER stay visible. all_toggleable = [c for c in COLS if c not in ALWAYS_VISIBLE_COLS] columns_to_hide = set(all_toggleable) - set(selected_columns) filtered_df = filtered_df[[c for c in filtered_df.columns if c not in columns_to_hide]] raw_rows = _active_raw_rows if _active_raw_rows is not None else load_raw_results() for r in raw_rows: normalize_legacy_csv_row(r) raw_by_mid = {r["model_id"]: r for r in raw_rows} visible_metric_keys = [ SCENARIO_DISPLAY_TO_KEY[c] for c in SCENARIO_DISPLAY_COLS if c in selected_columns ] def _row_avg_wer(rdict: dict) -> float | None: if not visible_metric_keys: return None vals = [] for k in visible_metric_keys: try: v = rdict.get(k, "") if v is None or str(v).strip() == "": continue vals.append(float(v)) except (TypeError, ValueError): continue if not vals: return None return sum(vals) / len(vals) def _avg_wer_key(row) -> tuple[int, float]: mid = model_id_from_leaderboard_cell(row[MODEL_COL]) rdict = raw_by_mid.get(mid) if not rdict: return (1, float("inf")) avg = _row_avg_wer(rdict) if avg is None: return (1, float("inf")) return (0, avg) if AVG_WER_COL in filtered_df.columns: def _display_avg(row): mid = model_id_from_leaderboard_cell(row[MODEL_COL]) rdict = raw_by_mid.get(mid) if not rdict: return "NA" avg = _row_avg_wer(rdict) if avg is None: return "NA" return format_wer_percent(avg) filtered_df[AVG_WER_COL] = filtered_df.apply(_display_avg, axis=1) filtered_df = filtered_df.assign(_sort=filtered_df.apply(_avg_wer_key, axis=1)) filtered_df = filtered_df.sort_values(by="_sort").drop(columns="_sort") return filtered_df def _leaderboard_table_update(search_query, selected_columns): """Return a Gradio update with table data and matching column widths.""" df = filter_main_table(search_query, selected_columns) return gr.update(value=df, column_widths=column_widths_for(list(df.columns))) def _format_version_recorded_at(ts: str) -> str: from datetime import datetime try: dt = datetime.fromisoformat(ts.replace("Z", "+00:00")) return dt.strftime("%Y-%m-%d %H:%M UTC") except Exception: return str(ts) def _version_choices() -> list[tuple[str, str]]: """(label, value) pairs for the version dropdown; first entry is always 'Latest'.""" choices: list[tuple[str, str]] = [("Latest", LATEST_VERSION)] try: for v in list_leaderboard_versions(): label = f"{_format_version_recorded_at(v['recorded_at'])} — {v['label']}" choices.append((label, v["version"])) except Exception: pass return choices # --------------------------------------------------------------------------- # Submission handler # --------------------------------------------------------------------------- def _wer_cell(row: dict, key: str) -> str: from utils_display import format_wer_percent normalize_legacy_csv_row(row) v = row.get(key, "") if v is None or (isinstance(v, str) and str(v).strip() == ""): return "N/A" pct = format_wer_percent(v) return f"{pct}%" if pct != "NA" else "N/A" def _model_hub_page_link(model_id: str) -> str: mid = (model_id or "").strip().replace(" ", "") if not mid or "/" not in mid: return ( "

" "Enter a model id (author/name) to open its Hub page.

" ) url = f"https://huggingface.co/{mid}" return ( f"

" f"" f"Open model page on the Hub — request access if the repo is gated.

" ) def submit_model( model_id: str, submission_notes: str, contact_email: str, extra_requirements: str, setup_script: str, custom_script: str, recipe_id: str, is_gated: bool, ) -> str: """Validate and enqueue evaluation; a background worker runs approved jobs (up to 4 parallel Hub Jobs when remote mode is on). The backend always uses the "Auto" family (pipeline → Granite → universal → CTC), which covers every checkpoint we currently support, so there is no per-submission family choice. """ model_id = model_id.strip() if not model_id: return styled_error("Please enter a model ID.") family_id = default_family_id() existing = load_raw_results() for row in existing: if row["model_id"] == model_id: return styled_message( f"Model '{model_id}' has already been evaluated. " f"Near Field Speech: {_wer_cell(row, 'wer_anechoic_speech')} | " f"Lab Measured: {_wer_cell(row, 'wer_lab_measured')} | " f"Lab Simulated: {_wer_cell(row, 'wer_lab_simulated')} | " f"High SNR: {_wer_cell(row, 'wer_realistic_high_snr')} | " f"Mid SNR: {_wer_cell(row, 'wer_realistic_mid_snr')} | " f"Low SNR: {_wer_cell(row, 'wer_realistic_low_snr')} | " f"Moving Sources (Low SNR): {_wer_cell(row, 'wer_moving_sources')}" ) # API / recipe / custom-script models (e.g. zoom/scribe_v1) are not Hub repos, so the # Hub-existence check would wrongly reject them. Skip it whenever a recipe resolves or a # custom evaluator is supplied; otherwise validate the model id against the Hub. resolved_recipe = resolve_recipe_id(model_id, (recipe_id or "").strip() or None) has_custom_script = bool((custom_script or "").strip()) if not resolved_recipe and not has_custom_script: on_hub, err_msg = is_model_on_hub(model_id) if not on_hub: return styled_error(f"Model '{model_id}' {err_msg}") if not (contact_email or "").strip(): return styled_error("Please enter a contact email address.") try: job_queue.sanitize_contact_email(contact_email) except ValueError as exc: msg = str(exc) if msg == "Contact email is required.": return styled_error("Please enter a contact email address.") return styled_error("Please enter a valid email address.") script_hint = "" if (custom_script or "").strip(): dep_warn = job_queue.custom_script_deprecated_api_warning(custom_script) if dep_warn: script_hint += f"

Note: {dep_warn}

" argparse_warn = job_queue.custom_script_argparse_warning(custom_script) if argparse_warn: script_hint += f"

Note: {argparse_warn}

" if not job_queue.custom_script_defines_evaluate(custom_script): script_hint += ( "

Note: Your custom script should define " "evaluate(file: pathlib.Path) -> str at the top level. " "It will be called once per sample during evaluation.

" ) try: job_id, position, err, awaiting_mod = job_queue.enqueue( model_id, family_id, submission_notes=submission_notes or "", contact_email=contact_email or "", extra_requirements=extra_requirements or "", setup_script=setup_script or "", custom_script=custom_script or "", recipe_id=(recipe_id or "").strip(), is_gated=bool(is_gated), ) except Exception as exc: return styled_warning(f"Could not enqueue submission: {exc}") if err == "remote_jobs_required": return styled_error( "Hub Jobs are required. Set Space secrets " "FFASR_REMOTE_JOBS=1, token_for_ffasr_jobs, " "HF_TOKEN, and FFASR_REMOTE_EVAL_REPO_URL." ) if err == "moderation_misconfigured": return styled_error( "This Space has moderation enabled but FFASR_MODERATOR_SECRET is not set. " "Ask the owner to add it under Space Settings → Secrets." ) if err == "pending_moderation_full": return styled_warning( "Too many models are awaiting moderator approval. Please try again later." ) if err == "already_in_csv": existing = load_raw_results() for row in existing: if row["model_id"] == model_id: return styled_message( f"Model '{model_id}' has already been evaluated. " f"Near Field Speech: {_wer_cell(row, 'wer_anechoic_speech')} | " f"Lab Measured: {_wer_cell(row, 'wer_lab_measured')} | " f"Lab Simulated: {_wer_cell(row, 'wer_lab_simulated')} | " f"High SNR: {_wer_cell(row, 'wer_realistic_high_snr')} | " f"Mid SNR: {_wer_cell(row, 'wer_realistic_mid_snr')} | " f"Low SNR: {_wer_cell(row, 'wer_realistic_low_snr')} | " f"Moving Sources (Low SNR): {_wer_cell(row, 'wer_moving_sources')}" ) return styled_error("Could not enqueue; please try again.") if err == "queue_full": return styled_warning( "The evaluation queue is full. Please try again later." ) if err: return styled_warning(err) if awaiting_mod: return ( script_hint + styled_message( f"Request {job_id} recorded for {model_id}. " f"A moderator must approve it before evaluation starts " f"(see the Moderate tab). " f"Approx. backlog awaiting approval: {position}." ) ) return ( script_hint + styled_message( f"Queued job {job_id} for {model_id}. " f"Approx. position in queue: {position}. " f"Evaluation runs in the background; refresh the Leaderboard tab " f"after a few minutes to see WER when the job finishes." ) ) # --------------------------------------------------------------------------- # Analysis tab: radar + line + bar charts # --------------------------------------------------------------------------- _METRIC_BY_KEY = {m.key: m for m in SCENARIO_METRICS} _METRIC_CHOICES = [ ( f"{_METRIC_BY_KEY[k].short}: {_METRIC_BY_KEY[k].label}" + (" (planned)" if _METRIC_BY_KEY[k].status == "planned" else ""), k, ) for k in HEATMAP_SCENARIO_KEYS if k in _METRIC_BY_KEY ] def _analytics_initial(): raw = load_raw_results() df = analytics._raw_to_analytics_df(raw) ids = df["model_id"].tolist() if not df.empty else [] avail = analytics.available_metric_keys(df) line_default = [k for k in HEATMAP_SCENARIO_KEYS if k in avail] or list(avail) tn = 10 fig_intel = analytics.plot_avg_wer_bars(df, top_n=tn) fig_speed = analytics.plot_speed_bars(df, top_n=tn) fig_hm = analytics.plot_scenario_heatmap(df, line_default, top_n=tn) fig_pareto = analytics.plot_pareto_frontier(df) # fig_radar = analytics.plot_robustness_radar(df, sel) fig_b = analytics.plot_scenario_bar_summary(df, 8) return ( gr.update(value=line_default), fig_intel, fig_speed, fig_hm, fig_pareto, fig_b, ) def _analytics_apply(line_keys, top_n): raw = load_raw_results() df = analytics._raw_to_analytics_df(raw) avail = analytics.available_metric_keys(df) lk = resolve_scenario_metric_keys(line_keys, _METRIC_CHOICES) if not lk: lk = [k for k in HEATMAP_SCENARIO_KEYS if k in avail] or list(avail) tn = int(top_n) if top_n else 10 fig_intel = analytics.plot_avg_wer_bars(df, top_n=tn) fig_speed = analytics.plot_speed_bars(df, top_n=tn) fig_hm = analytics.plot_scenario_heatmap(df, lk, top_n=tn) fig_pareto = analytics.plot_pareto_frontier(df) # fig_radar = analytics.plot_robustness_radar(df, valid_models) fig_b = analytics.plot_scenario_bar_summary(df, tn) return fig_intel, fig_speed, fig_hm, fig_pareto, fig_b # --------------------------------------------------------------------------- # Gradio App # --------------------------------------------------------------------------- _theme = treble_gradio_theme() with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo: gr.HTML(BANNER, elem_id="banner") gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem(tab_label("leaderboard", "Leaderboard"), elem_id="od-benchmark-tab-table", id=0): with gr.Row(): search_box = gr.Textbox( label="Search models", placeholder="Filter by model name -- separate multiple terms with commas, e.g. whisper, nvidia", scale=4, ) version_dropdown = gr.Dropdown( choices=_version_choices(), value=LATEST_VERSION, label="Version", scale=2, min_width=200, ) leaderboard_update_btn = gr.Button( "Update", variant="primary", scale=1, min_width=120 ) toggleable_columns = [c for c in COLS if c not in ALWAYS_VISIBLE_COLS] _default_visible_cols = [ c for c in toggleable_columns if c != AutoEvalColumn.wer_lab_measured.name and c != AutoEvalColumn.wer_lab_simulated.name and c != AutoEvalColumn.wer_moving_low.name and c != AutoEvalColumn.wer_moving_mid.name and c != AutoEvalColumn.wer_moving_high.name ] column_checkboxes = gr.CheckboxGroup( choices=toggleable_columns, value=_default_visible_cols, label="Select columns to display", ) leaderboard_table = gr.components.Dataframe( value=original_df, datatype=TYPES, elem_id="leaderboard-table", interactive=False, visible=True, wrap=False, column_widths=column_widths_for(COLS), ) gr.Markdown( "\* Moving-source evaluations are a beta feature.", elem_classes="markdown-text", ) def _set_active_version(version): """Load the chosen version into the module globals used by filter_main_table.""" global original_df, _active_raw_rows if version: _active_raw_rows = raw_rows_for_version(version) original_df = load_results(version) else: _active_raw_rows = None original_df = load_results() def _leaderboard_refresh(search, cols, version): """Reload results from storage for the selected version, refresh version list.""" from init import invalidate_results_cache try: invalidate_results_cache() choices = _version_choices() valid = {v for _, v in choices} if version not in valid: version = LATEST_VERSION _set_active_version(version) return ( _leaderboard_table_update(search, cols), gr.update(choices=choices, value=version), ) except Exception: return gr.update( value=original_df, column_widths=column_widths_for(list(original_df.columns)), ), gr.update() def _on_version_change(version, search, cols): """Swap the displayed leaderboard to the selected version, keep filters.""" try: _set_active_version(version) return _leaderboard_table_update(search, cols) except Exception: return gr.update( value=original_df, column_widths=column_widths_for(list(original_df.columns)), ) def _on_startup(): """Load leaderboard + next-up list after UI is up (keeps Space prepare step fast).""" global original_df, _active_raw_rows from init import invalidate_results_cache version_update = gr.update() try: invalidate_results_cache() _active_raw_rows = None original_df = load_results() table = _leaderboard_table_update("", _default_visible_cols) version_update = gr.update(choices=_version_choices(), value=LATEST_VERSION) except Exception as exc: table = gr.update( value=original_df, column_widths=column_widths_for(list(original_df.columns)), ) next_html = ( "

" f"Startup warning: {exc}

" ) return table, next_html, version_update try: job_queue.ensure_worker_started() next_html = job_queue.next_up_html() except Exception as exc: next_html = ( "

" f"Queue unavailable: {exc}

" ) return table, next_html, version_update filter_inputs = [search_box, column_checkboxes] search_box.change(_leaderboard_table_update, inputs=filter_inputs, outputs=leaderboard_table) column_checkboxes.change(_leaderboard_table_update, inputs=filter_inputs, outputs=leaderboard_table) version_dropdown.change( fn=_on_version_change, inputs=[version_dropdown, search_box, column_checkboxes], outputs=leaderboard_table, ) leaderboard_update_btn.click( fn=_leaderboard_refresh, inputs=[search_box, column_checkboxes, version_dropdown], outputs=[leaderboard_table, version_dropdown], ) with gr.TabItem(tab_label("submit", "Submit"), elem_id="submit-tab", id=1): gr.Markdown("## Submit a model for evaluation") gr.Markdown( "Paste a Hugging Face model id. The server loads every checkpoint with an automatic backend, so " "**Whisper, IBM Granite speech, Cohere Transcribe, efficient‑speech / custom Whisper variants, " "Wav2Vec2 / HuBERT CTC heads, and SpeechBrain ASR**, plus most other ASR stacks on the Hub are " "supported without a per‑submission choice.\n\n" "Submissions are **queued**; when Hub Jobs are enabled, up to **four** models evaluate in parallel. " "Otherwise jobs run **one at a time** on this Space. " "When moderation is enabled, new requests wait for approval before they run." ) with gr.Row(): model_input = gr.Textbox( label="Model ID", placeholder="e.g. openai/whisper-tiny", scale=3, ) submit_btn = gr.Button("Evaluate", variant="primary", scale=1) status_output = gr.HTML() recipe_input = gr.Dropdown( label="Complex model recipe (optional)", choices=RECIPE_CHOICES, value="", info="Pre-fills setup script, evaluate(), and deps for install-heavy stacks (e.g. Mega-ASR).", ) requirements_input = gr.Textbox( label="Optional extra Python requirements (one per line, requirements.txt format)", placeholder="transformers @ git+https://github.com/huggingface/transformers.git\nsentencepiece", lines=4, max_length=8000, ) setup_script_input = gr.Textbox( label="Optional one-time setup script (shell or Python, runs once per Hub Job)", placeholder="git clone …\npip install -r requirements.txt\npython scripts/download.py", lines=6, max_length=8000, ) script_input = gr.Code( label="Optional custom evaluator (Python)", language="python", lines=16, value=DEFAULT_CUSTOM_EVAL_EXAMPLE, ) gr.Markdown( "Edit the example below: define **`evaluate(file: Path) -> str`** to transcribe one WAV. " "Load your model **once** at module level (as in the Cohere example). " "Use **soundfile** (or numpy) to read each WAV (see the example). " "Hub Jobs disable torchcodec and fall back to librosa if you use " "``transformers.audio_utils.load_audio`` on file paths. " "The function is called **once per audio sample** inside the eval loop. " "**Model size:** expose your loaded model as a module-level `model` " "(or set `NUM_PARAMS = `) so the leaderboard can report its " "parameter count; otherwise size is left blank. " "Put extra Python dependencies in the requirements box above. " "Use the **setup script** for git clone / weight download (runs once before evaluation). " "**No maintainer recipe is needed for a new model:** if your `evaluate()` imports a " "cloned package (e.g. `from MyASR.model import MyASR`), the worker auto-discovers it " "under `/tmp`. For full control, export paths from your setup script via " "`echo \"PYTHONPATH=/tmp/MyASR/src\" >> \"$FFASR_ENV_FILE\"` (also accepts " "`FFASR_IMPORT_PATHS` and any `KEY=VALUE`). " "See [Mega-ASR recipe](docs/recipes/mega_asr.md) for an example. " "Custom evaluators run on a Hub Job **only** after a moderator approves them." ) with gr.Row(): is_gated_input = gr.Checkbox(label="This is a gated repo", value=False) gated_link = gr.HTML(value=_model_hub_page_link("")) model_input.change(fn=_model_hub_page_link, inputs=[model_input], outputs=[gated_link]) notes_input = gr.Textbox( label="Optional free-form notes for moderators", placeholder="Repo link, paper, eval caveats, hardware notes…", lines=2, max_length=4000, ) email_input = gr.Textbox( label="Contact email", placeholder="you@example.com", max_length=254, ) gr.Markdown( "### Contact\n\n" "Questions about submissions, gated repos, or evaluation issues? " "Email [contact@treble.tech](mailto:contact@treble.tech)." ) gr.Markdown("### Next models to evaluate") next_up_panel = gr.HTML(value=_NEXT_UP_PLACEHOLDER, elem_classes="next-up-panel") def _apply_recipe_fields(model_id, recipe_id, reqs, setup, script): from recipes.registry import apply_recipe_to_submission extra, setup_out, custom, _rid = apply_recipe_to_submission( (model_id or "").strip(), (recipe_id or "").strip() or None, reqs or "", setup or "", script or "", ) return extra, setup_out, custom def _submit_and_refresh(model_id, notes, email, reqs, setup, script, recipe, gated): status = submit_model(model_id, notes, email, reqs, setup, script, recipe, gated) try: nxt = job_queue.next_up_html() except Exception as exc: nxt = f"

Could not load queue: {exc}

" return status, nxt recipe_input.change( fn=_apply_recipe_fields, inputs=[ model_input, recipe_input, requirements_input, setup_script_input, script_input, ], outputs=[requirements_input, setup_script_input, script_input], ) submit_btn.click( fn=_submit_and_refresh, inputs=[ model_input, notes_input, email_input, requirements_input, setup_script_input, script_input, recipe_input, is_gated_input, ], outputs=[status_output, next_up_panel], ) with gr.TabItem(tab_label("moderate", "Moderate"), elem_id="moderate-tab", id=2): from benchmark.dataset import CONDITION_UI_CHOICES _MOD_COND_CHOICES = [(lbl, key) for lbl, key in CONDITION_UI_CHOICES] _MOD_COND_DEFAULT = [key for _, key in CONDITION_UI_CHOICES] gr.Markdown("## Moderator access") mod_unlock_feedback = gr.HTML() mod_secret = gr.Textbox( label="Moderator secret", type="password", placeholder="FFASR_MODERATOR_SECRET", ) mod_unlock_btn = gr.Button("Unlock moderator tools", variant="primary") mod_unlocked = gr.State(False) with gr.Column(visible=False) as mod_panel: gr.Markdown( "When FFASR_MODERATION=1 is set, new submissions **wait here** until you approve them. " "Use per-row **Check**, **Approve**, **Retry**, and **Remove** buttons — popups open for " "submission review, dataset selection, and approval.\n\n" "When FFASR_REMOTE_JOBS=1, use **Open Hub Job logs** while a job runs. " "The bucket results/remote_artifacts/ folder only gets a JSON file **after** a successful run.\n\n" "**Retry** re-queues failed, done, or queued jobs (not while running). " "Uncheck datasets in the retry popup to run only selected packed splits; " "other leaderboard WER columns are left unchanged on success. " "A full re-run (all datasets selected) **replaces** the row when the model is already listed." ) gr.Markdown("### Current job progress") mod_progress = gr.HTML( value=job_queue.moderation_locked_placeholder_html(), elem_classes="ffasr-card", ) mod_refresh_tick = gr.Number(value=0, visible=False) mod_refresh = gr.Button("Refresh lists") mod_feedback = gr.HTML() mod_check_job_id = gr.State("") mod_retry_job_id = gr.State("") mod_approve_job_id = gr.State("") with gr.Column(visible=False) as check_panel: check_header = gr.Markdown("### Check submission") check_notes = gr.Textbox( label="Submitter notes", lines=2, interactive=False, ) check_recipe = gr.Dropdown( label="Recipe", choices=RECIPE_CHOICES, value="", ) check_reqs = gr.Textbox( label="Extra Python requirements (one per line)", lines=4, max_length=8000, ) check_setup = gr.Textbox( label="One-time setup script (shell or Python)", lines=6, max_length=8000, ) check_script = gr.Code( label="Custom evaluator (Python)", language="python", lines=16, ) with gr.Row(): check_save_btn = gr.Button("Save", variant="primary") check_cancel_btn = gr.Button("Cancel") with gr.Column(visible=False) as retry_panel: retry_header = gr.Markdown("### Retry job") retry_eval_conditions = gr.CheckboxGroup( choices=_MOD_COND_CHOICES, value=_MOD_COND_DEFAULT, label="Datasets to evaluate (all selected = full benchmark)", ) with gr.Row(): retry_confirm_btn = gr.Button( "Confirm retry", variant="primary" ) retry_cancel_btn = gr.Button("Cancel") with gr.Column(visible=False) as approve_panel: approve_header = gr.Markdown("### Approve job") approve_run_custom = gr.Checkbox( label="Use submitter's custom evaluate() function (if provided)", value=False, ) approve_eval_conditions = gr.CheckboxGroup( choices=_MOD_COND_CHOICES, value=_MOD_COND_DEFAULT, label="Datasets to evaluate (all selected = full benchmark)", ) with gr.Row(): approve_confirm_btn = gr.Button("Approve", variant="primary") approve_cancel_btn = gr.Button("Cancel") _MOD_POPUP_HIDE = ( gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), ) def _mod_bump_tick(tick): return int(tick or 0) + 1 def _mod_feedback_html(ok: bool, msg: str) -> str: color = "green" if ok else "red" return f"

{msg}

" def _mod_open_check(jid: str, secret: str, unlocked: bool): hidden = gr.update(visible=False) if not unlocked: return ( gr.update(visible=False), hidden, hidden, "", "### Check submission", "", "", "", "", "", ) job = job_queue.peek_job(str(jid).strip()) if not job: return ( gr.update(visible=False), hidden, hidden, "", "### Check submission", "", "", "", "", "", ) return ( gr.update(visible=True), hidden, hidden, job.id, f"### Check submission: `{job.model_id}`", job.submission_notes or "", job.recipe_id or "", job.extra_requirements or "", job.setup_script or "", job.custom_script or "", ) def _mod_open_retry(jid: str, secret: str, unlocked: bool): hidden = gr.update(visible=False) if not unlocked: return ( hidden, gr.update(visible=False), hidden, "", "### Retry job", ) job = job_queue.peek_job(str(jid).strip()) if not job: return ( hidden, gr.update(visible=False), hidden, "", "### Retry job", ) return ( hidden, gr.update(visible=True), hidden, job.id, f"### Retry: `{job.model_id}` (`{job.id}`)", ) def _mod_open_approve(jid: str, secret: str, unlocked: bool): hidden = gr.update(visible=False) if not unlocked: return ( hidden, hidden, gr.update(visible=False), "", "### Approve job", False, ) job = job_queue.peek_job(str(jid).strip()) if not job: return ( hidden, hidden, gr.update(visible=False), "", "### Approve job", False, ) has_script = bool( (job.custom_script or "").strip() or (job.setup_script or "").strip() or (job.recipe_id or "").strip() ) return ( hidden, hidden, gr.update(visible=True), job.id, f"### Approve: `{job.model_id}` (`{job.id}`)", has_script, ) def _mod_reject_row(jid, secret, unlocked, tick): if not unlocked: return ( "

Unlock moderator tools first.

", tick, job_queue.progress_html(), ) ok, msg = job_queue.reject_job(str(jid).strip(), secret) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), job_queue.progress_html(), ) def _mod_remove_row(jid, secret, unlocked, tick): if not unlocked: return ( "

Unlock moderator tools first.

", tick, job_queue.progress_html(), ) ok, msg = job_queue.remove_job_entry(str(jid).strip(), secret) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), job_queue.progress_html(), ) def _mod_save_check(jid, secret, unlocked, tick, recipe, reqs, setup, script): if not unlocked: return ( "

Unlock moderator tools first.

", tick, *_MOD_POPUP_HIDE, ) ok, msg = job_queue.update_job_script_and_requirements( str(jid).strip(), secret, extra_requirements=reqs or "", setup_script=setup or "", custom_script=script or "", recipe_id=(recipe or "").strip(), ) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), *_MOD_POPUP_HIDE, ) def _mod_confirm_retry(jid, secret, unlocked, tick, eval_conds): if not unlocked: return ( "

Unlock moderator tools first.

", tick, *_MOD_POPUP_HIDE, job_queue.progress_html(), ) ok, msg = job_queue.retry_failed_job( str(jid).strip(), secret, eval_conditions=list(eval_conds or []), ) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), *_MOD_POPUP_HIDE, job_queue.progress_html(), ) def _mod_confirm_approve( jid, secret, unlocked, tick, run_custom, eval_conds ): if not unlocked: return ( "

Unlock moderator tools first.

", tick, *_MOD_POPUP_HIDE, job_queue.progress_html(), ) ok, msg = job_queue.approve_job( str(jid).strip(), secret, run_custom_script=bool(run_custom), eval_conditions=list(eval_conds or []), ) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), *_MOD_POPUP_HIDE, job_queue.progress_html(), ) def _mod_cancel_popup(tick): return tick, *_MOD_POPUP_HIDE def _mod_retry_all(secret, unlocked, tick): if not unlocked: return ( "

Unlock moderator tools first.

", tick, job_queue.progress_html(), ) ok, msg = job_queue.retry_all_eligible_jobs(secret) return ( _mod_feedback_html(ok, msg), _mod_bump_tick(tick), job_queue.progress_html(), ) def _mod_import_artifact( secret, artifact_ref, replace_existing, import_notes, unlocked, tick ): if not unlocked: return ( "

Unlock moderator tools first.

", tick, ) ok, msg = job_queue.import_artifact_to_leaderboard( artifact_ref, secret, replace_existing=bool(replace_existing), submission_notes=import_notes or "", ) return (_mod_feedback_html(ok, msg), _mod_bump_tick(tick)) def _mod_refresh_lists(secret, unlocked, tick): if not unlocked: return tick, job_queue.moderation_locked_placeholder_html() ok, msg = job_queue.verify_moderator_secret(secret) if not ok: return tick, job_queue.moderation_locked_placeholder_html() return _mod_bump_tick(tick), job_queue.progress_html() gr.Markdown("### Pending moderation") @gr.render(inputs=[mod_refresh_tick]) def _render_pending_jobs(_tick): try: jobs = job_queue.pending_jobs_for_render() except Exception as exc: gr.Markdown(f"*Queue unavailable: {exc}*") return if not jobs: gr.Markdown("*No jobs awaiting approval.*") return for job in jobs: jid = job["id"] row_cls = job_queue.job_row_elem_classes(job["status"]) gated = " · **gated**" if job["is_gated"] else "" script_bit = ( " · custom/setup/recipe" if job["has_custom_script"] else "" ) extras = ( f"{job['req_count']} extra req(s)" if job["req_count"] else "no extra reqs" ) email_bit = ( f" · 📧 `{job['contact_email']}`" if job.get("contact_email") else " · 📧 *(none)*" ) with gr.Row(elem_classes=row_cls): gr.Markdown( f"**`{jid}`** · `{job['model_id']}` · " f"`{job['family_id']}`{gated}{script_bit} · {extras}{email_bit} · " f"{job['notes_preview'] or '—'} · {job['created_at']}", elem_classes="ffasr-job-info", ) check_btn = gr.Button("Check", size="sm") approve_btn = gr.Button( "Approve", size="sm", variant="primary" ) reject_btn = gr.Button("Reject", size="sm", variant="stop") check_btn.click( fn=_mod_open_check, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_check_job_id, check_header, check_notes, check_recipe, check_reqs, check_setup, check_script, ], ) approve_btn.click( fn=_mod_open_approve, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_approve_job_id, approve_header, approve_run_custom, ], ) reject_btn.click( fn=_mod_reject_row, inputs=[gr.State(jid), mod_secret, mod_unlocked, mod_refresh_tick], outputs=[mod_feedback, mod_refresh_tick, mod_progress], ) gr.Markdown("### Recent job activity") retry_all_btn = gr.Button( "Retry ALL eligible jobs (full benchmark)", variant="secondary", ) gr.Markdown( "Retry ALL re-queues every failed / done / queued " "job against the full dataset set (all packed conditions). Running, dispatching, and " "pending-moderation jobs are skipped. Successful re-runs replace the existing " "leaderboard row for each model." ) @gr.render(inputs=[mod_refresh_tick]) def _render_recent_jobs(_tick): try: jobs = job_queue.recent_jobs_for_render(30) except Exception as exc: gr.Markdown(f"*Queue unavailable: {exc}*") return if not jobs: gr.Markdown("*No job history loaded yet.*") return for job in jobs: jid = job["id"] err = job["error"] or "N/A" hub = job["hub_link_html"] or "" row_cls = job_queue.job_row_elem_classes(job["status"]) email_bit = ( f" · 📧 `{job['contact_email']}`" if job.get("contact_email") else " · 📧 *(none)*" ) info_md = ( f"**`{jid}`** · `{job['model_id']}` · " f"**{job['status']}**{email_bit} · {err} · {job['updated_at']}" ) if hub: info_md += f" · {hub}" with gr.Row(elem_classes=row_cls): gr.Markdown( info_md, elem_classes="ffasr-job-info", ) check_btn = gr.Button("Check", size="sm") if job["can_retry"]: retry_btn = gr.Button( "Retry", size="sm", variant="secondary" ) if job["can_remove"]: remove_btn = gr.Button("Remove", size="sm", variant="stop") check_btn.click( fn=_mod_open_check, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_check_job_id, check_header, check_notes, check_recipe, check_reqs, check_setup, check_script, ], ) if job["can_retry"]: retry_btn.click( fn=_mod_open_retry, inputs=[gr.State(jid), mod_secret, mod_unlocked], outputs=[ check_panel, retry_panel, approve_panel, mod_retry_job_id, retry_header, ], ) if job["can_remove"]: remove_btn.click( fn=_mod_remove_row, inputs=[ gr.State(jid), mod_secret, mod_unlocked, mod_refresh_tick, ], outputs=[mod_feedback, mod_refresh_tick, mod_progress], ) gr.Markdown("### Import result from bucket artifact") gr.Markdown( "If a Hub Job finished but the leaderboard CSV was not updated, paste the artifact " "file name from results/remote_artifacts/ (e.g. a1b2c3d4.json) " "to merge WER/RTFx into the CSV." ) mod_artifact_ref = gr.Textbox( label="Artifact JSON file name or bucket path", placeholder="e.g. a1b2c3d4.json or results/remote_artifacts/a1b2c3d4.json", ) mod_replace_existing = gr.Checkbox( label="Replace existing leaderboard row for this model", value=False, ) mod_import_notes = gr.Textbox( label="Optional submission notes override", placeholder="Leave empty to use notes from the matched queue job, if any", lines=1, max_length=4000, ) import_artifact_btn = gr.Button("Import artifact to CSV", variant="secondary") def _mod_unlock(secret: str): ok, msg = job_queue.verify_moderator_secret(secret) if not ok: return ( False, gr.update(visible=False), f"

{msg}

", 0, ) return ( True, gr.update(visible=True), "

Moderator tools unlocked for this session.

", 1, ) def _mod_progress_gated(unlocked: bool): if not unlocked: return job_queue.moderation_locked_placeholder_html() return job_queue.progress_html() mod_unlock_btn.click( fn=_mod_unlock, inputs=[mod_secret], outputs=[mod_unlocked, mod_panel, mod_unlock_feedback, mod_refresh_tick], ) mod_refresh.click( fn=_mod_refresh_lists, inputs=[mod_secret, mod_unlocked, mod_refresh_tick], outputs=[mod_refresh_tick, mod_progress], ) check_save_btn.click( fn=_mod_save_check, inputs=[ mod_check_job_id, mod_secret, mod_unlocked, mod_refresh_tick, check_recipe, check_reqs, check_setup, check_script, ], outputs=[ mod_feedback, mod_refresh_tick, check_panel, retry_panel, approve_panel, ], ) check_cancel_btn.click( fn=_mod_cancel_popup, inputs=[mod_refresh_tick], outputs=[mod_refresh_tick, check_panel, retry_panel, approve_panel], ) retry_confirm_btn.click( fn=_mod_confirm_retry, inputs=[ mod_retry_job_id, mod_secret, mod_unlocked, mod_refresh_tick, retry_eval_conditions, ], outputs=[ mod_feedback, mod_refresh_tick, check_panel, retry_panel, approve_panel, mod_progress, ], ) retry_cancel_btn.click( fn=_mod_cancel_popup, inputs=[mod_refresh_tick], outputs=[mod_refresh_tick, check_panel, retry_panel, approve_panel], ) approve_confirm_btn.click( fn=_mod_confirm_approve, inputs=[ mod_approve_job_id, mod_secret, mod_unlocked, mod_refresh_tick, approve_run_custom, approve_eval_conditions, ], outputs=[ mod_feedback, mod_refresh_tick, check_panel, retry_panel, approve_panel, mod_progress, ], ) approve_cancel_btn.click( fn=_mod_cancel_popup, inputs=[mod_refresh_tick], outputs=[mod_refresh_tick, check_panel, retry_panel, approve_panel], ) retry_all_btn.click( fn=_mod_retry_all, inputs=[mod_secret, mod_unlocked, mod_refresh_tick], outputs=[mod_feedback, mod_refresh_tick, mod_progress], ) import_artifact_btn.click( fn=_mod_import_artifact, inputs=[ mod_secret, mod_artifact_ref, mod_replace_existing, mod_import_notes, mod_unlocked, mod_refresh_tick, ], outputs=[mod_feedback, mod_refresh_tick], ) with gr.TabItem(tab_label("analysis", "Analysis"), elem_id="analysis-tab", id=3) as analysis_tab: gr.Markdown("## Scenario analysis") gr.Markdown( "Charts support standard Plotly interaction (legend, zoom, pan, hover).\n\n" "**Leaderboard order** is by **Average WER** (mean across checked scenario columns; lower is better).\n\n" "**Average WER** ranks the top N models by mean WER across live scenarios (lower is better). " "**Speed** ranks the top N models by RTFx (audio sec / inference sec, higher is better). " "Bars are colored per company (e.g. all NVIDIA models share one color).\n\n" "**WER heatmap** shows WER (%) by model and scenario for the top N models (sorted by Average WER); " "lower WER corresponds to greener cells.\n\n" "**Pareto Front** plots Average WER on X and RTFx on Y (log scale). Models on the frontier are " "labeled and connected by the dashed blue line; other models render as faint dots.\n\n" "**WER by scenario** compares raw WER across conditions for the selected top models." ) # with gr.Row(): # an_models = gr.Dropdown( # label="Models on radar (up to 8)", # choices=[], # value=None, # multiselect=True, # max_choices=8, # scale=2, # ) an_topn = gr.Slider( 5, 80, value=10, step=1, label="Top‑N models (WER bars, heatmap & grouped WER)", scale=1, ) an_line_metrics = gr.CheckboxGroup( label="Scenarios shown in the heatmap", choices=_METRIC_CHOICES, value=[k for k in HEATMAP_SCENARIO_KEYS if k in LIVE_SCENARIO_KEYS], ) an_apply = gr.Button("Apply / refresh charts", variant="primary") gr.Markdown( "### Pareto Front: Average WER vs RTFx\n\n" "Models on the Pareto frontier achieve the best trade-off between WER " "and speed (RTFx). Names are shown for frontier models; hover over " "other points to see their names." ) an_plot_pareto = gr.Plot( label="Pareto Front: Average WER vs RTFx", elem_id="analysis-pareto-plot", ) with gr.Row(): an_plot_intelligence = gr.Plot(label="Average WER") an_plot_speed = gr.Plot(label="Speed") an_plot_compare = gr.Plot(label="WER heatmap") # with gr.Row(): # an_plot_radar = gr.Plot(label="Robustness radar") an_plot_bar = gr.Plot(label="WER by scenario") an_apply.click( fn=_analytics_apply, inputs=[ an_line_metrics, an_topn, ], outputs=[ an_plot_intelligence, an_plot_speed, an_plot_compare, an_plot_pareto, an_plot_bar, ], ) _analytics_outputs = [ an_line_metrics, an_plot_intelligence, an_plot_speed, an_plot_compare, an_plot_pareto, an_plot_bar, ] _analytics_loaded = gr.State(False) def _analytics_on_tab_select(loaded: bool): """Load charts once the Analysis tab is visible so Plotly gets the real width.""" if loaded: return (loaded,) + (gr.skip(),) * 6 ( metrics_upd, fig_intel, fig_speed, fig_hm, fig_pareto, fig_b, ) = _analytics_initial() return ( True, metrics_upd, fig_intel, fig_speed, fig_hm, fig_pareto, fig_b, ) analysis_tab.select( fn=_analytics_on_tab_select, inputs=[_analytics_loaded], outputs=[_analytics_loaded, *_analytics_outputs], ) with gr.TabItem(tab_label("examples", "Examples"), elem_id="examples-tab", id=4): examples_renderer.ensure_assets() _dry = examples_renderer.sample_paths("dry") _hi = examples_renderer.sample_paths("high_snr") _mid = examples_renderer.sample_paths("mid_snr") _lo = examples_renderer.sample_paths("low_snr") _scene = examples_renderer.scene_image_path() def _example_audio(path: str | None, label: str): return gr.Audio( value=path, type="filepath", label=label, interactive=False, editable=False, show_download_button=False, show_share_button=False, waveform_options=gr.WaveformOptions( waveform_color="#5c6670", waveform_progress_color=TREBLE_TEAL, show_recording_waveform=True, ), elem_classes="examples-audio", ) with gr.Column(elem_classes="examples-scene-wrap"): gr.Image( value=_scene, type="filepath", label="Treble scene", interactive=False, show_label=False, show_download_button=False, show_fullscreen_button=False, height=300, container=False, ) gr.Markdown("### Near Field Speech") with gr.Row(): _example_audio(_dry[0], "Example") gr.Markdown("### High SNR") with gr.Row(): _example_audio(_hi[0], "Example") gr.Markdown("### Mid SNR") with gr.Row(): _example_audio(_mid[0], "Example") gr.Markdown("### Low SNR") with gr.Row(): _example_audio(_lo[0], "Example") with gr.TabItem(tab_label("about", "About"), elem_id="about-tab", id=5): gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text") _footer_html = ( "
" "Leaderboard data loads from storage after the page opens · " "Evaluation runs on Hub Jobs against a held-out set · " "" "Models loaded from the Hugging Face Hub" "
" ) gr.HTML(_footer_html) with gr.Row(): with gr.Accordion("Citation", open=False): gr.Textbox( value=CITATION_TEXT, lines=7, label="Copy the BibTeX snippet to cite this source", elem_id="citation-button", show_label=True, ) # Periodic queue refresh (Timer added in Gradio 4.44). Avoid demo.load(..., every=); unsupported on some builds. if hasattr(gr, "Timer"): with gr.Row(visible=False): _next_up_timer = gr.Timer(15) _progress_timer = gr.Timer(2) _next_up_timer.tick(fn=job_queue.next_up_html, outputs=[next_up_panel]) _progress_timer.tick(fn=_mod_progress_gated, inputs=[mod_unlocked], outputs=[mod_progress]) demo.load(fn=_on_startup, outputs=[leaderboard_table, next_up_panel, version_dropdown]) def _launch_gradio_demo() -> None: """ Avoid OSError: Cannot find empty port in range: 7860-7860. Hugging Face / tooling sometimes sets ``GRADIO_NUM_PORTS=1``, so only 7860 is tried. After a Space restart (e.g. post pip install), that port can still be in use briefly. We drop an overly small ``GRADIO_NUM_PORTS`` so Gradio's default port sweep applies, then retry with short backoff. """ raw_np = os.environ.get("GRADIO_NUM_PORTS", "").strip() if raw_np.isdigit() and int(raw_np) < 10: os.environ.pop("GRADIO_NUM_PORTS", None) server_name = os.environ.get("GRADIO_SERVER_NAME", "0.0.0.0") last_err: OSError | None = None for attempt in range(24): try: demo.launch(server_name=server_name, server_port=None) return except OSError as e: last_err = e msg = str(e).lower() if "cannot find empty port" not in msg and "address already in use" not in msg: raise time.sleep(0.45 + attempt * 0.12) if last_err is not None: raise last_err _launch_gradio_demo()