"""UI text, CSS, and configuration constants for the FFASR leaderboard.""" import os from pathlib import Path DIR_OUTPUT_REQUESTS = Path("requested_models") # Space id shown in the ``gradio_client`` example in the About tab. Override with # ``FFASR_SPACE_ID`` when the Space is moved to another user / organisation. SPACE_ID = os.environ.get("FFASR_SPACE_ID", "treble-technologies/FFASR_Leaderboard-storage") # Browser tab / , passed to gr.Blocks(title=...). APP_TITLE = "FFASR: ASR Evaluation Leaderboard" # Emoji prefixes for the top tab buttons. Kept in one place so labels stay consistent. TAB_ICONS = { "leaderboard": "🏆", "submit": "🚀", "moderate": "đŸ›Ąïž", "analysis": "📊", "examples": "🎧", "about": "â„č", } def tab_label(key: str, name: str) -> str: """`TAB_ICONS[key]` + `name` with a non-breaking space so the icon hugs the label.""" icon = TAB_ICONS.get(key, "") return f"{icon}\u00a0{name}" if icon else name ########################## # Text definitions # ########################## # Banner: title.png (embedded as a data URI in branding.py to avoid binary commits) # is the wordmark. A visually hidden <h1> keeps "FFASR" for screen readers / SEO. from branding import TITLE_IMAGE_DATA_URI as _TITLE_IMAGE_DATA_URI BANNER = ( "<div id='ffasr-banner' class='ffasr-banner'>" "<h1 class='ffasr-sr-only'>FFASR Leaderboard</h1>" f"<img class='ffasr-title-img' src='{_TITLE_IMAGE_DATA_URI}' " "alt='FFASR Leaderboard — Far-Field Automatic Speech Recognition' />" "<div class='ffasr-subtitle'>Far‑Field Automatic Speech Recognition, multi‑condition leaderboard</div>" "<div class='ffasr-badges'>" "<span class='ffasr-badge'>Far‑field ASR</span>" "<span class='ffasr-badge'>Noise & reverberation</span>" "<span class='ffasr-badge'>Measured & simulated RIRs</span>" "<span class='ffasr-badge'>WER · RTFx</span>" "</div>" "</div>" ) INTRODUCTION_TEXT = ( "**FFASR** benchmarks speech‑recognition models on audio representative of **far‑field** use (noisy rooms, " "reverberation, adverse SNRs), not only studio‑dry speech. Every model runs on the **same held‑out set** " "and the **same text normalization**, so numbers are directly comparable. " "The simulated room impulse responses are similar to those in the " "[Treble10 dataset](https://huggingface.co/datasets/treble-technologies/Treble10-RIR).\n\n" "Each submission reports **WER** for nine scenarios (**Near Field Speech**, **Lab Measured**, **Lab Simulated**, " "**High SNR**, **Mid SNR**, **Low SNR**, and three **Moving** SNR splits) plus **RTFx** and **parameter count**. " "The leaderboard ranks models by **Average WER** over the scenario columns you have checked (lower is better). " "The Analysis tab visualises the **Pareto Front of Average WER vs RTFx** for WER/speed trade-offs.\n\n" "Paste a Hugging Face model id in the **Submit** tab; scoring runs server‑side and evaluation audio is not exposed " "to submitters. The **Analysis** tab provides WER rankings, heatmaps, speed views, " "and scenario‑wise comparisons.\n\n" "The FFASR Leaderboard is powered by acoustic data simulated with Treble Technologies" ) CITATION_TEXT = """@misc{ffasr_leaderboard_2026, title = {FFASR Evaluation Leaderboard}, year = {2026}, note = {Multi-condition ASR evaluation benchmark (clean / noisy / reverberant)}, } """ ABOUT_TEXT = """ ## About FFASR Far‑field speech recognition degrades under reverberation, noise, and moving‑source conditions. While speech enhancement and far‑field fine‑tuning can improve robustness, evaluating systems across many acoustic scenarios is difficult to do with measured data alone. Room acoustics simulation has proven to be a practical and scalable approach to train far‑field speech recognition systems [1, 2]. To quantify the sim‑to‑real gap, we measure three room conditions, yielding 48 source–receiver combinations, and simulate the same configurations. The small performance gap between the equivalent measured and simulated sets supports the validity of the other simulated conditions for evaluating far‑field performance. ### Scenario columns (WER · lower is better) The leaderboard reports WER on nine complementary conditions: * **Near Field Speech**: dry, anechoic speech with very little reverberation (clean baseline). * **Lab Measured**: measured RIRs from the Treble office room conditions (sim‑to‑real reference; hidden by default). * **Lab Simulated**: Treble‑simulated RIRs matching the measured office room conditions (hidden by default). * **High / Mid / Low SNR**: simulated far‑field mixtures grouped by SNR (high > 14 dB, mid 8–12 dB, low < 6 dB). * **Moving Low / Mid / High SNR** *(beta)*: moving talker with varying geometry at different SNRs (hidden by default). **Lower WER is always better.** Every submission uses the **same private held‑out set**, reference transcripts, and Whisper‑style text normalization (lowercase, punctuation/whitespace normalized). **Leaderboard order** is by **Average WER** over the scenario columns currently checked in the column selector (lower is better). By default the aggregate averages the WER of **Near Field Speech (dry), High SNR, Mid SNR, and Low SNR**; unchecking or checking a column updates the recomputed average. ## Leaderboard columns | Column | Meaning | |---|---| | **Avg WER (%)** | Mean WER (percent) over **checked** scenario columns (lower is better). Primary ranking key. | | **Near Field Speech** | WER (%) on dry / anechoic speech. | | **Lab Measured** | WER (%) on measured Treble office room conditions (hidden by default). | | **Lab Simulated** | WER (%) on simulated Treble office room conditions (hidden by default). | | **High / Mid / Low SNR** | WER (%) on simulated far‑field mixtures at the named SNR. | | **Moving Low / Mid / High SNR *** | WER (%) on moving-source splits *(beta)*. | | **RTFx** | *Audio seconds Ă· inference seconds* at batch size 1. Higher is faster; >1 means faster than real time. | | **Params (B)** | Trainable parameters (billions). | RTFx depends on the hardware this Space uses. Treat it as a **relative comparison between submissions**, not an absolute throughput number. ## Dataset All simulation datasets are created by convolving dry speech recordings with room impulse responses (RIRs) simulated with Treble Technologies' SDK. For the dry speech, 2000 original samples (~15 s each) are measured in an anechoic chamber. Recording original speech ensures no test‑set contamination for the evaluated models and that the signals carry very little reverberation. For the RIRs, a hybrid wave‑based and geometrical‑acoustics approach simulates acoustic scenes in 14 fully‑furnished rooms spanning 20 to 470 mÂł, including bathrooms, living rooms, meeting rooms, offices, classrooms, and restaurant spaces. The simulation captures phenomena such as diffraction, scattering, interference, and modal behavior that simpler simulations miss. They are similar to the RIRs of the Treble10 dataset [3]. Each acoustic scene includes one target speaker and up to three noise sources from the AID dataset [4], always including a transient noise (e.g. coughing) and a continuous noise (e.g. HVAC). Pink microphone noise is also added to each scene for further realism. Noise conditions are grouped into low, mid, and high SNRs, computed after rendering from the convolved speech and noise signals. The **measured** dataset is collected from a room in the Treble office under three considerably different acoustic conditions. (1) Piles of 40 × 40 cm concrete bricks placed around the room introduce geometric complexity, scattering, and diffraction. (2) The room is furnished with typical objects (a couch, an armchair, a shelf, and a table) mimicking a living room. (3) The room is treated with a varying number of acoustic absorbers, yielding three sub‑conditions with reverberation times of 0.3 s, 0.6 s, and 0.9 s. After the measurements, all room conditions are modeled within the Treble SDK to simulate matching synthetic RIRs. In summary, the following datasets are prepared: | Dataset | Duration | |---|---| | Dry speech | 8 hours | | High / Mid / Low SNR | 8 hours each | | Moving sources | 8 hours each (same config as high/mid/low SNR) | | Treble office rooms (simulated and measured) | 2 hours each | The datasets are **privately hosted on the Hugging Face Hub** to prevent test‑set contamination while participants prepare their solutions. Participants submit via this Hugging Face Space, specifying the model checkpoint, software requirements, and custom evaluation code (which can include speech enhancement). From the maintainer page, the challenge coordinators launch evaluation on a standardized hardware setup (important for latency comparisons). All models are evaluated on an **NVIDIA L4 GPU** using Hugging Face Jobs [8]. Participants may use the Treble10 datasets [3] or room‑acoustics simulators of their choice [5, 6, 7] for preparing and/or evaluating their solutions. External datasets and transfer learning are allowed. We encourage participants to clearly document their datasets and/or publish the source code that generates them to improve reproducibility. ## How to submit 1. Open the **Submit** tab and paste a Hugging Face model id (e.g. `openai/whisper-tiny`). 2. Optionally add notes (gated repos, custom inference, eval caveats) and custom evaluation code. 3. Your request is **queued** and evaluated in the background (up to four parallel Hub Jobs when enabled). Refresh the **Leaderboard** tab after the job finishes. Submissions are loaded with an automatic backend (SpeechBrain → Granite speech → `transformers` ASR pipeline → universal seq2seq loader → CTC). Gated Hub repos require a token with accepted license on the Space. ## Evaluation method & metrics Performance on each acoustic condition is measured with **word error rate (WER)**. An aggregate score is computed by averaging the WER of dry speech, high SNR, mid SNR, and low SNR. We also compute the **inverse real‑time factor (RTFx)** of each system (seconds of audio inferred divided by compute time in seconds) at **batch size 1** to quantify latency. The balance between performance and latency is of practical importance, as many far‑field systems may run as part of a real‑time interaction with users. A Pareto plot helps visualize this tradeoff (a single metric balancing Avg WER and RTFx, e.g. area under the curve, could also be considered). Scoring is performed by [`benchmark/dataset.py`](https://huggingface.co/spaces/treble-technologies/ffasr/blob/main/benchmark/dataset.py), which calls `evaluate_condition_wer_timed` for each inference to compute the per‑condition WER and timing used for RTFx. ### Word Error Rate (WER) WER is the fraction of reference words that are substituted, inserted, or deleted: ``` WER = (S + I + D) / N ``` Example: one substitution and one deletion in six reference words → WER ≈ 0.33 (33%). | WER | Typical quality | |---|---| | < 5% | Excellent | | 5–10% | Good / production‑ready | | 10–30% | Usable with post‑processing | | > 30% | Poor to unreliable | See the **Examples** tab for **Near Field Speech**, **High SNR**, **Mid SNR**, and **Low SNR** audio clips plus a Treble scene screenshot. ## Privacy * Only the model id, optional submitter notes, and produced scores are stored. * Custom `evaluate()` scripts run on **Hub Jobs** after moderator approval when moderation is enabled, not directly on this Space UI process. ## References 1. Kim, Chanwoo, et al. "Generation of Large‑Scale Simulated Utterances in Virtual Rooms to Train Deep‑Neural Networks for Far‑Field Speech Recognition in Google Home." *Interspeech*, 2017. 2. Doire, ClĂ©ment and Eric Bezzam. "Reproducing On‑Device Data Accurately for Private‑by‑Design Voice Control." 2023. <https://tech-blog.sonos.com/posts/reproducing-on-device-data-accurately-for-private-by-design-voice-control/> 3. Mullins, S.S., Götz, G., Bezzam, E., Zheng, S. and Nielsen, D.G., 2025. *Treble10: A high‑quality dataset for far‑field speech recognition, dereverberation, and enhancement.* arXiv preprint arXiv:2510.23141. 4. Götz, Philipp, et al. "AID: Open‑source anechoic interferer dataset." *IEEE IWAENC*, 2022. 5. Scheibler, Robin, Eric Bezzam, and Ivan Dokmanić. "Pyroomacoustics: A python package for audio room simulation and array processing algorithms." *IEEE ICASSP*, 2018. 6. Diaz‑Guerra, D., Miguel, A., and Beltran, J. R. "gpuRIR: A python library for room impulse response simulation with GPU acceleration." *Multimed. Tools Appl.*, vol. 80, no. 4, pp. 5653–5671, 2021. 7. Treble SDK. <https://www.treble.tech/software-development-kit> 8. Hugging Face Jobs. <https://huggingface.co/docs/huggingface_hub/en/guides/jobs> ## Programmatic submission Use the Gradio client against this Space (see the Submit tab for the current API signature): ```python from gradio_client import Client client = Client("__SPACE_ID__") # Illustrative: pass all fields the Submit tab exposes. result = client.predict( "openai/whisper-tiny", # model id "", # optional notes "you@example.com", # contact email (required) "", # extra requirements "", # setup script "", # custom evaluator "", # recipe id False, # gated repo api_name="/submit_model", ) print(result) ``` """ # Substitute the (env-overridable) current Space id into the docs. ABOUT_TEXT = ABOUT_TEXT.replace("__SPACE_ID__", SPACE_ID) # Audio condition folders are an internal detail of the evaluation pipeline and are intentionally not surfaced # in the UI copy above. AUDIO_CONDITIONS = { "clean": {"folder": "dry_wavs", "label": "WER Clean"}, "noisy": {"folder": "dry_noisy_wavs", "label": "WER Noisy"}, "reverberant": {"folder": "wavs", "label": "WER Reverberant"}, } # Treble brand palette (dark UI — teal accent instead of purple/indigo) TREBLE_TEAL = "#38BFA1" TREBLE_TEAL_RGB = "56, 191, 161" TREBLE_MINT = "#3DFFA3" TREBLE_CYAN = "#2D9BF0" TREBLE_BG = "#121212" TREBLE_SURFACE = "#1A1A1E" TREBLE_BORDER = "rgba(160, 160, 160, 0.18)" TREBLE_TEXT_MUTED = "#A0A0A0" def treble_gradio_theme(): """Dark Gradio theme with Treble teal primary (replaces indigo/purple).""" import gradio as gr treble_primary = gr.themes.colors.Color( name="treble", c50="#e8faf6", c100="#c5f0e8", c200="#9ae4d4", c300="#6fd7c0", c400="#52cdb0", c500=TREBLE_TEAL, c600="#2da88a", c700="#238a72", c800="#1a6c5a", c900="#124e43", c950="#0a3029", ) return ( gr.themes.Base( primary_hue=treble_primary, secondary_hue="cyan", neutral_hue="gray", ) .set( # Light mode: faint gray page background with white blocks so boxes # pop, mirroring the dark theme's body/surface contrast. Dark mode # keeps the Treble dark palette. body_background_fill="#eef0f3", body_background_fill_dark=TREBLE_BG, block_background_fill="#ffffff", block_background_fill_dark=TREBLE_SURFACE, block_border_color=f"rgba({TREBLE_TEAL_RGB}, 0.18)", block_border_color_dark=f"rgba({TREBLE_TEAL_RGB}, 0.14)", border_color_primary=f"rgba({TREBLE_TEAL_RGB}, 0.28)", border_color_primary_dark=f"rgba({TREBLE_TEAL_RGB}, 0.22)", color_accent=TREBLE_TEAL, color_accent_soft=f"rgba({TREBLE_TEAL_RGB}, 0.14)", link_text_color=TREBLE_TEAL, link_text_color_hover=TREBLE_MINT, link_text_color_active=TREBLE_CYAN, link_text_color_dark=TREBLE_TEAL, link_text_color_hover_dark=TREBLE_MINT, button_primary_background_fill="*primary_500", button_primary_background_fill_hover="*primary_400", button_primary_text_color="#0d1412", button_primary_text_color_hover="#0d1412", button_large_radius="999px", button_medium_radius="999px", button_small_radius="999px", block_radius="12px", input_background_fill="#f1f3f6", input_background_fill_dark="#141418", input_border_color="rgba(15, 23, 42, 0.12)", input_border_color_focus=TREBLE_TEAL, # Leaderboard / dataframe zebra striping. Light mode collapsed to a # single color before; give odd/even distinct light shades. Dark # mode keeps its existing alternating fills. table_odd_background_fill="#ffffff", table_even_background_fill="#f0f2f5", # Input/component titles: black + bold in light mode so headings like # "Complex model recipe (optional)" stand out. Dark mode keeps its # default light title color; bold applies to both. block_title_text_color="#111111", block_title_text_weight="600", block_label_text_color="#111111", block_label_text_weight="600", ) ) LEADERBOARD_CSS = """ /* Hide Gradio footer logo */ footer { display: none !important; } /* ---- Banner (gradient title + subtitle + badges) ---- */ .ffasr-banner { text-align: center; padding: 1.4rem 0 0.8rem 0; border-bottom: 1px solid var(--border-color-primary, rgba(160, 160, 160, 0.18)); margin-bottom: 0.6rem; } /* Visually hide the <h1> but keep it for screen readers / SEO. */ .ffasr-sr-only { position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; overflow: hidden; clip: rect(0, 0, 0, 0); white-space: nowrap; border: 0; } .ffasr-title-img { display: block; width: 100%; max-width: 880px; height: auto; margin: 0 auto; border-radius: 12px; } .ffasr-subtitle { margin-top: 0.35rem; font-size: 1rem; opacity: 0.72; color: __TREBLE_TEXT_MUTED__; } .ffasr-badges { margin-top: 0.7rem; display: flex; justify-content: center; flex-wrap: wrap; gap: 0.4rem; } .ffasr-badge { font-size: 0.78rem; padding: 0.18rem 0.65rem; border-radius: 999px; background: rgba(__TREBLE_TEAL_RGB__, 0.12); color: __TREBLE_TEAL__; border: 1px solid rgba(__TREBLE_TEAL_RGB__, 0.32); font-weight: 500; letter-spacing: 0.01em; } /* ---- Top tab row ---- */ .tab-buttons button { font-size: 0.88rem !important; padding: 0.55rem 1rem !important; border-radius: 10px 10px 0 0 !important; font-weight: 500 !important; } .tab-buttons button.selected { border-bottom: 2px solid __TREBLE_TEAL__ !important; background: var(--block-background-fill, __TREBLE_SURFACE__) !important; color: __TREBLE_TEAL__ !important; } /* ---- Leaderboard table ---- */ /* Gradio's Dataframe scrolls in TWO nested containers (the component block/wrappers AND the inner grid), producing two vertical scrollbars. Let the block + wrappers grow to fit (no scroll of their own) and keep a single vertical scrollbar on the inner grid. ``min-height: 0`` is required so the grid's max-height can actually clamp (flex/grid children default to min-height: auto, which overrides max-height). */ #leaderboard-table, #leaderboard-table .table-wrap { height: auto !important; max-height: none !important; overflow-y: visible !important; min-height: 0 !important; } #leaderboard-table .table { height: 480px !important; max-height: 480px !important; overflow-y: auto !important; min-height: 0 !important; } #leaderboard-table .table-wrap { overflow-x: auto !important; } #leaderboard-table th, #leaderboard-table td { min-width: 72px; } #leaderboard-table th .header-content { white-space: nowrap; } #leaderboard-table td { white-space: nowrap; overflow: visible !important; text-overflow: clip !important; max-width: none !important; } #leaderboard-table { width: 100% !important; table-layout: fixed !important; } #leaderboard-table tbody tr:hover td { background: rgba(__TREBLE_TEAL_RGB__, 0.08); } /* ---- Card-like panels ---- */ .queue-status, .next-up-panel, .ffasr-card { padding: 0.8rem 1rem; border-radius: 12px; background: var(--block-background-fill, __TREBLE_SURFACE__); border: 1px solid var(--border-color-primary, rgba(__TREBLE_TEAL_RGB__, 0.15)); } .queue-status p, .next-up-panel p { margin: 0.15rem 0; } /* ---- Links (markdown, footnote) ---- */ .markdown-text a, .ffasr-footnote a { color: __TREBLE_TEAL__; text-decoration-color: rgba(__TREBLE_TEAL_RGB__, 0.45); } .markdown-text a:hover, .ffasr-footnote a:hover { color: __TREBLE_MINT__; } /* ---- Sub-status footer line ---- */ .ffasr-footnote { text-align: center; font-size: 0.82rem; opacity: 0.65; margin-top: 0.8rem; color: __TREBLE_TEXT_MUTED__; } /* ---- Primary button emphasis ---- */ button.primary, .primary button { font-weight: 600 !important; } /* ---- Examples tab ---- */ #examples-tab .examples-scene-wrap .image-container, #examples-tab .examples-scene-wrap .wrap { max-height: 300px; justify-content: center; } #examples-tab .examples-scene-wrap img { object-fit: contain !important; max-height: 300px; width: auto !important; max-width: 100%; margin: 0 auto; display: block; } #examples-tab .examples-scene-wrap { max-width: 720px; margin: 0 auto 0.5rem auto; } /* Hide playback-speed (1x / 1.5x) control; keep play/pause in .play-pause-wrapper. */ #examples-tab .examples-audio button.playback.icon { display: none !important; } #examples-tab .examples-audio .play-pause-button, #examples-tab .examples-audio .rewind, #examples-tab .examples-audio .skip { color: __TREBLE_TEXT_MUTED__ !important; } #examples-tab .examples-audio .play-pause-button { width: 2rem !important; height: 2rem !important; min-width: 2rem !important; min-height: 2rem !important; } #examples-tab .examples-audio .play-pause-button svg, #examples-tab .examples-audio .rewind svg, #examples-tab .examples-audio .skip svg { display: block !important; width: 1.35rem !important; height: 1.35rem !important; /* Safari/WebKit collapses an SVG flex item (the icon ships with width/height: 100%) to 0x0 despite the explicit size above; pin a definite basis so it can't be shrunk away. */ flex: 0 0 auto !important; min-width: 1.35rem !important; min-height: 1.35rem !important; color: __TREBLE_TEXT_MUTED__ !important; fill: currentColor !important; stroke: currentColor !important; opacity: 1 !important; visibility: visible !important; } #examples-tab .examples-audio .play-pause-button:hover, #examples-tab .examples-audio .play-pause-button:focus, #examples-tab .examples-audio .rewind:hover, #examples-tab .examples-audio .skip:hover { color: __TREBLE_TEAL__ !important; } /* ---- Analysis tab: Plotly should fill the plot container ---- */ #analysis-tab .plot-container, #analysis-tab .plot-container > div, #analysis-tab .plot-container .js-plotly-plot, #analysis-tab .plot-container .plotly-graph-div { width: 100% !important; max-width: 100%; } #analysis-tab #analysis-pareto-plot.plot-container { min-height: __FIG_HEIGHT__px; } /* ---- Moderate tab: per-job rows ---- */ #moderate-tab .ffasr-job-row { align-items: flex-start !important; flex-wrap: nowrap !important; gap: 0.35rem !important; padding: 0.4rem 0.55rem !important; margin: 0.2rem 0 !important; border-radius: 8px; border: 1px solid var(--border-color-primary, rgba(128, 128, 128, 0.22)); } /* Text cell: fill remaining width and wrap whole words onto new lines. Use flex-basis:auto (NOT 0) so the column sizes to the available row width and shrinks with min-width:0; flex-basis:0 collapses it to min-content (one char per line). Only overflow-wrap:anywhere is used so unbreakable tokens (URLs) can still break, while normal words stay intact. */ #moderate-tab .ffasr-job-row .ffasr-job-info, #moderate-tab .ffasr-job-row > .ffasr-job-info.ffasr-job-info { flex: 1 1 auto !important; min-width: 0 !important; width: auto !important; overflow: visible !important; white-space: normal !important; overflow-wrap: anywhere !important; } #moderate-tab .ffasr-job-row .ffasr-job-info p, #moderate-tab .ffasr-job-row .ffasr-job-info code, #moderate-tab .ffasr-job-row .ffasr-job-info a, #moderate-tab .ffasr-job-row .ffasr-job-info span { margin: 0 !important; font-size: 0.88em; line-height: 1.35; white-space: normal !important; overflow-wrap: anywhere !important; } /* Buttons/controls stay fixed-width on the right. Exclude the text cell (which Gradio also tags with .block) so this never overrides its flexible sizing. */ #moderate-tab .ffasr-job-row > .block:not(.ffasr-job-info), #moderate-tab .ffasr-job-row > .form:not(.ffasr-job-info) { flex: 0 0 auto !important; min-width: unset !important; width: auto !important; align-self: flex-start !important; } #moderate-tab .ffasr-job-row button { flex: 0 0 auto !important; min-width: 4.25rem !important; max-width: 5.25rem !important; padding: 0.28rem 0.45rem !important; font-size: 0.8rem !important; white-space: nowrap !important; } #moderate-tab .ffasr-job-status-pending { background: rgba(245, 158, 11, 0.14); border-color: rgba(245, 158, 11, 0.35); } #moderate-tab .ffasr-job-status-queued { background: rgba(59, 130, 246, 0.12); border-color: rgba(59, 130, 246, 0.32); } #moderate-tab .ffasr-job-status-active { background: rgba(__TREBLE_TEAL_RGB__, 0.14); border-color: rgba(__TREBLE_TEAL_RGB__, 0.38); } #moderate-tab .ffasr-job-status-done { background: rgba(34, 197, 94, 0.12); border-color: rgba(34, 197, 94, 0.32); } #moderate-tab .ffasr-job-status-failed { background: rgba(239, 68, 68, 0.12); border-color: rgba(239, 68, 68, 0.35); } #moderate-tab .ffasr-job-status-unknown { background: var(--block-background-fill, __TREBLE_SURFACE__); } """.replace("__TREBLE_TEAL__", TREBLE_TEAL).replace( "__TREBLE_CYAN__", TREBLE_CYAN ).replace("__TREBLE_MINT__", TREBLE_MINT).replace( "__TREBLE_TEAL_RGB__", TREBLE_TEAL_RGB ).replace("__TREBLE_SURFACE__", TREBLE_SURFACE).replace( "__TREBLE_TEXT_MUTED__", TREBLE_TEXT_MUTED ).replace("__FIG_HEIGHT__", "460") DEFAULT_CUSTOM_EVAL_EXAMPLE = """from pathlib import Path import soundfile as sf import torch from transformers import AutoProcessor, CohereAsrForConditionalGeneration processor = AutoProcessor.from_pretrained("CohereLabs/cohere-transcribe-03-2026") # Expose the loaded model as a module-level `model` (or set NUM_PARAMS = <int>) so # the leaderboard can report its parameter count / size. model = CohereAsrForConditionalGeneration.from_pretrained( "CohereLabs/cohere-transcribe-03-2026", device_map="auto" ) def evaluate(file: Path) -> str: audio, sr = sf.read(str(file), dtype="float32", always_2d=True) audio = audio.mean(axis=1) inputs = processor(audio, sampling_rate=int(sr), return_tensors="pt", language="en") inputs.to(model.device, dtype=model.dtype) outputs = model.generate(**inputs, max_new_tokens=256) return processor.decode(outputs, skip_special_tokens=True) """