| """FonBench — leaderboard ASR pour le fongbe.
|
|
|
| Vitrine publique du classement, formulaire de soumission, et lancement de
|
| l'évaluateur de fond (voir evaluator.py). Cette interface ne lit que des
|
| scores agrégés : ni l'audio ni les transcriptions du corpus de test n'y
|
| transitent jamais.
|
|
|
| Interface en anglais (public international) ; commentaires en français,
|
| comme le reste du dépôt.
|
| """
|
|
|
| from __future__ import annotations
|
|
|
| import time
|
|
|
| import gradio as gr
|
| import pandas as pd
|
| import requests
|
|
|
| import evaluator
|
|
|
|
|
|
|
| SUPABASE_URL = evaluator.SUPABASE_URL
|
| ANON_KEY = evaluator.ANON_KEY
|
| REST = f"{SUPABASE_URL}/rest/v1"
|
| HEADERS = {"apikey": ANON_KEY}
|
|
|
| CACHE_SECONDS = 60
|
| _cache: dict = {}
|
|
|
|
|
| CSS = """
|
| :root {
|
| --fb-indigo:#3b3b7a; --fb-ocre:#c8873b; --fb-ink:#1b1b2e;
|
| --fb-muted:#6b6b7b; --fb-line:rgba(120,120,160,.22);
|
| }
|
| .dark { --fb-indigo:#a9a9f0; --fb-ink:#ececf6; --fb-muted:#9a9ab0; }
|
|
|
| .fb-head { padding:1.6rem 0 .5rem; border-bottom:1px solid var(--fb-line);
|
| margin-bottom:1rem; }
|
| .fb-title { display:flex; align-items:center; gap:.55rem;
|
| font-size:2.4rem; font-weight:800; letter-spacing:-.03em;
|
| color:var(--fb-indigo); line-height:1.05; margin:0; }
|
| .fb-title span { color:var(--fb-ocre); }
|
| .fb-title svg { flex:0 0 auto; box-shadow:0 0 0 1px rgba(0,0,0,.12); }
|
| .fb-sub { color:var(--fb-muted); margin:.35rem 0 0; font-size:1.02rem; }
|
|
|
| .fb-note { font-size:.9rem; line-height:1.6; color:var(--fb-ink); }
|
| .fb-note h3 { margin-top:1.4rem; font-size:1.05rem; letter-spacing:-.01em; }
|
| .fb-note table { font-size:.86rem; }
|
|
|
| .fb-badge { display:inline-block; padding:.12rem .55rem; border-radius:999px;
|
| font-size:.72rem; font-weight:700; letter-spacing:.02em;
|
| background:var(--fb-ocre); color:#fff; vertical-align:middle; }
|
|
|
| /* Bandeau de tête du classement : chiffres clés, pas un paragraphe. */
|
| .fb-meta { display:flex; flex-wrap:wrap; gap:.45rem 1.4rem; align-items:center;
|
| padding:.7rem .9rem; margin:.2rem 0 .7rem;
|
| border:1px solid var(--fb-line); border-radius:12px;
|
| font-size:.88rem; color:var(--fb-muted); }
|
| /* Chaque item est lui-même un flex : sans ça le SVG du drapeau se cale sur
|
| la ligne de base du texte et paraît remonté. */
|
| .fb-meta > span { display:inline-flex; align-items:center; gap:.35rem; }
|
| .fb-meta b { color:var(--fb-ink); font-weight:650; }
|
| .fb-meta .fb-sep { color:var(--fb-line); }
|
|
|
| .fb-legend { font-size:.82rem; color:var(--fb-muted); margin:.15rem 0 .9rem; }
|
|
|
| /* Filtres : chaque menu est un champ à part, avec son étiquette au-dessus.
|
| Sans ça, Gradio fusionne les champs voisins d'une rangée en un seul bloc
|
| gris où l'on ne distingue plus où commence quoi. */
|
| .fb-filters { gap:.7rem !important; margin-bottom:.6rem; }
|
| .fb-filters .block { padding:0 !important; }
|
| .fb-filters label > span:first-child,
|
| .fb-filters span[data-testid='block-info'] {
|
| display:block; margin-bottom:.28rem;
|
| font-size:.7rem !important; font-weight:700 !important;
|
| letter-spacing:.07em; text-transform:uppercase;
|
| color:var(--fb-muted) !important; }
|
| .fb-filters input, .fb-filters .wrap-inner, .fb-filters .secondary-wrap {
|
| border-radius:9px !important; }
|
| .fb-row2 { align-items:center; gap:.6rem !important; margin-bottom:.4rem; }
|
|
|
| /* Sélecteur de vue : des pastilles cliquables, pas des boutons radio. */
|
| .fb-views .wrap { gap:.35rem !important; }
|
| .fb-views label { border-radius:999px !important;
|
| padding:.32rem .9rem !important; font-weight:600;
|
| font-size:.86rem; border:1px solid var(--fb-line) !important;
|
| cursor:pointer; transition:background .12s; }
|
| .fb-views label:hover { background:rgba(120,120,180,.12) !important; }
|
| /* Le cercle du radio n'apporte rien à côté de l'état sélectionné : on le
|
| masque sans le retirer du parcours clavier. */
|
| .fb-views input[type=radio] { position:absolute; opacity:0;
|
| width:1px; height:1px; }
|
|
|
| /* Le tableau est large : on le laisse défiler plutôt que compresser. */
|
| .fb-table table { font-size:.86rem; }
|
| .fb-table td, .fb-table th { padding:.42rem .55rem !important; }
|
|
|
| footer { display:none !important; }
|
| """
|
|
|
|
|
|
|
|
|
| def fetch(path: str, params: dict, ttl: int = CACHE_SECONDS):
|
| key = (path, tuple(sorted(params.items())))
|
| hit = _cache.get(key)
|
| if hit and time.time() - hit[0] < ttl:
|
| return hit[1]
|
| r = requests.get(f"{REST}/{path}", headers=HEADERS, params=params, timeout=30)
|
| r.raise_for_status()
|
| data = r.json()
|
| _cache[key] = (time.time(), data)
|
| return data
|
|
|
|
|
| def get_benchmarks() -> list[dict]:
|
| return fetch("benchmarks", {"is_active": "eq.true", "select": "*",
|
| "order": "id"}, ttl=600)
|
|
|
|
|
| def get_results(benchmark_id: str) -> list[dict]:
|
| return fetch("results", {"is_hidden": "eq.false",
|
| "benchmark_id": f"eq.{benchmark_id}",
|
| "select": "*"})
|
|
|
|
|
|
|
|
|
| TYPES = {
|
| "base": "🌍 General",
|
| "maison": "🔧 FonBench",
|
| "tiers": "👥 Community",
|
| }
|
|
|
|
|
| VUES: dict[str, set[str] | None] = {
|
| "All models": None,
|
| "🎯 Fine-tuned on Fon": {"maison", "tiers"},
|
| "🔧 By FonBench": {"maison"},
|
| "👥 By the community": {"tiers"},
|
| "🌍 General-purpose": {"base"},
|
| }
|
|
|
|
|
| def categorie(r: dict) -> str:
|
| """Catégorie d'un modèle, tolérante aux schémas intermédiaires.
|
|
|
| `model_kind` a d'abord valu 'fongbe' ou 'base' avant de passer à trois
|
| valeurs. Entre les deux migrations, une lecture au pied de la lettre
|
| ferait disparaître tous les modèles fongbe : on retombe alors sur le
|
| préfixe du dépôt, qui dit la même chose.
|
| """
|
| k = r.get("model_kind")
|
| if k in ("base", "maison", "tiers"):
|
| return k
|
| if k == "fongbe":
|
| return "maison" if r["model_id"].startswith("fonbench/") else "tiers"
|
| return "base"
|
|
|
|
|
|
|
|
|
| def flag(h: int = 15) -> str:
|
| w = round(h * 1.5)
|
| return (
|
| f"<svg viewBox='0 0 30 20' width='{w}' height='{h}' "
|
| "style='vertical-align:-.12em;border-radius:2px' "
|
| "role='img' aria-label='Benin'>"
|
| "<rect width='30' height='20' fill='#FCD116'/>"
|
| "<rect y='10' width='30' height='10' fill='#E8112D'/>"
|
| "<rect width='12' height='20' fill='#008751'/></svg>"
|
| )
|
|
|
|
|
| def pct(x) -> str:
|
| return "—" if x is None else f"{float(x) * 100:.1f}"
|
|
|
|
|
| def params_txt(n) -> str:
|
| if not n:
|
| return "—"
|
| n = int(n)
|
| return f"{n / 1e9:.1f}B" if n >= 1e9 else f"{n / 1e6:.0f}M"
|
|
|
|
|
| def train_txt(r: dict) -> str:
|
| data = r.get("train_data")
|
| if not data:
|
| return "not declared"
|
| if data == "aucun fongbe":
|
| return "no Fon data"
|
| h = r.get("train_hours")
|
| return f"{data} · {float(h):g}h" if h else data
|
|
|
|
|
| def lien(model_id: str | None) -> str:
|
| if not model_id:
|
| return "—"
|
| return f"[{model_id}](https://huggingface.co/{model_id})"
|
|
|
|
|
| COLONNES = ["#", "Model", "Type", "T-WER", "WER_seg", "WER_ton", "WER", "CER",
|
| "MER", "WIL", "RTFx", "Size", "Base model", "Trained on",
|
| "Architecture", "Decoder"]
|
|
|
| DTYPES = (["str", "markdown"] + ["str"] * 10 + ["markdown"] + ["str"] * 3)
|
|
|
|
|
| def build_table(benchmark_id: str, vue: str, archs: list[str],
|
| decoders: list[str], hide_contaminated: bool, sort_by: str):
|
| try:
|
| rows = get_results(benchmark_id)
|
| except Exception as exc:
|
| return (pd.DataFrame({"Error": [f"Database unreachable: {exc}"]}),
|
| "", "", gr.update(), gr.update())
|
|
|
| all_archs = sorted({r["architecture"] for r in rows if r.get("architecture")})
|
| all_decs = sorted({r["decoder_type"] for r in rows if r.get("decoder_type")})
|
|
|
| sel = rows
|
| vises = VUES.get(vue)
|
| if vises:
|
| sel = [r for r in sel if categorie(r) in vises]
|
| if archs:
|
| sel = [r for r in sel if r.get("architecture") in archs]
|
| if decoders:
|
| sel = [r for r in sel if r.get("decoder_type") in decoders]
|
| if hide_contaminated:
|
| sel = [r for r in sel if not r.get("contamination_flag")]
|
|
|
|
|
|
|
|
|
| tonal = any(r.get("twer") is not None for r in rows)
|
| metric = "twer" if tonal else "wer_notone"
|
| if sort_by == "Speed (RTFx)":
|
| sel = sorted(sel, key=lambda r: -(r.get("rtfx") or 0))
|
| elif sort_by == "Model size":
|
| sel = sorted(sel, key=lambda r: (r.get("model_params") or 0))
|
| else:
|
| sel = sorted(sel,
|
| key=lambda r: (r.get(metric) is None, r.get(metric) or 9e9))
|
|
|
| data = []
|
| for i, r in enumerate(sel, 1):
|
| medaille = {1: "🥇", 2: "🥈", 3: "🥉"}.get(i, "")
|
| nom = lien(r["model_id"])
|
| if r.get("contamination_flag"):
|
| nom += " ⚠️"
|
| data.append({
|
| "#": f"{i} {medaille}".strip(),
|
| "Model": nom,
|
| "Type": TYPES[categorie(r)],
|
| "T-WER": pct(r.get("twer")),
|
| "WER_seg": pct(r.get("wer_notone")),
|
| "WER_ton": pct(r.get("wer_ton")),
|
| "WER": pct(r.get("wer")),
|
| "CER": pct(r.get("cer")),
|
| "MER": pct(r.get("mer")),
|
| "WIL": pct(r.get("wil")),
|
| "RTFx": "—" if not r.get("rtfx") else f"{float(r['rtfx']):.1f}×",
|
| "Size": params_txt(r.get("model_params")),
|
| "Base model": lien(r.get("base_model")),
|
| "Trained on": train_txt(r),
|
| "Architecture": r.get("architecture") or "—",
|
| "Decoder": r.get("decoder_type") or "—",
|
| })
|
|
|
| bench = next((b for b in get_benchmarks() if b["id"] == benchmark_id), {})
|
| compte = {c: sum(1 for r in sel if categorie(r) == c)
|
| for c in ("maison", "tiers", "base")}
|
| meta = (
|
| "<div class='fb-meta'>"
|
| f"<span>{flag(13)} <b>{bench.get('name', benchmark_id)}</b></span>"
|
| f"<span><b>{bench.get('num_utterances', '?')}</b> utterances</span>"
|
| f"<span><b>{bench.get('duration_hours', '?')}</b> hours</span>"
|
| + ("<span class='fb-badge'>private test set</span>"
|
| if bench.get("is_private") else "")
|
| + f"<span>ranked by <b>{'T-WER' if tonal else 'WER_seg'}</b></span>"
|
| + f"<span><b>{len(sel)}</b> of {len(rows)} models shown</span>"
|
|
|
|
|
| + ("<span>"
|
| f"🔧 <b>{compte['maison']}</b> fine-tuned by FonBench"
|
| "<span class='fb-sep'> · </span>"
|
| f"👥 <b>{compte['tiers']}</b> fine-tuned by the community"
|
| "<span class='fb-sep'> · </span>"
|
| f"🌍 <b>{compte['base']}</b> general-purpose"
|
| "</span>")
|
| + "</div>"
|
| )
|
| legende = (
|
| "All error rates in **%**, lower is better — except **RTFx**, where "
|
| "higher means faster. **⚠️** marks a model likely trained on this "
|
| "test set: its score is not comparable."
|
| )
|
| return (pd.DataFrame(data, columns=COLONNES), meta, legende,
|
| gr.update(choices=all_archs), gr.update(choices=all_decs))
|
|
|
|
|
| NOTE_METRIQUES = """
|
| ### Reading the table
|
|
|
| Fon is a **tonal** language: tones are written with diacritics (á, ɔ́, ě…)
|
| and change the meaning of words. But Fon corpora don't follow the same
|
| convention — some mark no tone at all. A raw WER therefore isn't comparable
|
| from one corpus to the next. Hence three families of measures.
|
|
|
| | Metric | What it tells you |
|
| |---|---|
|
| | **WER_seg** | Word errors with **tones stripped**. Measures phonetic accuracy and stays comparable across every corpus. |
|
| | **WER_ton** | Errors on tone marks alone. Shown as “—” when the corpus doesn't annotate tones, so the figure is never misleading. |
|
| | **T-WER** | `WER_seg + 2 × WER_ton`. The headline metric: it penalises tone mistakes twice over. |
|
| | WER, CER | Word and character errors, tones included. The classic reference points. |
|
| | MER, WIL | *Match error rate* and *word information lost* — more robust when a model produces many insertions. |
|
| | RTFx | Seconds of audio processed per second of compute. **Higher is faster.** |
|
|
|
| **Model types.** 🌍 *General-purpose* models were not built for Fon: some
|
| are multilingual systems that do cover it among a thousand other languages
|
| (the MMS family ships a Fon adapter), others are multilingual without Fon,
|
| and others again are monolingual systems for English or French. They are not
|
| *base* models in the pretrained sense — `wav2vec2-large-960h-lv60-self` is a
|
| finished English recogniser, not a starting point. The **Trained on** column
|
| says exactly what each one saw. 🔧 *FonBench* and 👥 *Community* models were
|
| fine-tuned on Fon; comparing them against a general-purpose model that shares
|
| their base is what actually measures the value of fine-tuning.
|
|
|
| Truly *base* models — raw pretrained checkpoints such as
|
| `facebook/wav2vec2-large-xlsr-53` — cannot appear here at all: without a CTC
|
| head or a vocabulary they transcribe nothing. They show up only in the
|
| **Base model** column, as the starting point of the models that were
|
| fine-tuned from them.
|
|
|
| **Speed.** RTFx depends on the hardware, which is recorded with each score.
|
| Models measured on the Space's shared GPU are not directly comparable to
|
| models measured on a dedicated L4 — only compare speeds at equal hardware.
|
| """
|
|
|
|
|
|
|
|
|
| def build_queue():
|
| try:
|
| rows = fetch("public_queue", {"select": "*", "order": "created_at.desc",
|
| "limit": "50"}, ttl=15)
|
| except Exception as exc:
|
| return pd.DataFrame({"Error": [f"Database unreachable: {exc}"]}), ""
|
|
|
| etats = {"pending": "⏳ pending", "running": "⚙️ running",
|
| "done": "✅ done", "failed": "❌ failed",
|
| "rejected": "🚫 rejected"}
|
| data = []
|
| for r in rows:
|
| total = r.get("progress_total") or 0
|
| done = r.get("progress_done") or 0
|
| avance = f"{done}/{total}" if total else "—"
|
| if total and r["status"] == "running":
|
| avance += f" ({done * 100 // total}%)"
|
| data.append({
|
| "Model": r["model_id"],
|
| "Status": etats.get(r["status"], r["status"]),
|
| "Progress": avance,
|
| "Details": (r.get("error_message") or "")[:120],
|
| "Submitted": (r.get("created_at") or "")[:10],
|
| })
|
|
|
| etat = evaluator.status()
|
| ligne = f"**Evaluator:** {etat['message']}"
|
| if etat.get("model"):
|
| ligne += f" — {etat['model']} ({etat['done']}/{etat['total']})"
|
| return pd.DataFrame(data), ligne
|
|
|
|
|
|
|
|
|
| def submit(model_id: str, hf_username: str, contact: str, note: str,
|
| benchmark_id: str, train_data: str, train_hours, base_model: str):
|
| model_id = (model_id or "").strip()
|
| if "/" not in model_id or len(model_id.split("/")) != 2 or \
|
| not all(model_id.split("/")):
|
| return ("### ❌ Invalid format\n"
|
| "The identifier must look like `organisation/name`, exactly as "
|
| "it appears in the model's URL on Hugging Face.")
|
|
|
| payload = {"model_id": model_id, "benchmark_id": benchmark_id}
|
| for champ, valeur in (("hf_username", hf_username), ("contact", contact),
|
| ("note", note), ("train_data", train_data),
|
| ("base_model", base_model)):
|
| if (valeur or "").strip():
|
| payload[champ] = valeur.strip()
|
| if train_hours:
|
| payload["train_hours"] = float(train_hours)
|
|
|
| try:
|
| r = requests.post(
|
| f"{REST}/public_requests",
|
| headers={**HEADERS, "Content-Type": "application/json",
|
| "Prefer": "return=minimal"},
|
| json=payload, timeout=30,
|
| )
|
| except Exception as exc:
|
| return f"### ❌ Could not submit\nDatabase unreachable: {exc}"
|
|
|
| if r.status_code >= 400:
|
|
|
|
|
| try:
|
| detail = r.json().get("message") or r.text
|
| except Exception:
|
| detail = r.text
|
| return f"### ❌ Submission rejected\n{detail}"
|
|
|
| _cache.clear()
|
| return (
|
| f"### ✅ `{model_id}` is queued\n\n"
|
| "Evaluation runs in slices on the Space's shared GPU. Depending on the "
|
| "available quota, expect anywhere from a few minutes to a few hours. "
|
| "The score will appear in the leaderboard once computed — follow "
|
| "progress in the **Queue** tab."
|
| )
|
|
|
|
|
| FINE_TUNING = """
|
| ## The FonBench fine-tuning runs
|
|
|
| Four pretrained models were fine-tuned on Fon under strictly identical
|
| conditions, so that any gap between them comes from the starting model
|
| alone.
|
|
|
| ### The data
|
|
|
| The training corpus holds **44,225 utterances**, of which **13,716 were
|
| kept — 30.00 hours from 471 speakers**. The sample is not drawn at random:
|
| it is built by taking turns across speakers, each contributing a little
|
| before any one of them dominates. For generalisation, the diversity of
|
| voices matters more than raw volume.
|
|
|
| The 471 training speakers and the 45 test speakers are **strictly
|
| disjoint** — verified, zero in common. A temporal cutoff separates the
|
| transcripts; 20 sentences out of 2,555 (0.8%) do appear in both, too few to
|
| move a score but worth stating rather than claiming a perfect separation.
|
|
|
| ### The protocol
|
|
|
| Identical across all four: **3 epochs**, i.e. 2,574 steps with 257 of
|
| warmup. CTC decoding, character vocabulary built from the training corpus,
|
| feature extractor taken from the base model. A single 24 GB L4, `fp16`,
|
| gradient checkpointing.
|
|
|
| | Resulting model | Base | Learning rate | Batch | Wall time | Final dev loss | WER_seg |
|
| |---|---|---|---|---|---|---|
|
| | `wav2vec2-large-xlsr-53-fon-30h` | facebook/wav2vec2-large-xlsr-53 | 3·10⁻⁴ | 8×2 | 54 min | **0.468** | **38.6%** |
|
| | `mms-300m-fon-30h` | facebook/mms-300m | 3·10⁻⁴ | 8×2 | not retained | not retained | **42.8%** |
|
| | `w2v-bert-2.0-fon-30h` | facebook/w2v-bert-2.0 | 3·10⁻⁵ | 4×4 | 116 min | 3.102 | 96.9% |
|
| | `AfriHuBERT-fon-30h` | ajesujoba/AfriHuBERT | 3·10⁻⁵ | 8×2 | 24 min | 3.078 | 100.0% |
|
|
|
| ### What this shows
|
|
|
| **Fine-tuning makes the difference, not the base.** `chrisjay/fonxlsr`
|
| starts from exactly the same model as our best run —
|
| `facebook/wav2vec2-large-xlsr-53`, confirmed in its configuration — and
|
| reaches 69.3% where we reach 38.6%. The error is cut by a factor of 1.8
|
| with 30 hours of well-chosen data.
|
|
|
| **Two runs out of four failed**, and that deserves saying plainly.
|
| `w2v-bert-2.0` and `AfriHuBERT` do learn: their loss drops clearly during
|
| training. But it plateaus around 3.1 against 0.47 for the run that
|
| succeeds — the signature of settling into CTC's trivial solution, emitting
|
| the blank symbol everywhere. This is not a learning-rate problem: both were
|
| first run at 3·10⁻⁴, then rerun at 3·10⁻⁵ — the figures in the table — for
|
| exactly the same outcome. They would need far more epochs, or more data.
|
|
|
| The `mms-300m` run predates per-model logging, so its wall time and final
|
| loss are genuinely lost. They are shown as *not retained* rather than
|
| reconstructed from memory.
|
|
|
| Both failures stay in the leaderboard. Removing them would paint a
|
| flattering and false picture of what fine-tuning guarantees.
|
|
|
| ### Reproducing
|
|
|
| `train_ctc.py` is published in the **Files** tab of this Space, under
|
| `repro/`, together with the evaluation script and the scoring code. See the
|
| **Reproduce** tab.
|
| """
|
|
|
|
|
| REPRODUIRE = """
|
| ## Reproduce every number
|
|
|
| A leaderboard nobody can check is just a poster. Everything needed to
|
| recompute any row is published in the **Files** tab of this Space, under
|
| `repro/` — no server of ours involved, no key, no hidden step.
|
|
|
| | File | What it is |
|
| |---|---|
|
| | `repro/fonbench_eval.py` | The scoring code: normalisation, WER/CER/MER/WIL, tone stripping, WER_ton, T-WER. The whole definition of every metric. Runs its own self-tests. |
|
| | `repro/evaluate.py` | Standalone evaluation. Loads a model from the Hub, transcribes a corpus, prints the metrics as JSON. |
|
| | `repro/train_ctc.py` | The fine-tuning script that produced the `fonbench/*` models. |
|
| | `repro/README.md` | Full instructions and reference values. |
|
|
|
| ```
|
| pip install torch transformers "datasets>=3" av jiwer huggingface_hub
|
| python evaluate.py --model chrisjay/fonxlsr --dataset alaleye/fon --split test
|
| ```
|
|
|
| ### Checking a row of this leaderboard
|
|
|
| The main test set is private — a test set that circulates stops being a
|
| test set. It is not secret: request access to `JMLdata/fon-test-v1` and you
|
| can recompute any row yourself.
|
|
|
| ```
|
| export HF_TOKEN=hf_...
|
| python evaluate.py --model chrisjay/fonxlsr \
|
| --dataset JMLdata/fon-test-v1 --split test \
|
| --revision b1c2db22604e76763cd850c2b473e80bd84b4059 --batch 16
|
| ```
|
|
|
| On an L4 this returns, for `chrisjay/fonxlsr`:
|
|
|
| | Metric | Leaderboard | `evaluate.py` |
|
| |---|---|---|
|
| | WER_seg | 0.6932 | **0.6932** |
|
| | WER_ton | 0.4568 | **0.4568** |
|
| | T-WER | 1.6068 | **1.6068** |
|
| | WER | 0.8730 | 0.8729 |
|
| | CER | 0.4214 | 0.4215 |
|
|
|
| ### Why some figures differ in the fourth decimal
|
|
|
| CTC inference pads every utterance in a batch to the longest one, and that
|
| padding shifts a handful of output tokens. Measured on `chrisjay/fonxlsr`,
|
| same corpus revision: WER_seg is 0.6930 at `--batch 4`, 0.6932 at `--batch
|
| 16` and 0.6932 at `--batch 32`. **Expect agreement within ±0.0002, not
|
| bit-identity.** A larger gap means something real
|
| differs — check the corpus revision, the model revision and the batch size
|
| first. We would rather document this than quietly round the published
|
| figures to three decimals.
|
|
|
| RTFx is a hardware measurement and only reproduces on the same GPU; the
|
| hardware is recorded with every score.
|
|
|
| If you get a materially different number, tell us. That is the point of
|
| publishing this.
|
| """
|
|
|
|
|
| A_PROPOS = """
|
| ## FonBench
|
|
|
| The first public speech-recognition leaderboard for **Fon**, a tonal
|
| language spoken by over two million people in Benin. The project answers
|
| a concrete difficulty: until now, published Fon results were not comparable
|
| with one another, for lack of a shared protocol and a shared test set.
|
|
|
| ### Why the test set is not published
|
|
|
| Public Fon corpora have been circulating since 2016, and several released
|
| models were trained on them. Their WER on that data then looks remarkable —
|
| while saying nothing about their real ability to transcribe an unseen
|
| voice. We measured it: one of the test sets we were using shared **83% of
|
| its utterances** with the training split of a public dataset.
|
|
|
| The main test set (**2,555 utterances, 4.98 hours, 45 speakers**) is
|
| therefore not distributed. Its speakers are disjoint from training
|
| speakers, and a temporal cutoff separates the transcripts. Only aggregate
|
| scores are made public: nobody — not even through this Space — can download
|
| its audio or its transcriptions.
|
|
|
| ### The protocol
|
|
|
| - **Pinned revision.** Every score is tied to the exact commit hash of the
|
| repository evaluated. Republishing a model under the same name does not
|
| change a score already obtained.
|
| - **Shared normalisation.** The same normalisation and scoring code for
|
| every model, tones included. It is open: `fonbench_eval.py`.
|
| - **No arbitrary code.** Models are loaded with `trust_remote_code=False`,
|
| so custom code shipped in a repository is never executed.
|
| - **No duplicate work.** A (model, revision, corpus) triple is never
|
| re-evaluated: the existing score is reused.
|
|
|
| ### Submitting a model
|
|
|
| The repository must be **public** on the Hub. Recognised architectures:
|
| wav2vec2, wav2vec2-BERT, HuBERT, MMS, Whisper. No account required.
|
| """
|
|
|
|
|
|
|
|
|
| def build_ui() -> gr.Blocks:
|
| try:
|
| benchs = get_benchmarks()
|
| except Exception:
|
| benchs = []
|
| choix = [(f"{b['name']} ({b.get('num_utterances', '?')} utterances)",
|
| b["id"]) for b in benchs]
|
| defaut = "jml-test-v1" if any(b["id"] == "jml-test-v1" for b in benchs) \
|
| else (benchs[0]["id"] if benchs else "")
|
|
|
|
|
| with gr.Blocks(title="FonBench — Fon ASR leaderboard") as demo:
|
| gr.HTML(
|
| "<div class='fb-head'>"
|
| "<p class='fb-title'>Fon<span>Bench</span> "
|
| f"{flag(26)}</p>"
|
| "<p class='fb-sub'>The public speech-recognition leaderboard for "
|
| "Fon, a tonal language of Benin</p>"
|
| "</div>"
|
| )
|
|
|
| with gr.Tabs():
|
| with gr.Tab("Leaderboard"):
|
|
|
|
|
|
|
| with gr.Column(elem_classes="fb-filters"):
|
| with gr.Row(equal_height=True):
|
| with gr.Column(scale=5, min_width=190):
|
| b_sel = gr.Dropdown(choix, value=defaut,
|
| label="Test set")
|
| with gr.Column(scale=3, min_width=160):
|
| tri = gr.Dropdown(
|
| ["Quality (ranking metric)", "Speed (RTFx)",
|
| "Model size"],
|
| value="Quality (ranking metric)",
|
| label="Sort by")
|
| with gr.Column(scale=3, min_width=160):
|
| f_arch = gr.Dropdown([], multiselect=True,
|
| label="Architecture")
|
| with gr.Column(scale=3, min_width=160):
|
| f_dec = gr.Dropdown([], multiselect=True,
|
| label="Decoder")
|
| with gr.Row(elem_classes="fb-row2"):
|
| f_type = gr.Radio(list(VUES), value="All models",
|
| show_label=False, container=False,
|
| elem_classes="fb-views", scale=8)
|
| f_cont = gr.Checkbox(label="Hide contaminated",
|
| container=False, scale=2)
|
| rafraichir = gr.Button("Refresh", size="sm", scale=1)
|
|
|
| meta = gr.HTML()
|
| legende = gr.Markdown(elem_classes="fb-legend")
|
| table = gr.Dataframe(interactive=False, wrap=False,
|
| datatype=DTYPES, elem_classes="fb-table")
|
| gr.Markdown(NOTE_METRIQUES, elem_classes="fb-note")
|
|
|
| entrees = [b_sel, f_type, f_arch, f_dec, f_cont, tri]
|
| sorties = [table, meta, legende, f_arch, f_dec]
|
| for widget in (b_sel, tri, f_type, f_arch, f_dec, f_cont):
|
| widget.change(build_table, entrees, sorties)
|
| rafraichir.click(lambda: _cache.clear(), None, None).then(
|
| build_table, entrees, sorties)
|
| demo.load(build_table, entrees, sorties)
|
|
|
| with gr.Tab("Queue"):
|
| etat_line = gr.Markdown()
|
| q_table = gr.Dataframe(interactive=False, wrap=True,
|
| elem_classes="fb-table")
|
| gr.Button("Refresh").click(build_queue, None,
|
| [q_table, etat_line])
|
| gr.Markdown(
|
| "Evaluation runs in slices on the Space's shared GPU. The "
|
| "compute itself is fast — a few minutes for a CTC model — "
|
| "but the daily GPU quota is limited, so a run may spread "
|
| "over several hours, pausing between slices. An "
|
| "interrupted run resumes exactly where it stopped.",
|
| elem_classes="fb-note")
|
| demo.load(build_queue, None, [q_table, etat_line])
|
|
|
| with gr.Tab("Submit a model"):
|
| gr.Markdown(
|
| "The repository must be **public** and must not rely on "
|
| "custom code: `trust_remote_code` is disabled. Both "
|
| "`safetensors` and `.bin` weights are accepted. No account "
|
| "required.", elem_classes="fb-note")
|
| s_model = gr.Textbox(label="Hugging Face model ID",
|
| placeholder="organisation/model-name")
|
| with gr.Row():
|
| s_user = gr.Textbox(label="Your HF username (optional)")
|
| s_contact = gr.Textbox(label="Contact (optional)")
|
| with gr.Row():
|
| s_base = gr.Textbox(
|
| label="Base model (optional)",
|
| placeholder="facebook/wav2vec2-large-xlsr-53")
|
| s_data = gr.Textbox(
|
| label="Training data (optional)",
|
| placeholder="ALFFA, Zenodo, private corpus…",
|
| info="Leave empty for a model never fine-tuned on Fon.")
|
| s_hours = gr.Number(label="Hours of Fon audio (optional)",
|
| precision=1, minimum=0)
|
| s_note = gr.Textbox(label="Note (optional)", lines=2)
|
| s_bench = gr.Dropdown(choix, value=defaut, label="Test set")
|
| s_out = gr.Markdown()
|
| gr.Button("Submit", variant="primary").click(
|
| submit,
|
| [s_model, s_user, s_contact, s_note, s_bench,
|
| s_data, s_hours, s_base],
|
| s_out)
|
|
|
| with gr.Tab("Fine-tuning"):
|
| gr.Markdown(FINE_TUNING, elem_classes="fb-note")
|
|
|
| with gr.Tab("Reproduce"):
|
| gr.Markdown(REPRODUIRE, elem_classes="fb-note")
|
|
|
| with gr.Tab("About"):
|
| gr.Markdown(A_PROPOS, elem_classes="fb-note")
|
|
|
| return demo
|
|
|
|
|
|
|
|
|
|
|
| THEME = gr.themes.Soft(primary_hue="indigo", secondary_hue="amber").set(
|
| block_label_background_fill="transparent",
|
| block_label_background_fill_dark="transparent",
|
| block_label_border_width="0px",
|
| block_label_text_color="*neutral_500",
|
| block_label_text_color_dark="*neutral_400",
|
| block_label_text_size="*text_xs",
|
| block_label_text_weight="600",
|
| block_background_fill="transparent",
|
| block_background_fill_dark="transparent",
|
| block_border_width="0px",
|
| block_shadow="none",
|
| panel_background_fill="transparent",
|
| panel_background_fill_dark="transparent",
|
| panel_border_width="0px",
|
| form_gap_width="0px",
|
| input_background_fill="*neutral_50",
|
| input_background_fill_dark="*neutral_800",
|
| input_border_width="1px",
|
| input_border_color="*neutral_200",
|
| input_border_color_dark="*neutral_700",
|
| )
|
|
|
|
|
|
|
| evaluator.start()
|
| demo = build_ui()
|
|
|
| if __name__ == "__main__":
|
| demo.launch(css=CSS, theme=THEME)
|
|
|
