app.js

/* FonBench — version statique du classement.
 *
 * Un Space statique ne peut pas exécuter de Python : cette copie affiche le
 * classement, la file et le formulaire, mais l'évaluation reste sur le Space
 * Gradio qui détient le GPU et l'accès au corpus privé. Les données sont lues
 * en direct depuis la même base : rien ici n'est une copie figée.
 *
 * La clé ci-dessous est publique par conception — c'est la Row Level Security
 * qui décide de ce qui est lisible, pas le secret de la clé.
 */
const SUPABASE = "https://cqdimvcnmhrsdcoobkmd.supabase.co/rest/v1";
const ANON = "sb_publishable_MapYll-_Y0hNoLYOfaDR3w_tsTEHyiz";
const H = { apikey: ANON };

const EVALUATEUR = "https://huggingface.co/spaces/Kimyayd/FonBench";

const TYPES = { base: "🌍 General", maison: "🔧 FonBench", tiers: "👥 Community" };
// « Fine-tuned » réunit les deux origines : c'est la question la plus
// fréquente — quels modèles ont vu du fongbe.
const VUES = [
  ["All models", null],
  ["🎯 Fine-tuned on Fon", ["maison", "tiers"]],
  ["🔧 By FonBench", ["maison"]],
  ["👥 By the community", ["tiers"]],
  ["🌍 General-purpose", ["base"]],
];

const ONGLETS = [
  ["leaderboard", "Leaderboard"], ["queue", "Queue"],
  ["submit", "Submit a model"], ["finetuning", "Fine-tuning"], ["about", "About"],
];

let etat = { vue: 0, benchmarks: [], resultats: [], tri: null, sens: 1 };

/* --- accès aux données ------------------------------------------------ */

async function lire(chemin) {
  const r = await fetch(`${SUPABASE}/${chemin}`, { headers: H });
  if (!r.ok) throw new Error(`${r.status} ${await r.text()}`);
  return r.json();
}

/* --- mise en forme ----------------------------------------------------- */

const pct = (x) => (x === null || x === undefined ? "—" : (x * 100).toFixed(1));

function taille(n) {
  if (!n) return "—";
  return n >= 1e9 ? `${(n / 1e9).toFixed(1)}B` : `${Math.round(n / 1e6)}M`;
}

/* La colonne model_kind a d'abord valu « fongbe » ou « base » avant de passer
 * à trois valeurs. On retombe sur le préfixe du dépôt le cas échéant, sinon
 * une base à l'ancien schéma ferait disparaître des lignes entières. */
function categorie(r) {
  const k = r.model_kind;
  if (["base", "maison", "tiers"].includes(k)) return k;
  if (k === "fongbe") return r.model_id.startsWith("fonbench/") ? "maison" : "tiers";
  return "base";
}

function lienModele(id, checkpoint) {
  if (!id) return "—";
  const url = `https://huggingface.co/${id}`;
  const base = `<a href="${url}" target="_blank" rel="noopener">${id}</a>`;
  return checkpoint ? `${base}<br><small>↳ ${checkpoint}</small>` : base;
}

const esc = (s) =>
  String(s).replace(/[&<>"]/g, (c) =>
    ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));

/* --- classement -------------------------------------------------------- */

const COLONNES = [
  ["#", null], ["Model", null], ["Type", null],
  ["T-WER", "twer"], ["WER_seg", "wer_notone"], ["WER_ton", "wer_ton"],
  ["WER", "wer"], ["CER", "cer"], ["MER", "mer"], ["WIL", "wil"],
  ["RTFx", "rtfx"], ["Size", "model_params"],
  ["Base model", null], ["Trained on", null],
  ["Architecture", null], ["Decoder", null],
];

function entraineSur(r) {
  if (!r.train_data) return "not declared";
  if (r.train_data === "aucun fongbe") return "no Fon data";
  return r.train_hours ? `${r.train_data} · ${+r.train_hours}h` : r.train_data;
}

function dessinerClassement() {
  const rows = etat.resultats;
  const bench = etat.benchmarks.find((b) => b.id === document.getElementById("bench").value) || {};

  // Renseigne les listes d'architectures et de décodeurs à partir des données.
  for (const [id, champ] of [["arch", "architecture"], ["dec", "decoder_type"]]) {
    const sel = document.getElementById(id);
    const vals = [...new Set(rows.map((r) => r[champ]).filter(Boolean))].sort();
    const courant = sel.value;
    sel.innerHTML = '<option value="">all</option>' +
      vals.map((v) => `<option${v === courant ? " selected" : ""}>${esc(v)}</option>`).join("");
  }

  let sel = rows.slice();
  const vises = VUES[etat.vue][1];
  if (vises) sel = sel.filter((r) => vises.includes(categorie(r)));
  const a = document.getElementById("arch").value;
  const d = document.getElementById("dec").value;
  if (a) sel = sel.filter((r) => r.architecture === a);
  if (d) sel = sel.filter((r) => r.decoder_type === d);
  if (document.getElementById("contam").checked)
    sel = sel.filter((r) => !r.contamination_flag);

  // Le corpus annote-t-il les tons ? Si oui le T-WER fait foi, sinon le
  // WER_seg : comparer des WER bruts entre conventions tonales différentes
  // n'aurait aucun sens.
  const tonal = rows.some((r) => r.twer !== null && r.twer !== undefined);
  const metrique = tonal ? "twer" : "wer_notone";
  const grand = 9e9;

  if (etat.tri) {
    sel.sort((x, y) => etat.sens * ((x[etat.tri] ?? grand) - (y[etat.tri] ?? grand)));
  } else {
    const mode = document.getElementById("sort").value;
    if (mode === "speed") sel.sort((x, y) => (y.rtfx ?? 0) - (x.rtfx ?? 0));
    else if (mode === "size") sel.sort((x, y) => (x.model_params ?? 0) - (y.model_params ?? 0));
    else sel.sort((x, y) => (x[metrique] ?? grand) - (y[metrique] ?? grand));
  }

  const medaille = { 1: "🥇", 2: "🥈", 3: "🥉" };
  const corps = sel.map((r, i) => {
    const ck = (r.model_revision || "").split(":");
    const nom = lienModele(r.model_id, ck.length > 1 ? esc(ck.slice(1).join(":")) : null)
      + (r.contamination_flag ? " ⚠️" : "");
    return `<tr>
      <td class="num">${i + 1} ${medaille[i + 1] || ""}</td>
      <td>${nom}</td><td>${TYPES[categorie(r)]}</td>
      <td class="num">${pct(r.twer)}</td><td class="num">${pct(r.wer_notone)}</td>
      <td class="num">${pct(r.wer_ton)}</td><td class="num">${pct(r.wer)}</td>
      <td class="num">${pct(r.cer)}</td><td class="num">${pct(r.mer)}</td>
      <td class="num">${pct(r.wil)}</td>
      <td class="num">${r.rtfx ? (+r.rtfx).toFixed(1) + "×" : "—"}</td>
      <td class="num">${taille(r.model_params)}</td>
      <td>${lienModele(r.base_model, null)}</td>
      <td>${esc(entraineSur(r))}</td>
      <td>${esc(r.architecture || "—")}</td><td>${esc(r.decoder_type || "—")}</td>
    </tr>`;
  }).join("");

  document.getElementById("board").innerHTML =
    `<thead><tr>${COLONNES.map(([t, c], i) =>
      `<th data-col="${c || ""}" title="${c ? "Trier" : ""}">${t}${
        etat.tri === c && c ? (etat.sens > 0 ? " ▲" : " ▼") : ""}</th>`).join("")}
     </tr></thead><tbody>${corps || '<tr><td colspan="16">Aucun modèle.</td></tr>'}</tbody>`;

  document.querySelectorAll("#board th[data-col]").forEach((th) => {
    const c = th.dataset.col;
    if (!c) return;
    th.onclick = () => {
      etat.sens = etat.tri === c ? -etat.sens : 1;
      etat.tri = c;
      dessinerClassement();
    };
  });

  const compte = { maison: 0, tiers: 0, base: 0 };
  sel.forEach((r) => compte[categorie(r)]++);
  const drapeau = `<svg viewBox="0 0 30 20" width="20" height="13"
      style="border-radius:2px"><rect width="30" height="20" fill="#FCD116"/>
      <rect y="10" width="30" height="10" fill="#E8112D"/>
      <rect width="12" height="20" fill="#008751"/></svg>`;
  document.getElementById("meta").innerHTML =
    `<span>${drapeau} <b>${esc(bench.name || "")}</b></span>
     <span><b>${bench.num_utterances ?? "?"}</b> utterances</span>
     <span><b>${bench.duration_hours ?? "?"}</b> hours</span>
     ${bench.is_private ? '<span class="badge">private test set</span>' : ""}
     <span>ranked by <b>${tonal ? "T-WER" : "WER_seg"}</b></span>
     <span><b>${sel.length}</b> of ${rows.length} models shown</span>
     <span>🔧 <b>${compte.maison}</b>&nbsp;fine-tuned by FonBench · 👥
       <b>${compte.tiers}</b>&nbsp;fine-tuned by the community · 🌍
       <b>${compte.base}</b>&nbsp;general-purpose</span>`;

  document.getElementById("legend").innerHTML =
    "All error rates in <b>%</b>, lower is better — except <b>RTFx</b>, where " +
    "higher means faster. <b>⚠️</b> marks a model likely trained on this test " +
    "set: its score is not comparable. Click a column header to sort.";
}

async function chargerClassement() {
  const id = document.getElementById("bench").value;
  try {
    etat.resultats = await lire(
      `results?is_hidden=eq.false&benchmark_id=eq.${encodeURIComponent(id)}&select=*`);
    dessinerClassement();
  } catch (e) {
    document.getElementById("board").innerHTML =
      `<tbody><tr><td>Database unreachable: ${esc(e.message)}</td></tr></tbody>`;
  }
}

/* --- file d'attente ---------------------------------------------------- */

async function chargerFile() {
  const etats = {
    pending: "⏳ pending", running: "⚙️ running", done: "✅ done",
    failed: "❌ failed", rejected: "🚫 rejected",
  };
  try {
    const rows = await lire("public_queue?select=*&order=created_at.desc&limit=50");
    const corps = rows.map((r) => {
      const t = r.progress_total || 0, f = r.progress_done || 0;
      let av = t ? `${f}/${t}` : "—";
      if (t && r.status === "running") av += ` (${Math.floor((f * 100) / t)}%)`;
      return `<tr><td>${esc(r.model_id)}</td><td>${etats[r.status] || esc(r.status)}</td>
        <td class="num">${av}</td><td>${esc((r.error_message || "").slice(0, 120))}</td>
        <td>${esc((r.created_at || "").slice(0, 10))}</td></tr>`;
    }).join("");
    document.getElementById("queue").innerHTML =
      `<thead><tr><th>Model</th><th>Status</th><th>Progress</th><th>Details</th>
       <th>Submitted</th></tr></thead>
       <tbody>${corps || '<tr><td colspan="5">Queue empty.</td></tr>'}</tbody>`;
  } catch (e) {
    document.getElementById("queue").innerHTML =
      `<tbody><tr><td>Database unreachable: ${esc(e.message)}</td></tr></tbody>`;
  }
}

/* --- soumission -------------------------------------------------------- */

async function soumettre(ev) {
  ev.preventDefault();
  const sortie = document.getElementById("submit-out");
  const val = (id) => document.getElementById(id).value.trim();
  const modele = val("f-model");
  const bouts = modele.split("/");
  if (bouts.length !== 2 || !bouts[0] || !bouts[1]) {
    sortie.innerHTML = '<div class="msg"><b>❌ Invalid format</b><br>The identifier ' +
      "must look like <code>organisation/name</code>, exactly as it appears in " +
      "the model URL on Hugging Face.</div>";
    return;
  }
  const corps = { model_id: modele, benchmark_id: val("f-bench") };
  for (const [champ, id] of [["hf_username", "f-user"], ["contact", "f-contact"],
                             ["note", "f-note"], ["train_data", "f-data"],
                             ["base_model", "f-base"]]) {
    if (val(id)) corps[champ] = val(id);
  }
  if (val("f-hours")) corps.train_hours = parseFloat(val("f-hours"));

  try {
    const r = await fetch(`${SUPABASE}/public_requests`, {
      method: "POST",
      headers: { ...H, "Content-Type": "application/json", Prefer: "return=minimal" },
      body: JSON.stringify(corps),
    });
    if (!r.ok) {
      // La base applique elle-même les garde-fous (doublon, débit, file
      // pleine) et renvoie un message déjà rédigé.
      let detail = await r.text();
      try { detail = JSON.parse(detail).message || detail; } catch (_) {}
      sortie.innerHTML = `<div class="msg"><b>❌ Submission rejected</b><br>${esc(detail)}</div>`;
      return;
    }
    sortie.innerHTML = `<div class="msg"><b>✅ <code>${esc(modele)}</code> is queued</b><br>
      Evaluation runs in slices on the GPU of
      <a href="${EVALUATEUR}" target="_blank" rel="noopener">Kimyayd/FonBench</a>.
      Depending on the available quota, expect anywhere from a few minutes to a
      few hours. The score will appear in the leaderboard once computed — follow
      progress in the <b>Queue</b> tab.</div>`;
    chargerFile();
  } catch (e) {
    sortie.innerHTML = `<div class="msg"><b>❌ Could not submit</b><br>${esc(e.message)}</div>`;
  }
}

/* --- textes longs ------------------------------------------------------ */

const NOTE_METRIQUES = `
<h3>Reading the table</h3>
<p>Fon is a <b>tonal</b> language: tones are written with diacritics (á, ɔ́, ě…)
and change the meaning of words. But Fon corpora don't follow the same
convention — some mark no tone at all. A raw WER therefore isn't comparable
from one corpus to the next. Hence three families of measures.</p>
<table>
<tr><th>Metric</th><th>What it tells you</th></tr>
<tr><td><b>WER_seg</b></td><td>Word errors with <b>tones stripped</b>. Measures
 phonetic accuracy and stays comparable across every corpus.</td></tr>
<tr><td><b>WER_ton</b></td><td>Errors on tone marks alone. Shown as “—” when the
 corpus doesn't annotate tones, so the figure is never misleading.</td></tr>
<tr><td><b>T-WER</b></td><td><code>WER_seg + 2 × WER_ton</code>. The headline
 metric: it penalises tone mistakes twice over.</td></tr>
<tr><td>WER, CER</td><td>Word and character errors, tones included.</td></tr>
<tr><td>MER, WIL</td><td><i>Match error rate</i> and <i>word information lost</i>
 — more robust when a model produces many insertions.</td></tr>
<tr><td>RTFx</td><td>Seconds of audio per second of compute. <b>Higher is
 faster.</b></td></tr>
</table>
<p><b>Model types.</b> 🌍 <i>General-purpose</i> models were not built for Fon:
some are multilingual systems that do cover it among a thousand other languages
(the MMS family ships a Fon adapter), others are multilingual without Fon, and
others again are monolingual systems for English or French. They are not
<i>base</i> models in the pretrained sense — <code>wav2vec2-large-960h-lv60-self</code>
is a finished English recogniser, not a starting point. The <b>Trained on</b>
column says exactly what each one saw. 🔧 <i>FonBench</i> and 👥 <i>Community</i>
models were fine-tuned on Fon.</p>
<p>Truly <i>base</i> models — raw pretrained checkpoints such as
<code>facebook/wav2vec2-large-xlsr-53</code> — cannot appear here at all: without
a CTC head or a vocabulary they transcribe nothing. They show up only in the
<b>Base model</b> column, as the starting point of the models fine-tuned from
them.</p>
<p><b>Speed.</b> RTFx depends on the hardware, recorded with each score. Only
compare speeds at equal hardware.</p>`;

const FINE_TUNING = `
<h3>The FonBench fine-tuning runs</h3>
<p>Four pretrained models were fine-tuned on Fon under strictly identical
conditions, so that any gap between them comes from the starting model alone.</p>
<p><b>The data.</b> The training corpus holds 44,225 utterances, of which
<b>13,716 were kept — 30.00 hours from 471 speakers</b>. The sample is not drawn
at random: it is built by taking turns across speakers, each contributing a
little before any one of them dominates. For generalisation, the diversity of
voices matters more than raw volume.</p>
<p>The 471 training speakers and the 45 test speakers are <b>strictly
disjoint</b> — verified, zero in common. A temporal cutoff separates the
transcripts; 20 sentences out of 2,555 (0.8%) do appear in both, too few to move
a score but worth stating rather than claiming a perfect separation.</p>
<p><b>The protocol.</b> Identical across all four: 3 epochs, i.e. 2,574 steps
with 257 of warmup. CTC decoding, character vocabulary built from the training
corpus, feature extractor taken from the base model. A single 24 GB L4,
<code>fp16</code>, gradient checkpointing.</p>
<table>
<tr><th>Resulting model</th><th>Base</th><th>LR</th><th>Batch</th>
    <th>Wall time</th><th>Dev loss</th><th>WER_seg</th></tr>
<tr><td><code>wav2vec2-large-xlsr-53-fon-30h</code></td><td>wav2vec2-large-xlsr-53</td>
    <td>3·10⁻⁴</td><td>8×2</td><td>54 min</td><td><b>0.468</b></td><td><b>38.6%</b></td></tr>
<tr><td><code>mms-300m-fon-30h</code></td><td>mms-300m</td><td>3·10⁻⁴</td>
    <td>8×2</td><td>not retained</td><td>not retained</td><td><b>42.8%</b></td></tr>
<tr><td><code>w2v-bert-2.0-fon-30h</code></td><td>w2v-bert-2.0</td><td>3·10⁻⁵</td>
    <td>4×4</td><td>116 min</td><td>3.102</td><td>96.9%</td></tr>
<tr><td><code>AfriHuBERT-fon-30h</code></td><td>ajesujoba/AfriHuBERT</td>
    <td>3·10⁻⁵</td><td>8×2</td><td>24 min</td><td>3.078</td><td>100.0%</td></tr>
</table>
<p><b>Fine-tuning makes the difference, not the base.</b>
<code>chrisjay/fonxlsr</code> starts from exactly the same model as our best run
— <code>facebook/wav2vec2-large-xlsr-53</code>, confirmed in its configuration —
and reaches 69.3% where we reach 38.6%. The error is cut by a factor of 1.8 with
30 hours of well-chosen data.</p>
<p><b>Two runs out of four failed</b>, and that deserves saying plainly.
<code>w2v-bert-2.0</code> and <code>AfriHuBERT</code> do learn: their loss drops
clearly during training. But it plateaus around 3.1 against 0.47 for the run that
succeeds — the signature of settling into the trivial CTC solution, emitting the
blank symbol everywhere. This is not a learning-rate problem: both were first run
at 3·10⁻⁴, then rerun at 3·10⁻⁵ — the figures in the table — for exactly the same
outcome. The same collapse was later reproduced independently by three
<code>afrihubert-fon-asr-vanilla</code> runs from another team, all between 97.8%
and 99.7%; the same encoder preceded by continued pretraining on 960 h of Fon
reaches 18.3%. Four failures and one success point at the same cause.</p>
<p>Both failures stay in the leaderboard. Removing them would paint a flattering
and false picture of what fine-tuning guarantees.</p>
<p><b>Reproducing.</b> The training and evaluation code is published at
<a href="https://github.com/Izzoudine/EvalScripts" target="_blank" rel="noopener">
github.com/Izzoudine/EvalScripts</a> — one script per evaluated model.</p>`;

const A_PROPOS = `
<h3>FonBench</h3>
<p>The first public speech-recognition leaderboard for <b>Fon</b>, a tonal
language spoken by over two million people in Benin. The project answers a
concrete difficulty: until now, published Fon results were not comparable with
one another, for lack of a shared protocol and a shared test set.</p>
<h3>Why the test set is not published</h3>
<p>Public Fon corpora have been circulating since 2016, and several released
models were trained on them. Their WER on that data then looks remarkable —
while saying nothing about their real ability to transcribe an unseen voice. We
measured it: one of the test sets we were using shared <b>83% of its
utterances</b> with the training split of a public dataset.</p>
<p>The main test set (<b>2,555 utterances, 4.98 hours, 45 speakers</b>) is
therefore not distributed. Only aggregate scores are made public. It is not
secret, though: request access to <code>JMLdata/fon-test-v1</code> and you can
recompute any row yourself.</p>
<h3>The protocol</h3>
<ul>
<li><b>Pinned revision.</b> Every score is tied to the exact commit hash of the
 repository evaluated.</li>
<li><b>Shared normalisation.</b> The same scoring code for every model, tones
 included. It is open: <code>fonbench_eval.py</code>.</li>
<li><b>No arbitrary code.</b> Models are loaded with
 <code>trust_remote_code=False</code>.</li>
<li><b>No duplicate work.</b> A (model, revision, corpus) triple is never
 re-evaluated.</li>
</ul>
<h3>Where things run</h3>
<p>This page is a <i>static</i> Space: it renders the leaderboard and accepts
submissions, but cannot execute Python. Evaluation runs on
<a href="${EVALUATEUR}" target="_blank" rel="noopener">Kimyayd/FonBench</a>,
which holds the GPU and the read access to the private test set. Both read and
write the same database, so what you see here is live, not a copy.</p>
<h3>Verify any number</h3>
<p>Scoring code, a standalone evaluator and one script per evaluated model:
<a href="https://github.com/Izzoudine/EvalScripts" target="_blank" rel="noopener">
github.com/Izzoudine/EvalScripts</a>. Expect agreement within ±0.0002 — CTC
padding depends on batch composition, and we would rather document that than
round the published figures to three decimals.</p>`;

/* --- démarrage --------------------------------------------------------- */

function onglets() {
  const nav = document.getElementById("tabs");
  nav.innerHTML = ONGLETS.map(([id, titre], i) =>
    `<button data-tab="${id}" aria-selected="${i === 0}">${titre}</button>`).join("");
  nav.querySelectorAll("button").forEach((b) => {
    b.onclick = () => {
      nav.querySelectorAll("button").forEach((x) =>
        x.setAttribute("aria-selected", x === b));
      ONGLETS.forEach(([id]) =>
        document.getElementById(`tab-${id}`).classList.toggle("hidden", id !== b.dataset.tab));
      if (b.dataset.tab === "queue") chargerFile();
    };
  });
}

async function demarrer() {
  onglets();
  document.getElementById("metrics-note").innerHTML = NOTE_METRIQUES;
  document.getElementById("ft").innerHTML = FINE_TUNING;
  document.getElementById("about").innerHTML = A_PROPOS;

  document.getElementById("views").innerHTML = VUES.map(([t], i) =>
    `<button data-vue="${i}" aria-pressed="${i === 0}">${t}</button>`).join("");
  document.querySelectorAll("#views button").forEach((b) => {
    b.onclick = () => {
      etat.vue = +b.dataset.vue;
      document.querySelectorAll("#views button").forEach((x) =>
        x.setAttribute("aria-pressed", x === b));
      dessinerClassement();
    };
  });

  try {
    etat.benchmarks = await lire("benchmarks?is_active=eq.true&select=*&order=id");
  } catch (e) {
    document.getElementById("meta").textContent = `Database unreachable: ${e.message}`;
    return;
  }
  const options = etat.benchmarks.map((b) =>
    `<option value="${esc(b.id)}"${b.id === "jml-test-v1" ? " selected" : ""}>
      ${esc(b.name)} (${b.num_utterances ?? "?"} utterances)</option>`).join("");
  document.getElementById("bench").innerHTML = options;
  document.getElementById("f-bench").innerHTML = options;

  ["bench", "sort", "arch", "dec"].forEach((id) => {
    document.getElementById(id).onchange = () => {
      etat.tri = null;
      if (id === "bench") chargerClassement(); else dessinerClassement();
    };
  });
  document.getElementById("contam").onchange = dessinerClassement;
  document.getElementById("refresh").onclick = chargerClassement;
  document.getElementById("refresh-queue").onclick = chargerFile;
  document.getElementById("submit-form").onsubmit = soumettre;

  chargerClassement();
}

demarrer();