Spaces:

nutrientdocs
/

grounding-leaderboard

Running

File size: 11,456 Bytes

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Grounding Leaderboard — document value grounding / NLI</title>
<style>
  :root{
    color-scheme:light dark;
    --paper:#eef1f4; --surface:#ffffff; --ink:#17222c; --muted:#5c6b78;
    --hair:#d7dee4; --hair-strong:#c2ccd4;
    --accent:#1d5fa8; --accent-soft:#e5eefa;
    --good:#2f8f6f; --good-soft:#dcefe8; --bad:#c0473b; --warn:#c98a2b;
    --serif:"Iowan Old Style","Palatino Linotype",Palatino,"Book Antiqua",Georgia,serif;
    --sans:system-ui,-apple-system,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
    --mono:ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
  }
  @media (prefers-color-scheme:dark){
    :root{--paper:#12171c;--surface:#1a2229;--ink:#e6ecf1;--muted:#93a2ae;
      --hair:#2b353e;--hair-strong:#38444e;--accent:#6aa9e6;--accent-soft:#1e2b3a;
      --good:#57b895;--good-soft:#173029;--bad:#e0796d;--warn:#d9a856;}
  }
  *{box-sizing:border-box}
  body{margin:0;background:var(--paper);color:var(--ink);font-family:var(--sans);
    line-height:1.6;font-size:16px;-webkit-font-smoothing:antialiased}
  .wrap{max-width:1100px;margin:0 auto;padding:clamp(20px,5vw,60px) clamp(18px,6vw,64px)}
  .eyebrow{font-family:var(--mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;
    color:var(--accent);font-weight:600;margin:0 0 14px}
  h1{font-family:var(--serif);font-weight:600;font-size:clamp(28px,5vw,44px);line-height:1.08;
    letter-spacing:-.01em;margin:0 0 14px}
  .lede{font-size:18px;color:var(--muted);max-width:80ch;margin:0 0 8px}
  a{color:var(--accent)}
  code{font-family:var(--mono);font-size:.86em;background:var(--accent-soft);padding:.1em .38em;border-radius:4px}

  .tabs{display:flex;gap:8px;margin:30px 0 4px}
  .tab{font-family:var(--mono);font-size:13px;letter-spacing:.04em;padding:8px 16px;border-radius:20px;
    border:1px solid var(--hair-strong);background:var(--surface);color:var(--muted);cursor:pointer}
  .tab.active{background:var(--accent);color:#fff;border-color:var(--accent)}
  .banner{background:#fbf5e9;border-left:3px solid var(--warn);color:#7a5512;padding:10px 16px;
    border-radius:0 8px 8px 0;margin:16px 0;font-size:13.5px;font-family:var(--mono)}
  @media (prefers-color-scheme:dark){.banner{background:#2a2312;color:var(--warn)}}

  .tablebox{overflow-x:auto;margin:14px 0}
  table{border-collapse:collapse;width:100%;font-size:14px;min-width:640px}
  th,td{text-align:left;padding:11px 12px;border-bottom:1px solid var(--hair)}
  th{font-family:var(--mono);font-size:11px;letter-spacing:.08em;text-transform:uppercase;color:var(--muted);font-weight:600}
  td.n,th.n{text-align:right;font-variant-numeric:tabular-nums;font-family:var(--mono)}
  tbody tr:last-child td{border-bottom:none}
  tr.ours{background:var(--accent-soft)}
  .rank{font-family:var(--mono);color:var(--muted);width:2ch}
  .name{font-weight:650}
  .chip{display:inline-block;font-family:var(--mono);font-size:10.5px;font-weight:600;letter-spacing:.04em;
    padding:2px 8px;border-radius:20px;white-space:nowrap;margin-left:6px}
  .c-open{background:var(--good-soft);color:var(--good)}
  .c-closed{background:var(--accent-soft);color:var(--accent)}
  .bar{position:relative;height:16px;background:var(--hair);border-radius:4px;min-width:120px}
  .bar>span{position:absolute;left:0;top:0;bottom:0;background:var(--accent);border-radius:4px}
  .bar.hl>span{background:var(--good)}
  .barnum{font-family:var(--mono);font-variant-numeric:tabular-nums;font-size:12.5px;margin-left:8px;color:var(--muted)}
  .barcell{display:flex;align-items:center;min-width:220px}

  .submit{background:var(--surface);border:1px solid var(--hair-strong);border-radius:12px;padding:22px 26px;margin:40px 0}
  .submit h2{font-family:var(--serif);font-size:22px;margin:0 0 8px}
  pre{background:var(--accent-soft);border:1px solid var(--hair);border-radius:8px;padding:14px 16px;
    overflow-x:auto;font-family:var(--mono);font-size:13px}
  footer{margin-top:56px;border-top:1px solid var(--hair-strong);padding-top:16px;color:var(--muted);
    font-size:12.5px;font-family:var(--mono)}
  .muted{color:var(--muted)}
</style>
</head>
<body>
<div class="wrap">
  <p class="eyebrow">Document value grounding · benchmark</p>
  <h1>Grounding Leaderboard</h1>
  <p class="lede">Is a claim actually supported by the document it cites? Models are ranked by
    <strong>ROC-AUC</strong> on the open
    <a href="https://huggingface.co/datasets/nutrientdocs/grounding-benchmark">grounding-benchmark</a> —
    with the headline metric being <strong>number grounding</strong>, where general NLI models fail.</p>

  <div class="tabs" id="tabs">
    <button class="tab active" data-config="en">English</button>
    <button class="tab" data-config="multi">Multilingual</button>
  </div>

  <div id="banner"></div>
  <div class="tablebox"><table>
    <thead id="head"></thead>
    <tbody id="rows"></tbody>
  </table></div>
  <p class="muted" style="font-size:13px">Highlighted rows are Nutrient models.
    <span class="chip c-open">open</span> = downloadable weights;
    <span class="chip c-closed">closed</span> = commercial (contact us). Ties broken by number AUC.
    <strong>Ctx</strong> = the model's context window in tokens — models with a shorter window truncate
    long premises (tables/filings), a real disadvantage reflected in the scores.
    <strong>Speed</strong> = inference throughput (pairs/s) on the standard hardware, a single
    <strong>NVIDIA RTX A6000</strong> (batch 64, max&nbsp;len&nbsp;1024).</p>

  <div class="submit">
    <h2>Submit your model</h2>
    <p>Scoring is self-service and reproducible — no server in the loop. Score any NLI cross-encoder and
      open a PR with the results JSON.</p>
    <pre>pip install torch transformers datasets
python benchmark/score.py --model &lt;your-model&gt; --config en --name "Your Model" --open true</pre>
    <p>Full instructions &amp; rules: <a href="https://huggingface.co/datasets/nutrientdocs/grounding-benchmark/blob/main/SUBMISSION.md">SUBMISSION.md</a>.
      The reference scorer <code>score.py</code> lives alongside the dataset.</p>
  </div>

  <p id="models-benchmarked" class="muted" style="font-size:12.5px;margin-top:28px">
    <strong>Models benchmarked:</strong>
    <a href="https://huggingface.co/nutrientdocs/grounding-en">nutrientdocs/grounding-en</a> ·
    <a href="https://huggingface.co/nutrientdocs/grounding-multilingual">nutrientdocs/grounding-multilingual</a> ·
    <a href="https://huggingface.co/cross-encoder/nli-deberta-v3-base">cross-encoder/nli-deberta-v3-base</a> ·
    <a href="https://huggingface.co/cross-encoder/nli-MiniLM2-L6-H768">cross-encoder/nli-MiniLM2-L6-H768</a> ·
    <a href="https://huggingface.co/facebook/bart-large-mnli">facebook/bart-large-mnli</a> ·
    <a href="https://huggingface.co/FacebookAI/roberta-large-mnli">FacebookAI/roberta-large-mnli</a> ·
    <a href="https://huggingface.co/joeddav/xlm-roberta-large-xnli">joeddav/xlm-roberta-large-xnli</a> ·
    <a href="https://huggingface.co/MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli">MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli</a> ·
    <a href="https://huggingface.co/MoritzLaurer/deberta-v3-large-zeroshot-v2.0">MoritzLaurer/deberta-v3-large-zeroshot-v2.0</a> ·
    <a href="https://huggingface.co/MoritzLaurer/mDeBERTa-v3-base-mnli-xnli">MoritzLaurer/mDeBERTa-v3-base-mnli-xnli</a> ·
    <a href="https://huggingface.co/MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7">MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7</a>
  </p>

  <footer>
    ROC-AUC on nutrientdocs/grounding-benchmark (CC-BY-SA-4.0) · reproducible via benchmark/score.py ·
    Nutrient models: <a href="https://huggingface.co/nutrientdocs/grounding-en">grounding-en</a>, <a href="https://huggingface.co/nutrientdocs/grounding-multilingual">grounding-multilingual</a> ·
    demo: <a href="https://huggingface.co/spaces/nutrientdocs/grounding-demo">grounding-demo</a>
  </footer>
</div>

<script>
let DATA = [];
// All possible facet columns; only those with data for the active config are rendered.
const FACETS = [["number","Number"],["date","Date"],["string","String"],["table","Table"],["prose","Prose"]];
// The commercial model's weights live in a private repo; its row must link to the public showcase page.
const REPO_ALIAS = {"nutrientdocs/grounding-multilingual-private":"nutrientdocs/grounding-multilingual"};
function hfUrl(m){ return m ? "https://huggingface.co/" + (REPO_ALIAS[m] || m) : null; }
function fmt(x){ return (x===null||x===undefined) ? "—" : x.toFixed(3).replace(/^0/,""); }
function render(config){
  const rows = DATA.filter(d => d.config === config)
    .sort((a,b) => (b.overall_auc - a.overall_auc) || ((b.auc?.number||0) - (a.auc?.number||0)));
  const active = FACETS.filter(([k]) => rows.some(r => r.auc && r.auc[k] != null));  // config-aware columns
  document.getElementById("head").innerHTML =
    '<tr><th class="rank">#</th><th>Model</th><th class="n">n</th><th class="barcell">Overall AUC</th>'
    + active.map(([,h]) => `<th class="n">${h}</th>`).join('')
    + '<th class="n">Ctx</th><th class="n">Speed</th></tr>';
  const tb = document.getElementById("rows");
  tb.innerHTML = "";
  const ncols = 4 + active.length + 2;
  if(!rows.length){
    tb.innerHTML = `<tr><td colspan="${ncols}" class="muted" style="padding:24px">No results yet for this `
      + `config. Be the first — see “Submit your model”.</td></tr>`;
    return;
  }
  const max = Math.max(...rows.map(r => r.overall_auc || 0));
  rows.forEach((r,i) => {
    const chip = r.open ? '<span class="chip c-open">open</span>' : '<span class="chip c-closed">closed</span>';
    const w = max ? Math.round((r.overall_auc/max)*100) : 0;
    const tr = document.createElement("tr");
    if(r.ours) tr.className = "ours";
    tr.innerHTML =
      `<td class="rank">${i+1}</td>`+
      `<td>${hfUrl(r.model) ? `<a class="name" href="${hfUrl(r.model)}" target="_blank" rel="noopener">${r.name}</a>` : `<span class="name">${r.name}</span>`}${chip}</td>`+
      `<td class="n">${(r.n||0).toLocaleString()}</td>`+
      `<td><div class="barcell"><div class="bar ${r.ours?'hl':''}" style="flex:1"><span style="width:${w}%"></span></div>`+
        `<span class="barnum">${fmt(r.overall_auc)}</span></div></td>`+
      active.map(([k]) => `<td class="n">${fmt(r.auc?.[k])}</td>`).join('')+
      `<td class="n">${r.ctx ? r.ctx.toLocaleString() : '—'}</td>`+
      `<td class="n">${r.pairs_per_s ? r.pairs_per_s.toLocaleString()+'/s' : '—'}</td>`;
    tb.appendChild(tr);
  });
}
function setBanner(){
  const b = document.getElementById("banner");
  if(DATA.length && DATA.some(d => d.provisional)){
    b.className = "banner";
    b.textContent = "Provisional numbers — public-slice re-scoring in progress; figures shown are from the internal full test set.";
  } else { b.className=""; b.textContent=""; }
}
document.querySelectorAll(".tab").forEach(t => t.addEventListener("click", () => {
  document.querySelectorAll(".tab").forEach(x => x.classList.remove("active"));
  t.classList.add("active");
  render(t.dataset.config);
}));
fetch("leaderboard.json").then(r => r.ok ? r.json() : []).then(d => {
  DATA = Array.isArray(d) ? d : (d.results || []);
  setBanner(); render("en");
}).catch(() => { render("en"); });
</script>
</body>
</html>