Spaces:

nutrientdocs
/

grounding-leaderboard

Running

App Files Files Community

hung-k-nguyen commited on 2 days ago

Commit

b62a42e

0 Parent(s):

Public release

Browse files

Files changed (20) hide show

.gitattributes +35 -0
README.md +24 -0
index.html +172 -0
leaderboard.json +287 -0
results/bart-mnli-en.json +30 -0
results/ce-deberta-en.json +30 -0
results/ce-minilm-en.json +30 -0
results/deberta-anli-en.json +30 -0
results/deberta-zs-en.json +30 -0
results/grounding-en-en.json +30 -0
results/grounding-multi-en.json +30 -0
results/grounding-multi-multi.json +30 -0
results/mdeberta-2mil7-en.json +30 -0
results/mdeberta-2mil7-multi.json +30 -0
results/mdeberta-mnli-en.json +30 -0
results/mdeberta-mnli-multi.json +30 -0
results/roberta-mnli-en.json +30 -0
results/xlmr-xnli-en.json +30 -0
results/xlmr-xnli-multi.json +30 -0
style.css +28 -0

.gitattributes ADDED Viewed

	@@ -0,0 +1,35 @@

+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md ADDED Viewed

	@@ -0,0 +1,24 @@

+---
+title: Grounding Leaderboard
+emoji: 📊
+colorFrom: blue
+colorTo: green
+sdk: static
+app_file: index.html
+pinned: false
+license: cc-by-sa-4.0
+---
+# Grounding Leaderboard
+A static leaderboard for the [`grounding-benchmark`](https://huggingface.co/datasets/nutrientdocs/grounding-benchmark):
+which NLI cross-encoders best tell whether a claim is **actually supported by the document it cites**,
+ranked by ROC-AUC (headline = number grounding), with inference speed on a standard **NVIDIA RTX A6000**.
+- `index.html` — the page (fetches `leaderboard.json`).
+- `leaderboard.json` — aggregated results (array of `score.py` outputs). Regenerated by
+  `scripts/score_leaderboard.sh` on the standard RTX A6000.
+- `results/` — individual submission JSONs (one per model×config), for provenance and PRs.
+**Submit a model:** score it with the reference scorer and open a PR adding your `results/*.json`. See
+[SUBMISSION.md](https://huggingface.co/datasets/nutrientdocs/grounding-benchmark/blob/main/SUBMISSION.md).

index.html ADDED Viewed

	@@ -0,0 +1,172 @@

+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Grounding Leaderboard — document value grounding / NLI</title>
+<style>
+  :root{
+    color-scheme:light dark;
+    --paper:#eef1f4; --surface:#ffffff; --ink:#17222c; --muted:#5c6b78;
+    --hair:#d7dee4; --hair-strong:#c2ccd4;
+    --accent:#1d5fa8; --accent-soft:#e5eefa;
+    --good:#2f8f6f; --good-soft:#dcefe8; --bad:#c0473b; --warn:#c98a2b;
+    --serif:"Iowan Old Style","Palatino Linotype",Palatino,"Book Antiqua",Georgia,serif;
+    --sans:system-ui,-apple-system,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
+    --mono:ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
+  }
+  @media (prefers-color-scheme:dark){
+    :root{--paper:#12171c;--surface:#1a2229;--ink:#e6ecf1;--muted:#93a2ae;
+      --hair:#2b353e;--hair-strong:#38444e;--accent:#6aa9e6;--accent-soft:#1e2b3a;
+      --good:#57b895;--good-soft:#173029;--bad:#e0796d;--warn:#d9a856;}
+  }
+  *{box-sizing:border-box}
+  body{margin:0;background:var(--paper);color:var(--ink);font-family:var(--sans);
+    line-height:1.6;font-size:16px;-webkit-font-smoothing:antialiased}
+  .wrap{max-width:1100px;margin:0 auto;padding:clamp(20px,5vw,60px) clamp(18px,6vw,64px)}
+  .eyebrow{font-family:var(--mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;
+    color:var(--accent);font-weight:600;margin:0 0 14px}
+  h1{font-family:var(--serif);font-weight:600;font-size:clamp(28px,5vw,44px);line-height:1.08;
+    letter-spacing:-.01em;margin:0 0 14px}
+  .lede{font-size:18px;color:var(--muted);max-width:80ch;margin:0 0 8px}
+  a{color:var(--accent)}
+  code{font-family:var(--mono);font-size:.86em;background:var(--accent-soft);padding:.1em .38em;border-radius:4px}
+  .tabs{display:flex;gap:8px;margin:30px 0 4px}
+  .tab{font-family:var(--mono);font-size:13px;letter-spacing:.04em;padding:8px 16px;border-radius:20px;
+    border:1px solid var(--hair-strong);background:var(--surface);color:var(--muted);cursor:pointer}
+  .tab.active{background:var(--accent);color:#fff;border-color:var(--accent)}
+  .banner{background:#fbf5e9;border-left:3px solid var(--warn);color:#7a5512;padding:10px 16px;
+    border-radius:0 8px 8px 0;margin:16px 0;font-size:13.5px;font-family:var(--mono)}
+  @media (prefers-color-scheme:dark){.banner{background:#2a2312;color:var(--warn)}}
+  .tablebox{overflow-x:auto;margin:14px 0}
+  table{border-collapse:collapse;width:100%;font-size:14px;min-width:640px}
+  th,td{text-align:left;padding:11px 12px;border-bottom:1px solid var(--hair)}
+  th{font-family:var(--mono);font-size:11px;letter-spacing:.08em;text-transform:uppercase;color:var(--muted);font-weight:600}
+  td.n,th.n{text-align:right;font-variant-numeric:tabular-nums;font-family:var(--mono)}
+  tbody tr:last-child td{border-bottom:none}
+  tr.ours{background:var(--accent-soft)}
+  .rank{font-family:var(--mono);color:var(--muted);width:2ch}
+  .name{font-weight:650}
+  .chip{display:inline-block;font-family:var(--mono);font-size:10.5px;font-weight:600;letter-spacing:.04em;
+    padding:2px 8px;border-radius:20px;white-space:nowrap;margin-left:6px}
+  .c-open{background:var(--good-soft);color:var(--good)}
+  .c-closed{background:var(--accent-soft);color:var(--accent)}
+  .bar{position:relative;height:16px;background:var(--hair);border-radius:4px;min-width:120px}
+  .bar>span{position:absolute;left:0;top:0;bottom:0;background:var(--accent);border-radius:4px}
+  .bar.hl>span{background:var(--good)}
+  .barnum{font-family:var(--mono);font-variant-numeric:tabular-nums;font-size:12.5px;margin-left:8px;color:var(--muted)}
+  .barcell{display:flex;align-items:center;min-width:220px}
+  .submit{background:var(--surface);border:1px solid var(--hair-strong);border-radius:12px;padding:22px 26px;margin:40px 0}
+  .submit h2{font-family:var(--serif);font-size:22px;margin:0 0 8px}
+  pre{background:var(--accent-soft);border:1px solid var(--hair);border-radius:8px;padding:14px 16px;
+    overflow-x:auto;font-family:var(--mono);font-size:13px}
+  footer{margin-top:56px;border-top:1px solid var(--hair-strong);padding-top:16px;color:var(--muted);
+    font-size:12.5px;font-family:var(--mono)}
+  .muted{color:var(--muted)}
+</style>
+</head>
+<body>
+<div class="wrap">
+  <p class="eyebrow">Document value grounding · benchmark</p>
+  <h1>Grounding Leaderboard</h1>
+  <p class="lede">Is a claim actually supported by the document it cites? Models are ranked by
+    <strong>ROC-AUC</strong> on the open
+    <a href="https://huggingface.co/datasets/nutrientdocs/grounding-benchmark">grounding-benchmark</a> —
+    with the headline metric being <strong>number grounding</strong>, where general NLI models fail.</p>
+  <div class="tabs" id="tabs">
+    <button class="tab active" data-config="en">English</button>
+    <button class="tab" data-config="multi">Multilingual</button>
+  </div>
+  <div id="banner"></div>
+  <div class="tablebox"><table>
+    <thead id="head"></thead>
+    <tbody id="rows"></tbody>
+  </table></div>
+  <p class="muted" style="font-size:13px">Highlighted rows are Nutrient models.
+    <span class="chip c-open">open</span> = downloadable weights;
+    <span class="chip c-closed">closed</span> = commercial (contact us). Ties broken by number AUC.
+    <strong>Ctx</strong> = the model's context window in tokens — models with a shorter window truncate
+    long premises (tables/filings), a real disadvantage reflected in the scores.
+    <strong>Speed</strong> = inference throughput (pairs/s) on the standard hardware, a single
+    <strong>NVIDIA RTX A6000</strong> (batch 64, max&nbsp;len&nbsp;1024).</p>
+  <div class="submit">
+    <h2>Submit your model</h2>
+    <p>Scoring is self-service and reproducible — no server in the loop. Score any NLI cross-encoder and
+      open a PR with the results JSON.</p>
+    <pre>pip install torch transformers datasets
+python benchmark/score.py --model &lt;your-model&gt; --config en --name "Your Model" --open true</pre>
+    <p>Full instructions &amp; rules: <a href="https://huggingface.co/datasets/nutrientdocs/grounding-benchmark/blob/main/SUBMISSION.md">SUBMISSION.md</a>.
+      The reference scorer <code>score.py</code> lives alongside the dataset.</p>
+  </div>
+  <footer>
+    ROC-AUC on nutrientdocs/grounding-benchmark (CC-BY-SA-4.0) · reproducible via benchmark/score.py ·
+    open models: <a href="https://huggingface.co/nutrientdocs/grounding-en">grounding-en</a> ·
+    demo: <a href="https://huggingface.co/spaces/nutrientdocs/grounding-demo">grounding-demo</a>
+  </footer>
+</div>
+<script>
+let DATA = [];
+// All possible facet columns; only those with data for the active config are rendered.
+const FACETS = [["number","Number"],["date","Date"],["string","String"],["table","Table"],["prose","Prose"]];
+function fmt(x){ return (x===null||x===undefined) ? "—" : x.toFixed(3).replace(/^0/,""); }
+function render(config){
+  const rows = DATA.filter(d => d.config === config)
+    .sort((a,b) => (b.overall_auc - a.overall_auc) || ((b.auc?.number||0) - (a.auc?.number||0)));
+  const active = FACETS.filter(([k]) => rows.some(r => r.auc && r.auc[k] != null));  // config-aware columns
+  document.getElementById("head").innerHTML =
+    '<tr><th class="rank">#</th><th>Model</th><th class="n">n</th><th class="barcell">Overall AUC</th>'
+    + active.map(([,h]) => `<th class="n">${h}</th>`).join('')
+    + '<th class="n">Ctx</th><th class="n">Speed</th></tr>';
+  const tb = document.getElementById("rows");
+  tb.innerHTML = "";
+  const ncols = 4 + active.length + 2;
+  if(!rows.length){
+    tb.innerHTML = `<tr><td colspan="${ncols}" class="muted" style="padding:24px">No results yet for this `
+      + `config. Be the first — see “Submit your model”.</td></tr>`;
+    return;
+  }
+  const max = Math.max(...rows.map(r => r.overall_auc || 0));
+  rows.forEach((r,i) => {
+    const chip = r.open ? '<span class="chip c-open">open</span>' : '<span class="chip c-closed">closed</span>';
+    const w = max ? Math.round((r.overall_auc/max)*100) : 0;
+    const tr = document.createElement("tr");
+    if(r.ours) tr.className = "ours";
+    tr.innerHTML =
+      `<td class="rank">${i+1}</td>`+
+      `<td><span class="name">${r.name}</span>${chip}</td>`+
+      `<td class="n">${(r.n||0).toLocaleString()}</td>`+
+      `<td><div class="barcell"><div class="bar ${r.ours?'hl':''}" style="flex:1"><span style="width:${w}%"></span></div>`+
+        `<span class="barnum">${fmt(r.overall_auc)}</span></div></td>`+
+      active.map(([k]) => `<td class="n">${fmt(r.auc?.[k])}</td>`).join('')+
+      `<td class="n">${r.ctx ? r.ctx.toLocaleString() : '—'}</td>`+
+      `<td class="n">${r.pairs_per_s ? r.pairs_per_s.toLocaleString()+'/s' : '—'}</td>`;
+    tb.appendChild(tr);
+  });
+}
+function setBanner(){
+  const b = document.getElementById("banner");
+  if(DATA.length && DATA.some(d => d.provisional)){
+    b.className = "banner";
+    b.textContent = "Provisional numbers — public-slice re-scoring in progress; figures shown are from the internal full test set.";
+  } else { b.className=""; b.textContent=""; }
+}
+document.querySelectorAll(".tab").forEach(t => t.addEventListener("click", () => {
+  document.querySelectorAll(".tab").forEach(x => x.classList.remove("active"));
+  t.classList.add("active");
+  render(t.dataset.config);
+}));
+fetch("leaderboard.json").then(r => r.ok ? r.json() : []).then(d => {
+  DATA = Array.isArray(d) ? d : (d.results || []);
+  setBanner(); render("en");
+}).catch(() => { render("en"); });
+</script>
+</body>
+</html>

leaderboard.json ADDED Viewed

	@@ -0,0 +1,287 @@

+[
+  {
+    "name": "BART-large MNLI",
+    "model": "facebook/bart-large-mnli",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.6362744923589948,
+    "auc": {
+      "number": 0.4784391111111111,
+      "date": 0.9242776612148942,
+      "string": 0.7570571789321789,
+      "table": 0.6105985609760144,
+      "prose": 0.891619842839355
+    },
+    "pairs_per_s": 24.5,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 1024,
+    "ours": false
+  },
+  {
+    "name": "cross-encoder DeBERTa-v3-base",
+    "model": "cross-encoder/nli-deberta-v3-base",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.644738021188759,
+    "auc": {
+      "number": 0.44977955555555554,
+      "date": 0.9497017113678303,
+      "string": 0.6665088383838383,
+      "table": 0.631454675168817,
+      "prose": 0.8665808665808665
+    },
+    "pairs_per_s": 96.2,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "cross-encoder MiniLM-L6",
+    "model": "cross-encoder/nli-MiniLM2-L6-H768",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.5927182536915374,
+    "auc": {
+      "number": 0.44786488888888887,
+      "date": 0.8190866806542126,
+      "string": 0.5469651875901876,
+      "table": 0.5710080357488511,
+      "prose": 0.8484582143118728
+    },
+    "pairs_per_s": 318.4,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "DeBERTa-v3-large MNLI/FEVER/ANLI",
+    "model": "MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.7693892982804592,
+    "auc": {
+      "number": 0.6419208888888889,
+      "date": 0.9954498344245305,
+      "string": 0.9134875541125541,
+      "table": 0.7472576140517762,
+      "prose": 0.9447217252095301
+    },
+    "pairs_per_s": 88.4,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "DeBERTa-v3-large zero-shot",
+    "model": "MoritzLaurer/deberta-v3-large-zeroshot-v2.0",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.7856283280264451,
+    "auc": {
+      "number": 0.6575431111111111,
+      "date": 0.994580879193104,
+      "string": 0.9412315115440115,
+      "table": 0.7659018786950342,
+      "prose": 0.9286289774094653
+    },
+    "pairs_per_s": 89.0,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "grounding-en (ours)",
+    "model": "nutrientdocs/grounding-en",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.8815400924541517,
+    "auc": {
+      "number": 0.9231893333333333,
+      "date": 0.998429560909022,
+      "string": 0.9549625721500722,
+      "table": 0.8631714938307792,
+      "prose": 0.9637233051867198
+    },
+    "pairs_per_s": 148.2,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 8192,
+    "ours": true
+  },
+  {
+    "name": "grounding-multilingual (ours)",
+    "model": "nutrientdocs/grounding-multilingual-private",
+    "config": "en",
+    "open": false,
+    "n": 5000,
+    "overall_auc": 0.9249458851297964,
+    "auc": {
+      "number": 0.9689866666666667,
+      "date": 0.9985970322808969,
+      "string": 0.9487170815295816,
+      "table": 0.9148366580575942,
+      "prose": 0.9703832752613241
+    },
+    "pairs_per_s": 243.5,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 8192,
+    "ours": true
+  },
+  {
+    "name": "grounding-multilingual (ours)",
+    "model": "nutrientdocs/grounding-multilingual-private",
+    "config": "multi",
+    "open": false,
+    "n": 8421,
+    "overall_auc": 0.9652247630640813,
+    "auc": {
+      "number": 0.9987015555555555,
+      "date": null,
+      "string": null,
+      "table": 0.9985981272248878,
+      "prose": 0.9261168118990385
+    },
+    "pairs_per_s": 337.7,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 8192,
+    "ours": true
+  },
+  {
+    "name": "mDeBERTa-v3-base XNLI-2mil7",
+    "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.6998874981384613,
+    "auc": {
+      "number": 0.4928408888888889,
+      "date": 0.9457582345357567,
+      "string": 0.8369633838383839,
+      "table": 0.664911263874525,
+      "prose": 0.9090885310397505
+    },
+    "pairs_per_s": 215.0,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "mDeBERTa-v3-base XNLI-2mil7",
+    "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
+    "config": "multi",
+    "open": true,
+    "n": 8421,
+    "overall_auc": 0.8937062502752443,
+    "auc": {
+      "number": 0.7750426666666667,
+      "date": null,
+      "string": null,
+      "table": 0.7274667234174277,
+      "prose": 0.9609073768028846
+    },
+    "pairs_per_s": 301.2,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "mDeBERTa-v3-base MNLI-XNLI",
+    "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.6294148925341116,
+    "auc": {
+      "number": 0.43475644444444445,
+      "date": 0.8523755656108597,
+      "string": 0.8139204545454546,
+      "table": 0.6084399143885807,
+      "prose": 0.8505352041937407
+    },
+    "pairs_per_s": 214.6,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "mDeBERTa-v3-base MNLI-XNLI",
+    "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
+    "config": "multi",
+    "open": true,
+    "n": 8421,
+    "overall_auc": 0.8305958149319597,
+    "auc": {
+      "number": 0.8063175555555555,
+      "date": null,
+      "string": null,
+      "table": 0.7796536913790435,
+      "prose": 0.8636295072115384
+    },
+    "pairs_per_s": 304.8,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "RoBERTa-large MNLI",
+    "model": "FacebookAI/roberta-large-mnli",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.5881571031267598,
+    "auc": {
+      "number": 0.4420177777777778,
+      "date": 0.887693066053237,
+      "string": 0.5542929292929293,
+      "table": 0.5558180923253898,
+      "prose": 0.857927614025175
+    },
+    "pairs_per_s": 56.8,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "XLM-R-large XNLI",
+    "model": "joeddav/xlm-roberta-large-xnli",
+    "config": "en",
+    "open": true,
+    "n": 5000,
+    "overall_auc": 0.5716570972756371,
+    "auc": {
+      "number": 0.44068444444444443,
+      "date": 0.7327852069061402,
+      "string": 0.5111381673881674,
+      "table": 0.5335998274498526,
+      "prose": 0.8582833704784925
+    },
+    "pairs_per_s": 55.7,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  },
+  {
+    "name": "XLM-R-large XNLI",
+    "model": "joeddav/xlm-roberta-large-xnli",
+    "config": "multi",
+    "open": true,
+    "n": 8421,
+    "overall_auc": 0.8107369827807182,
+    "auc": {
+      "number": 0.7651982222222222,
+      "date": null,
+      "string": null,
+      "table": 0.7150696486611979,
+      "prose": 0.8689075270432692
+    },
+    "pairs_per_s": 80.1,
+    "hardware": "NVIDIA RTX A6000",
+    "ctx": 512,
+    "ours": false
+  }
+]

results/bart-mnli-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "BART-large MNLI",
+  "model": "facebook/bart-large-mnli",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.6362744923589948,
+  "auc": {
+    "number": 0.4784391111111111,
+    "date": 0.9242776612148942,
+    "string": 0.7570571789321789,
+    "table": 0.6105985609760144,
+    "prose": 0.891619842839355
+  },
+  "pairs_per_s": 24.5,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 1024,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 1024,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 204.16,
+    "ctx": 1024,
+    "pairs_per_s": 24.5
+  }
+}

results/ce-deberta-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "cross-encoder DeBERTa-v3-base",
+  "model": "cross-encoder/nli-deberta-v3-base",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.644738021188759,
+  "auc": {
+    "number": 0.44977955555555554,
+    "date": 0.9497017113678303,
+    "string": 0.6665088383838383,
+    "table": 0.631454675168817,
+    "prose": 0.8665808665808665
+  },
+  "pairs_per_s": 96.2,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 51.97,
+    "ctx": 512,
+    "pairs_per_s": 96.2
+  }
+}

results/ce-minilm-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "cross-encoder MiniLM-L6",
+  "model": "cross-encoder/nli-MiniLM2-L6-H768",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.5927182536915374,
+  "auc": {
+    "number": 0.44786488888888887,
+    "date": 0.8190866806542126,
+    "string": 0.5469651875901876,
+    "table": 0.5710080357488511,
+    "prose": 0.8484582143118728
+  },
+  "pairs_per_s": 318.4,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 15.7,
+    "ctx": 512,
+    "pairs_per_s": 318.4
+  }
+}

results/deberta-anli-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "DeBERTa-v3-large MNLI/FEVER/ANLI",
+  "model": "MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.7693892982804592,
+  "auc": {
+    "number": 0.6419208888888889,
+    "date": 0.9954498344245305,
+    "string": 0.9134875541125541,
+    "table": 0.7472576140517762,
+    "prose": 0.9447217252095301
+  },
+  "pairs_per_s": 88.4,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 56.55,
+    "ctx": 512,
+    "pairs_per_s": 88.4
+  }
+}

results/deberta-zs-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "DeBERTa-v3-large zero-shot",
+  "model": "MoritzLaurer/deberta-v3-large-zeroshot-v2.0",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.7856283280264451,
+  "auc": {
+    "number": 0.6575431111111111,
+    "date": 0.994580879193104,
+    "string": 0.9412315115440115,
+    "table": 0.7659018786950342,
+    "prose": 0.9286289774094653
+  },
+  "pairs_per_s": 89.0,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 56.17,
+    "ctx": 512,
+    "pairs_per_s": 89.0
+  }
+}

results/grounding-en-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "grounding-en (ours)",
+  "model": "nutrientdocs/grounding-en",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.8815400924541517,
+  "auc": {
+    "number": 0.9231893333333333,
+    "date": 0.998429560909022,
+    "string": 0.9549625721500722,
+    "table": 0.8631714938307792,
+    "prose": 0.9637233051867198
+  },
+  "pairs_per_s": 148.2,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 8192,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 33.74,
+    "ctx": 512,
+    "pairs_per_s": 148.2
+  }
+}

results/grounding-multi-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "grounding-multilingual (ours)",
+  "model": "nutrientdocs/grounding-multilingual-private",
+  "config": "en",
+  "open": false,
+  "n": 5000,
+  "overall_auc": 0.9249458851297964,
+  "auc": {
+    "number": 0.9689866666666667,
+    "date": 0.9985970322808969,
+    "string": 0.9487170815295816,
+    "table": 0.9148366580575942,
+    "prose": 0.9703832752613241
+  },
+  "pairs_per_s": 243.5,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 8192,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 20.53,
+    "ctx": 512,
+    "pairs_per_s": 243.5
+  }
+}

results/grounding-multi-multi.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "grounding-multilingual (ours)",
+  "model": "nutrientdocs/grounding-multilingual-private",
+  "config": "multi",
+  "open": false,
+  "n": 8421,
+  "overall_auc": 0.9652247630640813,
+  "auc": {
+    "number": 0.9987015555555555,
+    "date": null,
+    "string": null,
+    "table": 0.9985981272248878,
+    "prose": 0.9261168118990385
+  },
+  "pairs_per_s": 337.7,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 8192,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 8421,
+    "elapsed_s": 24.93,
+    "ctx": 512,
+    "pairs_per_s": 337.7
+  }
+}

results/mdeberta-2mil7-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "mDeBERTa-v3-base XNLI-2mil7",
+  "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.6998874981384613,
+  "auc": {
+    "number": 0.4928408888888889,
+    "date": 0.9457582345357567,
+    "string": 0.8369633838383839,
+    "table": 0.664911263874525,
+    "prose": 0.9090885310397505
+  },
+  "pairs_per_s": 215.0,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 23.25,
+    "ctx": 512,
+    "pairs_per_s": 215.0
+  }
+}

results/mdeberta-2mil7-multi.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "mDeBERTa-v3-base XNLI-2mil7",
+  "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
+  "config": "multi",
+  "open": true,
+  "n": 8421,
+  "overall_auc": 0.8937062502752443,
+  "auc": {
+    "number": 0.7750426666666667,
+    "date": null,
+    "string": null,
+    "table": 0.7274667234174277,
+    "prose": 0.9609073768028846
+  },
+  "pairs_per_s": 301.2,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 8421,
+    "elapsed_s": 27.96,
+    "ctx": 512,
+    "pairs_per_s": 301.2
+  }
+}

results/mdeberta-mnli-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "mDeBERTa-v3-base MNLI-XNLI",
+  "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.6294148925341116,
+  "auc": {
+    "number": 0.43475644444444445,
+    "date": 0.8523755656108597,
+    "string": 0.8139204545454546,
+    "table": 0.6084399143885807,
+    "prose": 0.8505352041937407
+  },
+  "pairs_per_s": 214.6,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 23.3,
+    "ctx": 512,
+    "pairs_per_s": 214.6
+  }
+}

results/mdeberta-mnli-multi.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "mDeBERTa-v3-base MNLI-XNLI",
+  "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
+  "config": "multi",
+  "open": true,
+  "n": 8421,
+  "overall_auc": 0.8305958149319597,
+  "auc": {
+    "number": 0.8063175555555555,
+    "date": null,
+    "string": null,
+    "table": 0.7796536913790435,
+    "prose": 0.8636295072115384
+  },
+  "pairs_per_s": 304.8,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 8421,
+    "elapsed_s": 27.63,
+    "ctx": 512,
+    "pairs_per_s": 304.8
+  }
+}

results/roberta-mnli-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "RoBERTa-large MNLI",
+  "model": "FacebookAI/roberta-large-mnli",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.5881571031267598,
+  "auc": {
+    "number": 0.4420177777777778,
+    "date": 0.887693066053237,
+    "string": 0.5542929292929293,
+    "table": 0.5558180923253898,
+    "prose": 0.857927614025175
+  },
+  "pairs_per_s": 56.8,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 87.96,
+    "ctx": 512,
+    "pairs_per_s": 56.8
+  }
+}

results/xlmr-xnli-en.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "XLM-R-large XNLI",
+  "model": "joeddav/xlm-roberta-large-xnli",
+  "config": "en",
+  "open": true,
+  "n": 5000,
+  "overall_auc": 0.5716570972756371,
+  "auc": {
+    "number": 0.44068444444444443,
+    "date": 0.7327852069061402,
+    "string": 0.5111381673881674,
+    "table": 0.5335998274498526,
+    "prose": 0.8582833704784925
+  },
+  "pairs_per_s": 55.7,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 5000,
+    "elapsed_s": 89.72,
+    "ctx": 512,
+    "pairs_per_s": 55.7
+  }
+}

results/xlmr-xnli-multi.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "XLM-R-large XNLI",
+  "model": "joeddav/xlm-roberta-large-xnli",
+  "config": "multi",
+  "open": true,
+  "n": 8421,
+  "overall_auc": 0.8107369827807182,
+  "auc": {
+    "number": 0.7651982222222222,
+    "date": null,
+    "string": null,
+    "table": 0.7150696486611979,
+    "prose": 0.8689075270432692
+  },
+  "pairs_per_s": 80.1,
+  "hardware": "NVIDIA RTX A6000",
+  "ctx": 512,
+  "perf": {
+    "device": "cuda",
+    "batch_size": 64,
+    "max_length": 512,
+    "torch": "2.4.1+cu124",
+    "platform": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+    "accelerator": "NVIDIA RTX A6000",
+    "n_pairs": 8421,
+    "elapsed_s": 105.17,
+    "ctx": 512,
+    "pairs_per_s": 80.1
+  }
+}

style.css ADDED Viewed

	@@ -0,0 +1,28 @@

+body {
+	padding: 2rem;
+	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
+}
+h1 {
+	font-size: 16px;
+	margin-top: 0;
+}
+p {
+	color: rgb(107, 114, 128);
+	font-size: 15px;
+	margin-bottom: 10px;
+	margin-top: 5px;
+}
+.card {
+	max-width: 620px;
+	margin: 0 auto;
+	padding: 16px;
+	border: 1px solid lightgray;
+	border-radius: 16px;
+}
+.card p:last-child {
+	margin-bottom: 0;
+}