Spaces:

nutrientdocs
/

doc-split-leaderboard

Running

App Files Files Community

hung-k-nguyen commited on 3 days ago

Commit

0c5549b

0 Parent(s):

Release snapshot

Browse files

Files changed (4) hide show

.gitattributes +35 -0
README.md +16 -0
index.html +210 -0
style.css +28 -0

.gitattributes ADDED Viewed

	@@ -0,0 +1,35 @@

+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md ADDED Viewed

	@@ -0,0 +1,16 @@

+---
+title: Doc-Split Leaderboard
+emoji: 🏆
+colorFrom: indigo
+colorTo: blue
+sdk: static
+app_file: index.html
+pinned: false
+---
+# Doc-Split Leaderboard
+Unified page-stream-segmentation leaderboard — our flagship + open-weight model vs cloud VLMs and the
+field's self-declared numbers, one metric (boundary F1 + κ) across six cuts. Static page; see
+[doc-split-benchmark](https://huggingface.co/datasets/nutrientdocs/doc-split-benchmark) to reproduce and
+[SUBMISSION.md](https://huggingface.co/datasets/nutrientdocs/doc-split-benchmark/blob/main/SUBMISSION.md) to add a row.

index.html ADDED Viewed

	@@ -0,0 +1,210 @@

+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Page Stream Segmentation — Unified Leaderboard</title>
+<style>
+  :root{
+    --bg:#f6f8fb; --surface:#ffffff; --surface-2:#f0f4f8;
+    --text:#111820; --muted:#5c6773; --faint:#8a96a2; --border:#e2e8ef;
+    --accent:#1d5fa8; --accent-soft:#e9eefb; --accent-line:#c8d5f6;
+    --good:#0f7a58; --good-bg:#e2f3ec; --ok:#6b7280; --ok-bg:#eef1f4;
+    --warn:#9a6a12; --warn-bg:#f7edda; --poor:#a83a3a; --poor-bg:#f6e3e3;
+    --shadow:0 1px 2px rgba(16,32,64,.05),0 4px 16px rgba(16,32,64,.05);
+    --mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace;
+    --sans:system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;
+  }
+  @media (prefers-color-scheme:dark){:root{
+    --bg:#0c1016; --surface:#141a22; --surface-2:#1a222c;
+    --text:#e6edf4; --muted:#94a1af; --faint:#6a7684; --border:#232c37;
+    --accent:#6aa9e6; --accent-soft:#182238; --accent-line:#2c3d63;
+    --good:#3bbd8e; --good-bg:#122a22; --ok:#9aa5b1; --ok-bg:#1b232d;
+    --warn:#d6a24a; --warn-bg:#2b2416; --poor:#e07373; --poor-bg:#2c1a1a;
+    --shadow:0 1px 2px rgba(0,0,0,.3),0 6px 22px rgba(0,0,0,.35);
+  }}
+  :root[data-theme="light"]{
+    --bg:#f6f8fb; --surface:#ffffff; --surface-2:#f0f4f8; --text:#111820; --muted:#5c6773; --faint:#8a96a2; --border:#e2e8ef;
+    --accent:#1d5fa8; --accent-soft:#e9eefb; --accent-line:#c8d5f6;
+    --good:#0f7a58; --good-bg:#e2f3ec; --ok:#6b7280; --ok-bg:#eef1f4; --warn:#9a6a12; --warn-bg:#f7edda; --poor:#a83a3a; --poor-bg:#f6e3e3;
+    --shadow:0 1px 2px rgba(16,32,64,.05),0 4px 16px rgba(16,32,64,.05);
+  }
+  :root[data-theme="dark"]{
+    --bg:#0c1016; --surface:#141a22; --surface-2:#1a222c; --text:#e6edf4; --muted:#94a1af; --faint:#6a7684; --border:#232c37;
+    --accent:#6aa9e6; --accent-soft:#182238; --accent-line:#2c3d63;
+    --good:#3bbd8e; --good-bg:#122a22; --ok:#9aa5b1; --ok-bg:#1b232d; --warn:#d6a24a; --warn-bg:#2b2416; --poor:#e07373; --poor-bg:#2c1a1a;
+    --shadow:0 1px 2px rgba(0,0,0,.3),0 6px 22px rgba(0,0,0,.35);
+  }
+  *{box-sizing:border-box}
+  body{margin:0;background:var(--bg);color:var(--text);font-family:var(--sans);line-height:1.5;
+    -webkit-font-smoothing:antialiased;font-size:15px}
+  .wrap{max-width:1120px;margin:0 auto;padding:48px 24px 72px}
+  header .eyebrow{font-size:12px;letter-spacing:.14em;text-transform:uppercase;color:var(--accent);font-weight:600}
+  h1{font-size:clamp(26px,4vw,38px);line-height:1.12;margin:.35em 0 .25em;letter-spacing:-.02em;text-wrap:balance;font-weight:700}
+  .lede{color:var(--muted);max-width:64ch;margin:0}
+  .meta{margin-top:14px;font-size:12.5px;color:var(--faint);font-family:var(--mono)}
+  .findings{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:14px;margin:32px 0 8px}
+  .finding{background:var(--surface);border:1px solid var(--border);border-radius:10px;padding:16px 18px;box-shadow:var(--shadow)}
+  .finding b{display:block;font-size:22px;letter-spacing:-.01em;font-family:var(--mono);font-variant-numeric:tabular-nums}
+  .finding span{color:var(--muted);font-size:13px}
+  .finding .k{color:var(--accent);font-weight:600}
+  h2{font-size:13px;letter-spacing:.12em;text-transform:uppercase;color:var(--muted);font-weight:600;margin:40px 0 12px}
+  .tablecard{background:var(--surface);border:1px solid var(--border);border-radius:12px;box-shadow:var(--shadow);overflow:hidden}
+  .scroll{overflow-x:auto}
+  table{border-collapse:collapse;width:100%;min-width:820px}
+  thead th{position:sticky;top:0;background:var(--surface);z-index:1}
+  th,td{text-align:center;padding:10px 12px;border-bottom:1px solid var(--border);white-space:nowrap}
+  th.model,td.model{text-align:left;white-space:normal;min-width:230px;position:sticky;left:0;background:var(--surface);z-index:2}
+  thead th{font-size:11.5px;letter-spacing:.04em;color:var(--muted);font-weight:600;text-transform:uppercase;border-bottom:1.5px solid var(--border)}
+  thead th .sub{display:block;font-size:10px;color:var(--faint);text-transform:none;letter-spacing:0;font-weight:400}
+  .grouprow td{background:var(--surface-2);font-size:11px;letter-spacing:.1em;text-transform:uppercase;color:var(--faint);font-weight:600;text-align:left;padding:7px 12px}
+  td .f1{font-family:var(--mono);font-variant-numeric:tabular-nums;font-weight:600;font-size:14.5px}
+  td .kap{display:block;font-family:var(--mono);font-size:10.5px;color:var(--faint);margin-top:1px}
+  td.na{color:var(--faint)}
+  .modelname{font-weight:600;font-size:14px}
+  .modelsub{color:var(--muted);font-size:12px;margin-top:1px}
+  .chip{display:inline-block;font-size:10px;font-weight:600;letter-spacing:.03em;padding:2px 7px;border-radius:20px;margin-top:5px;text-transform:uppercase}
+  .chip.flag{background:var(--accent-soft);color:var(--accent);border:1px solid var(--accent-line)}
+  .chip.pub{background:var(--good-bg);color:var(--good)}
+  .chip.int{background:var(--ok-bg);color:var(--ok)}
+  .chip.cloud{background:var(--warn-bg);color:var(--warn)}
+  .chip.pubd{background:var(--ok-bg);color:var(--ok)}
+  tr.flagrow td.model{box-shadow:inset 3px 0 0 var(--accent)}
+  /* value heatmap (semantic, not the accent) */
+  .g1{background:var(--good-bg);color:var(--good)}     /* >=.85 */
+  .g2{color:var(--text)}                                /* .70-.85 */
+  .g3{background:var(--warn-bg);color:var(--warn)}      /* .50-.70 */
+  .g4{background:var(--poor-bg);color:var(--poor)}      /* <.50 */
+  .foot{margin-top:34px;display:grid;gap:9px}
+  .foot p{margin:0;color:var(--muted);font-size:12.5px;max-width:88ch;padding-left:16px;position:relative}
+  .foot p::before{content:"";position:absolute;left:0;top:8px;width:6px;height:6px;border-radius:50%;background:var(--accent-line)}
+  .legend{display:flex;flex-wrap:wrap;gap:14px;margin:10px 0 0;font-size:11.5px;color:var(--muted)}
+  .legend span{display:inline-flex;align-items:center;gap:6px}
+  .sw{width:12px;height:12px;border-radius:3px;border:1px solid var(--border)}
+  a{color:var(--accent)}
+</style>
+</head>
+<body>
+<div class="wrap">
+  <header>
+    <div class="eyebrow">Nutrient · Document Intelligence</div>
+    <h1>Page Stream Segmentation — Unified Leaderboard</h1>
+    <p class="lede">One boundary metric, one harness, every contender re-measured: our flagship and open-weight
+      models against cloud VLMs and the field's self-declared numbers. Cells show boundary <b>F1</b>; the small
+      figure is chance-corrected <b>κ</b>.</p>
+    <div class="meta">updated 2026-08-11 · metric: boundary page-F1 (page 0 forced) + Cohen's κ</div>
+  </header>
+  <section class="findings">
+    <div class="finding"><b>0.891</b><span>flagship <span class="k">OpenPSS-long</span> F1 — vs best cloud <b style="font-size:13px">0.244</b></span></div>
+    <div class="finding"><b>1 model</b><span>beats OpenPSS's <span class="k">two</span> specialists across both slices</span></div>
+    <div class="finding"><b>~0.0007</b><span>USD / 1k pages (A40) — cloud VLMs ≈ <span class="k">$0.014 / stream</span></span></div>
+    <div class="finding"><b>4.5×</b><span>lighter open model (<span class="k">doc-split-v1</span>) at near-flagship quality</span></div>
+  </section>
+  <h2>Boundary F1 · κ  —  across six evaluation cuts</h2>
+  <div class="tablecard"><div class="scroll">
+    <table>
+      <thead><tr>
+        <th class="model">Model</th>
+        <th>our-200<span class="sub">easy, saturated</span></th>
+        <th>OpenPSS-short<span class="sub">sparse · hardest</span></th>
+        <th>OpenPSS-long<span class="sub">long streams</span></th>
+        <th>TABME++<span class="sub">test</span></th>
+        <th>Tobacco800<span class="sub">test</span></th>
+        <th>val-full<span class="sub">our real-doc</span></th>
+      </tr></thead>
+      <tbody>
+        <tr class="grouprow"><td colspan="7">Ours</td></tr>
+        <tr class="flagrow">
+          <td class="model"><div class="modelname">doc-split-v2</div><div class="modelsub">flagship · one model for short + long streams · ~1.0B · on-prem</div><span class="chip flag">Commercial</span></td>
+          <td class="g1"><span class="f1">0.944</span><span class="kap">.79</span></td>
+          <td class="g3"><span class="f1">0.652</span><span class="kap">.60</span></td>
+          <td class="g1"><span class="f1">0.891</span><span class="kap">.86</span></td>
+          <td class="g1"><span class="f1">0.943</span><span class="kap">.91</span></td>
+          <td class="g1"><span class="f1">0.969</span><span class="kap">.93</span></td>
+          <td class="g1"><span class="f1">0.917</span><span class="kap">.86</span></td>
+        </tr>
+        <tr>
+          <td class="model"><div class="modelname">doc-split-v1</div><div class="modelsub">open-weight · ~4.5× faster</div><span class="chip pub">Open-weight</span></td>
+          <td class="g1"><span class="f1">0.936</span><span class="kap">.78</span></td>
+          <td class="g3"><span class="f1">0.585</span><span class="kap">.53</span></td>
+          <td class="g1"><span class="f1">0.859</span><span class="kap">.82</span></td>
+          <td class="g2"><span class="f1">0.704</span><span class="kap">.56</span></td>
+          <td class="g2"><span class="f1">0.820</span><span class="kap">.60</span></td>
+          <td class="g1"><span class="f1">0.918</span><span class="kap">.86</span></td>
+        </tr>
+        <tr class="grouprow"><td colspan="7">Cloud VLM · image-only · single-prompt</td></tr>
+        <tr>
+          <td class="model"><div class="modelname">gemini-flash</div><div class="modelsub">best cloud on OpenPSS · ~$0.014/stream</div><span class="chip cloud">Cloud</span></td>
+          <td class="g1"><span class="f1">0.917</span></td>
+          <td class="g3"><span class="f1">0.598</span><span class="kap">.53</span></td>
+          <td class="g4"><span class="f1">0.244</span><span class="kap">.16</span></td>
+          <td class="na">—</td><td class="na">—</td><td class="na">—</td>
+        </tr>
+        <tr>
+          <td class="model"><div class="modelname">gemini-pro</div><span class="chip cloud">Cloud</span></td>
+          <td class="g1"><span class="f1">0.936</span></td>
+          <td class="g3"><span class="f1">0.530</span><span class="kap">.45</span></td>
+          <td class="g4"><span class="f1">0.196</span><span class="kap">.11</span></td>
+          <td class="na">—</td><td class="na">—</td><td class="na">—</td>
+        </tr>
+        <tr>
+          <td class="model"><div class="modelname">gpt-sol</div><div class="modelsub">best cloud on our-200</div><span class="chip cloud">Cloud</span></td>
+          <td class="g1"><span class="f1">0.942</span></td>
+          <td class="g4"><span class="f1">0.193</span><span class="kap">.17</span></td>
+          <td class="g4"><span class="f1">0.025</span><span class="kap">.02</span></td>
+          <td class="na">—</td><td class="na">—</td><td class="na">—</td>
+        </tr>
+        <tr>
+          <td class="model"><div class="modelname">claude-opus</div><span class="chip cloud">Cloud</span></td>
+          <td class="na">—</td>
+          <td class="g4"><span class="f1">0.318</span><span class="kap">.28</span></td>
+          <td class="g4"><span class="f1">0.047</span><span class="kap">.03</span></td>
+          <td class="na">—</td><td class="na">—</td><td class="na">—</td>
+        </tr>
+        <tr class="grouprow"><td colspan="7">Research — self-declared (their metric / in-domain)</td></tr>
+        <tr>
+          <td class="model"><div class="modelname">OpenPSS SHORT-specialist</div><div class="modelsub">BERT-EffNet ensemble · one of two models</div><span class="chip pubd">Published</span></td>
+          <td class="na">—</td>
+          <td class="g2"><span class="f1">0.76</span></td>
+          <td class="g4"><span class="f1">0.50</span></td>
+          <td class="na">—</td><td class="na">—</td><td class="na">—</td>
+        </tr>
+        <tr>
+          <td class="model"><div class="modelname">OpenPSS LONG-specialist</div><div class="modelsub">separate model · cross-slice drops</div><span class="chip pubd">Published</span></td>
+          <td class="na">—</td>
+          <td class="g3"><span class="f1">0.62</span></td>
+          <td class="g2"><span class="f1">0.83</span></td>
+          <td class="na">—</td><td class="na">—</td><td class="na">—</td>
+        </tr>
+        <tr>
+          <td class="model"><div class="modelname">bert-pss (agiagoulas)</div><div class="modelsub">only released PSS specialist · text-only</div><span class="chip pubd">Released</span></td>
+          <td class="na">—</td><td class="na">—</td><td class="na">—</td><td class="na">—</td>
+          <td class="g4"><span class="f1">0.915<span style="font-size:10px">*</span></span><span class="kap">κ .00 run</span></td>
+          <td class="na">—</td>
+        </tr>
+      </tbody>
+    </table>
+  </div></div>
+  <div class="legend">
+    <span><i class="sw" style="background:var(--good-bg)"></i>≥ 0.85</span>
+    <span><i class="sw" style="background:var(--surface)"></i>0.70–0.85</span>
+    <span><i class="sw" style="background:var(--warn-bg)"></i>0.50–0.70</span>
+    <span><i class="sw" style="background:var(--poor-bg)"></i>&lt; 0.50</span>
+  </div>
+  <div class="foot">
+    <p><b>One balanced model vs two specialists.</b> No single OpenPSS model wins both slices — their short-specialist craters on long (0.50), their long-specialist on short (0.62). the v4 flagship does short + long with one model, and its long (0.891) tops even their long-specialist (0.83).</p>
+    <p><b>Cloud VLMs can't ingest long streams.</b> Per-request image caps (Anthropic ~100 / OpenAI ~200 / Gemini ~500) force predict-none on streams over the cap, which dominates OpenPSS-long → best cloud 0.244 vs flagship 0.891, at ~20× the cost per page.</p>
+    <p><b>* bert-pss</b> self-declares ~0.915 accuracy / 0.825 κ on Tobacco800, but collapses to a single class when actually run (κ ≈ 0) — the only released PSS specialist is non-functional off its exact serving harness.</p>
+    <p><b>Metric.</b> Ours &amp; cloud: micro boundary-F1 + κ over internal pages, identical harness. OpenPSS rows: their published per-stream page-F1 (ballpark-comparable). Tobacco incumbents report accuracy — compared via κ. AI-Lab-Splitter omitted: data gated, absolute F1 paywalled.</p>
+  </div>
+</div>
+</body>
+</html>

style.css ADDED Viewed

	@@ -0,0 +1,28 @@

+body {
+	padding: 2rem;
+	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
+}
+h1 {
+	font-size: 16px;
+	margin-top: 0;
+}
+p {
+	color: rgb(107, 114, 128);
+	font-size: 15px;
+	margin-bottom: 10px;
+	margin-top: 5px;
+}
+.card {
+	max-width: 620px;
+	margin: 0 auto;
+	padding: 16px;
+	border: 1px solid lightgray;
+	border-radius: 16px;
+}
+.card p:last-child {
+	margin-bottom: 0;
+}