Commit ·
a8caea6
0
Parent(s):
document-classification-leaderboard
Browse files- .gitattributes +35 -0
- README.md +23 -0
- index.html +169 -0
- leaderboard.json +324 -0
- style.css +28 -0
.gitattributes
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Document Classification Leaderboard
|
| 3 |
+
emoji: 🖼️
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: green
|
| 6 |
+
sdk: static
|
| 7 |
+
pinned: false
|
| 8 |
+
license: other
|
| 9 |
+
short_description: Open-vocab document classification leaderboard (macro-F1)
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# Document Classification Leaderboard
|
| 13 |
+
|
| 14 |
+
Static leaderboard for open-vocabulary, zero-shot document-type classification — ranked by mean macro-F1
|
| 15 |
+
across four tracks (DocLayNet / Forms / Tobacco / OOD) on the held-out
|
| 16 |
+
[document-classification-benchmark](https://huggingface.co/datasets/nutrientdocs/document-classification-benchmark).
|
| 17 |
+
|
| 18 |
+
- 🎯 [Demo](https://huggingface.co/spaces/nutrientdocs/document-classification-demo) ·
|
| 19 |
+
🏵️ [Flagship model](https://huggingface.co/nutrientdocs/document-classification-v2) ·
|
| 20 |
+
📊 [Benchmark](https://huggingface.co/datasets/nutrientdocs/document-classification-benchmark)
|
| 21 |
+
|
| 22 |
+
Page is `index.html` + `leaderboard.json`. Numbers are provisional pending a clean scoring run on standard
|
| 23 |
+
hardware.
|
index.html
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>Document Classification Leaderboard</title>
|
| 7 |
+
<style>
|
| 8 |
+
:root{
|
| 9 |
+
color-scheme:light dark;
|
| 10 |
+
--paper:#eef1f4; --surface:#ffffff; --ink:#17222c; --muted:#5c6b78;
|
| 11 |
+
--hair:#d7dee4; --hair-strong:#c2ccd4;
|
| 12 |
+
--accent:#1d5fa8; --accent-soft:#e5eefa;
|
| 13 |
+
--good:#2f8f6f; --good-soft:#dcefe8; --bad:#c0473b; --warn:#c98a2b;
|
| 14 |
+
--serif:"Iowan Old Style","Palatino Linotype",Palatino,"Book Antiqua",Georgia,serif;
|
| 15 |
+
--sans:system-ui,-apple-system,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
|
| 16 |
+
--mono:ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
| 17 |
+
}
|
| 18 |
+
@media (prefers-color-scheme:dark){
|
| 19 |
+
:root{--paper:#12171c;--surface:#1a2229;--ink:#e6ecf1;--muted:#93a2ae;
|
| 20 |
+
--hair:#2b353e;--hair-strong:#38444e;--accent:#6aa9e6;--accent-soft:#1e2b3a;
|
| 21 |
+
--good:#57b895;--good-soft:#173029;--bad:#e0796d;--warn:#d9a856;}
|
| 22 |
+
}
|
| 23 |
+
*{box-sizing:border-box}
|
| 24 |
+
body{margin:0;background:var(--paper);color:var(--ink);font-family:var(--sans);line-height:1.6;font-size:16px;-webkit-font-smoothing:antialiased}
|
| 25 |
+
.wrap{max-width:1180px;margin:0 auto;padding:clamp(20px,5vw,60px) clamp(18px,6vw,64px)}
|
| 26 |
+
.eyebrow{font-family:var(--mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;color:var(--accent);font-weight:600;margin:0 0 14px}
|
| 27 |
+
h1{font-family:var(--serif);font-weight:600;font-size:clamp(28px,5vw,44px);line-height:1.08;letter-spacing:-.01em;margin:0 0 14px}
|
| 28 |
+
.lede{font-size:18px;color:var(--muted);max-width:82ch;margin:0 0 8px}
|
| 29 |
+
a{color:var(--accent)}
|
| 30 |
+
code{font-family:var(--mono);font-size:.86em;background:var(--accent-soft);padding:.1em .38em;border-radius:4px}
|
| 31 |
+
.tracknote{background:var(--surface);border:1px solid var(--hair);border-radius:12px;padding:16px 20px;margin:26px 0 6px;font-size:14px}
|
| 32 |
+
.tracknote b{font-family:var(--mono);font-size:12px;letter-spacing:.04em;color:var(--accent)}
|
| 33 |
+
.tracknote ul{margin:8px 0 0;padding-left:18px}.tracknote li{margin:4px 0}
|
| 34 |
+
.tablebox{overflow-x:auto;margin:14px 0}
|
| 35 |
+
table{border-collapse:collapse;width:100%;font-size:14px;min-width:860px}
|
| 36 |
+
th,td{text-align:left;padding:10px 12px;border-bottom:1px solid var(--hair)}
|
| 37 |
+
th{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted);font-weight:600}
|
| 38 |
+
td.n,th.n{text-align:right;font-variant-numeric:tabular-nums;font-family:var(--mono)}
|
| 39 |
+
tbody tr:last-child td{border-bottom:none}
|
| 40 |
+
tr.ours{background:var(--accent-soft)}
|
| 41 |
+
.rank{font-family:var(--mono);color:var(--muted);width:2ch}
|
| 42 |
+
.name{font-weight:650}
|
| 43 |
+
.chip{display:inline-block;font-family:var(--mono);font-size:10px;font-weight:600;letter-spacing:.03em;padding:2px 7px;border-radius:20px;white-space:nowrap;margin-left:6px}
|
| 44 |
+
.c-com{background:var(--accent-soft);color:var(--accent)}
|
| 45 |
+
.c-open{background:var(--good-soft);color:var(--good)}
|
| 46 |
+
.c-cloud{background:#f0e9dc;color:var(--warn)}
|
| 47 |
+
@media (prefers-color-scheme:dark){.c-cloud{background:#2a2312;color:var(--warn)}}
|
| 48 |
+
.c-gen{background:var(--hair);color:var(--muted)}
|
| 49 |
+
.tabs{display:flex;gap:8px;margin:20px 0 4px}
|
| 50 |
+
.tab{font-family:var(--mono);font-size:12px;font-weight:600;letter-spacing:.03em;padding:7px 14px;border-radius:8px;
|
| 51 |
+
border:1px solid var(--hair-strong);background:var(--surface);color:var(--muted);cursor:pointer}
|
| 52 |
+
.tab.on{background:var(--accent);color:#fff;border-color:var(--accent)}
|
| 53 |
+
.oursonly{color:var(--muted)}
|
| 54 |
+
.bar{position:relative;height:15px;background:var(--hair);border-radius:4px;min-width:110px}
|
| 55 |
+
.bar>span{position:absolute;left:0;top:0;bottom:0;background:var(--accent);border-radius:4px}
|
| 56 |
+
.bar.hl>span{background:var(--good)}
|
| 57 |
+
.barnum{font-family:var(--mono);font-variant-numeric:tabular-nums;font-size:12.5px;margin-left:8px;color:var(--muted)}
|
| 58 |
+
.barcell{display:flex;align-items:center;min-width:200px}
|
| 59 |
+
footer{margin-top:52px;border-top:1px solid var(--hair-strong);padding-top:16px;color:var(--muted);font-size:12.5px;font-family:var(--mono)}
|
| 60 |
+
footer img{vertical-align:middle;margin-right:10px;border-radius:6px}
|
| 61 |
+
.muted{color:var(--muted)}
|
| 62 |
+
</style>
|
| 63 |
+
</head>
|
| 64 |
+
<body>
|
| 65 |
+
<div class="wrap">
|
| 66 |
+
<p class="eyebrow">Open-vocab document classification · benchmark</p>
|
| 67 |
+
<h1>Document Classification Leaderboard</h1>
|
| 68 |
+
<p class="lede">Zero-shot document-type classification — labels supplied at inference, not baked into a
|
| 69 |
+
head. Ranked by <strong>doc-type mean macro-F1</strong> across DocLayNet, Forms and Tobacco on the
|
| 70 |
+
held-out
|
| 71 |
+
<a href="https://huggingface.co/datasets/nutrientdocs/document-classification-benchmark">document-classification-benchmark</a>.
|
| 72 |
+
Nutrient's in-house small models vs. cloud frontier VLMs, all scored by the same
|
| 73 |
+
<code>score_classify.py</code> (unweighted macro-F1, argmax over candidate labels).</p>
|
| 74 |
+
|
| 75 |
+
<div class="tracknote">
|
| 76 |
+
<b>TRACKS · macro-F1</b>
|
| 77 |
+
<ul>
|
| 78 |
+
<li><strong>DocLayNet / Forms / Tobacco</strong> — document-type tracks (n=377 / 360 / 500); the
|
| 79 |
+
headline <strong>doc-type mean</strong> averages these three and rows are ranked by it.</li>
|
| 80 |
+
<li><strong>OOD</strong> (unseen types) & <strong>OOV</strong> (synonym label wording) — robustness
|
| 81 |
+
tracks, reported for the <em>open-vocab</em> models (Nutrient + the non-VLM generalists). Cloud VLMs
|
| 82 |
+
train on ~all data, so held-out/open-vocab generalization isn't a meaningful axis for them (shown —).</li>
|
| 83 |
+
</ul>
|
| 84 |
+
<p style="margin:10px 0 0"><strong>Reading it:</strong> the in-house models match/lead the cloud on the
|
| 85 |
+
<em>visual</em> tracks (DocLayNet, Forms) at zero API cost, and trail on <em>Tobacco</em> — a
|
| 86 |
+
read-the-header task where a VLM's text reading wins. A dash (—) = track not run for that model.</p>
|
| 87 |
+
</div>
|
| 88 |
+
|
| 89 |
+
<div class="tabs">
|
| 90 |
+
<button id="t-nonvlm" class="tab on" onclick="setTab('nonvlm')">vs open-vocab models (non-VLM)</button>
|
| 91 |
+
<button id="t-vlm" class="tab" onclick="setTab('vlm')">vs cloud VLMs</button>
|
| 92 |
+
</div>
|
| 93 |
+
<div id="banner"></div>
|
| 94 |
+
<div class="tablebox"><table><thead id="head"></thead><tbody id="rows"></tbody></table></div>
|
| 95 |
+
<p class="muted" style="font-size:13px">
|
| 96 |
+
<span class="chip c-com">Nutrient · commercial</span>
|
| 97 |
+
<span class="chip c-open">Nutrient · open-weight</span>
|
| 98 |
+
<span class="chip c-cloud">cloud VLM</span>
|
| 99 |
+
Highlighted rows are Nutrient models. Numbers are provisional pending a clean scoring pass.</p>
|
| 100 |
+
|
| 101 |
+
<!-- Plain static links so HuggingFace links these model pages (the table links are JS-rendered). -->
|
| 102 |
+
<p id="models-benchmarked" class="muted" style="font-size:12.5px;margin-top:22px">
|
| 103 |
+
<strong>Models benchmarked:</strong>
|
| 104 |
+
<a href="https://huggingface.co/nutrientdocs/document-classification-v2">nutrientdocs/document-classification-v2</a> ·
|
| 105 |
+
<a href="https://huggingface.co/nutrientdocs/document-classification-v1">nutrientdocs/document-classification-v1</a> ·
|
| 106 |
+
<a href="https://huggingface.co/google/siglip2-so400m-patch16-512">google/siglip2-so400m-patch16-512</a> ·
|
| 107 |
+
<a href="https://huggingface.co/google/siglip2-base-patch16-512">google/siglip2-base-patch16-512</a> ·
|
| 108 |
+
<a href="https://huggingface.co/google/siglip-base-patch16-224">google/siglip-base-patch16-224</a> ·
|
| 109 |
+
<a href="https://huggingface.co/openai/clip-vit-large-patch14-336">openai/clip-vit-large-patch14-336</a> ·
|
| 110 |
+
<a href="https://huggingface.co/openai/clip-vit-base-patch32">openai/clip-vit-base-patch32</a> ·
|
| 111 |
+
<a href="https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K">laion/CLIP-ViT-H-14</a> ·
|
| 112 |
+
<a href="https://huggingface.co/apple/DFN5B-CLIP-ViT-H-14-378">apple/DFN5B-CLIP-ViT-H-14-378</a> ·
|
| 113 |
+
<a href="https://huggingface.co/timm/eva02_large_patch14_clip_336.merged2b_s6b_b61k">timm/eva02_large_patch14_clip_336</a>.
|
| 114 |
+
Cloud VLMs (no public HF page): GPT-4o / 4o-mini / 4.1 / 4.1-mini · Gemini 2.5 Pro / Flash / Flash-Lite ·
|
| 115 |
+
Claude Opus / Sonnet / Haiku.</p>
|
| 116 |
+
|
| 117 |
+
<footer>
|
| 118 |
+
<a href="https://nutrient.io/"><img src="https://avatars2.githubusercontent.com/u/1527679?v=3&s=200" height="34" /></a>
|
| 119 |
+
Macro-F1 on nutrientdocs/document-classification-benchmark · same <code>score_classify.py</code> for all ·
|
| 120 |
+
maintained and funded by <a href="https://nutrient.io/">Nutrient</a> ·
|
| 121 |
+
flagship: <a href="https://huggingface.co/nutrientdocs/document-classification-v2">document-classification-v2</a>
|
| 122 |
+
</footer>
|
| 123 |
+
</div>
|
| 124 |
+
|
| 125 |
+
<script>
|
| 126 |
+
let DATA=[];
|
| 127 |
+
let TAB="nonvlm"; // default: the apples-to-apples open-vocab (non-VLM) fight
|
| 128 |
+
function setTab(t){TAB=t;
|
| 129 |
+
document.getElementById('t-nonvlm').className='tab'+(t==='nonvlm'?' on':'');
|
| 130 |
+
document.getElementById('t-vlm').className='tab'+(t==='vlm'?' on':'');
|
| 131 |
+
render();}
|
| 132 |
+
const CORE=[["doclaynet","DocLayNet"],["forms","Forms"],["tobacco","Tobacco"]];
|
| 133 |
+
const EXTRA=[["ood","OOD*"],["oov","OOV*"]];
|
| 134 |
+
const ALL=CORE.concat(EXTRA);
|
| 135 |
+
function hf(m){return (m&&m.includes("/"))?"https://huggingface.co/"+m:null;}
|
| 136 |
+
function fmt(x){return (x===null||x===undefined)?'<span class="muted">—</span>':x.toFixed(3).replace(/^0/,"");}
|
| 137 |
+
function coreMean(t){const v=CORE.map(c=>t[c[0]]).filter(x=>x!==null&&x!==undefined);
|
| 138 |
+
return v.length===CORE.length? v.reduce((a,b)=>a+b,0)/v.length : null;}
|
| 139 |
+
function chip(k){return k==="ours-commercial"?'<span class="chip c-com">commercial</span>'
|
| 140 |
+
:k==="ours-open"?'<span class="chip c-open">open</span>'
|
| 141 |
+
:k==="generalist"?'<span class="chip c-gen">generalist</span>':'<span class="chip c-cloud">cloud</span>';}
|
| 142 |
+
function render(){
|
| 143 |
+
// full-coverage (all 3 core tracks) ranked by doc-type mean; partial rows after, by DocLayNet
|
| 144 |
+
const D=DATA.filter(r=> TAB==="vlm" ? r.kind!=="generalist" : r.kind!=="cloud"); // ours anchor both tabs
|
| 145 |
+
const full=D.filter(r=>coreMean(r.tracks)!==null).sort((a,b)=>coreMean(b.tracks)-coreMean(a.tracks));
|
| 146 |
+
const part=D.filter(r=>coreMean(r.tracks)===null).sort((a,b)=>(b.tracks.doclaynet||0)-(a.tracks.doclaynet||0));
|
| 147 |
+
const rows=full.concat(part);
|
| 148 |
+
document.getElementById("head").innerHTML='<tr><th class="rank">#</th><th>Model</th>'
|
| 149 |
+
+ALL.map(t=>`<th class="n">${t[1]}</th>`).join("")+'<th class="barcell">Doc-type mean</th><th class="n">Latency</th></tr>';
|
| 150 |
+
const tb=document.getElementById("rows");tb.innerHTML="";
|
| 151 |
+
const max=Math.max(...full.map(r=>coreMean(r.tracks)));
|
| 152 |
+
rows.forEach((r,i)=>{
|
| 153 |
+
const m=coreMean(r.tracks), ours=r.kind&&r.kind.startsWith("ours");
|
| 154 |
+
const tr=document.createElement("tr");if(ours)tr.className="ours";
|
| 155 |
+
const nm=hf(r.model)?`<a class="name" href="${hf(r.model)}" target="_blank" rel="noopener">${r.name}</a>`:`<span class="name">${r.name}</span>`;
|
| 156 |
+
tr.innerHTML=`<td class="rank">${m!==null?i+1:''}</td><td>${nm}${chip(r.kind)}</td>`
|
| 157 |
+
+ALL.map(t=>`<td class="n ${EXTRA.some(e=>e[0]===t[0])?'oursonly':''}">${fmt(r.tracks[t[0]])}</td>`).join("")
|
| 158 |
+
+(m!==null?`<td><div class="barcell"><div class="bar ${ours?'hl':''}" style="flex:1"><span style="width:${Math.round(m/max*100)}%"></span></div><span class="barnum">${m.toFixed(3).replace(/^0/,"")}</span></div></td>`
|
| 159 |
+
:`<td class="muted" style="font-size:12px">partial coverage</td>`)
|
| 160 |
+
+`<td class="n">${r.latency||'<span class="muted">—</span>'}</td>`;
|
| 161 |
+
tb.appendChild(tr);
|
| 162 |
+
});
|
| 163 |
+
}
|
| 164 |
+
function setBanner(){const b=document.getElementById("banner");
|
| 165 |
+
if(DATA.some(d=>d.provisional)){b.className="tracknote";b.style.borderColor="var(--warn)";b.textContent="Provisional numbers — a clean scoring pass on standard hardware is pending.";}}
|
| 166 |
+
fetch("leaderboard.json").then(r=>r.ok?r.json():[]).then(d=>{DATA=Array.isArray(d)?d:(d.results||[]);render();}).catch(()=>render());
|
| 167 |
+
</script>
|
| 168 |
+
</body>
|
| 169 |
+
</html>
|
leaderboard.json
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"name": "claude-haiku",
|
| 4 |
+
"kind": "cloud",
|
| 5 |
+
"latency": "API",
|
| 6 |
+
"tracks": {
|
| 7 |
+
"doclaynet": 0.7516,
|
| 8 |
+
"forms": 0.9892,
|
| 9 |
+
"tobacco": 0.7773,
|
| 10 |
+
"ood": null,
|
| 11 |
+
"oov": null
|
| 12 |
+
}
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"name": "claude-opus",
|
| 16 |
+
"kind": "cloud",
|
| 17 |
+
"latency": "API",
|
| 18 |
+
"tracks": {
|
| 19 |
+
"doclaynet": 0.8273,
|
| 20 |
+
"forms": 0.9776,
|
| 21 |
+
"tobacco": 0.836,
|
| 22 |
+
"ood": null,
|
| 23 |
+
"oov": null
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"name": "claude-opus-4.5",
|
| 28 |
+
"kind": "cloud",
|
| 29 |
+
"latency": "API",
|
| 30 |
+
"tracks": {
|
| 31 |
+
"doclaynet": 0.8317,
|
| 32 |
+
"forms": 0.9737,
|
| 33 |
+
"tobacco": 0.8232,
|
| 34 |
+
"ood": null,
|
| 35 |
+
"oov": null
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"name": "claude-opus-5",
|
| 40 |
+
"kind": "cloud",
|
| 41 |
+
"latency": "API",
|
| 42 |
+
"tracks": {
|
| 43 |
+
"doclaynet": 0.8317,
|
| 44 |
+
"forms": 0.9577,
|
| 45 |
+
"tobacco": 0.852,
|
| 46 |
+
"ood": null,
|
| 47 |
+
"oov": null
|
| 48 |
+
}
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"name": "claude-sonnet",
|
| 52 |
+
"kind": "cloud",
|
| 53 |
+
"latency": "API",
|
| 54 |
+
"tracks": {
|
| 55 |
+
"doclaynet": 0.8299,
|
| 56 |
+
"forms": 0.8671,
|
| 57 |
+
"tobacco": 0.7971,
|
| 58 |
+
"ood": null,
|
| 59 |
+
"oov": null
|
| 60 |
+
}
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "claude-sonnet-4.5",
|
| 64 |
+
"kind": "cloud",
|
| 65 |
+
"latency": "API",
|
| 66 |
+
"tracks": {
|
| 67 |
+
"doclaynet": 0.8223,
|
| 68 |
+
"forms": 0.989,
|
| 69 |
+
"tobacco": 0.7725,
|
| 70 |
+
"ood": null,
|
| 71 |
+
"oov": null
|
| 72 |
+
}
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"name": "gemini-flash",
|
| 76 |
+
"kind": "cloud",
|
| 77 |
+
"latency": "API",
|
| 78 |
+
"tracks": {
|
| 79 |
+
"doclaynet": 0.8317,
|
| 80 |
+
"forms": 1.0,
|
| 81 |
+
"tobacco": 0.8416,
|
| 82 |
+
"ood": null,
|
| 83 |
+
"oov": null
|
| 84 |
+
}
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"name": "gemini-flash-lite",
|
| 88 |
+
"kind": "cloud",
|
| 89 |
+
"latency": "API",
|
| 90 |
+
"tracks": {
|
| 91 |
+
"doclaynet": 0.799,
|
| 92 |
+
"forms": 1.0,
|
| 93 |
+
"tobacco": 0.7981,
|
| 94 |
+
"ood": null,
|
| 95 |
+
"oov": null
|
| 96 |
+
}
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"name": "gemini-pro",
|
| 100 |
+
"kind": "cloud",
|
| 101 |
+
"latency": "API",
|
| 102 |
+
"tracks": {
|
| 103 |
+
"doclaynet": 0.8295,
|
| 104 |
+
"forms": 1.0,
|
| 105 |
+
"tobacco": 0.8497,
|
| 106 |
+
"ood": null,
|
| 107 |
+
"oov": null
|
| 108 |
+
}
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "gpt-4.1",
|
| 112 |
+
"kind": "cloud",
|
| 113 |
+
"latency": "API",
|
| 114 |
+
"tracks": {
|
| 115 |
+
"doclaynet": 0.7849,
|
| 116 |
+
"forms": 1.0,
|
| 117 |
+
"tobacco": 0.7547,
|
| 118 |
+
"ood": null,
|
| 119 |
+
"oov": null
|
| 120 |
+
}
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"name": "gpt-4.1-mini",
|
| 124 |
+
"kind": "cloud",
|
| 125 |
+
"latency": "API",
|
| 126 |
+
"tracks": {
|
| 127 |
+
"doclaynet": 0.8144,
|
| 128 |
+
"forms": 1.0,
|
| 129 |
+
"tobacco": 0.7843,
|
| 130 |
+
"ood": null,
|
| 131 |
+
"oov": null
|
| 132 |
+
}
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"name": "gpt-4o",
|
| 136 |
+
"kind": "cloud",
|
| 137 |
+
"latency": "API",
|
| 138 |
+
"tracks": {
|
| 139 |
+
"doclaynet": 0.8144,
|
| 140 |
+
"forms": 1.0,
|
| 141 |
+
"tobacco": 0.7693,
|
| 142 |
+
"ood": null,
|
| 143 |
+
"oov": null
|
| 144 |
+
}
|
| 145 |
+
},
|
| 146 |
+
{
|
| 147 |
+
"name": "gpt-4o-mini",
|
| 148 |
+
"kind": "cloud",
|
| 149 |
+
"latency": "API",
|
| 150 |
+
"tracks": {
|
| 151 |
+
"doclaynet": 0.8065,
|
| 152 |
+
"forms": 1.0,
|
| 153 |
+
"tobacco": 0.7357,
|
| 154 |
+
"ood": null,
|
| 155 |
+
"oov": null
|
| 156 |
+
}
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"name": "gpt-luna",
|
| 160 |
+
"kind": "cloud",
|
| 161 |
+
"latency": "API",
|
| 162 |
+
"tracks": {
|
| 163 |
+
"doclaynet": 0.8004,
|
| 164 |
+
"forms": 1.0,
|
| 165 |
+
"tobacco": 0.7661,
|
| 166 |
+
"ood": null,
|
| 167 |
+
"oov": null
|
| 168 |
+
}
|
| 169 |
+
},
|
| 170 |
+
{
|
| 171 |
+
"name": "gpt-sol",
|
| 172 |
+
"kind": "cloud",
|
| 173 |
+
"latency": "API",
|
| 174 |
+
"tracks": {
|
| 175 |
+
"doclaynet": 0.8317,
|
| 176 |
+
"forms": 1.0,
|
| 177 |
+
"tobacco": 0.8222,
|
| 178 |
+
"ood": null,
|
| 179 |
+
"oov": null
|
| 180 |
+
}
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"name": "gpt-terra",
|
| 184 |
+
"kind": "cloud",
|
| 185 |
+
"latency": "API",
|
| 186 |
+
"tracks": {
|
| 187 |
+
"doclaynet": 0.8223,
|
| 188 |
+
"forms": 1.0,
|
| 189 |
+
"tobacco": 0.8251,
|
| 190 |
+
"ood": null,
|
| 191 |
+
"oov": null
|
| 192 |
+
}
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"name": "document-classification-v2",
|
| 196 |
+
"kind": "ours-commercial",
|
| 197 |
+
"model": "nutrientdocs/document-classification-v2",
|
| 198 |
+
"latency": "~8/s (A40)",
|
| 199 |
+
"tracks": {
|
| 200 |
+
"doclaynet": 0.883,
|
| 201 |
+
"forms": 1.0,
|
| 202 |
+
"tobacco": 0.693,
|
| 203 |
+
"ood": 0.974,
|
| 204 |
+
"oov": 0.797
|
| 205 |
+
}
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"name": "document-classification-v1",
|
| 209 |
+
"kind": "ours-open",
|
| 210 |
+
"model": "nutrientdocs/document-classification-v1",
|
| 211 |
+
"latency": "~3-7/s (A40)",
|
| 212 |
+
"tracks": {
|
| 213 |
+
"doclaynet": 0.891,
|
| 214 |
+
"forms": 0.805,
|
| 215 |
+
"tobacco": 0.616,
|
| 216 |
+
"ood": 0.87,
|
| 217 |
+
"oov": 0.739
|
| 218 |
+
}
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
"name": "clip-vit-base-patch32",
|
| 222 |
+
"kind": "generalist",
|
| 223 |
+
"model": "openai/clip-vit-base-patch32",
|
| 224 |
+
"latency": null,
|
| 225 |
+
"tracks": {
|
| 226 |
+
"doclaynet": 0.2843,
|
| 227 |
+
"forms": 0.0067,
|
| 228 |
+
"tobacco": 0.356,
|
| 229 |
+
"ood": 0.1402,
|
| 230 |
+
"oov": 0.4209
|
| 231 |
+
}
|
| 232 |
+
},
|
| 233 |
+
{
|
| 234 |
+
"name": "clip-vit-large-patch14-336",
|
| 235 |
+
"kind": "generalist",
|
| 236 |
+
"model": "openai/clip-vit-large-patch14-336",
|
| 237 |
+
"latency": null,
|
| 238 |
+
"tracks": {
|
| 239 |
+
"doclaynet": 0.5949,
|
| 240 |
+
"forms": 0.0161,
|
| 241 |
+
"tobacco": 0.3748,
|
| 242 |
+
"ood": 0.1854,
|
| 243 |
+
"oov": 0.6084
|
| 244 |
+
}
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"name": "siglip-base-patch16-224",
|
| 248 |
+
"kind": "generalist",
|
| 249 |
+
"model": "google/siglip-base-patch16-224",
|
| 250 |
+
"latency": null,
|
| 251 |
+
"tracks": {
|
| 252 |
+
"doclaynet": 0.4346,
|
| 253 |
+
"forms": 0.2607,
|
| 254 |
+
"tobacco": 0.3711,
|
| 255 |
+
"ood": 0.2354,
|
| 256 |
+
"oov": 0.3677
|
| 257 |
+
}
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "siglip2-base-patch16-512",
|
| 261 |
+
"kind": "generalist",
|
| 262 |
+
"model": "google/siglip2-base-patch16-512",
|
| 263 |
+
"latency": null,
|
| 264 |
+
"tracks": {
|
| 265 |
+
"doclaynet": 0.5529,
|
| 266 |
+
"forms": 0.0245,
|
| 267 |
+
"tobacco": 0.4146,
|
| 268 |
+
"ood": 0.1901,
|
| 269 |
+
"oov": 0.4984
|
| 270 |
+
}
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"name": "siglip2-so400m-patch16-512",
|
| 274 |
+
"kind": "generalist",
|
| 275 |
+
"model": "google/siglip2-so400m-patch16-512",
|
| 276 |
+
"latency": null,
|
| 277 |
+
"tracks": {
|
| 278 |
+
"doclaynet": 0.6293,
|
| 279 |
+
"forms": 0.106,
|
| 280 |
+
"tobacco": 0.431,
|
| 281 |
+
"ood": 0.2428,
|
| 282 |
+
"oov": 0.5941
|
| 283 |
+
}
|
| 284 |
+
},
|
| 285 |
+
{
|
| 286 |
+
"name": "CLIP-ViT-H-14",
|
| 287 |
+
"kind": "generalist",
|
| 288 |
+
"model": "laion/CLIP-ViT-H-14",
|
| 289 |
+
"latency": null,
|
| 290 |
+
"tracks": {
|
| 291 |
+
"doclaynet": 0.5593,
|
| 292 |
+
"forms": 0.2715,
|
| 293 |
+
"tobacco": 0.5017,
|
| 294 |
+
"ood": 0.1769,
|
| 295 |
+
"oov": 0.4013
|
| 296 |
+
}
|
| 297 |
+
},
|
| 298 |
+
{
|
| 299 |
+
"name": "DFN5B-CLIP-ViT-H-14-378",
|
| 300 |
+
"kind": "generalist",
|
| 301 |
+
"model": "apple/DFN5B-CLIP-ViT-H-14-378",
|
| 302 |
+
"latency": null,
|
| 303 |
+
"tracks": {
|
| 304 |
+
"doclaynet": 0.4725,
|
| 305 |
+
"forms": 0.4533,
|
| 306 |
+
"tobacco": 0.3404,
|
| 307 |
+
"ood": 0.2224,
|
| 308 |
+
"oov": 0.3824
|
| 309 |
+
}
|
| 310 |
+
},
|
| 311 |
+
{
|
| 312 |
+
"name": "eva02_large_patch14_clip_336",
|
| 313 |
+
"kind": "generalist",
|
| 314 |
+
"model": "timm/eva02_large_patch14_clip_336",
|
| 315 |
+
"latency": null,
|
| 316 |
+
"tracks": {
|
| 317 |
+
"doclaynet": 0.5961,
|
| 318 |
+
"forms": 0.1136,
|
| 319 |
+
"tobacco": 0.4303,
|
| 320 |
+
"ood": 0.269,
|
| 321 |
+
"oov": 0.4878
|
| 322 |
+
}
|
| 323 |
+
}
|
| 324 |
+
]
|
style.css
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
body {
|
| 2 |
+
padding: 2rem;
|
| 3 |
+
font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
|
| 4 |
+
}
|
| 5 |
+
|
| 6 |
+
h1 {
|
| 7 |
+
font-size: 16px;
|
| 8 |
+
margin-top: 0;
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
p {
|
| 12 |
+
color: rgb(107, 114, 128);
|
| 13 |
+
font-size: 15px;
|
| 14 |
+
margin-bottom: 10px;
|
| 15 |
+
margin-top: 5px;
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
.card {
|
| 19 |
+
max-width: 620px;
|
| 20 |
+
margin: 0 auto;
|
| 21 |
+
padding: 16px;
|
| 22 |
+
border: 1px solid lightgray;
|
| 23 |
+
border-radius: 16px;
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
.card p:last-child {
|
| 27 |
+
margin-bottom: 0;
|
| 28 |
+
}
|