Hugging Face's logo

Spaces:
nutrientdocs
/
document-classification-leaderboard
Running

App Files Community
document-classification-leaderboard / index.html
hung-k-nguyen's picture
hung-k-nguyen
document-classification-leaderboard
a8caea6
Raw
History Blame Contribute Delete
11.6 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Document Classification Leaderboard</title>
<style>
 :root{
 color-scheme:light dark;
 --paper:#eef1f4; --surface:#ffffff; --ink:#17222c; --muted:#5c6b78;
 --hair:#d7dee4; --hair-strong:#c2ccd4;
 --accent:#1d5fa8; --accent-soft:#e5eefa;
 --good:#2f8f6f; --good-soft:#dcefe8; --bad:#c0473b; --warn:#c98a2b;
 --serif:"Iowan Old Style","Palatino Linotype",Palatino,"Book Antiqua",Georgia,serif;
 --sans:system-ui,-apple-system,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
 --mono:ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
 }
 @media (prefers-color-scheme:dark){
 :root{--paper:#12171c;--surface:#1a2229;--ink:#e6ecf1;--muted:#93a2ae;
 --hair:#2b353e;--hair-strong:#38444e;--accent:#6aa9e6;--accent-soft:#1e2b3a;
 --good:#57b895;--good-soft:#173029;--bad:#e0796d;--warn:#d9a856;}
 }
 *{box-sizing:border-box}
 body{margin:0;background:var(--paper);color:var(--ink);font-family:var(--sans);line-height:1.6;font-size:16px;-webkit-font-smoothing:antialiased}
 .wrap{max-width:1180px;margin:0 auto;padding:clamp(20px,5vw,60px) clamp(18px,6vw,64px)}
 .eyebrow{font-family:var(--mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;color:var(--accent);font-weight:600;margin:0 0 14px}
 h1{font-family:var(--serif);font-weight:600;font-size:clamp(28px,5vw,44px);line-height:1.08;letter-spacing:-.01em;margin:0 0 14px}
 .lede{font-size:18px;color:var(--muted);max-width:82ch;margin:0 0 8px}
 a{color:var(--accent)}
 code{font-family:var(--mono);font-size:.86em;background:var(--accent-soft);padding:.1em .38em;border-radius:4px}
 .tracknote{background:var(--surface);border:1px solid var(--hair);border-radius:12px;padding:16px 20px;margin:26px 0 6px;font-size:14px}
 .tracknote b{font-family:var(--mono);font-size:12px;letter-spacing:.04em;color:var(--accent)}
 .tracknote ul{margin:8px 0 0;padding-left:18px}.tracknote li{margin:4px 0}
 .tablebox{overflow-x:auto;margin:14px 0}
 table{border-collapse:collapse;width:100%;font-size:14px;min-width:860px}
 th,td{text-align:left;padding:10px 12px;border-bottom:1px solid var(--hair)}
 th{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted);font-weight:600}
 td.n,th.n{text-align:right;font-variant-numeric:tabular-nums;font-family:var(--mono)}
 tbody tr:last-child td{border-bottom:none}
 tr.ours{background:var(--accent-soft)}
 .rank{font-family:var(--mono);color:var(--muted);width:2ch}
 .name{font-weight:650}
 .chip{display:inline-block;font-family:var(--mono);font-size:10px;font-weight:600;letter-spacing:.03em;padding:2px 7px;border-radius:20px;white-space:nowrap;margin-left:6px}
 .c-com{background:var(--accent-soft);color:var(--accent)}
 .c-open{background:var(--good-soft);color:var(--good)}
 .c-cloud{background:#f0e9dc;color:var(--warn)}
 @media (prefers-color-scheme:dark){.c-cloud{background:#2a2312;color:var(--warn)}}
 .c-gen{background:var(--hair);color:var(--muted)}
 .tabs{display:flex;gap:8px;margin:20px 0 4px}
 .tab{font-family:var(--mono);font-size:12px;font-weight:600;letter-spacing:.03em;padding:7px 14px;border-radius:8px;
 border:1px solid var(--hair-strong);background:var(--surface);color:var(--muted);cursor:pointer}
 .tab.on{background:var(--accent);color:#fff;border-color:var(--accent)}
 .oursonly{color:var(--muted)}
 .bar{position:relative;height:15px;background:var(--hair);border-radius:4px;min-width:110px}
 .bar>span{position:absolute;left:0;top:0;bottom:0;background:var(--accent);border-radius:4px}
 .bar.hl>span{background:var(--good)}
 .barnum{font-family:var(--mono);font-variant-numeric:tabular-nums;font-size:12.5px;margin-left:8px;color:var(--muted)}
 .barcell{display:flex;align-items:center;min-width:200px}
 footer{margin-top:52px;border-top:1px solid var(--hair-strong);padding-top:16px;color:var(--muted);font-size:12.5px;font-family:var(--mono)}
 footer img{vertical-align:middle;margin-right:10px;border-radius:6px}
 .muted{color:var(--muted)}
</style>
</head>
<body>
<div class="wrap">
 <p class="eyebrow">Open-vocab document classification · benchmark</p>
 <h1>Document Classification Leaderboard</h1>
 <p class="lede">Zero-shot document-type classification — labels supplied at inference, not baked into a
 head. Ranked by <strong>doc-type mean macro-F1</strong> across DocLayNet, Forms and Tobacco on the
 held-out
 <a href="https://huggingface.co/datasets/nutrientdocs/document-classification-benchmark">document-classification-benchmark</a>.
 Nutrient's in-house small models vs. cloud frontier VLMs, all scored by the same
 <code>score_classify.py</code> (unweighted macro-F1, argmax over candidate labels).</p>
<div class="tracknote">
 <b>TRACKS · macro-F1</b>
 <ul>
 <li><strong>DocLayNet / Forms / Tobacco</strong> — document-type tracks (n=377 / 360 / 500); the
 headline <strong>doc-type mean</strong> averages these three and rows are ranked by it.</li>
 <li><strong>OOD</strong> (unseen types) &amp; <strong>OOV</strong> (synonym label wording) — robustness
 tracks, reported for the <em>open-vocab</em> models (Nutrient + the non-VLM generalists). Cloud VLMs
 train on ~all data, so held-out/open-vocab generalization isn't a meaningful axis for them (shown —).</li>
 </ul>
 <p style="margin:10px 0 0"><strong>Reading it:</strong> the in-house models match/lead the cloud on the
 <em>visual</em> tracks (DocLayNet, Forms) at zero API cost, and trail on <em>Tobacco</em> — a
 read-the-header task where a VLM's text reading wins. A dash (—) = track not run for that model.</p>
 </div>
<div class="tabs">
 <button id="t-nonvlm" class="tab on" onclick="setTab('nonvlm')">vs open-vocab models (non-VLM)</button>
 <button id="t-vlm" class="tab" onclick="setTab('vlm')">vs cloud VLMs</button>
 </div>
 <div id="banner"></div>
 <div class="tablebox"><table><thead id="head"></thead><tbody id="rows"></tbody></table></div>
 <p class="muted" style="font-size:13px">
 <span class="chip c-com">Nutrient · commercial</span>
 <span class="chip c-open">Nutrient · open-weight</span>
 <span class="chip c-cloud">cloud VLM</span>
 &nbsp; Highlighted rows are Nutrient models. Numbers are provisional pending a clean scoring pass.</p>
<!-- Plain static links so HuggingFace links these model pages (the table links are JS-rendered). -->
 <p id="models-benchmarked" class="muted" style="font-size:12.5px;margin-top:22px">
 <strong>Models benchmarked:</strong>
 <a href="https://huggingface.co/nutrientdocs/document-classification-v2">nutrientdocs/document-classification-v2</a> ·
 <a href="https://huggingface.co/nutrientdocs/document-classification-v1">nutrientdocs/document-classification-v1</a> ·
 <a href="https://huggingface.co/google/siglip2-so400m-patch16-512">google/siglip2-so400m-patch16-512</a> ·
 <a href="https://huggingface.co/google/siglip2-base-patch16-512">google/siglip2-base-patch16-512</a> ·
 <a href="https://huggingface.co/google/siglip-base-patch16-224">google/siglip-base-patch16-224</a> ·
 <a href="https://huggingface.co/openai/clip-vit-large-patch14-336">openai/clip-vit-large-patch14-336</a> ·
 <a href="https://huggingface.co/openai/clip-vit-base-patch32">openai/clip-vit-base-patch32</a> ·
 <a href="https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K">laion/CLIP-ViT-H-14</a> ·
 <a href="https://huggingface.co/apple/DFN5B-CLIP-ViT-H-14-378">apple/DFN5B-CLIP-ViT-H-14-378</a> ·
 <a href="https://huggingface.co/timm/eva02_large_patch14_clip_336.merged2b_s6b_b61k">timm/eva02_large_patch14_clip_336</a>.
 Cloud VLMs (no public HF page): GPT-4o / 4o-mini / 4.1 / 4.1-mini · Gemini 2.5 Pro / Flash / Flash-Lite ·
 Claude Opus / Sonnet / Haiku.</p>
<footer>
 <a href="https://nutrient.io/"><img src="https://avatars2.githubusercontent.com/u/1527679?v=3&s=200" height="34" /></a>
 Macro-F1 on nutrientdocs/document-classification-benchmark · same <code>score_classify.py</code> for all ·
 maintained and funded by <a href="https://nutrient.io/">Nutrient</a> ·
 flagship: <a href="https://huggingface.co/nutrientdocs/document-classification-v2">document-classification-v2</a>
 </footer>
</div>
<script>
let DATA=[];
let TAB="nonvlm"; // default: the apples-to-apples open-vocab (non-VLM) fight
function setTab(t){TAB=t;
 document.getElementById('t-nonvlm').className='tab'+(t==='nonvlm'?' on':'');
 document.getElementById('t-vlm').className='tab'+(t==='vlm'?' on':'');
 render();}
const CORE=[["doclaynet","DocLayNet"],["forms","Forms"],["tobacco","Tobacco"]];
const EXTRA=[["ood","OOD*"],["oov","OOV*"]];
const ALL=CORE.concat(EXTRA);
function hf(m){return (m&&m.includes("/"))?"https://huggingface.co/"+m:null;}
function fmt(x){return (x===null||x===undefined)?'<span class="muted">—</span>':x.toFixed(3).replace(/^0/,"");}
function coreMean(t){const v=CORE.map(c=>t[c[0]]).filter(x=>x!==null&&x!==undefined);
 return v.length===CORE.length? v.reduce((a,b)=>a+b,0)/v.length : null;}
function chip(k){return k==="ours-commercial"?'<span class="chip c-com">commercial</span>'
 :k==="ours-open"?'<span class="chip c-open">open</span>'
 :k==="generalist"?'<span class="chip c-gen">generalist</span>':'<span class="chip c-cloud">cloud</span>';}
function render(){
 // full-coverage (all 3 core tracks) ranked by doc-type mean; partial rows after, by DocLayNet
 const D=DATA.filter(r=> TAB==="vlm" ? r.kind!=="generalist" : r.kind!=="cloud"); // ours anchor both tabs
 const full=D.filter(r=>coreMean(r.tracks)!==null).sort((a,b)=>coreMean(b.tracks)-coreMean(a.tracks));
 const part=D.filter(r=>coreMean(r.tracks)===null).sort((a,b)=>(b.tracks.doclaynet||0)-(a.tracks.doclaynet||0));
 const rows=full.concat(part);
 document.getElementById("head").innerHTML='<tr><th class="rank">#</th><th>Model</th>'
 +ALL.map(t=>`<th class="n">${t[1]}</th>`).join("")+'<th class="barcell">Doc-type mean</th><th class="n">Latency</th></tr>';
 const tb=document.getElementById("rows");tb.innerHTML="";
 const max=Math.max(...full.map(r=>coreMean(r.tracks)));
 rows.forEach((r,i)=>{
 const m=coreMean(r.tracks), ours=r.kind&&r.kind.startsWith("ours");
 const tr=document.createElement("tr");if(ours)tr.className="ours";
 const nm=hf(r.model)?`<a class="name" href="${hf(r.model)}" target="_blank" rel="noopener">${r.name}</a>`:`<span class="name">${r.name}</span>`;
 tr.innerHTML=`<td class="rank">${m!==null?i+1:''}</td><td>${nm}${chip(r.kind)}</td>`
 +ALL.map(t=>`<td class="n ${EXTRA.some(e=>e[0]===t[0])?'oursonly':''}">${fmt(r.tracks[t[0]])}</td>`).join("")
 +(m!==null?`<td><div class="barcell"><div class="bar ${ours?'hl':''}" style="flex:1"><span style="width:${Math.round(m/max*100)}%"></span></div><span class="barnum">${m.toFixed(3).replace(/^0/,"")}</span></div></td>`
 :`<td class="muted" style="font-size:12px">partial coverage</td>`)
 +`<td class="n">${r.latency||'<span class="muted">—</span>'}</td>`;
 tb.appendChild(tr);
 });
}
function setBanner(){const b=document.getElementById("banner");
 if(DATA.some(d=>d.provisional)){b.className="tracknote";b.style.borderColor="var(--warn)";b.textContent="Provisional numbers — a clean scoring pass on standard hardware is pending.";}}
fetch("leaderboard.json").then(r=>r.ok?r.json():[]).then(d=>{DATA=Array.isArray(d)?d:(d.results||[]);render();}).catch(()=>render());
</script>
</body>
</html>