Researchers release a compact content extraction model
The encoder reads a long page in one pass and marks boilerplate for removal.
Benchmarks compare throughput, model size, and extraction quality.
- Small checkpoint
- Long context
- Block-level labels
The encoder reads a long page in one pass and marks boilerplate for removal.
Benchmarks compare throughput, model size, and extraction quality.
Install the package, tokenize simplified HTML blocks, then classify separator tokens.
pip install pulpie onnxruntime
The output can be reconstructed as clean Markdown.
Static Spaces can host complete machine learning demos without a Python server.
The browser downloads model weights once and keeps them in its cache for later visits.
Good progress indicators make large downloads feel predictable.
A compact EuroBERT model for separating article text from page chrome.
It scores simplified HTML blocks and keeps the pieces that read like main content.
${escapeHtml(text)}
`, original: rawHtml, tokenIds: [], prediction: 0, score: 0, }); } } return blocks; } function packChunks(blocks) { const chunks = []; let ids = [BOS_ID]; let sepPositions = []; let blockIndices = []; function flush() { if (blockIndices.length === 0) return; ids.push(EOS_ID); chunks.push({ ids, sepPositions, blockIndices }); ids = [BOS_ID]; sepPositions = []; blockIndices = []; } for (const block of blocks) { let tokenIds = block.tokenIds; const maxBlockTokens = MAX_TOKENS - 3; if (tokenIds.length > maxBlockTokens) { tokenIds = tokenIds.slice(0, maxBlockTokens); } if (ids.length + tokenIds.length + 2 > MAX_TOKENS) flush(); ids.push(...tokenIds); ids.push(SEP_ID); sepPositions.push(ids.length - 1); blockIndices.push(block.id); } flush(); return chunks; } function toBigIntTensor(values, dims) { return new ort.Tensor("int64", BigInt64Array.from(values, (value) => BigInt(value)), dims); } async function classifyBlocks(blocks) { for (const block of blocks) { block.tokenIds = state.tokenizer.encode(block.simplified, { add_special_tokens: false }); } const chunks = packChunks(blocks); const predictions = new Array(blocks.length).fill(0); const scores = new Array(blocks.length).fill(0); let totalTokens = 0; for (const chunk of chunks) { totalTokens += chunk.ids.length; const dims = [1, chunk.ids.length]; const feeds = { input_ids: toBigIntTensor(chunk.ids, dims), attention_mask: toBigIntTensor(new Array(chunk.ids.length).fill(1), dims), }; if (state.session.inputNames.includes("token_type_ids")) { feeds.token_type_ids = toBigIntTensor(new Array(chunk.ids.length).fill(0), dims); } const output = await state.session.run(feeds); const logits = output.logits || output[state.session.outputNames[0]]; const data = logits.data; for (let i = 0; i < chunk.sepPositions.length; i += 1) { const blockIndex = chunk.blockIndices[i]; const sep = chunk.sepPositions[i]; const other = Number(data[sep * 2]); const main = Number(data[sep * 2 + 1]); predictions[blockIndex] = main >= other ? 1 : 0; scores[blockIndex] = 1 / (1 + Math.exp(other - main)); } } blocks.forEach((block, index) => { block.prediction = predictions[index]; block.score = scores[index]; }); return { chunks: chunks.length, totalTokens }; } function renderClean(blocks) { const kept = blocks.filter((block) => block.prediction === 1); els.cleanOutput.replaceChildren(); if (kept.length === 0) { const empty = document.createElement("p"); empty.className = "empty-state"; empty.textContent = "No main content blocks were selected."; els.cleanOutput.append(empty); return ""; } let cleanText = ""; let list = null; for (const block of kept) { if (block.tag === "li") { if (!list) { list = document.createElement("ul"); els.cleanOutput.append(list); } const li = document.createElement("li"); li.textContent = block.text; list.append(li); cleanText += `- ${block.text}\n`; continue; } list = null; const element = /^h[1-6]$/.test(block.tag) ? document.createElement("h3") : document.createElement("p"); element.textContent = block.text; els.cleanOutput.append(element); cleanText += `${block.text}\n\n`; } return cleanText.trim(); } function renderBlocks(blocks) { els.blocksOutput.replaceChildren(); for (const block of blocks) { const item = document.createElement("article"); item.className = `block ${block.prediction === 1 ? "main" : "other"}`; const top = document.createElement("div"); top.className = "block-top"; const label = document.createElement("span"); label.textContent = block.prediction === 1 ? "Content" : "Boilerplate"; const score = document.createElement("span"); score.textContent = `${Math.round(block.score * 100)}%`; top.append(label, score); const text = document.createElement("p"); text.textContent = block.text; item.append(top, text); els.blocksOutput.append(item); } } function updateStats(blocks, timingMs, totalTokens, chunks) { const kept = blocks.filter((block) => block.prediction === 1).length; const dropped = blocks.length - kept; els.keptStat.textContent = String(kept); els.droppedStat.textContent = String(dropped); els.processingStat.textContent = formatTime(timingMs); els.blockMeta.textContent = `${blocks.length} blocks, ${totalTokens} tokens, ${chunks} chunk${chunks === 1 ? "" : "s"}`; } async function runExtraction() { const rawHtml = els.htmlInput.value.trim(); if (!rawHtml) { setStatus("Add HTML", "error"); return; } try { setBusy(true); setStatus("Running", "busy"); els.originalOutput.textContent = rawHtml; els.originalMeta.textContent = `${rawHtml.length.toLocaleString()} chars`; await ensureModel(); const started = performance.now(); const blocks = extractBlocks(rawHtml); const { chunks, totalTokens } = await classifyBlocks(blocks); const cleanText = renderClean(blocks); const timingMs = performance.now() - started; renderBlocks(blocks); updateStats(blocks, timingMs, totalTokens, chunks); els.cleanMeta.textContent = `${cleanText.length.toLocaleString()} chars`; setStatus(state.provider.toUpperCase(), "ready"); } catch (error) { console.error(error); setStatus("Error", "error"); els.cleanOutput.replaceChildren(); const message = document.createElement("p"); message.className = "empty-state"; message.textContent = error instanceof Error ? error.message : String(error); els.cleanOutput.append(message); } finally { setBusy(false); } } async function fetchUrl() { const url = els.urlInput.value.trim(); if (!url) return; setBusy(true); setStatus("Fetching", "busy"); try { let html; try { const response = await fetch(url); if (!response.ok) throw new Error(`HTTP ${response.status}`); html = await response.text(); } catch { const proxyUrl = `https://api.allorigins.win/raw?url=${encodeURIComponent(url)}`; const response = await fetch(proxyUrl); if (!response.ok) throw new Error(`Proxy HTTP ${response.status}`); html = await response.text(); } els.htmlInput.value = html; els.originalOutput.textContent = html; els.originalMeta.textContent = `${html.length.toLocaleString()} chars`; setStatus(state.session ? state.provider.toUpperCase() : "Fetched", state.session ? "ready" : ""); } catch (error) { console.error(error); setStatus("Fetch error", "error"); } finally { setBusy(false); } } function loadExample(example) { els.htmlInput.value = example.html; els.originalOutput.textContent = example.html; els.originalMeta.textContent = `${example.html.length.toLocaleString()} chars`; if (state.session && !state.busy) { runExtraction(); } else { setStatus("Example loaded"); } } function initExamples() { for (const example of examples) { const button = document.createElement("button"); button.className = "example-button"; button.type = "button"; button.textContent = example.name; button.addEventListener("click", () => loadExample(example)); els.examples.append(button); } loadExample(examples[0]); } els.extractButton.addEventListener("click", runExtraction); els.fetchButton.addEventListener("click", fetchUrl); els.urlInput.addEventListener("keydown", (event) => { if (event.key === "Enter") fetchUrl(); }); initExamples();