Hugging Face's logo

Spaces:
codelion
/
data-inventor
Running

App Files Community
1
data-inventor / static /curate.js
codelion's picture
codelion
Visualize: ask a dataset a question, get an exact chart with the rows behind every mark (small datasets first: instant glance charts, batched model labels, drill-down, scoped follow-ups)
917845f verified
Raw History Blame Contribute Delete
36.4 kB
"use strict";
/* Curate a corpus: register sources and targets, then clean, calibrate, create, compose.
The hosted run works on a sample; the recipe runs in full on your own machines. */
(function () {
const CSTEPS = ["Sources", "Clean", "Calibrate", "Create", "Compose", "Run"];
const BUDGETS = [["1B", 1e9], ["10B", 1e10], ["100B", 1e11], ["1T", 1e12]];
const BENCH_NAMES = { gsm8k: "GSM8K", math500: "MATH-500", humaneval: "HumanEval", mbpp: "MBPP", mmlu: "MMLU", arc_challenge: "ARC-Challenge",
arc_easy: "ARC-Easy", hellaswag: "HellaSwag", openbookqa: "OpenBookQA", winogrande: "WinoGrande", truthfulqa: "TruthfulQA", commonsenseqa: "CommonsenseQA" };
const KIND_NAMES = { web: "Web", educational: "Educational web", textbook: "Textbook-style", math: "Math", code: "Code", multilingual: "Multilingual", synthetic: "Synthetic", other: "Other" };
const RULES = [["min_words", "Fewest words in a document", 1], ["max_dup_lines", "Most repeated lines (share)", 0.01], ["max_dup10gram", "Most repeated passages (share)", 0.01], ["min_alpha_words", "Fewest words with letters (share)", 0.01]];
const PRESET = [
{ dataset: "codelion/finepdfs-1B", kind: "textbook" }, { dataset: "codelion/dclm-baseline-1B", kind: "web" }, { dataset: "codelion/fineweb-edu-1B", kind: "educational" }];
const C = {
step: 0, maxStep: 0, d: null, sources: [], targets: [], targetsDesc: "", add: "", addRole: "source", addBusy: false, addErr: "",
decon: ["gsm8k", "math500", "humaneval", "mmlu", "arc_challenge", "hellaswag"], rules: {},
focus: 0.3, keepTop: 0.6, near: true, threshold: 0.8, gradeDocs: 2000, scorer: "learned",
rephraseOn: true, styles: [], maxNew: 0.35, inventOn: false, inventReq: "", inventDocs: 0,
mixes: [{ name: "main", budget: 1e10, weights: {} }], maxEpochs: 4, maxSynth: 0.25,
tryText: "", tryBusy: false, tryErr: "", tryRes: null, tryFrom: "",
pid: null, pv: null, runErr: "",
};
const $ = (s, el = document) => el.querySelector(s);
const esc = (t) => String(t ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
const fmt = (n) => { n = +n || 0; for (const [u, d] of [["T", 1e12], ["B", 1e9], ["M", 1e6], ["K", 1e3]]) if (Math.abs(n) >= d) return `${+(n / d).toPrecision(3)}${u}`; return `${Math.round(n)}`; };
const pct = (x) => `${Math.round((+x || 0) * 100)}%`;
async function api(path, body, method) {
const r = await fetch(path, { method: method || (body ? "POST" : "GET"), headers: body ? { "Content-Type": "application/json" } : {}, body: body ? JSON.stringify(body) : undefined });
let data = null; try { data = await r.json(); } catch (e) { /* not json */ }
if (!r.ok) throw new Error((data && (data.detail || data.message)) || `Request failed (${r.status})`);
return data;
}
/* ------------------------------------------------------------ recipe */
function srcOut(s) { return { name: s.name, dataset: s.dataset, config: s.config, split: s.split, text_field: s.text_field, kind: s.kind, num_rows: s.num_rows || 0 }; }
function recipe() {
return {
name: "my-corpus", sources: C.sources.map(srcOut),
targets: { description: C.targetsDesc.trim(), datasets: C.targets.map(srcOut) },
clean: { decontaminate: C.decon, rules: C.rules },
calibrate: { dedup: { near: C.near, threshold: C.threshold }, grade_docs: C.gradeDocs, scorer: C.scorer, focus: C.focus, keep_top: C.keepTop },
create: { rephrase: { enabled: C.rephraseOn, styles: C.styles, max_new_share: C.maxNew },
invent: { enabled: C.inventOn && !!C.inventReq.trim(), request: C.inventReq.trim(), docs: C.inventDocs } },
compose: { mixes: C.mixes.map((m) => ({ name: m.name, budget_tokens: m.budget, weights: m.weights })), max_epochs: C.maxEpochs, max_synthetic_share: C.maxSynth },
};
}
/* mirrors datainventor.compose.default_weights */
function defaultWeights() {
const G = { textbook: "textbook", synthetic: "textbook", web: "web", multilingual: "web", other: "web", educational: "educational", math: "math", code: "code" };
const BASE = { textbook: 0.5, web: 0.3, educational: 0.2 };
const groups = {}; C.sources.forEach((s) => (groups[G[s.kind] || "web"] = groups[G[s.kind] || "web"] || []).push(s.name));
const fixed = {}; ["math", "code"].forEach((g) => { if (groups[g]) fixed[g] = 0.1; });
const base = Object.fromEntries(Object.entries(BASE).filter(([g]) => groups[g]));
const out = {};
if (!Object.keys(base).length) { C.sources.forEach((s) => (out[s.name] = 1 / C.sources.length)); return out; }
const rest = 1 - Object.values(fixed).reduce((a, b) => a + b, 0), tot = Object.values(base).reduce((a, b) => a + b, 0);
const shares = { ...Object.fromEntries(Object.entries(base).map(([g, v]) => [g, (rest * v) / tot])), ...fixed };
Object.entries(groups).forEach(([g, names]) => names.forEach((n) => (out[n] = shares[g] / names.length)));
return out;
}
const weightsOf = (m) => ({ ...defaultWeights(), ...m.weights });
/* ------------------------------------------------------------ chrome */
function ledger() {
$("#steps").innerHTML = CSTEPS.map((n, i) => {
const cls = i === C.step ? "on" : i < C.step ? "done" : "";
return `<li class="${cls}"><button data-cgo="${i}" ${i > C.maxStep ? "disabled" : ""} ${i === C.step ? 'aria-current="step"' : ""}><span class="n">${i + 1}</span><span>${n}</span></button></li>`;
}).join("");
}
function sheet() {
const line = (k, v) => `<div class="line"><dt>${k}</dt><dd>${v}</dd></div>`;
const m = C.mixes[0];
$("#sheet").innerHTML = `<h2>Your recipe</h2><dl>
${line("Sources", C.sources.length ? esc(C.sources.map((s) => s.name).join(", ")) : '<span class="hint">None yet</span>')}
${line("Targets", C.targetsDesc.trim() || C.targets.length ? esc([C.targetsDesc.trim().slice(0, 80), ...C.targets.map((t) => t.name)].filter(Boolean).join("; ")) : '<span class="hint">General quality only</span>')}
${line("Decontam", C.decon.length ? esc(C.decon.map((b) => BENCH_NAMES[b] || b).join(", ")) : "Off")}
${line("Focus", C.focus === 0 || !(C.targetsDesc.trim() || C.targets.length) ? "General quality" : `${pct(1 - C.focus)} quality, ${pct(C.focus)} targets`)}
${line("Keep", `Top ${pct(C.keepTop)} by score`)}
${line("Create", C.rephraseOn ? "Rephrase to fill gaps" + (C.inventOn ? ", invent" : "") : C.inventOn ? "Invent" : "Off")}
${line("Mixes", C.mixes.map((x) => `${esc(x.name)} ${fmt(x.budget)}`).join(", "))}
</dl>
<p class="est">The hosted run works on about <b>${C.d ? C.d.per_source : 200}</b> documents per source. The recipe runs the same steps on every document, on your machines.</p>
<div class="actions" style="margin-top:14px"><button class="btn" data-c="recipe" ${C.sources.length ? "" : "disabled"}>Download recipe</button></div>`;
}
function actions(back, next, ok = true, extra = "") {
return `<div class="actions">${back ? `<button class="btn" data-c="back">Back</button>` : ""}${next ? `<button class="btn go" data-c="next" ${ok ? "" : "disabled"}>${next}</button>` : ""}${extra}</div>`;
}
function check(on, attr, label, hint = "") {
return `<button class="choice" ${attr} aria-pressed="${on}"><span class="box"></span><span>${label}${hint ? ` <span class="hint">${hint}</span>` : ""}</span></button>`;
}
function slider(id, label, val, min, max, step, show) {
return `<label class="f"><span>${label} <b class="sv" id="${id}-v">${show(val)}</b></span><input type="range" class="range" id="${id}" min="${min}" max="${max}" step="${step}" value="${val}"></label>`;
}
/* ------------------------------------------------------------ steps */
function srcRow(s, i, role) {
const kinds = C.d ? C.d.kinds : Object.keys(KIND_NAMES);
return `<div class="srow">
<div class="sname"><b>${esc(s.dataset)}</b><span class="hint">${esc(s.config)} / ${esc(s.split)} / column <code>${esc(s.text_field)}</code>${s.num_rows ? ` / ${fmt(s.num_rows)} rows` : ""}</span>
${s.preview ? `<details><summary>Peek at a document</summary><div class="peek">${esc(s.preview)}</div></details>` : ""}</div>
<div class="sctl">${role === "source" ? `<select class="in sm" id="c-kind-${i}" aria-label="Kind of data">${kinds.map((k) => `<option value="${k}" ${s.kind === k ? "selected" : ""}>${KIND_NAMES[k] || k}</option>`).join("")}</select>` : ""}
${s.columns && s.columns.length > 1 ? `<select class="in sm" id="c-col-${role}-${i}" aria-label="Text column">${s.columns.map((c) => `<option ${c === s.text_field ? "selected" : ""}>${esc(c)}</option>`).join("")}</select>` : ""}
<button class="linkish" data-c="rm" data-role="${role}" data-i="${i}">Remove</button></div></div>`;
}
function stepSources() {
return `<h1>Bring the data you have</h1>
<p class="lede">Register each Hugging Face dataset as a source to curate, or as a target that shows what good looks like. Targets steer every later step toward your tasks.</p>
<div class="group"><h3>Add a dataset</h3>
<div class="addrow"><input class="in" id="c-add" placeholder="HuggingFaceFW/fineweb-edu" value="${esc(C.add)}" spellcheck="false" autocomplete="off">
<select class="in sm" id="c-addrole" aria-label="Register as"><option value="source" ${C.addRole === "source" ? "selected" : ""}>Source</option><option value="target" ${C.addRole === "target" ? "selected" : ""}>Target</option></select>
<button class="btn" data-c="add" ${C.addBusy || !C.add.trim() ? "disabled" : ""}>${C.addBusy ? "Reading" : "Add"}</button></div>
${C.addErr ? `<p class="note bad">${esc(C.addErr)}</p>` : ""}
${C.sources.length ? "" : `<p class="hint" style="margin-top:10px">Or start from <button class="linkish" data-c="preset">the 1B-token mix</button>: textbook-style PDFs, DCLM and FineWeb-Edu.</p>`}</div>
<div class="group"><h3>Sources <span class="hint">(up to ${C.d ? C.d.max_sources : 4} here)</span></h3>
${C.sources.length ? C.sources.map((s, i) => srcRow(s, i, "source")).join("") : '<p class="hint">None yet.</p>'}</div>
<div class="group"><h3>Targets</h3>
<label class="f" style="margin-top:6px"><span>What should the trained model be good at? <span class="hint">(optional)</span></span>
<textarea class="in" id="c-tdesc" style="min-height:80px" placeholder="Reading and drafting commercial contracts. Grade-school math word problems.">${esc(C.targetsDesc)}</textarea></label>
${C.targets.map((s, i) => srcRow(s, i, "target")).join("")}
<p class="hint" style="margin-top:8px">Target datasets are examples of the work you care about, such as a benchmark's training split or your own documents.</p></div>
${actions(false, "Continue", C.sources.length > 0)}`;
}
function stepClean() {
const b = C.d ? C.d.benchmarks : Object.keys(BENCH_NAMES), r = { ...(C.d ? C.d.clean_rules : {}), ...C.rules };
return `<h2>Clean</h2>
<p class="lede">Heuristic filters drop broken text, boilerplate, empty and repetitive documents. Math and code get their own looser rules. Then any document that shares a 13-word run with a test set you name is removed.</p>
<div class="group"><h3>Decontaminate against</h3><ul class="choices">${b.map((k) => `<li>${check(C.decon.includes(k), `data-bench="${k}"`, BENCH_NAMES[k] || k)}</li>`).join("")}</ul>
<p class="hint" style="margin-top:6px">To check against test sets of your own, list them under <code>clean.held_out</code> in the recipe, the same way as sources.</p></div>
<div class="group"><h3>Filters</h3><div class="row2">${RULES.map(([k, label, step]) => `<label class="f"><span>${label}</span><input class="in" type="number" id="c-rule-${k}" step="${step}" value="${r[k]}"></label>`).join("")}</div>
<p class="hint" style="margin-top:8px">The defaults follow the Gopher and C4 rules with FineWeb's thresholds. Every rule is in the recipe.</p></div>
${actions(true, "Continue")}`;
}
function stepCalibrate() {
const hasT = !!(C.targetsDesc.trim() || C.targets.length);
return `<h2>Calibrate</h2>
<p class="lede">A model grades a sample of each source for teaching value, clarity, accuracy and density${hasT ? ", and for relevance to your targets" : ""}. A small scorer learns those grades and scores every document. Redundant copies are dropped first.</p>
${hasT ? slider("c-focus", "Balance between general quality and your targets", C.focus, 0, 1, 0.05, (v) => (+v === 0 ? "general quality only" : `${pct(1 - v)} quality / ${pct(v)} targets`))
: `<p class="note">No targets, so documents are ranked by general quality. Add targets in step 1 to steer the ranking.</p>`}
${hasT ? '<p class="hint">Keep some weight on general quality. Data tuned only to the targets overfits them.</p>' : ""}
${slider("c-keep", "Keep the best", C.keepTop, 0.1, 1, 0.05, (v) => `${pct(v)} of each source`)}
<div class="group"><h3>Duplicates</h3><ul class="choices" style="columns:1">
<li>${check(true, "disabled", "Exact copies", "always removed")}</li>
<li>${check(C.near, 'data-c="near"', "Near copies", "MinHash, about 80% overlap")}</li></ul></div>
<div class="group"><h3>On your machines</h3>
<div class="row2"><label class="f"><span>Documents to grade with the model</span><input class="in" type="number" id="c-grade" min="100" step="100" value="${C.gradeDocs}"></label>
<label class="f"><span>Score every document by</span><select class="in" id="c-scorer"><option value="learned" ${C.scorer === "learned" ? "selected" : ""}>A scorer learned from the grades</option><option value="llm" ${C.scorer === "llm" ? "selected" : ""}>The model itself (slow, best)</option></select></label></div>
<p class="hint" style="margin-top:8px">The hosted run grades ${C.d ? 48 : "a few dozen"} documents, enough to see the rubric work but not to trust the learned scorer. In our test, 300 grades gave a rank agreement of about 0.5 with the model; more grades help.</p></div>
${actions(true, "Continue")}`;
}
function tryHtml() {
if (C.tryErr) return `<p class="note bad">${esc(C.tryErr)}</p>`;
if (!C.tryRes) return "";
return `<div class="pairs">${C.tryRes.results.map((o) => `<div class="pcard"><div class="ph"><b>${esc(o.name)}</b>${o.ok ? "" : `<span class="flag">${esc(o.why)}</span>`}</div>
<div class="ptext">${esc(o.text || "No rewrite. The model call failed.")}</div>
${o.fidelity && (o.fidelity.new_numbers.length || o.fidelity.new_names.length) ? `<p class="hint">Not in the source: ${esc([...o.fidelity.new_numbers, ...o.fidelity.new_names].join(", "))}</p>` : ""}</div>`).join("")}</div>`;
}
function stepCreate() {
const st = C.d ? C.d.styles : {}, picked = C.styles;
const first = C.sources.find((s) => s.preview);
return `<h2>Create</h2>
<p class="lede">Where a source is too small for its share of a mix, the best documents are rephrased into new ones: a dialogue, a tutorial, questions and answers, dense notes, a textbook section. Rewrites that add numbers or names the source does not contain are flagged and dropped.</p>
<ul class="choices" style="columns:1;margin-top:20px"><li>${check(C.rephraseOn, 'data-c="reph"', "Rephrase to fill gaps")}</li></ul>
${C.rephraseOn ? `<div class="group"><h3>Styles <span class="hint">(none picked = chosen per kind of data)</span></h3><ul class="choices">${Object.entries(st).map(([k, n]) => `<li>${check(picked.includes(k), `data-style="${k}"`, esc(n))}</li>`).join("")}</ul></div>
<div class="group"><h3>Try it on one document</h3>
<textarea class="in" id="c-try" style="min-height:120px" placeholder="Paste a passage of 60 words or more.">${esc(C.tryText)}</textarea>
<div class="actions" style="margin-top:12px">${first ? `<button class="btn" data-c="tryfill">Use a document from ${esc(first.name)}</button>` : ""}
<button class="btn go" data-c="try" ${C.tryBusy || C.tryText.trim().split(/\s+/).length < 60 ? "disabled" : ""}>${C.tryBusy ? "Rewriting" : "Rewrite it in four styles"}</button>${C.tryBusy ? '<span class="hint">This takes about a minute.</span>' : ""}</div>
<div id="c-tryout">${tryHtml()}</div></div>` : ""}
<ul class="choices" style="columns:1;margin-top:26px"><li>${check(C.inventOn, 'data-c="inv"', "Also write new documents on a subject", "for areas your sources barely cover")}</li></ul>
${C.inventOn ? `<div class="row2"><label class="f"><span>Subject</span><input class="in" id="c-invreq" value="${esc(C.inventReq)}" placeholder="Contract law for small businesses"></label>
<label class="f"><span>Documents on your machines</span><input class="in" type="number" id="c-invdocs" min="0" step="1000" value="${C.inventDocs}"></label></div>
<p class="hint" style="margin-top:8px">The hosted run writes four as a preview. Give invented documents a share in a mix with <code>invented_share</code> in the recipe.</p>` : ""}
${actions(true, "Continue")}`;
}
function stepCompose() {
return `<h2>Compose</h2>
<p class="lede">Build one or more training mixes from the prepared corpus. The default share is the static mix that won our 1B-token study: 50% textbook-style, 30% web, 20% educational web, with 10% each for math and code when present. It was measured at 70M parameters, so treat it as a starting point.</p>
${C.mixes.map((m, mi) => { const w = weightsOf(m), tot = Object.values(w).reduce((a, b) => a + b, 0) || 1;
return `<div class="group mix"><div class="mixhead"><input class="in sm" id="c-mname-${mi}" value="${esc(m.name)}" aria-label="Mix name" spellcheck="false">
<div class="seg">${BUDGETS.map(([l, v]) => `<button data-c="budget" data-m="${mi}" data-v="${v}" aria-pressed="${m.budget === v}">${l}</button>`).join("")}</div>
<label class="hint">or <input class="in sm num wide" type="number" id="c-mbud-${mi}" value="${m.budget}" min="1000000" step="1000000"> tokens</label>
${C.mixes.length > 1 ? `<button class="linkish" data-c="rmmix" data-m="${mi}">Remove</button>` : ""}</div>
<div class="wtable">${C.sources.map((s) => `<label><span>${esc(s.name)} <small class="hint">${KIND_NAMES[s.kind] || s.kind}</small></span>
<input class="in sm num" type="number" min="0" max="100" step="1" id="c-w-${mi}-${esc(s.name)}" value="${Math.round((w[s.name] / tot) * 100)}"><span class="hint">%</span></label>`).join("")}</div></div>`; }).join("")}
<p style="margin-top:14px"><button class="linkish" data-c="addmix">Add another mix</button> <span class="hint">for a different run, from the same prepared corpus</span></p>
<div class="row2">${slider("c-epochs", "Repeat a source at most", C.maxEpochs, 1, 8, 0.5, (v) => `${v} times`)}
${slider("c-synth", "Synthetic text at most", C.maxSynth, 0, 0.6, 0.05, (v) => `${pct(v)} of a mix`)}</div>
<p class="hint">A source short of its share is repeated first, then topped up by rephrasing its best documents. In our study a mix that leaned on synthetic-like text scored well on its own validation set but generalized worst.</p>
${actions(true, "Continue")}`;
}
/* ------------------------------------------------------------ run + report */
const isFinal = (p) => p && ["done", "stopped", "failed"].includes(p.state);
function hist(h) {
const mx = Math.max(1, ...h);
return `<div class="hist" aria-hidden="true">${h.map((v) => `<i style="height:${Math.round((v / mx) * 100)}%"></i>`).join("")}</div>`;
}
function report(rep) {
const src = rep.sources || [], cal = (rep.stages || {}).calibrate || {}, comp = (rep.stages || {}).compose || {}, crt = (rep.stages || {}).create || {};
const agree = cal.agreement || {};
const drops = (f) => Object.entries(f.dropped).filter(([, v]) => v).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${esc(k)} ${v}`).join(", ") || "none";
return `
<div class="group"><h3>What each source went through</h3><div class="tscroll"><table class="tbl"><thead><tr><th>Source</th><th>Read</th><th>Passed cleaning</th><th>Kept</th><th>Removed for test overlap</th><th>Tokens after curation</th></tr></thead><tbody>
${src.map((f) => `<tr><td><b>${esc(f.name)}</b><br><span class="hint">${drops(f)}</span></td><td>${f.read}</td><td>${f.kept}</td><td>${f.selected}</td>
<td>${Object.keys(f.decontaminated).length ? esc(Object.entries(f.decontaminated).map(([k, v]) => `${BENCH_NAMES[k] || k} ${v}`).join(", ")) : "0"}</td>
<td>${f.available_tokens ? "about " + fmt(f.available_tokens) : '<span class="hint">unknown size</span>'}</td></tr>`).join("")}</tbody></table></div>
<p class="hint" style="margin-top:8px">Tokens after curation scale each sample's keep rate to the full dataset. They are estimates at four characters per token.</p></div>
${src.some((f) => f.repeated_passages && f.repeated_passages.length) ? `<div class="group"><h3>Passages repeated across documents</h3>${src.filter((f) => f.repeated_passages.length).map((f) => f.repeated_passages.slice(0, 3).map((r) => `<p class="quote"><b>${esc(f.name)}</b>, in ${r.docs} documents of the sample: ${esc(r.text)}</p>`).join("")).join("")}</div>` : ""}
<div class="group"><h3>Scores</h3>
<p class="hint">The model graded ${cal.graded} documents and those keep their grades. The rest are ranked by a scorer learned from so few grades that its ranking is close to a guess${cal.graded >= 100 ? ` (rank agreement with the model on held-out documents: quality ${agree.quality ?? "n/a"}${cal.has_targets ? `, relevance ${agree.relevance ?? "n/a"}` : ""})` : ""}. The full run grades ${C.gradeDocs.toLocaleString()} documents and reports how well the scorer agrees with the model.</p>
<div class="hists">${src.map((f) => `<div><span>${esc(f.name)}</span>${hist(f.score_hist || [])}<small class="hint">low to high score</small></div>`).join("")}</div>
${src.map((f) => `<details class="ex"><summary>Best and worst kept in ${esc(f.name)}</summary>${[...(f.examples.top || []).map((e) => ["Best", e]), ...(f.examples.bottom || []).map((e) => ["Worst", e])].map(([l, e]) => `<div class="exd"><b>${l}</b> <span class="hint">score ${e.score}, quality ${e.quality}${cal.has_targets ? `, relevance ${e.relevance}` : ""}${e.reason ? ` / ${esc(e.reason)}` : ""}</span><div class="ptext">${esc(e.text)}</div></div>`).join("")}</details>`).join("")}</div>
${(rep.created_examples || []).length ? `<div class="group"><h3>Created</h3><p class="hint">${crt.kept} of ${crt.made} rewrites kept.${(crt.flagged || []).length ? ` ${crt.flagged.length} flagged for adding facts the source does not contain.` : ""}</p>
${rep.created_examples.map((c) => `<details class="ex"><summary>${esc(c.style ? (C.d.styles[c.style] || c.style) : "New document: " + (c.topic || ""))}${c.kept ? "" : ` <span class="flag">${esc(c.why)}</span>`}</summary>
<div class="pairs two">${c.origin_text ? `<div class="pcard"><div class="ph"><b>Source</b></div><div class="ptext">${esc(c.origin_text)}</div></div>` : ""}<div class="pcard"><div class="ph"><b>Rewrite</b></div><div class="ptext">${esc(c.text)}</div></div></div></details>`).join("")}</div>` : ""}
<div class="group"><h3>Mixes</h3>${(comp.plans || []).map((p) => `<p style="margin-top:12px"><b>${esc(p.name)}</b>, ${esc(p.budget)} tokens</p>
<div class="tscroll"><table class="tbl"><thead><tr><th>Source</th><th>Share</th><th>Needs</th><th>Available</th><th>Repeats</th><th>Rephrased</th></tr></thead><tbody>
${p.rows.map((r) => `<tr><td>${esc(r.name)}</td><td>${pct(r.share)}</td><td>${fmt(r.want_tokens)}</td><td>${fmt(r.available_tokens)}</td><td>${r.epochs}x</td><td>${r.synthetic_tokens ? fmt(r.synthetic_tokens) : "0"}${r.short_tokens ? ` <span class="flag">${fmt(r.short_tokens)} short</span>` : ""}</td></tr>`).join("")}</tbody></table></div>
${p.synthetic_tokens ? `<p class="hint">About ${p.synthetic_calls.toLocaleString()} model calls to rephrase ${fmt(p.synthetic_tokens)} tokens on your endpoint.</p>` : ""}
${p.warnings.map((w) => `<p class="note">${esc(w)}</p>`).join("")}`).join("")}</div>
${(rep.warnings || []).map((w) => `<p class="note">${esc(w)}</p>`).join("")}`;
}
function stepRun() {
const p = C.pv, fin = isFinal(p);
const stageIdx = p ? ["setup", "clean", "calibrate", "create", "done"].indexOf(p.stage) : -1;
const out = p && p.report && p.report.output;
return `<h2>${!p ? "Run it on a sample" : !fin ? "Curating a sample" : p.state === "failed" ? "The run stopped" : "Your sample is curated"}</h2>
<p class="lede">Every step runs here on about ${C.d ? C.d.per_source : 200} documents from each source. You get the kept and created documents, a report of what each step did, and the recipe to run the same steps on all of your data.</p>
${C.runErr ? `<p class="note bad">${esc(C.runErr)}</p>` : ""}
${!p ? `<div class="actions"><button class="btn" data-c="back">Back</button><button class="btn go" data-c="run">Run on a sample</button></div>` : ""}
${p && !fin ? `<ol class="stages">${["Reading", "Cleaning", "Calibrating", "Creating", "Composing"].map((s, i) => `<li class="${i < stageIdx ? "done" : i === stageIdx ? "on" : ""}">${s}</li>`).join("")}</ol>
<p class="state">${esc(p.message || "Starting")}. Elapsed ${Math.round(p.elapsed || 0)} s.</p><div class="actions"><button class="btn" data-c="stop">Stop</button></div>` : ""}
${p && p.state === "failed" ? `<p class="note bad">${esc(p.message)}</p><div class="actions"><button class="btn" data-c="again">Change the recipe</button><button class="btn go" data-c="run">Try again</button></div>` : ""}
${p && fin && p.report ? `<div class="stats"><div><b>${out.docs}</b><span>documents ready</span></div><div><b>${out.curated}</b><span>kept from sources</span></div><div><b>${out.created}</b><span>created</span></div><div><b>${fmt(out.tokens)}</b><span>tokens, about</span></div></div>
<div class="actions"><a class="btn go" href="/api/curate/previews/${p.id}/download" style="text-decoration:none">Download the sample (JSONL)</a><button class="btn" data-c="recipe">Download recipe</button><button class="btn" data-c="again">Change the recipe</button></div>
${runYourself()}${report(p.report)}` : ""}`;
}
function runYourself() {
const inst = C.d ? C.d.install : "pip install datainventor";
return `<div class="group"><h3>Run it on all of your data</h3>
<p class="hint" style="max-width:62ch">On your own machines, the same recipe reads every document, grades ${C.gradeDocs.toLocaleString()} with your model, and writes the curated corpus and each mix as JSONL shards with a report. Point it at any OpenAI-compatible server (vLLM, SGLang, Ollama or a hosted API) by editing <code>endpoint</code> in the recipe. Interrupted runs resume where they stopped.</p>
<pre class="code">${esc(inst)}
export DI_API_KEY=... # key for the endpoint in the recipe
datainventor preview recipe.yaml # optional: the hosted check, locally
datainventor run recipe.yaml --out ./curated --procs 32 --workers 64</pre></div>`;
}
/* ------------------------------------------------------------ render + events */
function render() {
if (window.DI_MODE !== "curate") return;
ledger(); sheet();
const fn = [stepSources, stepClean, stepCalibrate, stepCreate, stepCompose, stepRun][C.step];
const keep = document.activeElement && document.activeElement.id;
$("#panel").innerHTML = fn();
if (keep && keep.startsWith("c-")) { const el = document.getElementById(keep); if (el) { el.focus(); if (el.setSelectionRange && el.type === "text") { const l = el.value.length; try { el.setSelectionRange(l, l); } catch (e) { /* ok */ } } } }
}
function go(i) { C.step = i; C.maxStep = Math.max(C.maxStep, i); render(); $("#panel").focus({ preventScroll: true }); window.scrollTo({ top: 0 }); }
async function addDataset(id, role, kind) {
C.addBusy = true; C.addErr = ""; render();
try {
const info = await api(`/api/curate/inspect?dataset=${encodeURIComponent(id)}`);
const list = role === "target" ? C.targets : C.sources, max = role === "target" ? (C.d ? C.d.max_targets : 2) : (C.d ? C.d.max_sources : 4);
if (list.length >= max) throw new Error(`The hosted run takes up to ${max} ${role}s. The recipe takes any number.`);
let name = info.dataset.split("/")[1]; const taken = new Set([...C.sources, ...C.targets].map((s) => s.name));
for (let k = 2; taken.has(name); k++) name = `${info.dataset.split("/")[1]}-${k}`;
list.push({ ...info, name, kind: kind || info.kind });
C.add = ""; C.pv = null;
} catch (e) { C.addErr = e.message; }
C.addBusy = false; render();
}
async function download() {
try {
const r = await fetch("/api/curate/recipe", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ recipe: recipe() }) });
if (!r.ok) { const d = await r.json().catch(() => ({})); throw new Error(d.detail || "Could not build the recipe"); }
const url = URL.createObjectURL(await r.blob()), a = document.createElement("a");
a.href = url; a.download = "recipe.yaml"; document.body.appendChild(a); a.click(); a.remove(); setTimeout(() => URL.revokeObjectURL(url), 2000);
} catch (e) { C.runErr = e.message; render(); }
}
async function run() {
C.runErr = ""; C.pv = null;
try { const r = await api("/api/curate/previews", { recipe: recipe() }); C.pid = r.id; C.pv = { id: r.id, state: "queued", stage: "setup", message: "Starting" }; render(); poll(); }
catch (e) { C.runErr = e.message; render(); }
}
async function poll() {
const id = C.pid; if (!id) return;
try {
const p = await api(`/api/curate/previews/${id}`);
if (id !== C.pid) return;
C.pv = p; if (C.step === 5) render();
if (!isFinal(p)) setTimeout(poll, 1500);
} catch (e) { C.pv = { id, state: "failed", message: e.message }; if (C.step === 5) render(); }
}
async function tryIt() {
C.tryBusy = true; C.tryErr = ""; C.tryRes = null; render();
const kind = (C.sources.find((s) => s.name === C.tryFrom) || {}).kind || "web";
try { C.tryRes = await api("/api/curate/try", { text: C.tryText, kind, styles: C.styles.slice(0, 4) }); } catch (e) { C.tryErr = e.message; }
C.tryBusy = false; render();
}
document.addEventListener("click", async (ev) => {
if (window.DI_MODE !== "curate") return;
const t = ev.target.closest("button, a"); if (!t) return;
if (t.dataset.cgo !== undefined) return go(+t.dataset.cgo);
if (t.dataset.bench) { const b = t.dataset.bench; C.decon = C.decon.includes(b) ? C.decon.filter((x) => x !== b) : [...C.decon, b]; return render(); }
if (t.dataset.style) { const s = t.dataset.style; C.styles = C.styles.includes(s) ? C.styles.filter((x) => x !== s) : [...C.styles, s]; return render(); }
const a = t.dataset.c; if (!a) return;
if (a === "next") return go(C.step + 1);
if (a === "back") return go(Math.max(0, C.step - 1));
if (a === "add") return addDataset(C.add.trim(), C.addRole);
if (a === "preset") { for (const p of PRESET) await addDataset(p.dataset, "source", p.kind); if (!C.targetsDesc) C.targetsDesc = ""; return render(); }
if (a === "rm") { (t.dataset.role === "target" ? C.targets : C.sources).splice(+t.dataset.i, 1); C.pv = null; return render(); }
if (a === "near") { C.near = !C.near; return render(); }
if (a === "reph") { C.rephraseOn = !C.rephraseOn; return render(); }
if (a === "inv") { C.inventOn = !C.inventOn; return render(); }
if (a === "tryfill") { const s = C.sources.find((x) => x.preview); if (s) { C.tryText = s.preview; C.tryFrom = s.name; } return render(); }
if (a === "try") return tryIt();
if (a === "budget") { C.mixes[+t.dataset.m].budget = +t.dataset.v; return render(); }
if (a === "addmix") { C.mixes.push({ name: `mix-${C.mixes.length + 1}`, budget: 1e9, weights: {} }); return render(); }
if (a === "rmmix") { C.mixes.splice(+t.dataset.m, 1); return render(); }
if (a === "recipe") return download();
if (a === "run") return run();
if (a === "stop") { if (C.pid) await api(`/api/curate/previews/${C.pid}/stop`, {}); return; }
if (a === "again") { C.pv = null; C.pid = null; return go(0); }
});
document.addEventListener("input", (ev) => {
if (window.DI_MODE !== "curate") return;
const el = ev.target, id = el.id || ""; if (!id.startsWith("c-")) return;
const v = el.value;
if (id === "c-add") { C.add = v; const b = $('[data-c="add"]'); if (b) b.disabled = C.addBusy || !v.trim(); return; }
if (id === "c-tdesc") { C.targetsDesc = v; return sheet(); }
if (id === "c-try") { C.tryText = v; C.tryFrom = ""; const b = $('[data-c="try"]'); if (b) b.disabled = C.tryBusy || v.trim().split(/\s+/).length < 60; return; }
if (id === "c-invreq") { C.inventReq = v; return; }
const sv = (x) => { const o = document.getElementById(id + "-v"); if (o) o.textContent = x; };
if (id === "c-focus") { C.focus = +v; sv(+v === 0 ? "general quality only" : `${pct(1 - v)} quality / ${pct(v)} targets`); return sheet(); }
if (id === "c-keep") { C.keepTop = +v; sv(`${pct(v)} of each source`); return sheet(); }
if (id === "c-epochs") { C.maxEpochs = +v; sv(`${v} times`); return; }
if (id === "c-synth") { C.maxSynth = +v; sv(`${pct(v)} of a mix`); return; }
});
document.addEventListener("change", (ev) => {
if (window.DI_MODE !== "curate") return;
const el = ev.target, id = el.id || ""; if (!id.startsWith("c-")) return;
const v = el.value; let m;
if (id === "c-addrole") C.addRole = v;
else if ((m = id.match(/^c-kind-(\d+)$/))) { C.sources[+m[1]].kind = v; C.pv = null; }
else if ((m = id.match(/^c-col-(source|target)-(\d+)$/))) { (m[1] === "target" ? C.targets : C.sources)[+m[2]].text_field = v; C.pv = null; }
else if ((m = id.match(/^c-rule-(.+)$/))) { if (v !== "") C.rules[m[1]] = +v; }
else if (id === "c-grade") C.gradeDocs = Math.max(100, +v || 2000);
else if (id === "c-scorer") C.scorer = v;
else if (id === "c-invdocs") C.inventDocs = Math.max(0, +v || 0);
else if ((m = id.match(/^c-mname-(\d+)$/))) C.mixes[+m[1]].name = v.trim().replace(/[^\w.-]+/g, "-") || `mix-${+m[1] + 1}`;
else if ((m = id.match(/^c-mbud-(\d+)$/))) C.mixes[+m[1]].budget = Math.max(1e6, Math.round(+v || 0));
else if ((m = id.match(/^c-w-(\d+)-(.+)$/))) { const mix = C.mixes[+m[1]]; mix.weights = { ...weightsOf(mix), [m[2]]: Math.max(0, +v || 0) / 100 }; }
else return;
render();
});
document.addEventListener("keydown", (ev) => { if (window.DI_MODE === "curate" && ev.key === "Enter" && ev.target.id === "c-add" && C.add.trim()) addDataset(C.add.trim(), C.addRole); });
/* ------------------------------------------------------------ mode switch */
const TAGS = {
invent: "Describe the data you wish existed. Get training rows back. Bring any OpenAI-compatible model.",
curate: "Clean, score, rephrase and mix the pretraining data you already have. Preview it here on a sample, then run the recipe on your own machines.",
visualize: "Ask a question about a dataset and get a chart back, with the rows behind every bar one click away.",
};
function setMode(mode) {
window.DI_MODE = mode;
document.querySelectorAll("[data-mode]").forEach((b) => b.setAttribute("aria-pressed", String(b.dataset.mode === mode)));
$(".tag").textContent = TAGS[mode] || TAGS.invent;
if (mode === "curate") { if (!C.d) api("/api/curate/defaults").then((d) => { C.d = d; render(); }).catch(() => render()); render(); }
else if (mode === "visualize") { if (window.DI_renderVis) window.DI_renderVis(); }
else if (window.DI_renderInvent) window.DI_renderInvent();
const hash = mode === "invent" ? "" : "#" + mode;
if (location.hash !== hash) history.replaceState(null, "", hash || location.pathname + location.search);
}
window.DI_setMode = setMode;
document.addEventListener("click", (ev) => { const b = ev.target.closest("[data-mode]"); if (b) setMode(b.dataset.mode); });
window.DI_MODE = location.hash === "#curate" ? "curate" : location.hash === "#visualize" ? "visualize" : "invent";
if (window.DI_MODE !== "invent") document.addEventListener("DOMContentLoaded", () => setMode(window.DI_MODE));
})();