Register each Hugging Face dataset as a source to curate, or as a target that shows what good looks like. Targets steer every later step toward your tasks.
Add a dataset
${C.addErr ? `
${esc(C.addErr)}
` : ""}
${C.sources.length ? "" : `
Or start from : textbook-style PDFs, DCLM and FineWeb-Edu.
`}
Sources (up to ${C.d ? C.d.max_sources : 4} here)
${C.sources.length ? C.sources.map((s, i) => srcRow(s, i, "source")).join("") : '
None yet.
'}
Targets
${C.targets.map((s, i) => srcRow(s, i, "target")).join("")}
Target datasets are examples of the work you care about, such as a benchmark's training split or your own documents.
${actions(false, "Continue", C.sources.length > 0)}`;
}
function stepClean() {
const b = C.d ? C.d.benchmarks : Object.keys(BENCH_NAMES), r = { ...(C.d ? C.d.clean_rules : {}), ...C.rules };
return `
Clean
Heuristic filters drop broken text, boilerplate, empty and repetitive documents. Math and code get their own looser rules. Then any document that shares a 13-word run with a test set you name is removed.
A model grades a sample of each source for teaching value, clarity, accuracy and density${hasT ? ", and for relevance to your targets" : ""}. A small scorer learns those grades and scores every document. Redundant copies are dropped first.
${hasT ? slider("c-focus", "Balance between general quality and your targets", C.focus, 0, 1, 0.05, (v) => (+v === 0 ? "general quality only" : `${pct(1 - v)} quality / ${pct(v)} targets`))
: `
No targets, so documents are ranked by general quality. Add targets in step 1 to steer the ranking.
`}
${hasT ? '
Keep some weight on general quality. Data tuned only to the targets overfits them.
' : ""}
${slider("c-keep", "Keep the best", C.keepTop, 0.1, 1, 0.05, (v) => `${pct(v)} of each source`)}
${check(C.near, 'data-c="near"', "Near copies", "MinHash, about 80% overlap")}
On your machines
The hosted run grades ${C.d ? 48 : "a few dozen"} documents, enough to see the rubric work but not to trust the learned scorer. In our test, 300 grades gave a rank agreement of about 0.5 with the model; more grades help.
${actions(true, "Continue")}`;
}
function tryHtml() {
if (C.tryErr) return `
${esc(C.tryErr)}
`;
if (!C.tryRes) return "";
return `
${C.tryRes.results.map((o) => `
${esc(o.name)}${o.ok ? "" : `${esc(o.why)}`}
${esc(o.text || "No rewrite. The model call failed.")}
Not in the source: ${esc([...o.fidelity.new_numbers, ...o.fidelity.new_names].join(", "))}
` : ""}
`).join("")}
`;
}
function stepCreate() {
const st = C.d ? C.d.styles : {}, picked = C.styles;
const first = C.sources.find((s) => s.preview);
return `
Create
Where a source is too small for its share of a mix, the best documents are rephrased into new ones: a dialogue, a tutorial, questions and answers, dense notes, a textbook section. Rewrites that add numbers or names the source does not contain are flagged and dropped.
${check(C.rephraseOn, 'data-c="reph"', "Rephrase to fill gaps")}
Build one or more training mixes from the prepared corpus. The default share is the static mix that won our 1B-token study: 50% textbook-style, 30% web, 20% educational web, with 10% each for math and code when present. It was measured at 70M parameters, so treat it as a starting point.
${C.mixes.map((m, mi) => { const w = weightsOf(m), tot = Object.values(w).reduce((a, b) => a + b, 0) || 1;
return `
${BUDGETS.map(([l, v]) => ``).join("")}
${C.mixes.length > 1 ? `` : ""}
${C.sources.map((s) => ``).join("")}
`; }).join("")}
for a different run, from the same prepared corpus
${slider("c-epochs", "Repeat a source at most", C.maxEpochs, 1, 8, 0.5, (v) => `${v} times`)}
${slider("c-synth", "Synthetic text at most", C.maxSynth, 0, 0.6, 0.05, (v) => `${pct(v)} of a mix`)}
A source short of its share is repeated first, then topped up by rephrasing its best documents. In our study a mix that leaned on synthetic-like text scored well on its own validation set but generalized worst.
${actions(true, "Continue")}`;
}
/* ------------------------------------------------------------ run + report */
const isFinal = (p) => p && ["done", "stopped", "failed"].includes(p.state);
function hist(h) {
const mx = Math.max(1, ...h);
return `
${esc(f.name)}, in ${r.docs} documents of the sample: ${esc(r.text)}
`).join("")).join("")}
` : ""}
Scores
The model graded ${cal.graded} documents and those keep their grades. The rest are ranked by a scorer learned from so few grades that its ranking is close to a guess${cal.graded >= 100 ? ` (rank agreement with the model on held-out documents: quality ${agree.quality ?? "n/a"}${cal.has_targets ? `, relevance ${agree.relevance ?? "n/a"}` : ""})` : ""}. The full run grades ${C.gradeDocs.toLocaleString()} documents and reports how well the scorer agrees with the model.
${src.map((f) => `
${esc(f.name)}${hist(f.score_hist || [])}low to high score
${crt.kept} of ${crt.made} rewrites kept.${(crt.flagged || []).length ? ` ${crt.flagged.length} flagged for adding facts the source does not contain.` : ""}
About ${p.synthetic_calls.toLocaleString()} model calls to rephrase ${fmt(p.synthetic_tokens)} tokens on your endpoint.
` : ""}
${p.warnings.map((w) => `
${esc(w)}
`).join("")}`).join("")}
${(rep.warnings || []).map((w) => `
${esc(w)}
`).join("")}`;
}
function stepRun() {
const p = C.pv, fin = isFinal(p);
const stageIdx = p ? ["setup", "clean", "calibrate", "create", "done"].indexOf(p.stage) : -1;
const out = p && p.report && p.report.output;
return `
${!p ? "Run it on a sample" : !fin ? "Curating a sample" : p.state === "failed" ? "The run stopped" : "Your sample is curated"}
Every step runs here on about ${C.d ? C.d.per_source : 200} documents from each source. You get the kept and created documents, a report of what each step did, and the recipe to run the same steps on all of your data.
On your own machines, the same recipe reads every document, grades ${C.gradeDocs.toLocaleString()} with your model, and writes the curated corpus and each mix as JSONL shards with a report. Point it at any OpenAI-compatible server (vLLM, SGLang, Ollama or a hosted API) by editing endpoint in the recipe. Interrupted runs resume where they stopped.
${esc(inst)}
export DI_API_KEY=... # key for the endpoint in the recipe
datainventor preview recipe.yaml # optional: the hosted check, locally
datainventor run recipe.yaml --out ./curated --procs 32 --workers 64
`;
}
/* ------------------------------------------------------------ render + events */
function render() {
if (window.DI_MODE !== "curate") return;
ledger(); sheet();
const fn = [stepSources, stepClean, stepCalibrate, stepCreate, stepCompose, stepRun][C.step];
const keep = document.activeElement && document.activeElement.id;
$("#panel").innerHTML = fn();
if (keep && keep.startsWith("c-")) { const el = document.getElementById(keep); if (el) { el.focus(); if (el.setSelectionRange && el.type === "text") { const l = el.value.length; try { el.setSelectionRange(l, l); } catch (e) { /* ok */ } } } }
}
function go(i) { C.step = i; C.maxStep = Math.max(C.maxStep, i); render(); $("#panel").focus({ preventScroll: true }); window.scrollTo({ top: 0 }); }
async function addDataset(id, role, kind) {
C.addBusy = true; C.addErr = ""; render();
try {
const info = await api(`/api/curate/inspect?dataset=${encodeURIComponent(id)}`);
const list = role === "target" ? C.targets : C.sources, max = role === "target" ? (C.d ? C.d.max_targets : 2) : (C.d ? C.d.max_sources : 4);
if (list.length >= max) throw new Error(`The hosted run takes up to ${max} ${role}s. The recipe takes any number.`);
let name = info.dataset.split("/")[1]; const taken = new Set([...C.sources, ...C.targets].map((s) => s.name));
for (let k = 2; taken.has(name); k++) name = `${info.dataset.split("/")[1]}-${k}`;
list.push({ ...info, name, kind: kind || info.kind });
C.add = ""; C.pv = null;
} catch (e) { C.addErr = e.message; }
C.addBusy = false; render();
}
async function download() {
try {
const r = await fetch("/api/curate/recipe", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ recipe: recipe() }) });
if (!r.ok) { const d = await r.json().catch(() => ({})); throw new Error(d.detail || "Could not build the recipe"); }
const url = URL.createObjectURL(await r.blob()), a = document.createElement("a");
a.href = url; a.download = "recipe.yaml"; document.body.appendChild(a); a.click(); a.remove(); setTimeout(() => URL.revokeObjectURL(url), 2000);
} catch (e) { C.runErr = e.message; render(); }
}
async function run() {
C.runErr = ""; C.pv = null;
try { const r = await api("/api/curate/previews", { recipe: recipe() }); C.pid = r.id; C.pv = { id: r.id, state: "queued", stage: "setup", message: "Starting" }; render(); poll(); }
catch (e) { C.runErr = e.message; render(); }
}
async function poll() {
const id = C.pid; if (!id) return;
try {
const p = await api(`/api/curate/previews/${id}`);
if (id !== C.pid) return;
C.pv = p; if (C.step === 5) render();
if (!isFinal(p)) setTimeout(poll, 1500);
} catch (e) { C.pv = { id, state: "failed", message: e.message }; if (C.step === 5) render(); }
}
async function tryIt() {
C.tryBusy = true; C.tryErr = ""; C.tryRes = null; render();
const kind = (C.sources.find((s) => s.name === C.tryFrom) || {}).kind || "web";
try { C.tryRes = await api("/api/curate/try", { text: C.tryText, kind, styles: C.styles.slice(0, 4) }); } catch (e) { C.tryErr = e.message; }
C.tryBusy = false; render();
}
document.addEventListener("click", async (ev) => {
if (window.DI_MODE !== "curate") return;
const t = ev.target.closest("button, a"); if (!t) return;
if (t.dataset.cgo !== undefined) return go(+t.dataset.cgo);
if (t.dataset.bench) { const b = t.dataset.bench; C.decon = C.decon.includes(b) ? C.decon.filter((x) => x !== b) : [...C.decon, b]; return render(); }
if (t.dataset.style) { const s = t.dataset.style; C.styles = C.styles.includes(s) ? C.styles.filter((x) => x !== s) : [...C.styles, s]; return render(); }
const a = t.dataset.c; if (!a) return;
if (a === "next") return go(C.step + 1);
if (a === "back") return go(Math.max(0, C.step - 1));
if (a === "add") return addDataset(C.add.trim(), C.addRole);
if (a === "preset") { for (const p of PRESET) await addDataset(p.dataset, "source", p.kind); if (!C.targetsDesc) C.targetsDesc = ""; return render(); }
if (a === "rm") { (t.dataset.role === "target" ? C.targets : C.sources).splice(+t.dataset.i, 1); C.pv = null; return render(); }
if (a === "near") { C.near = !C.near; return render(); }
if (a === "reph") { C.rephraseOn = !C.rephraseOn; return render(); }
if (a === "inv") { C.inventOn = !C.inventOn; return render(); }
if (a === "tryfill") { const s = C.sources.find((x) => x.preview); if (s) { C.tryText = s.preview; C.tryFrom = s.name; } return render(); }
if (a === "try") return tryIt();
if (a === "budget") { C.mixes[+t.dataset.m].budget = +t.dataset.v; return render(); }
if (a === "addmix") { C.mixes.push({ name: `mix-${C.mixes.length + 1}`, budget: 1e9, weights: {} }); return render(); }
if (a === "rmmix") { C.mixes.splice(+t.dataset.m, 1); return render(); }
if (a === "recipe") return download();
if (a === "run") return run();
if (a === "stop") { if (C.pid) await api(`/api/curate/previews/${C.pid}/stop`, {}); return; }
if (a === "again") { C.pv = null; C.pid = null; return go(0); }
});
document.addEventListener("input", (ev) => {
if (window.DI_MODE !== "curate") return;
const el = ev.target, id = el.id || ""; if (!id.startsWith("c-")) return;
const v = el.value;
if (id === "c-add") { C.add = v; const b = $('[data-c="add"]'); if (b) b.disabled = C.addBusy || !v.trim(); return; }
if (id === "c-tdesc") { C.targetsDesc = v; return sheet(); }
if (id === "c-try") { C.tryText = v; C.tryFrom = ""; const b = $('[data-c="try"]'); if (b) b.disabled = C.tryBusy || v.trim().split(/\s+/).length < 60; return; }
if (id === "c-invreq") { C.inventReq = v; return; }
const sv = (x) => { const o = document.getElementById(id + "-v"); if (o) o.textContent = x; };
if (id === "c-focus") { C.focus = +v; sv(+v === 0 ? "general quality only" : `${pct(1 - v)} quality / ${pct(v)} targets`); return sheet(); }
if (id === "c-keep") { C.keepTop = +v; sv(`${pct(v)} of each source`); return sheet(); }
if (id === "c-epochs") { C.maxEpochs = +v; sv(`${v} times`); return; }
if (id === "c-synth") { C.maxSynth = +v; sv(`${pct(v)} of a mix`); return; }
});
document.addEventListener("change", (ev) => {
if (window.DI_MODE !== "curate") return;
const el = ev.target, id = el.id || ""; if (!id.startsWith("c-")) return;
const v = el.value; let m;
if (id === "c-addrole") C.addRole = v;
else if ((m = id.match(/^c-kind-(\d+)$/))) { C.sources[+m[1]].kind = v; C.pv = null; }
else if ((m = id.match(/^c-col-(source|target)-(\d+)$/))) { (m[1] === "target" ? C.targets : C.sources)[+m[2]].text_field = v; C.pv = null; }
else if ((m = id.match(/^c-rule-(.+)$/))) { if (v !== "") C.rules[m[1]] = +v; }
else if (id === "c-grade") C.gradeDocs = Math.max(100, +v || 2000);
else if (id === "c-scorer") C.scorer = v;
else if (id === "c-invdocs") C.inventDocs = Math.max(0, +v || 0);
else if ((m = id.match(/^c-mname-(\d+)$/))) C.mixes[+m[1]].name = v.trim().replace(/[^\w.-]+/g, "-") || `mix-${+m[1] + 1}`;
else if ((m = id.match(/^c-mbud-(\d+)$/))) C.mixes[+m[1]].budget = Math.max(1e6, Math.round(+v || 0));
else if ((m = id.match(/^c-w-(\d+)-(.+)$/))) { const mix = C.mixes[+m[1]]; mix.weights = { ...weightsOf(mix), [m[2]]: Math.max(0, +v || 0) / 100 }; }
else return;
render();
});
document.addEventListener("keydown", (ev) => { if (window.DI_MODE === "curate" && ev.key === "Enter" && ev.target.id === "c-add" && C.add.trim()) addDataset(C.add.trim(), C.addRole); });
/* ------------------------------------------------------------ mode switch */
const TAGS = {
invent: "Describe the data you wish existed. Get training rows back. Bring any OpenAI-compatible model.",
curate: "Clean, score, rephrase and mix the pretraining data you already have. Preview it here on a sample, then run the recipe on your own machines.",
visualize: "Ask a question about a dataset and get a chart back, with the rows behind every bar one click away.",
};
function setMode(mode) {
window.DI_MODE = mode;
document.querySelectorAll("[data-mode]").forEach((b) => b.setAttribute("aria-pressed", String(b.dataset.mode === mode)));
$(".tag").textContent = TAGS[mode] || TAGS.invent;
if (mode === "curate") { if (!C.d) api("/api/curate/defaults").then((d) => { C.d = d; render(); }).catch(() => render()); render(); }
else if (mode === "visualize") { if (window.DI_renderVis) window.DI_renderVis(); }
else if (window.DI_renderInvent) window.DI_renderInvent();
const hash = mode === "invent" ? "" : "#" + mode;
if (location.hash !== hash) history.replaceState(null, "", hash || location.pathname + location.search);
}
window.DI_setMode = setMode;
document.addEventListener("click", (ev) => { const b = ev.target.closest("[data-mode]"); if (b) setMode(b.dataset.mode); });
window.DI_MODE = location.hash === "#curate" ? "curate" : location.hash === "#visualize" ? "visualize" : "invent";
if (window.DI_MODE !== "invent") document.addEventListener("DOMContentLoaded", () => setMode(window.DI_MODE));
})();