"use strict"; const STEPS = ["Format", "Domains", "Subdomains", "Languages", "Summary", "Review", "Run"]; const LANGS = ["English", "Spanish", "French", "German", "Portuguese", "Italian", "Dutch", "Hindi", "Bengali", "Arabic", "Turkish", "Russian", "Polish", "Ukrainian", "Chinese", "Japanese", "Korean", "Vietnamese", "Indonesian", "Thai", "Swahili", "Serbian", "Greek", "Hebrew"]; const FORMATS = { instruction: { name: "Instruction pairs", blurb: "A prompt and the reply you want a model to learn. Use it for supervised fine-tuning.", eg: [["Prompt", "My hardcover arrived with a bent corner. Can I swap it?"], ["Completion", "Sorry about that. Yes, send a photo of the corner and we will ship a replacement today, no need to return the damaged copy."]] }, preference: { name: "Preference pairs", blurb: "A prompt with a chosen and a rejected reply, ranked by the model. Use it for preference tuning.", eg: [["Prompt", "Where is order 8824? It was due Monday."], ["Chosen", "It left our warehouse Saturday and is with the courier. Tracking shows delivery Thursday."], ["Rejected", "Orders usually take a while. Please be patient."]] }, }; const S = { step: 0, maxStep: 0, format: "instruction", domains: [], subs: {}, languages: ["English"], request: "", constraints: "", rows: 50, blueprint: null, bpBusy: false, bpError: "", model: "", job: null, me: null, pub: { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" }, jobRows: [], jobState: null, drawn: 0, printed: 0, pubShown: false, maxRows: 200, domainData: [], runError: "", }; const $ = (s, el = document) => el.querySelector(s); const esc = (t) => String(t ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); const sentence = (t) => (t ? t.charAt(0).toUpperCase() + t.slice(1) : t); async function api(path, body, method) { const r = await fetch(path, { method: method || (body ? "POST" : "GET"), headers: body ? { "Content-Type": "application/json" } : {}, body: body ? JSON.stringify(body) : undefined }); let data = null; try { data = await r.json(); } catch (e) { /* not json */ } if (!r.ok) throw new Error((data && (data.detail || data.message)) || `Request failed (${r.status})`); return data; } const spec = () => ({ format: S.format, request: S.request.trim(), domains: S.domains, subdomains: Object.values(S.subs).flat(), languages: S.languages, constraints: S.constraints, rows: S.rows, blueprint: S.blueprint }); /* ------------------------------------------------------------ chrome */ function renderLedger() { $("#steps").innerHTML = STEPS.map((n, i) => { const cls = i === S.step ? "on" : i < S.step ? "done" : ""; return `
  • `; }).join(""); } function estimate() { const per = S.format === "preference" ? 4.4 : 2.5, lat = 5500, workers = 4; const calls = Math.round(S.rows * per + 2); const secs = Math.max(10, Math.round((calls * lat) / 1000 / workers)); const t = secs >= 3600 ? `${(secs / 3600).toFixed(1)} hours` : secs >= 90 ? `${Math.round(secs / 60)} minutes` : `${secs} seconds`; return { calls, time: t }; } function renderSheet() { const e = estimate(), subs = Object.values(S.subs).flat(); const line = (k, v) => `
    ${k}
    ${v}
    `; $("#sheet").innerHTML = `

    Your dataset

    ${line("Format", esc(FORMATS[S.format].name))} ${line("Domains", S.domains.length ? esc(S.domains.join(", ")) : 'None chosen. The description will guide it.')} ${line("Subdomains", subs.length ? esc(subs.slice(0, 6).join(", ")) + (subs.length > 6 ? ` and ${subs.length - 6} more` : "") : 'All')} ${line("Languages", esc(S.languages.join(", ")))} ${line("Rows", S.rows.toLocaleString())} ${line("Model", esc(S.model || "Free model on Hugging Face"))}

    About ${e.calls.toLocaleString()} model calls and ${e.time} on this setup.

    `; } function go(i) { S.step = i; S.maxStep = Math.max(S.maxStep, i); render(); $("#panel").focus({ preventScroll: true }); window.scrollTo({ top: 0 }); } function actions(back, nextLabel, nextOk = true, extra = "") { return `
    ${back ? `` : ""}${nextLabel ? `` : ""}${extra}
    `; } /* ------------------------------------------------------------ steps */ function stepFormat() { return `

    What kind of rows do you need?

    Pick the shape of each example. You can describe the content in the steps that follow.

    ${Object.entries(FORMATS).map(([k, f]) => ` `).join("")}
    ${actions(false, "Continue")}`; } function stepDomains() { return `

    Which areas should the data cover?

    Choose any number, or skip this and describe the data yourself in the summary.

    ${actions(true, "Continue")}`; } function stepSubs() { if (!S.domains.length) return `

    Subdomains

    No domains chosen, so there is nothing to narrow. Continue to the languages, or go back and pick a domain.

    ${actions(true, "Continue")}`; return `

    Narrow it down

    Pick the subdomains you care about. Leave a domain empty to cover all of it.

    ${S.domains.map((name) => { const d = S.domainData.find((x) => x.name === name); const sel = S.subs[name] || []; return `

    ${esc(name)}

    `; }).join("")} ${actions(true, "Continue")}`; } function stepLangs() { return `

    Which languages?

    Rows are written directly in each language you choose, split evenly. English is the default.

    ${actions(true, "Continue", S.languages.length > 0)}`; } function bpView() { const bp = S.blueprint; if (!bp) return ""; return `
    Who answers${esc(bp.assistant_role)}
    Topics
    ${bp.topics.map((a) => `
    ${esc(a.area)} ${a.subtopics.map((s) => esc(s.subtopic)).join(", ")}
    `).join("")}
    Who asks
    Question shapes
    `; } function stepSummary() { const suggest = () => { const d = S.domains.length ? ` covering ${S.domains.join(", ").toLowerCase()}` : ""; return `A dataset of ${S.format === "preference" ? "prompts with a better and a worse reply" : "prompts and ideal replies"}${d}.`; }; return `

    Describe the dataset

    Say what you want in your own words. The more specific you are about who asks and who answers, the better the rows.

    How many rows?

    ${[25, 50, 100, 200].filter((n) => n <= S.maxRows).map((n) => ``).join("")}

    Hosted runs are capped at ${S.maxRows.toLocaleString()} rows. Run it yourself for more.

    Blueprint

    A plan for coverage: topics, who asks, who answers, and the shapes questions take. Preview it before you spend model calls on rows.

    ${S.bpError ? `

    ${esc(S.bpError)}

    ` : ""}${bpView()}
    ${actions(true, "Continue", !!S.request.trim())}`; } function stepReview() { const e = estimate(), subs = Object.values(S.subs).flat(); const row = (k, v) => `
    ${k}${v}
    `; return `

    Check it, then start writing

    Rows are written by a free model on Hugging Face Inference Providers. You can leave this page open or come back to it while the run goes.

    ${row("Format", esc(FORMATS[S.format].name))}${row("Description", esc(S.request))} ${row("Domains", S.domains.length ? esc(S.domains.join(", ")) : "Guided by the description")}${row("Subdomains", subs.length ? esc(subs.join(", ")) : "All")} ${row("Languages", esc(S.languages.join(", ")))}${S.constraints.trim() ? row("Rules", esc(S.constraints)) : ""}${row("Rows", S.rows.toLocaleString())} ${row("Blueprint", S.blueprint ? "Previewed. It will be used as shown." : "Drafted when the run starts.")}

    About ${e.calls.toLocaleString()} model calls and ${e.time}. Rows that fail the quality checks are rewritten, so a run can take a little longer.

    `; } function publishHtml() { const j = S.jobState; if (!isFinal(j) || !j.count) return ""; const p = S.pub, me = S.me || {}; if (p.url) return `

    Published

    Your dataset is on Hugging Face${p.priv ? " as a private dataset" : ""}. Open it.

    `; const head = `

    Publish to your Hugging Face account

    `; if (!me.user) { if (!me.can_sign_in) return `
    ${head}

    Publishing works on the hosted app. Download the CSV or JSONL instead.

    `; const embedded = window.top !== window.self, href = embedded && me.host ? `https://${me.host}/oauth/login?job=${j.id}` : `/oauth/login?job=${j.id}`; return `
    ${head}

    Sign in to save this dataset to your own account, with a card that says how it was made. DataInventor can only create datasets for you. It cannot see your other repos. It is private unless you choose public.

    Sign in with Hugging Face
    ${embedded ? '

    Sign-in opens in its own tab because browsers block it inside an embedded page.

    ' : ""}
    `; } if (!p.name) p.name = "datainventor-" + (j.slug || "dataset"); return `
    ${head}

    Signed in as ${esc(me.user)}.

    ${p.err ? `

    ${esc(p.err)}

    ` : ""}
    `; } async function loadMe() { try { S.me = await api("/api/me"); } catch (e) { S.me = { user: null, can_sign_in: false }; } const el = document.getElementById("run-publish"); if (el) el.innerHTML = publishHtml(); } async function doPublish() { S.pub.busy = true; S.pub.err = ""; refreshPublish(); try { const r = await api(`/api/jobs/${S.job}/publish`, { name: S.pub.name.trim(), private: S.pub.priv, license: S.pub.license, confirm: S.pub.ok }); S.pub.url = r.url; } catch (e) { S.pub.err = e.message; } S.pub.busy = false; refreshPublish(); } function refreshPublish() { const el = document.getElementById("run-publish"); if (el) el.innerHTML = publishHtml(); } async function restoreJob(id) { try { let since = 0, last = null; S.jobRows = []; for (let k = 0; k < 8; k++) { last = await api(`/api/jobs/${id}?since=${since}`); S.jobRows.push(...last.new_rows); since = S.jobRows.length; if (since >= last.count) break; } const c = last.config || {}; S.job = id; S.jobState = last; S.format = c.format || "instruction"; S.request = c.request || ""; S.domains = c.domains || []; S.languages = c.languages || ["English"]; S.constraints = c.constraints || ""; S.rows = c.rows || S.rows; S.maxStep = 6; S.step = 6; S.printed = S.jobRows.length ? S.jobRows[S.jobRows.length - 1].id : 0; render(); if (!isFinal(last)) poll(); } catch (e) { /* the job is gone; start fresh */ } } const isFinal = (j) => j && ["done", "stopped", "failed"].includes(j.state); function rowHtml(r) { const pref = S.format === "preference"; return `
    ${r.id}
    ${esc(r.prompt)}
    ${esc(r.area)}, ${esc(r.persona)}${r.language && r.language !== "English" ? ", " + esc(r.language) : ""}
    ${pref ? "Read both replies" : "Read the reply"}
    ${pref ? "Chosen\n" + esc(r.chosen) + "\n\nRejected\n" + esc(r.rejected) : esc(r.completion)}
    `; } function deckHtml() { const pref = S.format === "preference", latest = S.jobRows.slice(-3); const fresh = latest.length ? latest[latest.length - 1].id : 0; const out = latest.map((r, i) => `
    ${esc(r.persona)}, ${esc(r.topic)}
    ${esc(r.prompt)}
    ${esc(pref ? r.chosen : r.completion)}
    `).join(""); S.printed = Math.max(S.printed, fresh); return out; } function headParts() { const j = S.jobState, n = j ? j.count : 0, t = j ? j.target || S.rows : S.rows, pct = t ? Math.min(100, Math.round((n / t) * 100)) : 0, fin = isFinal(j); const st = j && j.stats && j.stats.rows ? j.stats : null; return { h2: fin ? (j.state === "failed" ? "The run stopped" : "Your dataset is ready") : "Writing your dataset", count: `${n.toLocaleString()}of ${t.toLocaleString()} rows`, pct, state: `${j ? esc(j.message) : "Starting"}${j && !fin ? ` Elapsed ${Math.round(j.elapsed)} s.` : ""}`, note: j && j.state === "failed" ? `

    ${esc(j.message)}

    ` : j && j.state === "done" && j.count < j.target ? `

    ${esc(j.message)}

    ` : "", actions: fin ? `${j.count ? `Download CSVDownload JSONL` : ""}${j.count ? `` : ""}` : ``, stats: st ? `
    ${st.rows.toLocaleString()}rows
    ${st.distinct3_prompts}distinct 3-grams in prompts
    ${st.avg_prompt_words}words per prompt
    ${st.avg_answer_words}words per reply
    ` : "", filtered: j && j.rejected && Object.keys(j.rejected).length ? `

    Filtered out: ${Object.entries(j.rejected).map(([k, v]) => `${v} ${esc(k)}`).join(", ")}.

    ` : "", }; } function stepRun() { if (S.runError && !S.jobState) return `

    The run did not start

    ${esc(S.runError)}

    ${actions(true, "Try again")}`; const h = headParts(); S.drawn = S.jobRows.length; return `

    ${h.h2}

    ${h.count}

    ${h.state}

    ${h.note}
    ${h.actions}
    ${h.stats}
    ${h.filtered}
    ${publishHtml()}
    ${S.jobRows.slice().reverse().map(rowHtml).join("")}
    `; } /* patch the run view in place so open replies stay open and only a new card prints */ function updateRun() { const h = headParts(), set = (id, html) => { const el = document.getElementById(id); if (el && el.innerHTML !== html) el.innerHTML = html; }; set("run-h2", h.h2); set("run-count", h.count); set("run-state", h.state); set("run-note", h.note); set("run-actions", h.actions); set("run-stats", h.stats); set("run-filtered", h.filtered); if (isFinal(S.jobState) && !S.pubShown) { S.pubShown = true; loadMe(); } const bar = document.getElementById("run-bar"); if (bar) { bar.style.width = h.pct + "%"; bar.parentElement.setAttribute("aria-valuenow", h.pct); } const deck = document.getElementById("run-deck"); if (deck) deck.innerHTML = deckHtml(); const rows = document.getElementById("run-rows"); if (rows && S.jobRows.length > S.drawn) { rows.insertAdjacentHTML("afterbegin", S.jobRows.slice(S.drawn).reverse().map(rowHtml).join("")); S.drawn = S.jobRows.length; } } /* ------------------------------------------------------------ render + events */ function render() { if (window.DI_MODE !== "invent") return; renderLedger(); renderSheet(); const fn = [stepFormat, stepDomains, stepSubs, stepLangs, stepSummary, stepReview, stepRun][S.step]; const keep = document.activeElement && document.activeElement.id; $("#panel").innerHTML = fn(); if (keep) { const el = document.getElementById(keep); if (el) { el.focus(); if (el.setSelectionRange && el.type !== "number") { const l = el.value.length; try { el.setSelectionRange(l, l); } catch (e) { /* ok */ } } } } } document.addEventListener("click", async (ev) => { const t = ev.target.closest("button, a"); if (!t) return; if (t.dataset.go !== undefined) return go(+t.dataset.go); if (t.dataset.format) { S.format = t.dataset.format; S.blueprint = null; return render(); } if (t.dataset.domain) { const d = t.dataset.domain; S.domains = S.domains.includes(d) ? S.domains.filter((x) => x !== d) : [...S.domains, d]; if (!S.domains.includes(d)) delete S.subs[d]; S.blueprint = null; return render(); } if (t.dataset.sub) { const [d, s] = t.dataset.sub.split("|"); const cur = S.subs[d] || []; S.subs[d] = cur.includes(s) ? cur.filter((x) => x !== s) : [...cur, s]; S.blueprint = null; return render(); } if (t.dataset.lang) { const l = t.dataset.lang; S.languages = S.languages.includes(l) ? S.languages.filter((x) => x !== l) : [...S.languages, l]; S.blueprint = null; return render(); } if (t.dataset.rows) { S.rows = +t.dataset.rows; return render(); } const act = t.dataset.act; if (!act) return; if (act === "next") return void go(S.step + 1); if (act === "back") return go(Math.max(0, S.step - 1)); if (act === "suggest") { S.request = S.request.trim() ? S.request : $("[data-suggest]").dataset.suggest; return render(); } if (act === "blueprint") return doBlueprint(); if (act === "publish") return doPublish(); if (act === "signout") { await api("/api/signout", {}); S.me = { ...S.me, user: null }; return refreshPublish(); } if (act === "visualize") { if (S.job && window.DI_visualizeJob) window.DI_visualizeJob(S.job); return; } if (act === "stop") { if (S.job) await api(`/api/jobs/${S.job}/stop`, {}); return; } if (act === "again") { S.job = null; S.jobState = null; S.jobRows = []; S.drawn = 0; S.printed = 0; S.pubShown = false; S.pub = { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" }; S.runError = ""; return go(4); } }); document.addEventListener("change", (ev) => { if (ev.target.id && ev.target.id.startsWith("pub-")) { const id = ev.target.id; if (id === "pub-private") S.pub.priv = ev.target.value === "1"; else if (id === "pub-license") S.pub.license = ev.target.value; else if (id === "pub-ok") S.pub.ok = ev.target.checked; } }); document.addEventListener("input", (ev) => { const id = ev.target.id; if (id === "pub-name") S.pub.name = ev.target.value; else if (id === "pub-private") S.pub.priv = ev.target.value === "1"; else if (id === "pub-license") S.pub.license = ev.target.value; else if (id === "pub-ok") S.pub.ok = ev.target.checked; if (id === "request") { S.request = ev.target.value; S.blueprint = null; const b = $('[data-act="next"]'); if (b) b.disabled = !S.request.trim(); const bp = $('[data-act="blueprint"]'); if (bp) bp.disabled = S.bpBusy || !S.request.trim(); } else if (id === "constraints") { S.constraints = ev.target.value; S.blueprint = null; } }); async function doBlueprint() { S.bpBusy = true; S.bpError = ""; render(); try { S.blueprint = await api("/api/blueprint", spec()); } catch (e) { S.bpError = e.message; } S.bpBusy = false; render(); } async function startRun() { S.runError = ""; S.jobRows = []; S.jobState = null; S.drawn = 0; S.printed = 0; S.pubShown = false; S.pub = { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" }; try { const r = await api("/api/jobs", spec()); S.job = r.id; poll(); } catch (e) { S.runError = e.message; } render(); } async function poll() { const id = S.job; if (!id) return; try { const j = await api(`/api/jobs/${id}?since=${S.jobRows.length}`); if (id !== S.job) return; const wasFinal = isFinal(S.jobState); S.jobRows.push(...j.new_rows); S.jobState = j; if (S.step === 6) { if (document.getElementById("runroot") && !wasFinal) updateRun(); else render(); } if (!["done", "stopped", "failed"].includes(j.state) || j.count > S.jobRows.length) return void setTimeout(poll, j.count > S.jobRows.length ? 200 : 1500); if (j.stats && j.stats.rows) S.maxStep = 6; } catch (e) { console.error(e.stack); S.jobState = { state: "failed", message: e.message, count: S.jobRows.length, target: S.rows, elapsed: 0, id }; if (S.step === 6) render(); } } window.DI_renderInvent = () => render(); window.DI_lastJob = () => (S.job && isFinal(S.jobState) && S.jobState.count ? { id: S.job, n: S.jobState.count, request: S.request || "" } : null); /* "Start writing" on the review step starts the run */ const _go = go; go = function (i) { const wasReview = S.step === 5; _go(i); if (i === 6 && wasReview && !S.jobState) startRun(); return true; }; (async function init() { try { const d = await api("/api/defaults"); S.model = d.model; S.maxRows = d.max_rows; S.configured = d.configured; if (S.rows > S.maxRows) S.rows = S.maxRows; } catch (e) { /* offline */ } try { S.domainData = await api("/api/domains"); } catch (e) { S.domainData = []; } const q = new URLSearchParams(location.search), jid = q.get("job"); if (jid) { await loadMe(); await restoreJob(jid); history.replaceState(null, "", "/"); } else render(); })();