codelion commited on
Commit
4d9dbcd
·
verified ·
1 Parent(s): 33e08f1

Add sign-in with Hugging Face and publish-to-my-account with a provenance card

Browse files
Files changed (5) hide show
  1. README.md +4 -0
  2. app/main.py +121 -3
  3. app/publish.py +106 -0
  4. requirements.txt +1 -0
  5. static/app.js +47 -5
README.md CHANGED
@@ -6,6 +6,10 @@ colorTo: pink
6
  sdk: docker
7
  app_port: 7860
8
  pinned: false
 
 
 
 
9
  short_description: Describe a dataset in plain words, get training rows back.
10
  ---
11
 
 
6
  sdk: docker
7
  app_port: 7860
8
  pinned: false
9
+ hf_oauth: true
10
+ hf_oauth_scopes:
11
+ - contribute-repos
12
+ hf_oauth_expiration_minutes: 120
13
  short_description: Describe a dataset in plain words, get training rows back.
14
  ---
15
 
app/main.py CHANGED
@@ -1,20 +1,25 @@
1
  """DataInventor: an open-source recreation of the Invent a Dataset workflow. Bring any OpenAI-compatible endpoint."""
2
  from __future__ import annotations
3
 
 
4
  import csv
5
  import io
6
  import json
7
  import os
 
8
  import threading
 
 
9
  import time
10
  import uuid
11
  from pathlib import Path
12
 
13
- from fastapi import FastAPI, HTTPException
14
- from fastapi.responses import FileResponse, JSONResponse, Response
15
  from fastapi.staticfiles import StaticFiles
16
  from pydantic import BaseModel
17
 
 
18
  from engine import HF_BASE, HF_MODEL, Endpoint, EndpointError, Job, make_blueprint, run_job, stats
19
 
20
  ROOT = Path(__file__).resolve().parent.parent
@@ -104,7 +109,7 @@ def status(jid: str, since: int = 0):
104
  with j.lock:
105
  rows = list(j.rows)
106
  out = {"id": j.id, "state": j.state, "message": j.message, "target": j.target, "count": len(rows), "attempts": j.attempts,
107
- "rejected": dict(j.rejected), "elapsed": round((j.finished or time.time()) - j.started, 1), "new_rows": rows[since:since + 50]}
108
  if j.state in ("done", "stopped", "failed"):
109
  out["stats"] = stats(rows)
110
  out["blueprint"] = j.blueprint
@@ -138,6 +143,119 @@ def download(jid: str, format: str = "csv"):
138
  return Response(buf.getvalue().encode("utf-8-sig"), media_type="text/csv", headers={"Content-Disposition": f'attachment; filename="{name}.csv"'})
139
 
140
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
  @app.get("/")
142
  def index():
143
  return FileResponse(STATIC / "index.html")
 
1
  """DataInventor: an open-source recreation of the Invent a Dataset workflow. Bring any OpenAI-compatible endpoint."""
2
  from __future__ import annotations
3
 
4
+ import base64
5
  import csv
6
  import io
7
  import json
8
  import os
9
+ import secrets
10
  import threading
11
+ import urllib.parse
12
+ import urllib.request
13
  import time
14
  import uuid
15
  from pathlib import Path
16
 
17
+ from fastapi import FastAPI, HTTPException, Request
18
+ from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, Response
19
  from fastapi.staticfiles import StaticFiles
20
  from pydantic import BaseModel
21
 
22
+ from publish import LICENSES, publish as hub_publish, slug
23
  from engine import HF_BASE, HF_MODEL, Endpoint, EndpointError, Job, make_blueprint, run_job, stats
24
 
25
  ROOT = Path(__file__).resolve().parent.parent
 
109
  with j.lock:
110
  rows = list(j.rows)
111
  out = {"id": j.id, "state": j.state, "message": j.message, "target": j.target, "count": len(rows), "attempts": j.attempts,
112
+ "rejected": dict(j.rejected), "config": {k: j.config.get(k) for k in ("format", "request", "domains", "languages", "constraints", "rows")}, "slug": slug(j.config.get("request", "")), "elapsed": round((j.finished or time.time()) - j.started, 1), "new_rows": rows[since:since + 50]}
113
  if j.state in ("done", "stopped", "failed"):
114
  out["stats"] = stats(rows)
115
  out["blueprint"] = j.blueprint
 
143
  return Response(buf.getvalue().encode("utf-8-sig"), media_type="text/csv", headers={"Content-Disposition": f'attachment; filename="{name}.csv"'})
144
 
145
 
146
+
147
+ # ---------------------------------------------------------------- sign in with Hugging Face (publishing)
148
+ OAUTH_ID = os.environ.get("OAUTH_CLIENT_ID", "")
149
+ OAUTH_SECRET = os.environ.get("OAUTH_CLIENT_SECRET", "")
150
+ OAUTH_URL = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co").rstrip("/")
151
+ SPACE_HOST = os.environ.get("SPACE_HOST", "")
152
+ DEV_TOKEN = os.environ.get("DI_DEV_USER_TOKEN", "") # local testing only
153
+ SESSIONS: dict[str, dict] = {}
154
+ PUBLISHES: dict[str, list] = {}
155
+ SESSION_SECONDS = 2 * 3600
156
+
157
+
158
+ def current_session(request: Request) -> dict | None:
159
+ sid = request.cookies.get("di_sid")
160
+ s = SESSIONS.get(sid or "")
161
+ if s and s["exp"] > time.time():
162
+ return s
163
+ if DEV_TOKEN:
164
+ from huggingface_hub import HfApi
165
+ return {"user": HfApi(token=DEV_TOKEN).whoami()["name"], "token": DEV_TOKEN, "exp": time.time() + 600, "sid": "dev"}
166
+ return None
167
+
168
+
169
+ def redirect_uri() -> str:
170
+ return f"https://{SPACE_HOST}/oauth/callback"
171
+
172
+
173
+ @app.get("/api/me")
174
+ def me(request: Request):
175
+ s = current_session(request)
176
+ return {"user": s["user"] if s else None, "can_sign_in": bool(OAUTH_ID and SPACE_HOST) or bool(DEV_TOKEN), "host": SPACE_HOST}
177
+
178
+
179
+ @app.get("/oauth/login")
180
+ def login(job: str = ""):
181
+ if not (OAUTH_ID and SPACE_HOST):
182
+ raise HTTPException(404, "Sign-in is not available on this host.")
183
+ state = secrets.token_urlsafe(24)
184
+ q = urllib.parse.urlencode({"client_id": OAUTH_ID, "redirect_uri": redirect_uri(), "response_type": "code", "scope": "openid profile contribute-repos", "state": state})
185
+ resp = RedirectResponse(f"{OAUTH_URL}/oauth/authorize?{q}")
186
+ resp.set_cookie("di_state", state + "|" + (job if job.isalnum() else ""), max_age=600, httponly=True, secure=True, samesite="lax")
187
+ return resp
188
+
189
+
190
+ @app.get("/oauth/callback")
191
+ def callback(request: Request, code: str = "", state: str = "", error: str = ""):
192
+ saved = request.cookies.get("di_state", "")
193
+ want, _, job = saved.partition("|")
194
+ if error or not code or not want or not secrets.compare_digest(want, state):
195
+ return RedirectResponse("/?signin=failed")
196
+ basic = base64.b64encode(f"{OAUTH_ID}:{OAUTH_SECRET}".encode()).decode()
197
+ try:
198
+ data = urllib.parse.urlencode({"grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri(), "client_id": OAUTH_ID}).encode()
199
+ tok = json.load(urllib.request.urlopen(urllib.request.Request(f"{OAUTH_URL}/oauth/token", data, {"Authorization": "Basic " + basic, "Content-Type": "application/x-www-form-urlencoded"}), timeout=30))
200
+ info = json.load(urllib.request.urlopen(urllib.request.Request(f"{OAUTH_URL}/oauth/userinfo", headers={"Authorization": "Bearer " + tok["access_token"]}), timeout=30))
201
+ except Exception:
202
+ return RedirectResponse("/?signin=failed")
203
+ sid = secrets.token_urlsafe(24)
204
+ SESSIONS[sid] = {"user": info.get("preferred_username") or info.get("name"), "token": tok["access_token"], "exp": time.time() + SESSION_SECONDS, "sid": sid}
205
+ for k in [k for k, v in SESSIONS.items() if v["exp"] < time.time()]:
206
+ SESSIONS.pop(k, None)
207
+ resp = RedirectResponse("/?signin=ok" + (f"&job={job}" if job else ""))
208
+ resp.set_cookie("di_sid", sid, max_age=SESSION_SECONDS, httponly=True, secure=True, samesite="lax")
209
+ resp.delete_cookie("di_state")
210
+ return resp
211
+
212
+
213
+ @app.post("/api/signout")
214
+ def signout(request: Request):
215
+ SESSIONS.pop(request.cookies.get("di_sid", ""), None)
216
+ resp = JSONResponse({"ok": True})
217
+ resp.delete_cookie("di_sid")
218
+ return resp
219
+
220
+
221
+ class PublishIn(BaseModel):
222
+ name: str
223
+ private: bool = True
224
+ license: str = "apache-2.0"
225
+ confirm: bool = False
226
+
227
+
228
+ @app.post("/api/jobs/{jid}/publish")
229
+ def publish(jid: str, body: PublishIn, request: Request):
230
+ s = current_session(request)
231
+ if not s:
232
+ raise HTTPException(401, "Sign in with Hugging Face to publish.")
233
+ if not body.confirm:
234
+ raise HTTPException(400, "Confirm that you have the right to publish this data.")
235
+ if body.license not in LICENSES:
236
+ raise HTTPException(400, "Choose one of the offered licenses.")
237
+ j = get_job(jid)
238
+ if j.state not in ("done", "stopped") or not j.rows:
239
+ raise HTTPException(400, "Wait for the run to finish, or stop it, before publishing.")
240
+ now = time.time()
241
+ recent = [t for t in PUBLISHES.get(s["user"], []) if now - t < 3600]
242
+ if len(recent) >= 5:
243
+ raise HTTPException(429, "You have published five datasets in the last hour. Try again later.")
244
+ try:
245
+ url = hub_publish(s["token"], s["user"], body.name.strip(), body.private, body.license, j.config, list(j.rows), stats(list(j.rows)), ENDPOINTS[jid].model if jid in ENDPOINTS else "")
246
+ except ValueError as e:
247
+ raise HTTPException(400, str(e))
248
+ except Exception as e:
249
+ msg = str(e)
250
+ if "409" in msg or "already" in msg.lower():
251
+ raise HTTPException(409, "You already have a dataset with that name. Pick another name.")
252
+ if "403" in msg or "401" in msg:
253
+ raise HTTPException(403, "Hugging Face refused the request. Sign in again and allow DataInventor to create datasets.")
254
+ raise HTTPException(502, "Publishing failed. Try again in a moment.")
255
+ PUBLISHES[s["user"]] = recent + [now]
256
+ return {"url": url}
257
+
258
+
259
  @app.get("/")
260
  def index():
261
  return FileResponse(STATIC / "index.html")
app/publish.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Publish a finished run to the user's own Hugging Face account, with a provenance card."""
2
+ from __future__ import annotations
3
+
4
+ import io
5
+ import json
6
+ import re
7
+
8
+ from huggingface_hub import HfApi
9
+
10
+ SPACE_URL = "https://huggingface.co/spaces/LocalLLaMA/data-inventor"
11
+ LICENSES = {"apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "mit", "cc0-1.0"}
12
+
13
+
14
+ def slug(text: str, fallback: str = "dataset") -> str:
15
+ s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:48].strip("-")
16
+ return s or fallback
17
+
18
+
19
+ def valid_name(name: str) -> bool:
20
+ return bool(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{1,90}", name))
21
+
22
+
23
+ def rows_to_jsonl(rows: list, pref: bool) -> bytes:
24
+ cols = ["prompt", "chosen", "rejected"] if pref else ["prompt", "completion"]
25
+ meta = ["area", "subtopic", "topic", "persona", "shape", "difficulty", "language"]
26
+ return ("\n".join(json.dumps({k: r.get(k) for k in cols + meta}, ensure_ascii=False) for r in rows) + "\n").encode()
27
+
28
+
29
+ def size_category(n: int) -> str:
30
+ return "n<1K" if n < 1000 else "1K<n<10K"
31
+
32
+
33
+ def dataset_card(cfg: dict, stats: dict, model: str, license_id: str, user: str, n: int) -> str:
34
+ pref = cfg.get("format") == "preference"
35
+ langs = cfg.get("languages") or ["English"]
36
+ domains = ", ".join(cfg.get("domains") or []) or "chosen by the description"
37
+ cols = "`prompt`, `chosen`, `rejected`" if pref else "`prompt`, `completion`"
38
+ constraints = (cfg.get("constraints") or "").strip()
39
+ return f"""---
40
+ license: {license_id}
41
+ task_categories:
42
+ - text-generation
43
+ tags:
44
+ - synthetic
45
+ - datainventor
46
+ size_categories:
47
+ - {size_category(n)}
48
+ configs:
49
+ - config_name: default
50
+ data_files:
51
+ - split: train
52
+ path: data/train.jsonl
53
+ ---
54
+
55
+ # Synthetic dataset built with DataInventor
56
+
57
+ This dataset was generated with [DataInventor]({SPACE_URL}), an open-source recreation of the Invent a Dataset workflow. Every row was written by a language model. Nothing in it comes from real people, and nothing in it has been checked by a human.
58
+
59
+ ## What it contains
60
+
61
+ - **Request:** {cfg.get("request", "").strip()}
62
+ - **Format:** {"preference pairs (a prompt with a chosen and a rejected reply, ranked by the model)" if pref else "instruction pairs (a prompt and a reply)"}
63
+ - **Columns:** {cols}, plus `area`, `subtopic`, `topic`, `persona`, `shape`, `difficulty` and `language` describing how each row was planned
64
+ - **Domains:** {domains}
65
+ - **Languages:** {", ".join(langs)}
66
+ - **Rows:** {n:,}
67
+ {f"- **Rules every row had to follow:** {constraints}" + chr(10) if constraints else ""}
68
+ ## How it was made
69
+
70
+ 1. A blueprint was drafted from the request: topics, who asks, who answers, and the shapes a question can take.
71
+ 2. Unique combinations of topic, persona, shape, difficulty, length and language were sampled.
72
+ 3. For each combination the model wrote one message, then the reply{" (two replies, then a ranking)" if pref else ""}.
73
+ 4. Rows were rejected for cut-off or degenerate text, for asking for many items at once, and for near-duplicate questions.
74
+
75
+ Model: `{model}`, served through Hugging Face Inference Providers.
76
+
77
+ ## Quick numbers
78
+
79
+ - Distinct 3-grams in prompts: {stats.get("distinct3_prompts", "n/a")}
80
+ - Average words per prompt: {stats.get("avg_prompt_words", "n/a")}
81
+ - Average words per reply: {stats.get("avg_answer_words", "n/a")}
82
+
83
+ ## Limits
84
+
85
+ - Replies can contain errors and invented facts. Check them before you train on this data or rely on it.
86
+ - Synthetic data repeats the habits and blind spots of the model that wrote it.
87
+ - {"The ranking of chosen against rejected comes from the same model, so it reflects that model's taste." if pref else "Questions may be less varied than real user messages."}
88
+
89
+ Published by @{user} through DataInventor. The data is shared under the license above.
90
+ """
91
+
92
+
93
+ def publish(token: str, user: str, name: str, private: bool, license_id: str, cfg: dict, rows: list, stats: dict, model: str) -> str:
94
+ if not valid_name(name):
95
+ raise ValueError("Use letters, numbers, dots, dashes or underscores for the dataset name.")
96
+ if license_id not in LICENSES:
97
+ raise ValueError("Choose one of the offered licenses.")
98
+ pref = "chosen" in rows[0]
99
+ repo = f"{user}/{name}"
100
+ api = HfApi(token=token)
101
+ api.create_repo(repo, repo_type="dataset", private=private, exist_ok=False)
102
+ api.upload_file(path_or_fileobj=io.BytesIO(rows_to_jsonl(rows, pref)), path_in_repo="data/train.jsonl", repo_id=repo, repo_type="dataset",
103
+ commit_message="Add rows generated with DataInventor")
104
+ api.upload_file(path_or_fileobj=io.BytesIO(dataset_card(cfg, stats, model, license_id, user, len(rows)).encode()), path_in_repo="README.md",
105
+ repo_id=repo, repo_type="dataset", commit_message="Add dataset card")
106
+ return f"https://huggingface.co/datasets/{repo}"
requirements.txt CHANGED
@@ -1,3 +1,4 @@
1
  fastapi==0.115.*
2
  uvicorn[standard]==0.32.*
3
  pydantic>=2.7
 
 
1
  fastapi==0.115.*
2
  uvicorn[standard]==0.32.*
3
  pydantic>=2.7
4
+ huggingface_hub>=0.30
static/app.js CHANGED
@@ -11,7 +11,7 @@ const FORMATS = {
11
 
12
  const S = {
13
  step: 0, maxStep: 0, format: "instruction", domains: [], subs: {}, languages: ["English"], request: "", constraints: "", rows: 50,
14
- blueprint: null, bpBusy: false, bpError: "", model: "", job: null, jobRows: [], jobState: null, drawn: 0, printed: 0, maxRows: 200, domainData: [], runError: "",
15
  };
16
  const $ = (s, el = document) => el.querySelector(s);
17
  const esc = (t) => String(t ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
@@ -123,6 +123,42 @@ function stepReview() {
123
  <p class="note">About <b>${e.calls.toLocaleString()}</b> model calls and <b>${e.time}</b>. Rows that fail the quality checks are rewritten, so a run can take a little longer.</p>
124
  <div class="actions"><button class="btn" data-act="back">Back</button><button class="btn go" data-act="next">Start writing</button></div>`;
125
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  const isFinal = (j) => j && ["done", "stopped", "failed"].includes(j.state);
127
  function rowHtml(r) {
128
  const pref = S.format === "preference";
@@ -156,7 +192,7 @@ function stepRun() {
156
  <div class="count" id="run-count" aria-live="polite">${h.count}</div>
157
  <div class="bar" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${h.pct}"><i id="run-bar" style="width:${h.pct}%"></i></div>
158
  <p class="state" id="run-state">${h.state}</p><div id="run-note">${h.note}</div>
159
- <div class="actions" id="run-actions">${h.actions}</div><div id="run-stats">${h.stats}</div><div id="run-filtered">${h.filtered}</div>
160
  <div class="press" id="run-deck" aria-hidden="true">${deckHtml()}</div>
161
  <div class="table" id="run-rows">${S.jobRows.slice().reverse().map(rowHtml).join("")}</div></div>`;
162
  }
@@ -164,6 +200,7 @@ function stepRun() {
164
  function updateRun() {
165
  const h = headParts(), set = (id, html) => { const el = document.getElementById(id); if (el && el.innerHTML !== html) el.innerHTML = html; };
166
  set("run-h2", h.h2); set("run-count", h.count); set("run-state", h.state); set("run-note", h.note); set("run-actions", h.actions); set("run-stats", h.stats); set("run-filtered", h.filtered);
 
167
  const bar = document.getElementById("run-bar"); if (bar) { bar.style.width = h.pct + "%"; bar.parentElement.setAttribute("aria-valuenow", h.pct); }
168
  const deck = document.getElementById("run-deck"); if (deck) deck.innerHTML = deckHtml();
169
  const rows = document.getElementById("run-rows");
@@ -191,11 +228,15 @@ document.addEventListener("click", async (ev) => {
191
  if (act === "back") return go(Math.max(0, S.step - 1));
192
  if (act === "suggest") { S.request = S.request.trim() ? S.request : $("[data-suggest]").dataset.suggest; return render(); }
193
  if (act === "blueprint") return doBlueprint();
 
 
194
  if (act === "stop") { if (S.job) await api(`/api/jobs/${S.job}/stop`, {}); return; }
195
- if (act === "again") { S.job = null; S.jobState = null; S.jobRows = []; S.drawn = 0; S.printed = 0; S.runError = ""; return go(4); }
196
  });
 
197
  document.addEventListener("input", (ev) => {
198
  const id = ev.target.id;
 
199
  if (id === "request") { S.request = ev.target.value; S.blueprint = null; const b = $('[data-act="next"]'); if (b) b.disabled = !S.request.trim(); const bp = $('[data-act="blueprint"]'); if (bp) bp.disabled = S.bpBusy || !S.request.trim(); }
200
  else if (id === "constraints") { S.constraints = ev.target.value; S.blueprint = null; }
201
  });
@@ -206,7 +247,7 @@ async function doBlueprint() {
206
  S.bpBusy = false; render();
207
  }
208
  async function startRun() {
209
- S.runError = ""; S.jobRows = []; S.jobState = null; S.drawn = 0; S.printed = 0;
210
  try { const r = await api("/api/jobs", spec()); S.job = r.id; poll(); } catch (e) { S.runError = e.message; }
211
  render();
212
  }
@@ -228,5 +269,6 @@ go = function (i) { const wasReview = S.step === 5; _go(i); if (i === 6 && wasRe
228
  (async function init() {
229
  try { const d = await api("/api/defaults"); S.model = d.model; S.maxRows = d.max_rows; S.configured = d.configured; if (S.rows > S.maxRows) S.rows = S.maxRows; } catch (e) { /* offline */ }
230
  try { S.domainData = await api("/api/domains"); } catch (e) { S.domainData = []; }
231
- render();
 
232
  })();
 
11
 
12
  const S = {
13
  step: 0, maxStep: 0, format: "instruction", domains: [], subs: {}, languages: ["English"], request: "", constraints: "", rows: 50,
14
+ blueprint: null, bpBusy: false, bpError: "", model: "", job: null, me: null, pub: { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" }, jobRows: [], jobState: null, drawn: 0, printed: 0, pubShown: false, maxRows: 200, domainData: [], runError: "",
15
  };
16
  const $ = (s, el = document) => el.querySelector(s);
17
  const esc = (t) => String(t ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
 
123
  <p class="note">About <b>${e.calls.toLocaleString()}</b> model calls and <b>${e.time}</b>. Rows that fail the quality checks are rewritten, so a run can take a little longer.</p>
124
  <div class="actions"><button class="btn" data-act="back">Back</button><button class="btn go" data-act="next">Start writing</button></div>`;
125
  }
126
+ function publishHtml() {
127
+ const j = S.jobState; if (!isFinal(j) || !j.count) return "";
128
+ const p = S.pub, me = S.me || {};
129
+ if (p.url) return `<div class="group"><h3>Published</h3><p class="note ok">Your dataset is on Hugging Face${p.priv ? " as a private dataset" : ""}. <a href="${esc(p.url)}" target="_blank" rel="noopener">Open it</a>.</p></div>`;
130
+ const head = `<h3>Publish to your Hugging Face account</h3>`;
131
+ if (!me.user) {
132
+ if (!me.can_sign_in) return `<div class="group">${head}<p class="hint">Publishing works on the hosted app. Download the CSV or JSONL instead.</p></div>`;
133
+ const embedded = window.top !== window.self, href = embedded && me.host ? `https://${me.host}/oauth/login?job=${j.id}` : `/oauth/login?job=${j.id}`;
134
+ return `<div class="group">${head}<p class="hint" style="max-width:60ch">Sign in to save this dataset to your own account, with a card that says how it was made. DataInventor can only create datasets for you. It cannot see your other repos. It is private unless you choose public.</p>
135
+ <div class="actions" style="margin-top:12px"><a class="btn go" href="${esc(href)}" ${embedded ? 'target="_blank" rel="noopener"' : ""} style="text-decoration:none">Sign in with Hugging Face</a></div>${embedded ? '<p class="hint" style="margin-top:8px">Sign-in opens in its own tab because browsers block it inside an embedded page.</p>' : ""}</div>`;
136
+ }
137
+ if (!p.name) p.name = "datainventor-" + (j.slug || "dataset");
138
+ return `<div class="group">${head}<p class="hint">Signed in as <b>${esc(me.user)}</b>. <button class="linkish" data-act="signout">Sign out</button></p>
139
+ <div class="row2"><label class="f"><span>Dataset name</span><input class="in" id="pub-name" value="${esc(p.name)}" spellcheck="false"></label>
140
+ <label class="f"><span>Who can see it</span><select class="in" id="pub-private"><option value="1" ${p.priv ? "selected" : ""}>Only me (private)</option><option value="0" ${p.priv ? "" : "selected"}>Everyone (public)</option></select></label></div>
141
+ <label class="f" style="max-width:320px"><span>License</span><select class="in" id="pub-license">${["apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "mit", "cc0-1.0"].map((l) => `<option ${p.license === l ? "selected" : ""}>${l}</option>`).join("")}</select></label>
142
+ <label class="check"><input type="checkbox" id="pub-ok" ${p.ok ? "checked" : ""}><span>I have the right to publish this data. I understand it was written by a model and has not been checked by a person.</span></label>
143
+ ${p.err ? `<p class="note bad">${esc(p.err)}</p>` : ""}
144
+ <div class="actions"><button class="btn go" data-act="publish" ${p.busy ? "disabled" : ""}>${p.busy ? "Publishing" : "Publish dataset"}</button></div></div>`;
145
+ }
146
+ async function loadMe() { try { S.me = await api("/api/me"); } catch (e) { S.me = { user: null, can_sign_in: false }; } const el = document.getElementById("run-publish"); if (el) el.innerHTML = publishHtml(); }
147
+ async function doPublish() {
148
+ S.pub.busy = true; S.pub.err = ""; refreshPublish();
149
+ try { const r = await api(`/api/jobs/${S.job}/publish`, { name: S.pub.name.trim(), private: S.pub.priv, license: S.pub.license, confirm: S.pub.ok }); S.pub.url = r.url; } catch (e) { S.pub.err = e.message; }
150
+ S.pub.busy = false; refreshPublish();
151
+ }
152
+ function refreshPublish() { const el = document.getElementById("run-publish"); if (el) el.innerHTML = publishHtml(); }
153
+ async function restoreJob(id) {
154
+ try {
155
+ let since = 0, last = null; S.jobRows = [];
156
+ for (let k = 0; k < 8; k++) { last = await api(`/api/jobs/${id}?since=${since}`); S.jobRows.push(...last.new_rows); since = S.jobRows.length; if (since >= last.count) break; }
157
+ const c = last.config || {}; S.job = id; S.jobState = last; S.format = c.format || "instruction"; S.request = c.request || ""; S.domains = c.domains || []; S.languages = c.languages || ["English"]; S.constraints = c.constraints || ""; S.rows = c.rows || S.rows;
158
+ S.maxStep = 6; S.step = 6; S.printed = S.jobRows.length ? S.jobRows[S.jobRows.length - 1].id : 0; render();
159
+ if (!isFinal(last)) poll();
160
+ } catch (e) { /* the job is gone; start fresh */ }
161
+ }
162
  const isFinal = (j) => j && ["done", "stopped", "failed"].includes(j.state);
163
  function rowHtml(r) {
164
  const pref = S.format === "preference";
 
192
  <div class="count" id="run-count" aria-live="polite">${h.count}</div>
193
  <div class="bar" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${h.pct}"><i id="run-bar" style="width:${h.pct}%"></i></div>
194
  <p class="state" id="run-state">${h.state}</p><div id="run-note">${h.note}</div>
195
+ <div class="actions" id="run-actions">${h.actions}</div><div id="run-stats">${h.stats}</div><div id="run-filtered">${h.filtered}</div><div id="run-publish">${publishHtml()}</div>
196
  <div class="press" id="run-deck" aria-hidden="true">${deckHtml()}</div>
197
  <div class="table" id="run-rows">${S.jobRows.slice().reverse().map(rowHtml).join("")}</div></div>`;
198
  }
 
200
  function updateRun() {
201
  const h = headParts(), set = (id, html) => { const el = document.getElementById(id); if (el && el.innerHTML !== html) el.innerHTML = html; };
202
  set("run-h2", h.h2); set("run-count", h.count); set("run-state", h.state); set("run-note", h.note); set("run-actions", h.actions); set("run-stats", h.stats); set("run-filtered", h.filtered);
203
+ if (isFinal(S.jobState) && !S.pubShown) { S.pubShown = true; loadMe(); }
204
  const bar = document.getElementById("run-bar"); if (bar) { bar.style.width = h.pct + "%"; bar.parentElement.setAttribute("aria-valuenow", h.pct); }
205
  const deck = document.getElementById("run-deck"); if (deck) deck.innerHTML = deckHtml();
206
  const rows = document.getElementById("run-rows");
 
228
  if (act === "back") return go(Math.max(0, S.step - 1));
229
  if (act === "suggest") { S.request = S.request.trim() ? S.request : $("[data-suggest]").dataset.suggest; return render(); }
230
  if (act === "blueprint") return doBlueprint();
231
+ if (act === "publish") return doPublish();
232
+ if (act === "signout") { await api("/api/signout", {}); S.me = { ...S.me, user: null }; return refreshPublish(); }
233
  if (act === "stop") { if (S.job) await api(`/api/jobs/${S.job}/stop`, {}); return; }
234
+ if (act === "again") { S.job = null; S.jobState = null; S.jobRows = []; S.drawn = 0; S.printed = 0; S.pubShown = false; S.pub = { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" }; S.runError = ""; return go(4); }
235
  });
236
+ document.addEventListener("change", (ev) => { if (ev.target.id && ev.target.id.startsWith("pub-")) { const id = ev.target.id; if (id === "pub-private") S.pub.priv = ev.target.value === "1"; else if (id === "pub-license") S.pub.license = ev.target.value; else if (id === "pub-ok") S.pub.ok = ev.target.checked; } });
237
  document.addEventListener("input", (ev) => {
238
  const id = ev.target.id;
239
+ if (id === "pub-name") S.pub.name = ev.target.value; else if (id === "pub-private") S.pub.priv = ev.target.value === "1"; else if (id === "pub-license") S.pub.license = ev.target.value; else if (id === "pub-ok") S.pub.ok = ev.target.checked;
240
  if (id === "request") { S.request = ev.target.value; S.blueprint = null; const b = $('[data-act="next"]'); if (b) b.disabled = !S.request.trim(); const bp = $('[data-act="blueprint"]'); if (bp) bp.disabled = S.bpBusy || !S.request.trim(); }
241
  else if (id === "constraints") { S.constraints = ev.target.value; S.blueprint = null; }
242
  });
 
247
  S.bpBusy = false; render();
248
  }
249
  async function startRun() {
250
+ S.runError = ""; S.jobRows = []; S.jobState = null; S.drawn = 0; S.printed = 0; S.pubShown = false; S.pub = { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" };
251
  try { const r = await api("/api/jobs", spec()); S.job = r.id; poll(); } catch (e) { S.runError = e.message; }
252
  render();
253
  }
 
269
  (async function init() {
270
  try { const d = await api("/api/defaults"); S.model = d.model; S.maxRows = d.max_rows; S.configured = d.configured; if (S.rows > S.maxRows) S.rows = S.maxRows; } catch (e) { /* offline */ }
271
  try { S.domainData = await api("/api/domains"); } catch (e) { S.domainData = []; }
272
+ const q = new URLSearchParams(location.search), jid = q.get("job");
273
+ if (jid) { await loadMe(); await restoreJob(jid); history.replaceState(null, "", "/"); } else render();
274
  })();