Spaces:
Running
Running
Add sign-in with Hugging Face and publish-to-my-account with a provenance card
Browse files- README.md +4 -0
- app/main.py +121 -3
- app/publish.py +106 -0
- requirements.txt +1 -0
- static/app.js +47 -5
README.md
CHANGED
|
@@ -6,6 +6,10 @@ colorTo: pink
|
|
| 6 |
sdk: docker
|
| 7 |
app_port: 7860
|
| 8 |
pinned: false
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
short_description: Describe a dataset in plain words, get training rows back.
|
| 10 |
---
|
| 11 |
|
|
|
|
| 6 |
sdk: docker
|
| 7 |
app_port: 7860
|
| 8 |
pinned: false
|
| 9 |
+
hf_oauth: true
|
| 10 |
+
hf_oauth_scopes:
|
| 11 |
+
- contribute-repos
|
| 12 |
+
hf_oauth_expiration_minutes: 120
|
| 13 |
short_description: Describe a dataset in plain words, get training rows back.
|
| 14 |
---
|
| 15 |
|
app/main.py
CHANGED
|
@@ -1,20 +1,25 @@
|
|
| 1 |
"""DataInventor: an open-source recreation of the Invent a Dataset workflow. Bring any OpenAI-compatible endpoint."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
|
|
|
| 4 |
import csv
|
| 5 |
import io
|
| 6 |
import json
|
| 7 |
import os
|
|
|
|
| 8 |
import threading
|
|
|
|
|
|
|
| 9 |
import time
|
| 10 |
import uuid
|
| 11 |
from pathlib import Path
|
| 12 |
|
| 13 |
-
from fastapi import FastAPI, HTTPException
|
| 14 |
-
from fastapi.responses import FileResponse, JSONResponse, Response
|
| 15 |
from fastapi.staticfiles import StaticFiles
|
| 16 |
from pydantic import BaseModel
|
| 17 |
|
|
|
|
| 18 |
from engine import HF_BASE, HF_MODEL, Endpoint, EndpointError, Job, make_blueprint, run_job, stats
|
| 19 |
|
| 20 |
ROOT = Path(__file__).resolve().parent.parent
|
|
@@ -104,7 +109,7 @@ def status(jid: str, since: int = 0):
|
|
| 104 |
with j.lock:
|
| 105 |
rows = list(j.rows)
|
| 106 |
out = {"id": j.id, "state": j.state, "message": j.message, "target": j.target, "count": len(rows), "attempts": j.attempts,
|
| 107 |
-
"rejected": dict(j.rejected), "elapsed": round((j.finished or time.time()) - j.started, 1), "new_rows": rows[since:since + 50]}
|
| 108 |
if j.state in ("done", "stopped", "failed"):
|
| 109 |
out["stats"] = stats(rows)
|
| 110 |
out["blueprint"] = j.blueprint
|
|
@@ -138,6 +143,119 @@ def download(jid: str, format: str = "csv"):
|
|
| 138 |
return Response(buf.getvalue().encode("utf-8-sig"), media_type="text/csv", headers={"Content-Disposition": f'attachment; filename="{name}.csv"'})
|
| 139 |
|
| 140 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
@app.get("/")
|
| 142 |
def index():
|
| 143 |
return FileResponse(STATIC / "index.html")
|
|
|
|
| 1 |
"""DataInventor: an open-source recreation of the Invent a Dataset workflow. Bring any OpenAI-compatible endpoint."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
+
import base64
|
| 5 |
import csv
|
| 6 |
import io
|
| 7 |
import json
|
| 8 |
import os
|
| 9 |
+
import secrets
|
| 10 |
import threading
|
| 11 |
+
import urllib.parse
|
| 12 |
+
import urllib.request
|
| 13 |
import time
|
| 14 |
import uuid
|
| 15 |
from pathlib import Path
|
| 16 |
|
| 17 |
+
from fastapi import FastAPI, HTTPException, Request
|
| 18 |
+
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, Response
|
| 19 |
from fastapi.staticfiles import StaticFiles
|
| 20 |
from pydantic import BaseModel
|
| 21 |
|
| 22 |
+
from publish import LICENSES, publish as hub_publish, slug
|
| 23 |
from engine import HF_BASE, HF_MODEL, Endpoint, EndpointError, Job, make_blueprint, run_job, stats
|
| 24 |
|
| 25 |
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
| 109 |
with j.lock:
|
| 110 |
rows = list(j.rows)
|
| 111 |
out = {"id": j.id, "state": j.state, "message": j.message, "target": j.target, "count": len(rows), "attempts": j.attempts,
|
| 112 |
+
"rejected": dict(j.rejected), "config": {k: j.config.get(k) for k in ("format", "request", "domains", "languages", "constraints", "rows")}, "slug": slug(j.config.get("request", "")), "elapsed": round((j.finished or time.time()) - j.started, 1), "new_rows": rows[since:since + 50]}
|
| 113 |
if j.state in ("done", "stopped", "failed"):
|
| 114 |
out["stats"] = stats(rows)
|
| 115 |
out["blueprint"] = j.blueprint
|
|
|
|
| 143 |
return Response(buf.getvalue().encode("utf-8-sig"), media_type="text/csv", headers={"Content-Disposition": f'attachment; filename="{name}.csv"'})
|
| 144 |
|
| 145 |
|
| 146 |
+
|
| 147 |
+
# ---------------------------------------------------------------- sign in with Hugging Face (publishing)
|
| 148 |
+
OAUTH_ID = os.environ.get("OAUTH_CLIENT_ID", "")
|
| 149 |
+
OAUTH_SECRET = os.environ.get("OAUTH_CLIENT_SECRET", "")
|
| 150 |
+
OAUTH_URL = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co").rstrip("/")
|
| 151 |
+
SPACE_HOST = os.environ.get("SPACE_HOST", "")
|
| 152 |
+
DEV_TOKEN = os.environ.get("DI_DEV_USER_TOKEN", "") # local testing only
|
| 153 |
+
SESSIONS: dict[str, dict] = {}
|
| 154 |
+
PUBLISHES: dict[str, list] = {}
|
| 155 |
+
SESSION_SECONDS = 2 * 3600
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def current_session(request: Request) -> dict | None:
|
| 159 |
+
sid = request.cookies.get("di_sid")
|
| 160 |
+
s = SESSIONS.get(sid or "")
|
| 161 |
+
if s and s["exp"] > time.time():
|
| 162 |
+
return s
|
| 163 |
+
if DEV_TOKEN:
|
| 164 |
+
from huggingface_hub import HfApi
|
| 165 |
+
return {"user": HfApi(token=DEV_TOKEN).whoami()["name"], "token": DEV_TOKEN, "exp": time.time() + 600, "sid": "dev"}
|
| 166 |
+
return None
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def redirect_uri() -> str:
|
| 170 |
+
return f"https://{SPACE_HOST}/oauth/callback"
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
@app.get("/api/me")
|
| 174 |
+
def me(request: Request):
|
| 175 |
+
s = current_session(request)
|
| 176 |
+
return {"user": s["user"] if s else None, "can_sign_in": bool(OAUTH_ID and SPACE_HOST) or bool(DEV_TOKEN), "host": SPACE_HOST}
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
@app.get("/oauth/login")
|
| 180 |
+
def login(job: str = ""):
|
| 181 |
+
if not (OAUTH_ID and SPACE_HOST):
|
| 182 |
+
raise HTTPException(404, "Sign-in is not available on this host.")
|
| 183 |
+
state = secrets.token_urlsafe(24)
|
| 184 |
+
q = urllib.parse.urlencode({"client_id": OAUTH_ID, "redirect_uri": redirect_uri(), "response_type": "code", "scope": "openid profile contribute-repos", "state": state})
|
| 185 |
+
resp = RedirectResponse(f"{OAUTH_URL}/oauth/authorize?{q}")
|
| 186 |
+
resp.set_cookie("di_state", state + "|" + (job if job.isalnum() else ""), max_age=600, httponly=True, secure=True, samesite="lax")
|
| 187 |
+
return resp
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
@app.get("/oauth/callback")
|
| 191 |
+
def callback(request: Request, code: str = "", state: str = "", error: str = ""):
|
| 192 |
+
saved = request.cookies.get("di_state", "")
|
| 193 |
+
want, _, job = saved.partition("|")
|
| 194 |
+
if error or not code or not want or not secrets.compare_digest(want, state):
|
| 195 |
+
return RedirectResponse("/?signin=failed")
|
| 196 |
+
basic = base64.b64encode(f"{OAUTH_ID}:{OAUTH_SECRET}".encode()).decode()
|
| 197 |
+
try:
|
| 198 |
+
data = urllib.parse.urlencode({"grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri(), "client_id": OAUTH_ID}).encode()
|
| 199 |
+
tok = json.load(urllib.request.urlopen(urllib.request.Request(f"{OAUTH_URL}/oauth/token", data, {"Authorization": "Basic " + basic, "Content-Type": "application/x-www-form-urlencoded"}), timeout=30))
|
| 200 |
+
info = json.load(urllib.request.urlopen(urllib.request.Request(f"{OAUTH_URL}/oauth/userinfo", headers={"Authorization": "Bearer " + tok["access_token"]}), timeout=30))
|
| 201 |
+
except Exception:
|
| 202 |
+
return RedirectResponse("/?signin=failed")
|
| 203 |
+
sid = secrets.token_urlsafe(24)
|
| 204 |
+
SESSIONS[sid] = {"user": info.get("preferred_username") or info.get("name"), "token": tok["access_token"], "exp": time.time() + SESSION_SECONDS, "sid": sid}
|
| 205 |
+
for k in [k for k, v in SESSIONS.items() if v["exp"] < time.time()]:
|
| 206 |
+
SESSIONS.pop(k, None)
|
| 207 |
+
resp = RedirectResponse("/?signin=ok" + (f"&job={job}" if job else ""))
|
| 208 |
+
resp.set_cookie("di_sid", sid, max_age=SESSION_SECONDS, httponly=True, secure=True, samesite="lax")
|
| 209 |
+
resp.delete_cookie("di_state")
|
| 210 |
+
return resp
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
@app.post("/api/signout")
|
| 214 |
+
def signout(request: Request):
|
| 215 |
+
SESSIONS.pop(request.cookies.get("di_sid", ""), None)
|
| 216 |
+
resp = JSONResponse({"ok": True})
|
| 217 |
+
resp.delete_cookie("di_sid")
|
| 218 |
+
return resp
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
class PublishIn(BaseModel):
|
| 222 |
+
name: str
|
| 223 |
+
private: bool = True
|
| 224 |
+
license: str = "apache-2.0"
|
| 225 |
+
confirm: bool = False
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
@app.post("/api/jobs/{jid}/publish")
|
| 229 |
+
def publish(jid: str, body: PublishIn, request: Request):
|
| 230 |
+
s = current_session(request)
|
| 231 |
+
if not s:
|
| 232 |
+
raise HTTPException(401, "Sign in with Hugging Face to publish.")
|
| 233 |
+
if not body.confirm:
|
| 234 |
+
raise HTTPException(400, "Confirm that you have the right to publish this data.")
|
| 235 |
+
if body.license not in LICENSES:
|
| 236 |
+
raise HTTPException(400, "Choose one of the offered licenses.")
|
| 237 |
+
j = get_job(jid)
|
| 238 |
+
if j.state not in ("done", "stopped") or not j.rows:
|
| 239 |
+
raise HTTPException(400, "Wait for the run to finish, or stop it, before publishing.")
|
| 240 |
+
now = time.time()
|
| 241 |
+
recent = [t for t in PUBLISHES.get(s["user"], []) if now - t < 3600]
|
| 242 |
+
if len(recent) >= 5:
|
| 243 |
+
raise HTTPException(429, "You have published five datasets in the last hour. Try again later.")
|
| 244 |
+
try:
|
| 245 |
+
url = hub_publish(s["token"], s["user"], body.name.strip(), body.private, body.license, j.config, list(j.rows), stats(list(j.rows)), ENDPOINTS[jid].model if jid in ENDPOINTS else "")
|
| 246 |
+
except ValueError as e:
|
| 247 |
+
raise HTTPException(400, str(e))
|
| 248 |
+
except Exception as e:
|
| 249 |
+
msg = str(e)
|
| 250 |
+
if "409" in msg or "already" in msg.lower():
|
| 251 |
+
raise HTTPException(409, "You already have a dataset with that name. Pick another name.")
|
| 252 |
+
if "403" in msg or "401" in msg:
|
| 253 |
+
raise HTTPException(403, "Hugging Face refused the request. Sign in again and allow DataInventor to create datasets.")
|
| 254 |
+
raise HTTPException(502, "Publishing failed. Try again in a moment.")
|
| 255 |
+
PUBLISHES[s["user"]] = recent + [now]
|
| 256 |
+
return {"url": url}
|
| 257 |
+
|
| 258 |
+
|
| 259 |
@app.get("/")
|
| 260 |
def index():
|
| 261 |
return FileResponse(STATIC / "index.html")
|
app/publish.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Publish a finished run to the user's own Hugging Face account, with a provenance card."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import io
|
| 5 |
+
import json
|
| 6 |
+
import re
|
| 7 |
+
|
| 8 |
+
from huggingface_hub import HfApi
|
| 9 |
+
|
| 10 |
+
SPACE_URL = "https://huggingface.co/spaces/LocalLLaMA/data-inventor"
|
| 11 |
+
LICENSES = {"apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "mit", "cc0-1.0"}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def slug(text: str, fallback: str = "dataset") -> str:
|
| 15 |
+
s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:48].strip("-")
|
| 16 |
+
return s or fallback
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def valid_name(name: str) -> bool:
|
| 20 |
+
return bool(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{1,90}", name))
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def rows_to_jsonl(rows: list, pref: bool) -> bytes:
|
| 24 |
+
cols = ["prompt", "chosen", "rejected"] if pref else ["prompt", "completion"]
|
| 25 |
+
meta = ["area", "subtopic", "topic", "persona", "shape", "difficulty", "language"]
|
| 26 |
+
return ("\n".join(json.dumps({k: r.get(k) for k in cols + meta}, ensure_ascii=False) for r in rows) + "\n").encode()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def size_category(n: int) -> str:
|
| 30 |
+
return "n<1K" if n < 1000 else "1K<n<10K"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def dataset_card(cfg: dict, stats: dict, model: str, license_id: str, user: str, n: int) -> str:
|
| 34 |
+
pref = cfg.get("format") == "preference"
|
| 35 |
+
langs = cfg.get("languages") or ["English"]
|
| 36 |
+
domains = ", ".join(cfg.get("domains") or []) or "chosen by the description"
|
| 37 |
+
cols = "`prompt`, `chosen`, `rejected`" if pref else "`prompt`, `completion`"
|
| 38 |
+
constraints = (cfg.get("constraints") or "").strip()
|
| 39 |
+
return f"""---
|
| 40 |
+
license: {license_id}
|
| 41 |
+
task_categories:
|
| 42 |
+
- text-generation
|
| 43 |
+
tags:
|
| 44 |
+
- synthetic
|
| 45 |
+
- datainventor
|
| 46 |
+
size_categories:
|
| 47 |
+
- {size_category(n)}
|
| 48 |
+
configs:
|
| 49 |
+
- config_name: default
|
| 50 |
+
data_files:
|
| 51 |
+
- split: train
|
| 52 |
+
path: data/train.jsonl
|
| 53 |
+
---
|
| 54 |
+
|
| 55 |
+
# Synthetic dataset built with DataInventor
|
| 56 |
+
|
| 57 |
+
This dataset was generated with [DataInventor]({SPACE_URL}), an open-source recreation of the Invent a Dataset workflow. Every row was written by a language model. Nothing in it comes from real people, and nothing in it has been checked by a human.
|
| 58 |
+
|
| 59 |
+
## What it contains
|
| 60 |
+
|
| 61 |
+
- **Request:** {cfg.get("request", "").strip()}
|
| 62 |
+
- **Format:** {"preference pairs (a prompt with a chosen and a rejected reply, ranked by the model)" if pref else "instruction pairs (a prompt and a reply)"}
|
| 63 |
+
- **Columns:** {cols}, plus `area`, `subtopic`, `topic`, `persona`, `shape`, `difficulty` and `language` describing how each row was planned
|
| 64 |
+
- **Domains:** {domains}
|
| 65 |
+
- **Languages:** {", ".join(langs)}
|
| 66 |
+
- **Rows:** {n:,}
|
| 67 |
+
{f"- **Rules every row had to follow:** {constraints}" + chr(10) if constraints else ""}
|
| 68 |
+
## How it was made
|
| 69 |
+
|
| 70 |
+
1. A blueprint was drafted from the request: topics, who asks, who answers, and the shapes a question can take.
|
| 71 |
+
2. Unique combinations of topic, persona, shape, difficulty, length and language were sampled.
|
| 72 |
+
3. For each combination the model wrote one message, then the reply{" (two replies, then a ranking)" if pref else ""}.
|
| 73 |
+
4. Rows were rejected for cut-off or degenerate text, for asking for many items at once, and for near-duplicate questions.
|
| 74 |
+
|
| 75 |
+
Model: `{model}`, served through Hugging Face Inference Providers.
|
| 76 |
+
|
| 77 |
+
## Quick numbers
|
| 78 |
+
|
| 79 |
+
- Distinct 3-grams in prompts: {stats.get("distinct3_prompts", "n/a")}
|
| 80 |
+
- Average words per prompt: {stats.get("avg_prompt_words", "n/a")}
|
| 81 |
+
- Average words per reply: {stats.get("avg_answer_words", "n/a")}
|
| 82 |
+
|
| 83 |
+
## Limits
|
| 84 |
+
|
| 85 |
+
- Replies can contain errors and invented facts. Check them before you train on this data or rely on it.
|
| 86 |
+
- Synthetic data repeats the habits and blind spots of the model that wrote it.
|
| 87 |
+
- {"The ranking of chosen against rejected comes from the same model, so it reflects that model's taste." if pref else "Questions may be less varied than real user messages."}
|
| 88 |
+
|
| 89 |
+
Published by @{user} through DataInventor. The data is shared under the license above.
|
| 90 |
+
"""
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def publish(token: str, user: str, name: str, private: bool, license_id: str, cfg: dict, rows: list, stats: dict, model: str) -> str:
|
| 94 |
+
if not valid_name(name):
|
| 95 |
+
raise ValueError("Use letters, numbers, dots, dashes or underscores for the dataset name.")
|
| 96 |
+
if license_id not in LICENSES:
|
| 97 |
+
raise ValueError("Choose one of the offered licenses.")
|
| 98 |
+
pref = "chosen" in rows[0]
|
| 99 |
+
repo = f"{user}/{name}"
|
| 100 |
+
api = HfApi(token=token)
|
| 101 |
+
api.create_repo(repo, repo_type="dataset", private=private, exist_ok=False)
|
| 102 |
+
api.upload_file(path_or_fileobj=io.BytesIO(rows_to_jsonl(rows, pref)), path_in_repo="data/train.jsonl", repo_id=repo, repo_type="dataset",
|
| 103 |
+
commit_message="Add rows generated with DataInventor")
|
| 104 |
+
api.upload_file(path_or_fileobj=io.BytesIO(dataset_card(cfg, stats, model, license_id, user, len(rows)).encode()), path_in_repo="README.md",
|
| 105 |
+
repo_id=repo, repo_type="dataset", commit_message="Add dataset card")
|
| 106 |
+
return f"https://huggingface.co/datasets/{repo}"
|
requirements.txt
CHANGED
|
@@ -1,3 +1,4 @@
|
|
| 1 |
fastapi==0.115.*
|
| 2 |
uvicorn[standard]==0.32.*
|
| 3 |
pydantic>=2.7
|
|
|
|
|
|
| 1 |
fastapi==0.115.*
|
| 2 |
uvicorn[standard]==0.32.*
|
| 3 |
pydantic>=2.7
|
| 4 |
+
huggingface_hub>=0.30
|
static/app.js
CHANGED
|
@@ -11,7 +11,7 @@ const FORMATS = {
|
|
| 11 |
|
| 12 |
const S = {
|
| 13 |
step: 0, maxStep: 0, format: "instruction", domains: [], subs: {}, languages: ["English"], request: "", constraints: "", rows: 50,
|
| 14 |
-
blueprint: null, bpBusy: false, bpError: "", model: "", job: null, jobRows: [], jobState: null, drawn: 0, printed: 0, maxRows: 200, domainData: [], runError: "",
|
| 15 |
};
|
| 16 |
const $ = (s, el = document) => el.querySelector(s);
|
| 17 |
const esc = (t) => String(t ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
|
@@ -123,6 +123,42 @@ function stepReview() {
|
|
| 123 |
<p class="note">About <b>${e.calls.toLocaleString()}</b> model calls and <b>${e.time}</b>. Rows that fail the quality checks are rewritten, so a run can take a little longer.</p>
|
| 124 |
<div class="actions"><button class="btn" data-act="back">Back</button><button class="btn go" data-act="next">Start writing</button></div>`;
|
| 125 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
const isFinal = (j) => j && ["done", "stopped", "failed"].includes(j.state);
|
| 127 |
function rowHtml(r) {
|
| 128 |
const pref = S.format === "preference";
|
|
@@ -156,7 +192,7 @@ function stepRun() {
|
|
| 156 |
<div class="count" id="run-count" aria-live="polite">${h.count}</div>
|
| 157 |
<div class="bar" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${h.pct}"><i id="run-bar" style="width:${h.pct}%"></i></div>
|
| 158 |
<p class="state" id="run-state">${h.state}</p><div id="run-note">${h.note}</div>
|
| 159 |
-
<div class="actions" id="run-actions">${h.actions}</div><div id="run-stats">${h.stats}</div><div id="run-filtered">${h.filtered}</div>
|
| 160 |
<div class="press" id="run-deck" aria-hidden="true">${deckHtml()}</div>
|
| 161 |
<div class="table" id="run-rows">${S.jobRows.slice().reverse().map(rowHtml).join("")}</div></div>`;
|
| 162 |
}
|
|
@@ -164,6 +200,7 @@ function stepRun() {
|
|
| 164 |
function updateRun() {
|
| 165 |
const h = headParts(), set = (id, html) => { const el = document.getElementById(id); if (el && el.innerHTML !== html) el.innerHTML = html; };
|
| 166 |
set("run-h2", h.h2); set("run-count", h.count); set("run-state", h.state); set("run-note", h.note); set("run-actions", h.actions); set("run-stats", h.stats); set("run-filtered", h.filtered);
|
|
|
|
| 167 |
const bar = document.getElementById("run-bar"); if (bar) { bar.style.width = h.pct + "%"; bar.parentElement.setAttribute("aria-valuenow", h.pct); }
|
| 168 |
const deck = document.getElementById("run-deck"); if (deck) deck.innerHTML = deckHtml();
|
| 169 |
const rows = document.getElementById("run-rows");
|
|
@@ -191,11 +228,15 @@ document.addEventListener("click", async (ev) => {
|
|
| 191 |
if (act === "back") return go(Math.max(0, S.step - 1));
|
| 192 |
if (act === "suggest") { S.request = S.request.trim() ? S.request : $("[data-suggest]").dataset.suggest; return render(); }
|
| 193 |
if (act === "blueprint") return doBlueprint();
|
|
|
|
|
|
|
| 194 |
if (act === "stop") { if (S.job) await api(`/api/jobs/${S.job}/stop`, {}); return; }
|
| 195 |
-
if (act === "again") { S.job = null; S.jobState = null; S.jobRows = []; S.drawn = 0; S.printed = 0; S.runError = ""; return go(4); }
|
| 196 |
});
|
|
|
|
| 197 |
document.addEventListener("input", (ev) => {
|
| 198 |
const id = ev.target.id;
|
|
|
|
| 199 |
if (id === "request") { S.request = ev.target.value; S.blueprint = null; const b = $('[data-act="next"]'); if (b) b.disabled = !S.request.trim(); const bp = $('[data-act="blueprint"]'); if (bp) bp.disabled = S.bpBusy || !S.request.trim(); }
|
| 200 |
else if (id === "constraints") { S.constraints = ev.target.value; S.blueprint = null; }
|
| 201 |
});
|
|
@@ -206,7 +247,7 @@ async function doBlueprint() {
|
|
| 206 |
S.bpBusy = false; render();
|
| 207 |
}
|
| 208 |
async function startRun() {
|
| 209 |
-
S.runError = ""; S.jobRows = []; S.jobState = null; S.drawn = 0; S.printed = 0;
|
| 210 |
try { const r = await api("/api/jobs", spec()); S.job = r.id; poll(); } catch (e) { S.runError = e.message; }
|
| 211 |
render();
|
| 212 |
}
|
|
@@ -228,5 +269,6 @@ go = function (i) { const wasReview = S.step === 5; _go(i); if (i === 6 && wasRe
|
|
| 228 |
(async function init() {
|
| 229 |
try { const d = await api("/api/defaults"); S.model = d.model; S.maxRows = d.max_rows; S.configured = d.configured; if (S.rows > S.maxRows) S.rows = S.maxRows; } catch (e) { /* offline */ }
|
| 230 |
try { S.domainData = await api("/api/domains"); } catch (e) { S.domainData = []; }
|
| 231 |
-
|
|
|
|
| 232 |
})();
|
|
|
|
| 11 |
|
| 12 |
const S = {
|
| 13 |
step: 0, maxStep: 0, format: "instruction", domains: [], subs: {}, languages: ["English"], request: "", constraints: "", rows: 50,
|
| 14 |
+
blueprint: null, bpBusy: false, bpError: "", model: "", job: null, me: null, pub: { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" }, jobRows: [], jobState: null, drawn: 0, printed: 0, pubShown: false, maxRows: 200, domainData: [], runError: "",
|
| 15 |
};
|
| 16 |
const $ = (s, el = document) => el.querySelector(s);
|
| 17 |
const esc = (t) => String(t ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
|
|
|
| 123 |
<p class="note">About <b>${e.calls.toLocaleString()}</b> model calls and <b>${e.time}</b>. Rows that fail the quality checks are rewritten, so a run can take a little longer.</p>
|
| 124 |
<div class="actions"><button class="btn" data-act="back">Back</button><button class="btn go" data-act="next">Start writing</button></div>`;
|
| 125 |
}
|
| 126 |
+
function publishHtml() {
|
| 127 |
+
const j = S.jobState; if (!isFinal(j) || !j.count) return "";
|
| 128 |
+
const p = S.pub, me = S.me || {};
|
| 129 |
+
if (p.url) return `<div class="group"><h3>Published</h3><p class="note ok">Your dataset is on Hugging Face${p.priv ? " as a private dataset" : ""}. <a href="${esc(p.url)}" target="_blank" rel="noopener">Open it</a>.</p></div>`;
|
| 130 |
+
const head = `<h3>Publish to your Hugging Face account</h3>`;
|
| 131 |
+
if (!me.user) {
|
| 132 |
+
if (!me.can_sign_in) return `<div class="group">${head}<p class="hint">Publishing works on the hosted app. Download the CSV or JSONL instead.</p></div>`;
|
| 133 |
+
const embedded = window.top !== window.self, href = embedded && me.host ? `https://${me.host}/oauth/login?job=${j.id}` : `/oauth/login?job=${j.id}`;
|
| 134 |
+
return `<div class="group">${head}<p class="hint" style="max-width:60ch">Sign in to save this dataset to your own account, with a card that says how it was made. DataInventor can only create datasets for you. It cannot see your other repos. It is private unless you choose public.</p>
|
| 135 |
+
<div class="actions" style="margin-top:12px"><a class="btn go" href="${esc(href)}" ${embedded ? 'target="_blank" rel="noopener"' : ""} style="text-decoration:none">Sign in with Hugging Face</a></div>${embedded ? '<p class="hint" style="margin-top:8px">Sign-in opens in its own tab because browsers block it inside an embedded page.</p>' : ""}</div>`;
|
| 136 |
+
}
|
| 137 |
+
if (!p.name) p.name = "datainventor-" + (j.slug || "dataset");
|
| 138 |
+
return `<div class="group">${head}<p class="hint">Signed in as <b>${esc(me.user)}</b>. <button class="linkish" data-act="signout">Sign out</button></p>
|
| 139 |
+
<div class="row2"><label class="f"><span>Dataset name</span><input class="in" id="pub-name" value="${esc(p.name)}" spellcheck="false"></label>
|
| 140 |
+
<label class="f"><span>Who can see it</span><select class="in" id="pub-private"><option value="1" ${p.priv ? "selected" : ""}>Only me (private)</option><option value="0" ${p.priv ? "" : "selected"}>Everyone (public)</option></select></label></div>
|
| 141 |
+
<label class="f" style="max-width:320px"><span>License</span><select class="in" id="pub-license">${["apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "mit", "cc0-1.0"].map((l) => `<option ${p.license === l ? "selected" : ""}>${l}</option>`).join("")}</select></label>
|
| 142 |
+
<label class="check"><input type="checkbox" id="pub-ok" ${p.ok ? "checked" : ""}><span>I have the right to publish this data. I understand it was written by a model and has not been checked by a person.</span></label>
|
| 143 |
+
${p.err ? `<p class="note bad">${esc(p.err)}</p>` : ""}
|
| 144 |
+
<div class="actions"><button class="btn go" data-act="publish" ${p.busy ? "disabled" : ""}>${p.busy ? "Publishing" : "Publish dataset"}</button></div></div>`;
|
| 145 |
+
}
|
| 146 |
+
async function loadMe() { try { S.me = await api("/api/me"); } catch (e) { S.me = { user: null, can_sign_in: false }; } const el = document.getElementById("run-publish"); if (el) el.innerHTML = publishHtml(); }
|
| 147 |
+
async function doPublish() {
|
| 148 |
+
S.pub.busy = true; S.pub.err = ""; refreshPublish();
|
| 149 |
+
try { const r = await api(`/api/jobs/${S.job}/publish`, { name: S.pub.name.trim(), private: S.pub.priv, license: S.pub.license, confirm: S.pub.ok }); S.pub.url = r.url; } catch (e) { S.pub.err = e.message; }
|
| 150 |
+
S.pub.busy = false; refreshPublish();
|
| 151 |
+
}
|
| 152 |
+
function refreshPublish() { const el = document.getElementById("run-publish"); if (el) el.innerHTML = publishHtml(); }
|
| 153 |
+
async function restoreJob(id) {
|
| 154 |
+
try {
|
| 155 |
+
let since = 0, last = null; S.jobRows = [];
|
| 156 |
+
for (let k = 0; k < 8; k++) { last = await api(`/api/jobs/${id}?since=${since}`); S.jobRows.push(...last.new_rows); since = S.jobRows.length; if (since >= last.count) break; }
|
| 157 |
+
const c = last.config || {}; S.job = id; S.jobState = last; S.format = c.format || "instruction"; S.request = c.request || ""; S.domains = c.domains || []; S.languages = c.languages || ["English"]; S.constraints = c.constraints || ""; S.rows = c.rows || S.rows;
|
| 158 |
+
S.maxStep = 6; S.step = 6; S.printed = S.jobRows.length ? S.jobRows[S.jobRows.length - 1].id : 0; render();
|
| 159 |
+
if (!isFinal(last)) poll();
|
| 160 |
+
} catch (e) { /* the job is gone; start fresh */ }
|
| 161 |
+
}
|
| 162 |
const isFinal = (j) => j && ["done", "stopped", "failed"].includes(j.state);
|
| 163 |
function rowHtml(r) {
|
| 164 |
const pref = S.format === "preference";
|
|
|
|
| 192 |
<div class="count" id="run-count" aria-live="polite">${h.count}</div>
|
| 193 |
<div class="bar" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${h.pct}"><i id="run-bar" style="width:${h.pct}%"></i></div>
|
| 194 |
<p class="state" id="run-state">${h.state}</p><div id="run-note">${h.note}</div>
|
| 195 |
+
<div class="actions" id="run-actions">${h.actions}</div><div id="run-stats">${h.stats}</div><div id="run-filtered">${h.filtered}</div><div id="run-publish">${publishHtml()}</div>
|
| 196 |
<div class="press" id="run-deck" aria-hidden="true">${deckHtml()}</div>
|
| 197 |
<div class="table" id="run-rows">${S.jobRows.slice().reverse().map(rowHtml).join("")}</div></div>`;
|
| 198 |
}
|
|
|
|
| 200 |
function updateRun() {
|
| 201 |
const h = headParts(), set = (id, html) => { const el = document.getElementById(id); if (el && el.innerHTML !== html) el.innerHTML = html; };
|
| 202 |
set("run-h2", h.h2); set("run-count", h.count); set("run-state", h.state); set("run-note", h.note); set("run-actions", h.actions); set("run-stats", h.stats); set("run-filtered", h.filtered);
|
| 203 |
+
if (isFinal(S.jobState) && !S.pubShown) { S.pubShown = true; loadMe(); }
|
| 204 |
const bar = document.getElementById("run-bar"); if (bar) { bar.style.width = h.pct + "%"; bar.parentElement.setAttribute("aria-valuenow", h.pct); }
|
| 205 |
const deck = document.getElementById("run-deck"); if (deck) deck.innerHTML = deckHtml();
|
| 206 |
const rows = document.getElementById("run-rows");
|
|
|
|
| 228 |
if (act === "back") return go(Math.max(0, S.step - 1));
|
| 229 |
if (act === "suggest") { S.request = S.request.trim() ? S.request : $("[data-suggest]").dataset.suggest; return render(); }
|
| 230 |
if (act === "blueprint") return doBlueprint();
|
| 231 |
+
if (act === "publish") return doPublish();
|
| 232 |
+
if (act === "signout") { await api("/api/signout", {}); S.me = { ...S.me, user: null }; return refreshPublish(); }
|
| 233 |
if (act === "stop") { if (S.job) await api(`/api/jobs/${S.job}/stop`, {}); return; }
|
| 234 |
+
if (act === "again") { S.job = null; S.jobState = null; S.jobRows = []; S.drawn = 0; S.printed = 0; S.pubShown = false; S.pub = { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" }; S.runError = ""; return go(4); }
|
| 235 |
});
|
| 236 |
+
document.addEventListener("change", (ev) => { if (ev.target.id && ev.target.id.startsWith("pub-")) { const id = ev.target.id; if (id === "pub-private") S.pub.priv = ev.target.value === "1"; else if (id === "pub-license") S.pub.license = ev.target.value; else if (id === "pub-ok") S.pub.ok = ev.target.checked; } });
|
| 237 |
document.addEventListener("input", (ev) => {
|
| 238 |
const id = ev.target.id;
|
| 239 |
+
if (id === "pub-name") S.pub.name = ev.target.value; else if (id === "pub-private") S.pub.priv = ev.target.value === "1"; else if (id === "pub-license") S.pub.license = ev.target.value; else if (id === "pub-ok") S.pub.ok = ev.target.checked;
|
| 240 |
if (id === "request") { S.request = ev.target.value; S.blueprint = null; const b = $('[data-act="next"]'); if (b) b.disabled = !S.request.trim(); const bp = $('[data-act="blueprint"]'); if (bp) bp.disabled = S.bpBusy || !S.request.trim(); }
|
| 241 |
else if (id === "constraints") { S.constraints = ev.target.value; S.blueprint = null; }
|
| 242 |
});
|
|
|
|
| 247 |
S.bpBusy = false; render();
|
| 248 |
}
|
| 249 |
async function startRun() {
|
| 250 |
+
S.runError = ""; S.jobRows = []; S.jobState = null; S.drawn = 0; S.printed = 0; S.pubShown = false; S.pub = { name: "", priv: true, license: "apache-2.0", ok: false, busy: false, err: "", url: "" };
|
| 251 |
try { const r = await api("/api/jobs", spec()); S.job = r.id; poll(); } catch (e) { S.runError = e.message; }
|
| 252 |
render();
|
| 253 |
}
|
|
|
|
| 269 |
(async function init() {
|
| 270 |
try { const d = await api("/api/defaults"); S.model = d.model; S.maxRows = d.max_rows; S.configured = d.configured; if (S.rows > S.maxRows) S.rows = S.maxRows; } catch (e) { /* offline */ }
|
| 271 |
try { S.domainData = await api("/api/domains"); } catch (e) { S.domainData = []; }
|
| 272 |
+
const q = new URLSearchParams(location.search), jid = q.get("job");
|
| 273 |
+
if (jid) { await loadMe(); await restoreJob(jid); history.replaceState(null, "", "/"); } else render();
|
| 274 |
})();
|