Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Scoring: names-only multi-label task with subject-area prompt, bf16, no truncation; fitted default thresholds; descriptions become notes; hide few-shot rescore
Browse files- app.py +23 -9
- index.html +13 -7
- thresholds.json +23 -0
app.py
CHANGED
|
@@ -21,6 +21,10 @@ api = HfApi()
|
|
| 21 |
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 22 |
CACHE = {} # source -> (time, n, rows)
|
| 23 |
CACHE_TTL = 30 * 60
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
AUTO_TAG = re.compile(r"^(region|format|size_categories|library|modality|arxiv|endpoints_compatible|"
|
| 26 |
r"text-generation-inference|autotrain_compatible|safetensors|license|base_model|doi|dataset|"
|
|
@@ -131,17 +135,25 @@ def get_rows(source, n):
|
|
| 131 |
return rows
|
| 132 |
|
| 133 |
|
| 134 |
-
@spaces.GPU(duration=
|
| 135 |
-
def run_model(texts,
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
|
| 138 |
|
| 139 |
def scores_for(texts, topics, examples):
|
| 140 |
-
|
|
|
|
|
|
|
| 141 |
if examples:
|
| 142 |
-
task["
|
| 143 |
-
preds = run_model(texts, task)
|
| 144 |
-
return [{d["label"]: round(float(d["confidence"]), 4) for d in (p["
|
| 145 |
|
| 146 |
|
| 147 |
app = Server()
|
|
@@ -161,14 +173,16 @@ def feed(source: str, n: int, topics_text: str, examples: list) -> dict:
|
|
| 161 |
t = time.time()
|
| 162 |
scores = scores_for([r["text"] for r in rows], topics, examples or [])
|
| 163 |
out = [{k: v for k, v in r.items() if k != "text"} | {"scores": s} for r, s in zip(rows, scores)]
|
| 164 |
-
return {"items": out, "topics": topics, "seconds": round(time.time() - t, 1), "n_examples": len(examples or [])
|
|
|
|
| 165 |
|
| 166 |
|
| 167 |
@app.api(name="score_text")
|
| 168 |
def score_text(text: str, topics_text: str) -> dict:
|
| 169 |
"""Score one text against every topic."""
|
| 170 |
topics = parse_topics(topics_text)
|
| 171 |
-
return {"scores": scores_for([text], topics, [])[0], "topics": topics
|
|
|
|
| 172 |
|
| 173 |
|
| 174 |
MIN_CONFIRMED = 20
|
|
|
|
| 21 |
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 22 |
CACHE = {} # source -> (time, n, rows)
|
| 23 |
CACHE_TTL = 30 * 60
|
| 24 |
+
with open(os.path.join(HERE, "thresholds.json"), encoding="utf-8") as _f:
|
| 25 |
+
_thr = json.load(_f)
|
| 26 |
+
DEFAULT_THRESHOLDS, FALLBACK_THRESHOLD = _thr["thresholds"], _thr["fallback"]
|
| 27 |
+
PROMPT = "Which subject areas is this model, dataset or paper substantially about"
|
| 28 |
|
| 29 |
AUTO_TAG = re.compile(r"^(region|format|size_categories|library|modality|arxiv|endpoints_compatible|"
|
| 30 |
r"text-generation-inference|autotrain_compatible|safetensors|license|base_model|doi|dataset|"
|
|
|
|
| 135 |
return rows
|
| 136 |
|
| 137 |
|
| 138 |
+
@spaces.GPU(duration=120)
|
| 139 |
+
def run_model(texts, schema):
|
| 140 |
+
"""bf16, no input truncation. Long cards can OOM at batch 8 (sometimes as a plain RuntimeError): retry at 2."""
|
| 141 |
+
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=torch.cuda.is_available()):
|
| 142 |
+
try:
|
| 143 |
+
return model.batch_classify_text(texts, schema, batch_size=8, include_confidence=True, max_len=None)
|
| 144 |
+
except RuntimeError: # includes torch.cuda.OutOfMemoryError
|
| 145 |
+
torch.cuda.empty_cache()
|
| 146 |
+
return model.batch_classify_text(texts, schema, batch_size=2, include_confidence=True, max_len=None)
|
| 147 |
|
| 148 |
|
| 149 |
def scores_for(texts, topics, examples):
|
| 150 |
+
"""Score every topic for every text. Topic NAMES only (descriptions hurt accuracy), fixed prompt,
|
| 151 |
+
cls_threshold 0 so every label comes back with a score. `examples` ([[text, topic], ...]) is optional few-shot."""
|
| 152 |
+
task = {"labels": list(topics), "multi_label": True, "cls_threshold": 0.0, "prompt": PROMPT}
|
| 153 |
if examples:
|
| 154 |
+
task["examples"] = [(e[0][:200], e[1]) for e in examples if e[1] in topics][:24]
|
| 155 |
+
preds = run_model(texts, {"topics": task})
|
| 156 |
+
return [{d["label"]: round(float(d["confidence"]), 4) for d in (p["topics"] or [])} for p in preds]
|
| 157 |
|
| 158 |
|
| 159 |
app = Server()
|
|
|
|
| 173 |
t = time.time()
|
| 174 |
scores = scores_for([r["text"] for r in rows], topics, examples or [])
|
| 175 |
out = [{k: v for k, v in r.items() if k != "text"} | {"scores": s} for r, s in zip(rows, scores)]
|
| 176 |
+
return {"items": out, "topics": topics, "seconds": round(time.time() - t, 1), "n_examples": len(examples or []),
|
| 177 |
+
"default_thresholds": {k: DEFAULT_THRESHOLDS.get(k, FALLBACK_THRESHOLD) for k in topics}}
|
| 178 |
|
| 179 |
|
| 180 |
@app.api(name="score_text")
|
| 181 |
def score_text(text: str, topics_text: str) -> dict:
|
| 182 |
"""Score one text against every topic."""
|
| 183 |
topics = parse_topics(topics_text)
|
| 184 |
+
return {"scores": scores_for([text], topics, [])[0], "topics": topics,
|
| 185 |
+
"default_thresholds": {k: DEFAULT_THRESHOLDS.get(k, FALLBACK_THRESHOLD) for k in topics}}
|
| 186 |
|
| 187 |
|
| 188 |
MIN_CONFIRMED = 20
|
index.html
CHANGED
|
@@ -146,7 +146,7 @@ input[type=range]{width:100%;margin:0;accent-color:var(--accent);border:0;paddin
|
|
| 146 |
<div class="summary" id="summary"><h2 style="margin-right:4px">Your feed</h2><span id="feedTopics" class="summary"></span><button class="link" id="editBtn" aria-expanded="false">edit topics</button></div>
|
| 147 |
<div class="editor" id="editor" hidden>
|
| 148 |
<div class="hd row" style="justify-content:space-between"><h2>Edit topics</h2></div>
|
| 149 |
-
<p class="note">Tick the topics you want in your feed.
|
| 150 |
<div id="erows" class="editor" style="padding:0"></div>
|
| 151 |
<div class="row">
|
| 152 |
<button class="btn ghost small" id="addTopic">+ Add topic</button>
|
|
@@ -236,7 +236,11 @@ const store = {
|
|
| 236 |
let source = store.get("source", "papers");
|
| 237 |
let topicDefs = store.get("topics", null) || migrateTopicsText(store.get("topicsText", null)) || DEFAULT_TOPICS.map(t => ({...t}));
|
| 238 |
let selected = store.get("selected", ["agents and tool use", "cultural heritage and history", "documents and OCR"]);
|
| 239 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
// confirmed: {itemKey: {title, text, src, url, topics: [confirmed positives], considered: [topics the user judged]}}
|
| 241 |
// A topic in `considered` but not in `topics` is a "doesn't belong". Topics never considered are unknown.
|
| 242 |
let confirmed = store.get("confirmed", null) || migrateMarks(store.get("marks", {}));
|
|
@@ -261,7 +265,7 @@ function migrateMarks(marks){
|
|
| 261 |
}
|
| 262 |
const saveConfirmed = () => store.set("confirmed", confirmed);
|
| 263 |
const key = it => fetchedSource + ":" + it.id;
|
| 264 |
-
const thr = t => thresholds[t] ??
|
| 265 |
const has = (it, t) => scored.includes(t); // scored topics below the model cutoff (0.02) are omitted, so treat as ~0
|
| 266 |
const sc = (it, t) => it.scores?.[t] ?? 0;
|
| 267 |
const judged = (k, t) => { const r = confirmed[k]; return r && r.considered.includes(t) ? r.topics.includes(t) : undefined; };
|
|
@@ -292,7 +296,7 @@ function renameTopic(from, to){
|
|
| 292 |
r.considered = r.considered.map(t => t === from ? to : t);
|
| 293 |
r.topics = r.topics.map(t => t === from ? to : t);
|
| 294 |
}
|
| 295 |
-
if (from in thresholds){ thresholds[to] = thresholds[from]; delete thresholds[from]; store.set("thr", thresholds); }
|
| 296 |
selected = selected.map(t => t === from ? to : t);
|
| 297 |
saveConfirmed();
|
| 298 |
}
|
|
@@ -303,7 +307,7 @@ function renderEditor(){
|
|
| 303 |
return `<div class="erow" data-i="${i}">
|
| 304 |
<input type="checkbox" data-f="on" aria-label="Show ${esc(nm)} in my feed" title="In my feed" ${ok && selected.includes(nm) ? "checked" : ""} ${ok ? "" : "disabled"}>
|
| 305 |
<input class="name" value="${esc(tp.name)}" placeholder="name" aria-label="Topic name" aria-invalid="${!!err}" data-f="name">
|
| 306 |
-
<input class="desc" value="${esc(tp.desc)}" placeholder="
|
| 307 |
<button class="x" title="Remove topic" aria-label="Remove ${esc(tp.name)}">×</button>
|
| 308 |
${err ? `<div class="err">${esc(err)}</div>` : ""}</div>`; }).join("");
|
| 309 |
$("presets").innerHTML = Object.keys(PRESETS).map(p => `<button class="link" data-p="${esc(p)}">${esc(p)}</button>`).join("") + `<button class="link" data-p="">reset to defaults</button>`;
|
|
@@ -393,6 +397,7 @@ async function run(useExamples){
|
|
| 393 |
const r = await c.predict("/feed", { source, n: +$("n").value, topics_text: topicsText(), examples });
|
| 394 |
const d = r.data[0];
|
| 395 |
items = d.items; scored = Object.keys(d.topics); fetchedSource = source;
|
|
|
|
| 396 |
byKey.clear(); for (const it of items) byKey.set(key(it), it);
|
| 397 |
for (const k of Object.keys(drafts)) delete drafts[k];
|
| 398 |
if (!selected.length) selected = scored.slice(0, 3);
|
|
@@ -405,6 +410,7 @@ async function run(useExamples){
|
|
| 405 |
}
|
| 406 |
$("go").onclick = () => run(false);
|
| 407 |
$("rescore").onclick = () => run(true);
|
|
|
|
| 408 |
|
| 409 |
/* ---------- feed ---------- */
|
| 410 |
function stateOf(k, it, t){
|
|
@@ -530,8 +536,8 @@ $("queue").addEventListener("click", e => {
|
|
| 530 |
if (!r.considered.length) delete confirmed[k];
|
| 531 |
delete drafts[k]; saveConfirmed(); afterMarks(k);
|
| 532 |
});
|
| 533 |
-
$("thr").oninput = e => { thresholds[calTopic] = +e.target.value; store.set("thr", thresholds); renderToggles(); renderFeed(); renderCal(); };
|
| 534 |
-
$("suggest").onclick = () => { const s = suggested(calTopic); if (s){ thresholds[calTopic] = s.th; store.set("thr", thresholds); renderAll(); } };
|
| 535 |
$("copy").onclick = () => navigator.clipboard.writeText(JSON.stringify(trainingPayload(), null, 1))
|
| 536 |
.then(() => { $("copy").textContent = "copied"; setTimeout(() => $("copy").textContent = "copy my data", 1500); })
|
| 537 |
.catch(() => { $("copy").textContent = "copy failed"; });
|
|
|
|
| 146 |
<div class="summary" id="summary"><h2 style="margin-right:4px">Your feed</h2><span id="feedTopics" class="summary"></span><button class="link" id="editBtn" aria-expanded="false">edit topics</button></div>
|
| 147 |
<div class="editor" id="editor" hidden>
|
| 148 |
<div class="hd row" style="justify-content:space-between"><h2>Edit topics</h2></div>
|
| 149 |
+
<p class="note">Tick the topics you want in your feed. Only the topic name is sent to the model. The note is for you and is not sent. Names cannot contain ( ) [ ] or a colon.</p>
|
| 150 |
<div id="erows" class="editor" style="padding:0"></div>
|
| 151 |
<div class="row">
|
| 152 |
<button class="btn ghost small" id="addTopic">+ Add topic</button>
|
|
|
|
| 236 |
let source = store.get("source", "papers");
|
| 237 |
let topicDefs = store.get("topics", null) || migrateTopicsText(store.get("topicsText", null)) || DEFAULT_TOPICS.map(t => ({...t}));
|
| 238 |
let selected = store.get("selected", ["agents and tool use", "cultural heritage and history", "documents and OCR"]);
|
| 239 |
+
// "thr-v2": scores changed scale with the names-only prompt strategy, so old overrides are dropped.
|
| 240 |
+
let thresholds = store.get("thr-v2", {}); // user overrides
|
| 241 |
+
let defaultThr = {}; // per-topic defaults sent by the backend (fitted on LLM labels)
|
| 242 |
+
const FALLBACK_THR = 0.31;
|
| 243 |
+
const FEW_SHOT = false; // few-shot rescoring lowered AUROC in experiments; kept behind this flag
|
| 244 |
// confirmed: {itemKey: {title, text, src, url, topics: [confirmed positives], considered: [topics the user judged]}}
|
| 245 |
// A topic in `considered` but not in `topics` is a "doesn't belong". Topics never considered are unknown.
|
| 246 |
let confirmed = store.get("confirmed", null) || migrateMarks(store.get("marks", {}));
|
|
|
|
| 265 |
}
|
| 266 |
const saveConfirmed = () => store.set("confirmed", confirmed);
|
| 267 |
const key = it => fetchedSource + ":" + it.id;
|
| 268 |
+
const thr = t => thresholds[t] ?? defaultThr[t] ?? FALLBACK_THR;
|
| 269 |
const has = (it, t) => scored.includes(t); // scored topics below the model cutoff (0.02) are omitted, so treat as ~0
|
| 270 |
const sc = (it, t) => it.scores?.[t] ?? 0;
|
| 271 |
const judged = (k, t) => { const r = confirmed[k]; return r && r.considered.includes(t) ? r.topics.includes(t) : undefined; };
|
|
|
|
| 296 |
r.considered = r.considered.map(t => t === from ? to : t);
|
| 297 |
r.topics = r.topics.map(t => t === from ? to : t);
|
| 298 |
}
|
| 299 |
+
if (from in thresholds){ thresholds[to] = thresholds[from]; delete thresholds[from]; store.set("thr-v2", thresholds); }
|
| 300 |
selected = selected.map(t => t === from ? to : t);
|
| 301 |
saveConfirmed();
|
| 302 |
}
|
|
|
|
| 307 |
return `<div class="erow" data-i="${i}">
|
| 308 |
<input type="checkbox" data-f="on" aria-label="Show ${esc(nm)} in my feed" title="In my feed" ${ok && selected.includes(nm) ? "checked" : ""} ${ok ? "" : "disabled"}>
|
| 309 |
<input class="name" value="${esc(tp.name)}" placeholder="name" aria-label="Topic name" aria-invalid="${!!err}" data-f="name">
|
| 310 |
+
<input class="desc" value="${esc(tp.desc)}" placeholder="note (not sent to the model)" title="Note for you (not sent to the model)" aria-label="Note, not sent to the model" data-f="desc">
|
| 311 |
<button class="x" title="Remove topic" aria-label="Remove ${esc(tp.name)}">×</button>
|
| 312 |
${err ? `<div class="err">${esc(err)}</div>` : ""}</div>`; }).join("");
|
| 313 |
$("presets").innerHTML = Object.keys(PRESETS).map(p => `<button class="link" data-p="${esc(p)}">${esc(p)}</button>`).join("") + `<button class="link" data-p="">reset to defaults</button>`;
|
|
|
|
| 397 |
const r = await c.predict("/feed", { source, n: +$("n").value, topics_text: topicsText(), examples });
|
| 398 |
const d = r.data[0];
|
| 399 |
items = d.items; scored = Object.keys(d.topics); fetchedSource = source;
|
| 400 |
+
defaultThr = {...defaultThr, ...(d.default_thresholds || {})};
|
| 401 |
byKey.clear(); for (const it of items) byKey.set(key(it), it);
|
| 402 |
for (const k of Object.keys(drafts)) delete drafts[k];
|
| 403 |
if (!selected.length) selected = scored.slice(0, 3);
|
|
|
|
| 410 |
}
|
| 411 |
$("go").onclick = () => run(false);
|
| 412 |
$("rescore").onclick = () => run(true);
|
| 413 |
+
$("rescore").hidden = !FEW_SHOT;
|
| 414 |
|
| 415 |
/* ---------- feed ---------- */
|
| 416 |
function stateOf(k, it, t){
|
|
|
|
| 536 |
if (!r.considered.length) delete confirmed[k];
|
| 537 |
delete drafts[k]; saveConfirmed(); afterMarks(k);
|
| 538 |
});
|
| 539 |
+
$("thr").oninput = e => { thresholds[calTopic] = +e.target.value; store.set("thr-v2", thresholds); renderToggles(); renderFeed(); renderCal(); };
|
| 540 |
+
$("suggest").onclick = () => { const s = suggested(calTopic); if (s){ thresholds[calTopic] = s.th; store.set("thr-v2", thresholds); renderAll(); } };
|
| 541 |
$("copy").onclick = () => navigator.clipboard.writeText(JSON.stringify(trainingPayload(), null, 1))
|
| 542 |
.then(() => { $("copy").textContent = "copied"; setTimeout(() => $("copy").textContent = "copy my data", 1500); })
|
| 543 |
.catch(() => { $("copy").textContent = "copy failed"; });
|
thresholds.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"source": "strat/v2/strategies/r2/r2_names_prompt_full_bf16.json",
|
| 3 |
+
"fit_on": "all rows, max F1 per topic vs DeepSeek-V4.1-Flash labels",
|
| 4 |
+
"fallback": 0.31,
|
| 5 |
+
"thresholds": {
|
| 6 |
+
"legal": 0.2227,
|
| 7 |
+
"medical and health": 0.2451,
|
| 8 |
+
"finance and business": 0.2334,
|
| 9 |
+
"coding and software": 0.2383,
|
| 10 |
+
"agents and tool use": 0.3086,
|
| 11 |
+
"robotics": 0.4316,
|
| 12 |
+
"mathematics": 0.2227,
|
| 13 |
+
"natural sciences": 0.6914,
|
| 14 |
+
"education": 0.8164,
|
| 15 |
+
"multilingual and low-resource languages": 0.5,
|
| 16 |
+
"cultural heritage and history": 0.3086,
|
| 17 |
+
"documents and OCR": 0.3926,
|
| 18 |
+
"safety and alignment": 0.1553,
|
| 19 |
+
"climate and earth observation": 0.5352,
|
| 20 |
+
"creative writing and roleplay": 0.2969,
|
| 21 |
+
"speech and music": 0.4473
|
| 22 |
+
}
|
| 23 |
+
}
|