davanstrien HF Staff commited on
Commit
8f325a4
·
verified ·
1 Parent(s): 8193284

Scoring: names-only multi-label task with subject-area prompt, bf16, no truncation; fitted default thresholds; descriptions become notes; hide few-shot rescore

Browse files
Files changed (3) hide show
  1. app.py +23 -9
  2. index.html +13 -7
  3. thresholds.json +23 -0
app.py CHANGED
@@ -21,6 +21,10 @@ api = HfApi()
21
  HERE = os.path.dirname(os.path.abspath(__file__))
22
  CACHE = {} # source -> (time, n, rows)
23
  CACHE_TTL = 30 * 60
 
 
 
 
24
 
25
  AUTO_TAG = re.compile(r"^(region|format|size_categories|library|modality|arxiv|endpoints_compatible|"
26
  r"text-generation-inference|autotrain_compatible|safetensors|license|base_model|doi|dataset|"
@@ -131,17 +135,25 @@ def get_rows(source, n):
131
  return rows
132
 
133
 
134
- @spaces.GPU(duration=90)
135
- def run_model(texts, task):
136
- return model.batch_classify_text(texts, task, batch_size=16, include_confidence=True, max_len=512)
 
 
 
 
 
 
137
 
138
 
139
  def scores_for(texts, topics, examples):
140
- task = {"topic": {"labels": topics, "multi_label": True, "cls_threshold": 0.02}}
 
 
141
  if examples:
142
- task["topic"]["examples"] = [(e[0][:200], e[1]) for e in examples if e[1] in topics][:24]
143
- preds = run_model(texts, task)
144
- return [{d["label"]: round(float(d["confidence"]), 4) for d in (p["topic"] or [])} for p in preds]
145
 
146
 
147
  app = Server()
@@ -161,14 +173,16 @@ def feed(source: str, n: int, topics_text: str, examples: list) -> dict:
161
  t = time.time()
162
  scores = scores_for([r["text"] for r in rows], topics, examples or [])
163
  out = [{k: v for k, v in r.items() if k != "text"} | {"scores": s} for r, s in zip(rows, scores)]
164
- return {"items": out, "topics": topics, "seconds": round(time.time() - t, 1), "n_examples": len(examples or [])}
 
165
 
166
 
167
  @app.api(name="score_text")
168
  def score_text(text: str, topics_text: str) -> dict:
169
  """Score one text against every topic."""
170
  topics = parse_topics(topics_text)
171
- return {"scores": scores_for([text], topics, [])[0], "topics": topics}
 
172
 
173
 
174
  MIN_CONFIRMED = 20
 
21
  HERE = os.path.dirname(os.path.abspath(__file__))
22
  CACHE = {} # source -> (time, n, rows)
23
  CACHE_TTL = 30 * 60
24
+ with open(os.path.join(HERE, "thresholds.json"), encoding="utf-8") as _f:
25
+ _thr = json.load(_f)
26
+ DEFAULT_THRESHOLDS, FALLBACK_THRESHOLD = _thr["thresholds"], _thr["fallback"]
27
+ PROMPT = "Which subject areas is this model, dataset or paper substantially about"
28
 
29
  AUTO_TAG = re.compile(r"^(region|format|size_categories|library|modality|arxiv|endpoints_compatible|"
30
  r"text-generation-inference|autotrain_compatible|safetensors|license|base_model|doi|dataset|"
 
135
  return rows
136
 
137
 
138
+ @spaces.GPU(duration=120)
139
+ def run_model(texts, schema):
140
+ """bf16, no input truncation. Long cards can OOM at batch 8 (sometimes as a plain RuntimeError): retry at 2."""
141
+ with torch.autocast("cuda", dtype=torch.bfloat16, enabled=torch.cuda.is_available()):
142
+ try:
143
+ return model.batch_classify_text(texts, schema, batch_size=8, include_confidence=True, max_len=None)
144
+ except RuntimeError: # includes torch.cuda.OutOfMemoryError
145
+ torch.cuda.empty_cache()
146
+ return model.batch_classify_text(texts, schema, batch_size=2, include_confidence=True, max_len=None)
147
 
148
 
149
  def scores_for(texts, topics, examples):
150
+ """Score every topic for every text. Topic NAMES only (descriptions hurt accuracy), fixed prompt,
151
+ cls_threshold 0 so every label comes back with a score. `examples` ([[text, topic], ...]) is optional few-shot."""
152
+ task = {"labels": list(topics), "multi_label": True, "cls_threshold": 0.0, "prompt": PROMPT}
153
  if examples:
154
+ task["examples"] = [(e[0][:200], e[1]) for e in examples if e[1] in topics][:24]
155
+ preds = run_model(texts, {"topics": task})
156
+ return [{d["label"]: round(float(d["confidence"]), 4) for d in (p["topics"] or [])} for p in preds]
157
 
158
 
159
  app = Server()
 
173
  t = time.time()
174
  scores = scores_for([r["text"] for r in rows], topics, examples or [])
175
  out = [{k: v for k, v in r.items() if k != "text"} | {"scores": s} for r, s in zip(rows, scores)]
176
+ return {"items": out, "topics": topics, "seconds": round(time.time() - t, 1), "n_examples": len(examples or []),
177
+ "default_thresholds": {k: DEFAULT_THRESHOLDS.get(k, FALLBACK_THRESHOLD) for k in topics}}
178
 
179
 
180
  @app.api(name="score_text")
181
  def score_text(text: str, topics_text: str) -> dict:
182
  """Score one text against every topic."""
183
  topics = parse_topics(topics_text)
184
+ return {"scores": scores_for([text], topics, [])[0], "topics": topics,
185
+ "default_thresholds": {k: DEFAULT_THRESHOLDS.get(k, FALLBACK_THRESHOLD) for k in topics}}
186
 
187
 
188
  MIN_CONFIRMED = 20
index.html CHANGED
@@ -146,7 +146,7 @@ input[type=range]{width:100%;margin:0;accent-color:var(--accent);border:0;paddin
146
  <div class="summary" id="summary"><h2 style="margin-right:4px">Your feed</h2><span id="feedTopics" class="summary"></span><button class="link" id="editBtn" aria-expanded="false">edit topics</button></div>
147
  <div class="editor" id="editor" hidden>
148
  <div class="hd row" style="justify-content:space-between"><h2>Edit topics</h2></div>
149
- <p class="note">Tick the topics you want in your feed. Each topic has a short name and a one-line description of what belongs. Names cannot contain ( ) [ ] or a colon.</p>
150
  <div id="erows" class="editor" style="padding:0"></div>
151
  <div class="row">
152
  <button class="btn ghost small" id="addTopic">+ Add topic</button>
@@ -236,7 +236,11 @@ const store = {
236
  let source = store.get("source", "papers");
237
  let topicDefs = store.get("topics", null) || migrateTopicsText(store.get("topicsText", null)) || DEFAULT_TOPICS.map(t => ({...t}));
238
  let selected = store.get("selected", ["agents and tool use", "cultural heritage and history", "documents and OCR"]);
239
- let thresholds = store.get("thr", {});
 
 
 
 
240
  // confirmed: {itemKey: {title, text, src, url, topics: [confirmed positives], considered: [topics the user judged]}}
241
  // A topic in `considered` but not in `topics` is a "doesn't belong". Topics never considered are unknown.
242
  let confirmed = store.get("confirmed", null) || migrateMarks(store.get("marks", {}));
@@ -261,7 +265,7 @@ function migrateMarks(marks){
261
  }
262
  const saveConfirmed = () => store.set("confirmed", confirmed);
263
  const key = it => fetchedSource + ":" + it.id;
264
- const thr = t => thresholds[t] ?? 0.5;
265
  const has = (it, t) => scored.includes(t); // scored topics below the model cutoff (0.02) are omitted, so treat as ~0
266
  const sc = (it, t) => it.scores?.[t] ?? 0;
267
  const judged = (k, t) => { const r = confirmed[k]; return r && r.considered.includes(t) ? r.topics.includes(t) : undefined; };
@@ -292,7 +296,7 @@ function renameTopic(from, to){
292
  r.considered = r.considered.map(t => t === from ? to : t);
293
  r.topics = r.topics.map(t => t === from ? to : t);
294
  }
295
- if (from in thresholds){ thresholds[to] = thresholds[from]; delete thresholds[from]; store.set("thr", thresholds); }
296
  selected = selected.map(t => t === from ? to : t);
297
  saveConfirmed();
298
  }
@@ -303,7 +307,7 @@ function renderEditor(){
303
  return `<div class="erow" data-i="${i}">
304
  <input type="checkbox" data-f="on" aria-label="Show ${esc(nm)} in my feed" title="In my feed" ${ok && selected.includes(nm) ? "checked" : ""} ${ok ? "" : "disabled"}>
305
  <input class="name" value="${esc(tp.name)}" placeholder="name" aria-label="Topic name" aria-invalid="${!!err}" data-f="name">
306
- <input class="desc" value="${esc(tp.desc)}" placeholder="what belongs, in one line" aria-label="Description" data-f="desc">
307
  <button class="x" title="Remove topic" aria-label="Remove ${esc(tp.name)}">×</button>
308
  ${err ? `<div class="err">${esc(err)}</div>` : ""}</div>`; }).join("");
309
  $("presets").innerHTML = Object.keys(PRESETS).map(p => `<button class="link" data-p="${esc(p)}">${esc(p)}</button>`).join("") + `<button class="link" data-p="">reset to defaults</button>`;
@@ -393,6 +397,7 @@ async function run(useExamples){
393
  const r = await c.predict("/feed", { source, n: +$("n").value, topics_text: topicsText(), examples });
394
  const d = r.data[0];
395
  items = d.items; scored = Object.keys(d.topics); fetchedSource = source;
 
396
  byKey.clear(); for (const it of items) byKey.set(key(it), it);
397
  for (const k of Object.keys(drafts)) delete drafts[k];
398
  if (!selected.length) selected = scored.slice(0, 3);
@@ -405,6 +410,7 @@ async function run(useExamples){
405
  }
406
  $("go").onclick = () => run(false);
407
  $("rescore").onclick = () => run(true);
 
408
 
409
  /* ---------- feed ---------- */
410
  function stateOf(k, it, t){
@@ -530,8 +536,8 @@ $("queue").addEventListener("click", e => {
530
  if (!r.considered.length) delete confirmed[k];
531
  delete drafts[k]; saveConfirmed(); afterMarks(k);
532
  });
533
- $("thr").oninput = e => { thresholds[calTopic] = +e.target.value; store.set("thr", thresholds); renderToggles(); renderFeed(); renderCal(); };
534
- $("suggest").onclick = () => { const s = suggested(calTopic); if (s){ thresholds[calTopic] = s.th; store.set("thr", thresholds); renderAll(); } };
535
  $("copy").onclick = () => navigator.clipboard.writeText(JSON.stringify(trainingPayload(), null, 1))
536
  .then(() => { $("copy").textContent = "copied"; setTimeout(() => $("copy").textContent = "copy my data", 1500); })
537
  .catch(() => { $("copy").textContent = "copy failed"; });
 
146
  <div class="summary" id="summary"><h2 style="margin-right:4px">Your feed</h2><span id="feedTopics" class="summary"></span><button class="link" id="editBtn" aria-expanded="false">edit topics</button></div>
147
  <div class="editor" id="editor" hidden>
148
  <div class="hd row" style="justify-content:space-between"><h2>Edit topics</h2></div>
149
+ <p class="note">Tick the topics you want in your feed. Only the topic name is sent to the model. The note is for you and is not sent. Names cannot contain ( ) [ ] or a colon.</p>
150
  <div id="erows" class="editor" style="padding:0"></div>
151
  <div class="row">
152
  <button class="btn ghost small" id="addTopic">+ Add topic</button>
 
236
  let source = store.get("source", "papers");
237
  let topicDefs = store.get("topics", null) || migrateTopicsText(store.get("topicsText", null)) || DEFAULT_TOPICS.map(t => ({...t}));
238
  let selected = store.get("selected", ["agents and tool use", "cultural heritage and history", "documents and OCR"]);
239
+ // "thr-v2": scores changed scale with the names-only prompt strategy, so old overrides are dropped.
240
+ let thresholds = store.get("thr-v2", {}); // user overrides
241
+ let defaultThr = {}; // per-topic defaults sent by the backend (fitted on LLM labels)
242
+ const FALLBACK_THR = 0.31;
243
+ const FEW_SHOT = false; // few-shot rescoring lowered AUROC in experiments; kept behind this flag
244
  // confirmed: {itemKey: {title, text, src, url, topics: [confirmed positives], considered: [topics the user judged]}}
245
  // A topic in `considered` but not in `topics` is a "doesn't belong". Topics never considered are unknown.
246
  let confirmed = store.get("confirmed", null) || migrateMarks(store.get("marks", {}));
 
265
  }
266
  const saveConfirmed = () => store.set("confirmed", confirmed);
267
  const key = it => fetchedSource + ":" + it.id;
268
+ const thr = t => thresholds[t] ?? defaultThr[t] ?? FALLBACK_THR;
269
  const has = (it, t) => scored.includes(t); // scored topics below the model cutoff (0.02) are omitted, so treat as ~0
270
  const sc = (it, t) => it.scores?.[t] ?? 0;
271
  const judged = (k, t) => { const r = confirmed[k]; return r && r.considered.includes(t) ? r.topics.includes(t) : undefined; };
 
296
  r.considered = r.considered.map(t => t === from ? to : t);
297
  r.topics = r.topics.map(t => t === from ? to : t);
298
  }
299
+ if (from in thresholds){ thresholds[to] = thresholds[from]; delete thresholds[from]; store.set("thr-v2", thresholds); }
300
  selected = selected.map(t => t === from ? to : t);
301
  saveConfirmed();
302
  }
 
307
  return `<div class="erow" data-i="${i}">
308
  <input type="checkbox" data-f="on" aria-label="Show ${esc(nm)} in my feed" title="In my feed" ${ok && selected.includes(nm) ? "checked" : ""} ${ok ? "" : "disabled"}>
309
  <input class="name" value="${esc(tp.name)}" placeholder="name" aria-label="Topic name" aria-invalid="${!!err}" data-f="name">
310
+ <input class="desc" value="${esc(tp.desc)}" placeholder="note (not sent to the model)" title="Note for you (not sent to the model)" aria-label="Note, not sent to the model" data-f="desc">
311
  <button class="x" title="Remove topic" aria-label="Remove ${esc(tp.name)}">×</button>
312
  ${err ? `<div class="err">${esc(err)}</div>` : ""}</div>`; }).join("");
313
  $("presets").innerHTML = Object.keys(PRESETS).map(p => `<button class="link" data-p="${esc(p)}">${esc(p)}</button>`).join("") + `<button class="link" data-p="">reset to defaults</button>`;
 
397
  const r = await c.predict("/feed", { source, n: +$("n").value, topics_text: topicsText(), examples });
398
  const d = r.data[0];
399
  items = d.items; scored = Object.keys(d.topics); fetchedSource = source;
400
+ defaultThr = {...defaultThr, ...(d.default_thresholds || {})};
401
  byKey.clear(); for (const it of items) byKey.set(key(it), it);
402
  for (const k of Object.keys(drafts)) delete drafts[k];
403
  if (!selected.length) selected = scored.slice(0, 3);
 
410
  }
411
  $("go").onclick = () => run(false);
412
  $("rescore").onclick = () => run(true);
413
+ $("rescore").hidden = !FEW_SHOT;
414
 
415
  /* ---------- feed ---------- */
416
  function stateOf(k, it, t){
 
536
  if (!r.considered.length) delete confirmed[k];
537
  delete drafts[k]; saveConfirmed(); afterMarks(k);
538
  });
539
+ $("thr").oninput = e => { thresholds[calTopic] = +e.target.value; store.set("thr-v2", thresholds); renderToggles(); renderFeed(); renderCal(); };
540
+ $("suggest").onclick = () => { const s = suggested(calTopic); if (s){ thresholds[calTopic] = s.th; store.set("thr-v2", thresholds); renderAll(); } };
541
  $("copy").onclick = () => navigator.clipboard.writeText(JSON.stringify(trainingPayload(), null, 1))
542
  .then(() => { $("copy").textContent = "copied"; setTimeout(() => $("copy").textContent = "copy my data", 1500); })
543
  .catch(() => { $("copy").textContent = "copy failed"; });
thresholds.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "source": "strat/v2/strategies/r2/r2_names_prompt_full_bf16.json",
3
+ "fit_on": "all rows, max F1 per topic vs DeepSeek-V4.1-Flash labels",
4
+ "fallback": 0.31,
5
+ "thresholds": {
6
+ "legal": 0.2227,
7
+ "medical and health": 0.2451,
8
+ "finance and business": 0.2334,
9
+ "coding and software": 0.2383,
10
+ "agents and tool use": 0.3086,
11
+ "robotics": 0.4316,
12
+ "mathematics": 0.2227,
13
+ "natural sciences": 0.6914,
14
+ "education": 0.8164,
15
+ "multilingual and low-resource languages": 0.5,
16
+ "cultural heritage and history": 0.3086,
17
+ "documents and OCR": 0.3926,
18
+ "safety and alignment": 0.1553,
19
+ "climate and earth observation": 0.5352,
20
+ "creative writing and roleplay": 0.2969,
21
+ "speech and music": 0.4473
22
+ }
23
+ }