evalstate HF Staff commited on
Commit
8cecfac
·
verified ·
1 Parent(s): 1886473

Cost estimate: only a fit that reproduces recorded costs prices trials; unpriced wording; PR-aware row page

Browse files
Files changed (4) hide show
  1. data/leaderboard.js +0 -0
  2. home.js +4 -3
  3. index.html +1 -1
  4. row.js +6 -4
data/leaderboard.js CHANGED
The diff for this file is too large to render. See raw diff
 
home.js CHANGED
@@ -47,8 +47,9 @@
47
  else if (m.foreignTrials) out.push(["warn", "fallback " + m.served.filter(s => !s.configured).map(s => shortModel(s.model).replace(/^claude-/, "")).join(", ") + " in " + m.foreignTrials + " trials"]);
48
  const c = h.cost;
49
  if (c.imputedTrials) out.push(["warn", c.imputedTrials + " trials priced from tokens"]);
50
- if (c.noUsage / r.scan.trials > 0.02) out.push(["note", c.noUsage + " trials with no usage"]);
51
  if (c.unbilled) out.push(["bad", "reasoning unbilled"]);
 
52
  return out;
53
  }
54
  const hasLb = D.rows.some(r => r.lb.source !== "recount");
@@ -205,11 +206,11 @@
205
  tr.append(start(el("td", "n", money(c.recorded))));
206
  const est = el("td", "n" + (c.estimate && c.estimate > c.recorded * 1.02 ? " bad" : ""));
207
  est.append(document.createTextNode(c.estimate === null ? "–" : (c.lowerBound ? "≥" : "") + money(c.estimate)));
208
- if (c.imputedTrials || c.unbilled || c.noUsage) {
209
  const parts = [];
210
  if (c.imputedTrials) parts.push("+" + money(c.imputed) + " " + c.imputedTrials + " priced");
211
  if (c.unbilled) parts.push("+" + money(c.unbilled) + " unbilled");
212
- if (c.noUsage) parts.push(c.noUsage + " no usage");
213
  est.append(el("small", null, parts.join(" · ")));
214
  }
215
  tr.append(est);
 
47
  else if (m.foreignTrials) out.push(["warn", "fallback " + m.served.filter(s => !s.configured).map(s => shortModel(s.model).replace(/^claude-/, "")).join(", ") + " in " + m.foreignTrials + " trials"]);
48
  const c = h.cost;
49
  if (c.imputedTrials) out.push(["warn", c.imputedTrials + " trials priced from tokens"]);
50
+ if (c.unpriced / r.scan.trials > 0.02) out.push(["note", c.unpriced + " trials unpriced"]);
51
  if (c.unbilled) out.push(["bad", "reasoning unbilled"]);
52
+ if (r.lb.disqualified) out.push(["bad", r.lb.disqualified + " disqualified by the PR pipeline"]);
53
  return out;
54
  }
55
  const hasLb = D.rows.some(r => r.lb.source !== "recount");
 
206
  tr.append(start(el("td", "n", money(c.recorded))));
207
  const est = el("td", "n" + (c.estimate && c.estimate > c.recorded * 1.02 ? " bad" : ""));
208
  est.append(document.createTextNode(c.estimate === null ? "–" : (c.lowerBound ? "≥" : "") + money(c.estimate)));
209
+ if (c.imputedTrials || c.unbilled || c.unpriced) {
210
  const parts = [];
211
  if (c.imputedTrials) parts.push("+" + money(c.imputed) + " " + c.imputedTrials + " priced");
212
  if (c.unbilled) parts.push("+" + money(c.unbilled) + " unbilled");
213
+ if (c.unpriced) parts.push(c.unpriced + " unpriced");
214
  est.append(el("small", null, parts.join(" · ")));
215
  }
216
  tr.append(est);
index.html CHANGED
@@ -55,7 +55,7 @@
55
  fallback model (a Gemini preview name served as another Gemini preview is an alias, not a fallback).
56
  <b>Output per trial</b>: the median trial's output tokens, one scale across rows; the <span class="ob-key"></span> part has no text in
57
  the trace: reasoning you can't read. <b>Cost</b>: recorded trial costs, uncosted trials priced from their tokens at the row's own
58
- rates, and reasoning left out of the output tokens; ≥ where some trials have no usage at all.</p>
59
  </section>
60
 
61
  <section id="gaps" class="block">
 
55
  fallback model (a Gemini preview name served as another Gemini preview is an alias, not a fallback).
56
  <b>Output per trial</b>: the median trial's output tokens, one scale across rows; the <span class="ob-key"></span> part has no text in
57
  the trace: reasoning you can't read. <b>Cost</b>: recorded trial costs, uncosted trials priced from their tokens at the row's own
58
+ rates, and reasoning left out of the output tokens; ≥ where some trials can't be priced (no usage, or no rates that fit the recorded costs).</p>
59
  </section>
60
 
61
  <section id="gaps" class="block">
row.js CHANGED
@@ -12,7 +12,7 @@
12
  const hubScript = document.createElement("script");
13
  hubScript.src = rowPath(r) + "hub.js";
14
  document.head.append(hubScript);
15
- document.title = "#" + r.rank + " " + title(r) + " · " + r.agent + " · TB2.1 annotated";
16
  const h = r.scan.holes, n = r.scan.trials;
17
  const full = Object.values(trialInfo[r.id]).filter(t => t.h).length;
18
 
@@ -61,13 +61,15 @@
61
 
62
  /* Header. */
63
  const head = el("header", "row-head");
64
- head.append(el("p", "kicker", "#" + r.rank + " on the Terminal-Bench 2.1 leaderboard · " + (r.date || "")));
 
65
  head.append(el("h1", null, title(r)));
66
  const meta = el("p", "row-meta");
67
  meta.append(el("span", "harn", r.agent), el("span", null, [r.modelOrg].filter(Boolean).join(" · ")));
68
- meta.append(link(r.source, "Submission ↗", "ext"));
69
  r.jobs.forEach(j => meta.append(link("https://hub.harborframework.com/jobs/" + j, "Hub job ↗", "ext")));
70
  head.append(meta);
 
71
  const probs = problems(r);
72
  if (probs.length) { const c = el("p", "flags"); probs.forEach(p => c.append(problemChip(p))); head.append(c); }
73
 
@@ -156,7 +158,7 @@
156
  stat("Our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate),
157
  [ce.imputedTrials ? "+" + money(ce.imputed) + " for " + ce.imputedTrials + " uncosted trials, from their tokens" : "",
158
  ce.unbilled ? "+" + money(ce.unbilled) + " unbilled reasoning" : "",
159
- ce.noUsage ? ce.noUsage + " trials with no usage: not priced" : ""].filter(Boolean).join(" · ") || "nothing to add"));
160
  cost.append(cstats);
161
  if (ce.rates) cost.append(el("p", "muted small", "Rates fitted to this row's own trial costs and tokens: $" + ce.rates.uncached + " / $" + ce.rates.cached +
162
  " / $" + ce.rates.output + " per million uncached input / cached input / output tokens (R² " + ce.rates.r2 + ")."));
 
12
  const hubScript = document.createElement("script");
13
  hubScript.src = rowPath(r) + "hub.js";
14
  document.head.append(hubScript);
15
+ document.title = "#" + r.rank + " " + title(r) + " · " + r.agent + " · " + (r.pr ? "TB2.1 submissions" : "TB2.1 annotated");
16
  const h = r.scan.holes, n = r.scan.trials;
17
  const full = Object.values(trialInfo[r.id]).filter(t => t.h).length;
18
 
 
61
 
62
  /* Header. */
63
  const head = el("header", "row-head");
64
+ head.append(el("p", "kicker", (r.pr ? "Terminal-Bench 2.1 submission · PR #" + r.pr + " (" + r.prState.toLowerCase() + ")"
65
+ : "#" + r.rank + " on the Terminal-Bench 2.1 leaderboard") + " · " + (r.date || "")));
66
  head.append(el("h1", null, title(r)));
67
  const meta = el("p", "row-meta");
68
  meta.append(el("span", "harn", r.agent), el("span", null, [r.modelOrg].filter(Boolean).join(" · ")));
69
+ meta.append(link(r.source, r.pr ? "PR #" + r.pr + " ↗" : "Submission ↗", "ext"));
70
  r.jobs.forEach(j => meta.append(link("https://hub.harborframework.com/jobs/" + j, "Hub job ↗", "ext")));
71
  head.append(meta);
72
+ if (r.why) head.append(el("p", "why-line", r.why));
73
  const probs = problems(r);
74
  if (probs.length) { const c = el("p", "flags"); probs.forEach(p => c.append(problemChip(p))); head.append(c); }
75
 
 
158
  stat("Our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate),
159
  [ce.imputedTrials ? "+" + money(ce.imputed) + " for " + ce.imputedTrials + " uncosted trials, from their tokens" : "",
160
  ce.unbilled ? "+" + money(ce.unbilled) + " unbilled reasoning" : "",
161
+ ce.unpriced ? ce.unpriced + " trials unpriced: no token usage" + (ce.rates ? "" : ", or no rates fit the recorded costs") : ""].filter(Boolean).join(" · ") || "nothing to add"));
162
  cost.append(cstats);
163
  if (ce.rates) cost.append(el("p", "muted small", "Rates fitted to this row's own trial costs and tokens: $" + ce.rates.uncached + " / $" + ce.rates.cached +
164
  " / $" + ce.rates.output + " per million uncached input / cached input / output tokens (R² " + ce.rates.r2 + ")."));