Spaces:
Running
Running
Cost estimate: only a fit that reproduces recorded costs prices trials; unpriced wording; PR-aware row page
Browse files- data/leaderboard.js +0 -0
- home.js +4 -3
- index.html +1 -1
- row.js +6 -4
data/leaderboard.js
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
home.js
CHANGED
|
@@ -47,8 +47,9 @@
|
|
| 47 |
else if (m.foreignTrials) out.push(["warn", "fallback " + m.served.filter(s => !s.configured).map(s => shortModel(s.model).replace(/^claude-/, "")).join(", ") + " in " + m.foreignTrials + " trials"]);
|
| 48 |
const c = h.cost;
|
| 49 |
if (c.imputedTrials) out.push(["warn", c.imputedTrials + " trials priced from tokens"]);
|
| 50 |
-
if (c.
|
| 51 |
if (c.unbilled) out.push(["bad", "reasoning unbilled"]);
|
|
|
|
| 52 |
return out;
|
| 53 |
}
|
| 54 |
const hasLb = D.rows.some(r => r.lb.source !== "recount");
|
|
@@ -205,11 +206,11 @@
|
|
| 205 |
tr.append(start(el("td", "n", money(c.recorded))));
|
| 206 |
const est = el("td", "n" + (c.estimate && c.estimate > c.recorded * 1.02 ? " bad" : ""));
|
| 207 |
est.append(document.createTextNode(c.estimate === null ? "–" : (c.lowerBound ? "≥" : "") + money(c.estimate)));
|
| 208 |
-
if (c.imputedTrials || c.unbilled || c.
|
| 209 |
const parts = [];
|
| 210 |
if (c.imputedTrials) parts.push("+" + money(c.imputed) + " " + c.imputedTrials + " priced");
|
| 211 |
if (c.unbilled) parts.push("+" + money(c.unbilled) + " unbilled");
|
| 212 |
-
if (c.
|
| 213 |
est.append(el("small", null, parts.join(" · ")));
|
| 214 |
}
|
| 215 |
tr.append(est);
|
|
|
|
| 47 |
else if (m.foreignTrials) out.push(["warn", "fallback " + m.served.filter(s => !s.configured).map(s => shortModel(s.model).replace(/^claude-/, "")).join(", ") + " in " + m.foreignTrials + " trials"]);
|
| 48 |
const c = h.cost;
|
| 49 |
if (c.imputedTrials) out.push(["warn", c.imputedTrials + " trials priced from tokens"]);
|
| 50 |
+
if (c.unpriced / r.scan.trials > 0.02) out.push(["note", c.unpriced + " trials unpriced"]);
|
| 51 |
if (c.unbilled) out.push(["bad", "reasoning unbilled"]);
|
| 52 |
+
if (r.lb.disqualified) out.push(["bad", r.lb.disqualified + " disqualified by the PR pipeline"]);
|
| 53 |
return out;
|
| 54 |
}
|
| 55 |
const hasLb = D.rows.some(r => r.lb.source !== "recount");
|
|
|
|
| 206 |
tr.append(start(el("td", "n", money(c.recorded))));
|
| 207 |
const est = el("td", "n" + (c.estimate && c.estimate > c.recorded * 1.02 ? " bad" : ""));
|
| 208 |
est.append(document.createTextNode(c.estimate === null ? "–" : (c.lowerBound ? "≥" : "") + money(c.estimate)));
|
| 209 |
+
if (c.imputedTrials || c.unbilled || c.unpriced) {
|
| 210 |
const parts = [];
|
| 211 |
if (c.imputedTrials) parts.push("+" + money(c.imputed) + " " + c.imputedTrials + " priced");
|
| 212 |
if (c.unbilled) parts.push("+" + money(c.unbilled) + " unbilled");
|
| 213 |
+
if (c.unpriced) parts.push(c.unpriced + " unpriced");
|
| 214 |
est.append(el("small", null, parts.join(" · ")));
|
| 215 |
}
|
| 216 |
tr.append(est);
|
index.html
CHANGED
|
@@ -55,7 +55,7 @@
|
|
| 55 |
fallback model (a Gemini preview name served as another Gemini preview is an alias, not a fallback).
|
| 56 |
<b>Output per trial</b>: the median trial's output tokens, one scale across rows; the <span class="ob-key"></span> part has no text in
|
| 57 |
the trace: reasoning you can't read. <b>Cost</b>: recorded trial costs, uncosted trials priced from their tokens at the row's own
|
| 58 |
-
rates, and reasoning left out of the output tokens; ≥ where some trials
|
| 59 |
</section>
|
| 60 |
|
| 61 |
<section id="gaps" class="block">
|
|
|
|
| 55 |
fallback model (a Gemini preview name served as another Gemini preview is an alias, not a fallback).
|
| 56 |
<b>Output per trial</b>: the median trial's output tokens, one scale across rows; the <span class="ob-key"></span> part has no text in
|
| 57 |
the trace: reasoning you can't read. <b>Cost</b>: recorded trial costs, uncosted trials priced from their tokens at the row's own
|
| 58 |
+
rates, and reasoning left out of the output tokens; ≥ where some trials can't be priced (no usage, or no rates that fit the recorded costs).</p>
|
| 59 |
</section>
|
| 60 |
|
| 61 |
<section id="gaps" class="block">
|
row.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
| 12 |
const hubScript = document.createElement("script");
|
| 13 |
hubScript.src = rowPath(r) + "hub.js";
|
| 14 |
document.head.append(hubScript);
|
| 15 |
-
document.title = "#" + r.rank + " " + title(r) + " · " + r.agent + " · TB2.1 annotated";
|
| 16 |
const h = r.scan.holes, n = r.scan.trials;
|
| 17 |
const full = Object.values(trialInfo[r.id]).filter(t => t.h).length;
|
| 18 |
|
|
@@ -61,13 +61,15 @@
|
|
| 61 |
|
| 62 |
/* Header. */
|
| 63 |
const head = el("header", "row-head");
|
| 64 |
-
head.append(el("p", "kicker",
|
|
|
|
| 65 |
head.append(el("h1", null, title(r)));
|
| 66 |
const meta = el("p", "row-meta");
|
| 67 |
meta.append(el("span", "harn", r.agent), el("span", null, [r.modelOrg].filter(Boolean).join(" · ")));
|
| 68 |
-
meta.append(link(r.source, "Submission ↗", "ext"));
|
| 69 |
r.jobs.forEach(j => meta.append(link("https://hub.harborframework.com/jobs/" + j, "Hub job ↗", "ext")));
|
| 70 |
head.append(meta);
|
|
|
|
| 71 |
const probs = problems(r);
|
| 72 |
if (probs.length) { const c = el("p", "flags"); probs.forEach(p => c.append(problemChip(p))); head.append(c); }
|
| 73 |
|
|
@@ -156,7 +158,7 @@
|
|
| 156 |
stat("Our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate),
|
| 157 |
[ce.imputedTrials ? "+" + money(ce.imputed) + " for " + ce.imputedTrials + " uncosted trials, from their tokens" : "",
|
| 158 |
ce.unbilled ? "+" + money(ce.unbilled) + " unbilled reasoning" : "",
|
| 159 |
-
ce.
|
| 160 |
cost.append(cstats);
|
| 161 |
if (ce.rates) cost.append(el("p", "muted small", "Rates fitted to this row's own trial costs and tokens: $" + ce.rates.uncached + " / $" + ce.rates.cached +
|
| 162 |
" / $" + ce.rates.output + " per million uncached input / cached input / output tokens (R² " + ce.rates.r2 + ")."));
|
|
|
|
| 12 |
const hubScript = document.createElement("script");
|
| 13 |
hubScript.src = rowPath(r) + "hub.js";
|
| 14 |
document.head.append(hubScript);
|
| 15 |
+
document.title = "#" + r.rank + " " + title(r) + " · " + r.agent + " · " + (r.pr ? "TB2.1 submissions" : "TB2.1 annotated");
|
| 16 |
const h = r.scan.holes, n = r.scan.trials;
|
| 17 |
const full = Object.values(trialInfo[r.id]).filter(t => t.h).length;
|
| 18 |
|
|
|
|
| 61 |
|
| 62 |
/* Header. */
|
| 63 |
const head = el("header", "row-head");
|
| 64 |
+
head.append(el("p", "kicker", (r.pr ? "Terminal-Bench 2.1 submission · PR #" + r.pr + " (" + r.prState.toLowerCase() + ")"
|
| 65 |
+
: "#" + r.rank + " on the Terminal-Bench 2.1 leaderboard") + " · " + (r.date || "")));
|
| 66 |
head.append(el("h1", null, title(r)));
|
| 67 |
const meta = el("p", "row-meta");
|
| 68 |
meta.append(el("span", "harn", r.agent), el("span", null, [r.modelOrg].filter(Boolean).join(" · ")));
|
| 69 |
+
meta.append(link(r.source, r.pr ? "PR #" + r.pr + " ↗" : "Submission ↗", "ext"));
|
| 70 |
r.jobs.forEach(j => meta.append(link("https://hub.harborframework.com/jobs/" + j, "Hub job ↗", "ext")));
|
| 71 |
head.append(meta);
|
| 72 |
+
if (r.why) head.append(el("p", "why-line", r.why));
|
| 73 |
const probs = problems(r);
|
| 74 |
if (probs.length) { const c = el("p", "flags"); probs.forEach(p => c.append(problemChip(p))); head.append(c); }
|
| 75 |
|
|
|
|
| 158 |
stat("Our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate),
|
| 159 |
[ce.imputedTrials ? "+" + money(ce.imputed) + " for " + ce.imputedTrials + " uncosted trials, from their tokens" : "",
|
| 160 |
ce.unbilled ? "+" + money(ce.unbilled) + " unbilled reasoning" : "",
|
| 161 |
+
ce.unpriced ? ce.unpriced + " trials unpriced: no token usage" + (ce.rates ? "" : ", or no rates fit the recorded costs") : ""].filter(Boolean).join(" · ") || "nothing to add"));
|
| 162 |
cost.append(cstats);
|
| 163 |
if (ce.rates) cost.append(el("p", "muted small", "Rates fitted to this row's own trial costs and tokens: $" + ce.rates.uncached + " / $" + ce.rates.cached +
|
| 164 |
" / $" + ce.rates.output + " per million uncached input / cached input / output tokens (R² " + ce.rates.r2 + ")."));
|