/* A row's page: the hub between the leaderboard and atif-scan's highlights and trajectories. */ (function () { "use strict"; const T = window.TB; const { D, el, link, pct0, fmt1, money, shortModel, rowPath, title, byId, OUTCOME, trialInfo, viewerHref, fullHref, reasoningChip, RZ, RZ_HELP, visibleShare, problems, problemChip, strip } = T; const DETAILS = window.TB21_DETAILS || {}; const compactN = n => (n >= 1e6 ? (n / 1e6).toFixed(1) + "M" : n >= 1e3 ? Math.round(n / 1e3) + "k" : String(n)); const main = document.getElementById("row"); const r = byId[new URLSearchParams(location.search).get("r")]; if (!r) { main.replaceChildren(el("p", null, "Unknown row. "), link("index.html", "Back to the leaderboard")); return; } const hubScript = document.createElement("script"); hubScript.src = rowPath(r) + "hub.js"; document.head.append(hubScript); document.title = "#" + r.rank + " " + title(r) + " · " + r.agent + " · " + (r.pr ? "TB2.1 submissions" : "TB2.1 annotated"); const h = r.scan.holes, n = r.scan.trials; const full = Object.values(trialInfo[r.id]).filter(t => t.h).length; /* A count that opens the trials behind it. */ function trials(key, label, note) { const items = (DETAILS[r.id] || {})[key] || []; const d = el("details", "trials"); d.id = "k-" + key; const s = el("summary"); s.append(el("span", "label", label), el("span", "count", items.length + (items.length === 1 ? " trial" : " trials"))); d.append(s); d.addEventListener("toggle", () => { if (!d.open || d.querySelector("ol")) return; if (note) d.append(el("p", "muted small", note)); const ol = el("ol", "trial-list"); items.map(([vid, fid]) => ({ vid, fid, t: trialInfo[r.id][vid] || {} })) .sort((a, b) => (a.t.task || "").localeCompare(b.t.task || "") || (a.t.attempt || 0) - (b.t.attempt || 0)) .forEach(({ vid, fid, t }) => { const li = el("li"); li.append(el("i", "sq sq--" + (t.c || "0") + ("chm".includes(t.s) ? " sq--" + t.s : "")), link(viewerHref(r, vid, fid), (t.task || "trial " + vid) + " · " + (t.attempt || "?")), el("small", null, OUTCOME[t.c] || "")); if (t.h) li.append(link(fullHref(r, vid), "full ↗", "full")); const hub = (window.TB21_HUB || {})[vid]; if (hub) { const a = link(hub, "Hub ↗", "full"); a.target = "_blank"; a.rel = "noopener noreferrer"; li.append(a); } ol.append(li); }); d.append(ol); }); return d; } function panel(id, heading, lede) { const p = el("section", "panel"); p.id = id; p.append(el("h2", null, heading)); if (lede) p.append(el("p", "lede-sm", lede)); return p; } function stat(label, value, note, cls) { const s = el("div", "stat" + (cls ? " " + cls : "")); s.append(el("span", "stat-label", label)); if (value instanceof Node) s.append(value); else s.append(el("b", null, value)); if (note) s.append(el("small", null, note)); return s; } /* Header. */ const head = el("header", "row-head"); head.append(el("p", "kicker", (r.pr ? "Terminal-Bench 2.1 submission · PR #" + r.pr + " (" + r.prState.toLowerCase() + ")" : "#" + r.rank + " on the Terminal-Bench 2.1 leaderboard") + " · " + (r.date || ""))); head.append(el("h1", null, title(r))); const meta = el("p", "row-meta"); meta.append(el("span", "harn", r.agent), el("span", null, [r.modelOrg].filter(Boolean).join(" · "))); meta.append(link(r.source, r.pr ? "PR #" + r.pr + " ↗" : "Submission ↗", "ext")); r.jobs.forEach(j => meta.append(link("https://hub.harborframework.com/jobs/" + j, "Hub job ↗", "ext"))); head.append(meta); if (r.why) head.append(el("p", "why-line", r.why)); const probs = problems(r); if (probs.length) { const c = el("p", "flags"); probs.forEach(p => c.append(problemChip(p))); head.append(c); } const adj = r.scan.adjusted, tk = h.tokens, ce = h.cost; const recount = r.lb.source === "recount"; const stats = el("div", "stats"); stats.append( stat(recount ? "Score" : "Leaderboard score", fmt1(r.lb.accuracy) + "%", "± " + fmt1(r.lb.ci95) + " · " + r.scan.rewarded + " of " + n + " trials rewarded"), stat("Our estimate", fmt1(adj.accuracy) + "%", adj.rewarded + " of " + n + (adj.leads ? " · less " + adj.leads + (adj.leads === 1 ? " review lead" : " review leads") : "") + (adj.fallback ? " · less " + adj.fallback + (adj.fallback === 1 ? " fallback trial" : " fallback trials") : ""), adj.accuracy < r.lb.accuracy - 1 ? "stat--hot" : ""), stat("Trace completeness", tk.mix ? pct0(1 - tk.mix.hidden) : "–", tk.mix ? "of output tokens have text in the trace · " + RZ[h.text.why] : "no token counts"), stat("Cost, our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate), (!recount && r.lb.cost !== null ? "leaderboard " + money(r.lb.cost) + " · " : "") + "recorded " + money(ce.recorded))); const actions = el("div", "actions"); actions.append(link(rowPath(r) + "highlights/index.html", "Highlights", "btn btn--primary"), link(rowPath(r) + "trajectories/index.html", "Full trajectories · " + full + " trials", "btn"), link(rowPath(r) + "viewer/index.html", "All " + n + " trials · findings view", "btn")); const stripWrap = el("section", "strip-wrap"); stripWrap.append(el("p", "muted small", "Every attempt: a column per task, easiest on the left. Click one to open its trajectory."), strip(r, { large: true })); /* What atif-scan found. */ const found = panel("found", "What atif-scan found", "Medium-or-higher findings not explained by an allowance. Each opens its excerpts."); const hits = Object.entries(D.matrix[r.id] || {}); const families = Object.keys(D.categories).concat([null]); if (!hits.length) found.append(el("p", "muted", "No medium-or-higher findings.")); families.forEach(cat => { const rows = hits.filter(([id]) => D.checks[id].category === cat).sort((a, b) => b[1].trials - a[1].trials); if (!rows.length) return; found.append(el("h3", null, cat ? D.categories[cat] : "Other")); const ul = el("ul", "checks"); rows.forEach(([id, v]) => { const c = D.checks[id], li = el("li"); li.append(el("span", "sev sev--" + c.severity, c.severity[0].toUpperCase()), link(rowPath(r) + "highlights/index.html#check=" + encodeURIComponent(id), c.title), el("span", "count", v.trials + (v.rewarded ? " · " + v.rewarded + " rewarded" : ""))); ul.append(li); }); found.append(ul); }); const quiet = Object.entries(r.scan.categories).filter(([k]) => !hits.some(([id]) => D.checks[id].category === k)); if (quiet.length) found.append(el("p", "muted small", "At low priority only: " + quiet.map(([k, v]) => D.categories[k] + " " + v.trials).join(" · ") + " trials.")); /* What ran. */ const ran = panel("ran", "What ran", "Models by share of the trajectories' steps."); const m = h.models; const models = el("ul", "models"); m.served.filter(x => x.share >= 0.001).forEach(x => { const li = el("li", x.configured ? "" : "off"); const bar = el("span", "bar"); const fill = el("i"); fill.style.width = Math.max(1, Math.round(100 * x.share)) + "%"; bar.append(fill); li.append(el("code", null, shortModel(x.model)), bar, el("span", "count", pct0(x.share) + " · " + x.trials + " trials")); models.append(li); }); ran.append(models, el("p", "muted small", "Configured: " + Object.keys(m.configured).join(", "))); if (m.foreignTrials) ran.append(trials("foreign", m.mismatch ? "Served by another model" : "Trials with fallback steps")); /* What the trace holds. */ const trace = panel("trace", "What the trace holds"); const rz = el("p", "rz-line"); rz.append(reasoningChip(h), document.createTextNode(": " + RZ_HELP[h.text.why] + ".")); trace.append(rz); const tstats = el("div", "stats stats--sm"); tstats.append( stat("Output tokens, median trial", tk.outputMedian ? compactN(tk.outputMedian) : "–", tk.outputTotal ? compactN(tk.outputTotal) + " over the row" : ""), stat("No text in trace", tk.mix ? pct0(tk.mix.hidden) : "–", tk.measured ? "counted: reasoning tokens reported, no text" : "inferred: recorded text at ≈3.2 chars/token"), stat("Text per output token", h.text.allCpt === null ? "–" : String(h.text.allCpt), "reasoning text in " + pct0(h.text.withText) + " of trials")); trace.append(tstats); if (tk.mix) { const mx = el("div", "mix-row"); mx.append(T.mixBar(tk, true), T.mixKey()); trace.append(mx); } if (h.sourceNote) { const q = el("p", "source-note"); q.append(el("b", null, "Source session files: "), document.createTextNode(h.sourceNote)); trace.append(q); } const HOLES = [ ["unseen", "Most output has no text in the trace", "Trials whose recorded text covers under 80% of their output tokens."], ["compacted", "History compacted"], ["webUnrecorded", "Web results not in the trace"], ["webUnresolved", "Web provenance unresolved"], ["unpaired", "Tool results not paired to calls"], ["callsMissing", "Model calls missing from the steps"], ["endsOnCall", "Ends on an unanswered tool call"], ["noTrajectory", "No trajectory"], ["timestamps", "Timestamps missing or out of order"], ]; const holeList = el("div", "holes"); HOLES.forEach(([key, label, note]) => { if (h[key]) holeList.append(trials(key, label, note)); }); trace.append(holeList); trace.append(el("p", "muted small", h.evidenceIncomplete + " of " + n + " trials have at least one check atif-scan couldn't decide: unknown, never clean.")); /* What it cost. */ const cost = panel("cost", "What it cost"); const cstats = el("div", "stats stats--sm"); if (!recount && r.lb.cost !== null) cstats.append(stat("Leaderboard", money(r.lb.cost))); cstats.append(stat("Recorded, sum of trials", money(ce.recorded), h.uncosted ? h.uncosted + " trials uncosted" : "every trial costed"), stat("Our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate), [ce.imputedTrials ? "+" + money(ce.imputed) + " for " + ce.imputedTrials + " uncosted trials, from their tokens" : "", ce.unbilled ? "+" + money(ce.unbilled) + " undercounted reasoning" : "", ce.unpriced ? ce.unpriced + " trials unpriced: no token usage" + (ce.rates ? "" : ", or no rates fit the recorded costs") : ""].filter(Boolean).join(" · ") || "nothing to add")); cost.append(cstats); if (ce.rates) cost.append(el("p", "muted small", "Rates fitted to this row's own trial costs and tokens: $" + ce.rates.uncached + " / $" + ce.rates.cached + " / $" + ce.rates.output + " per million uncached input / cached input / output tokens (R² " + ce.rates.r2 + ").")); if (h.text.uncounted) cost.append(trials("uncounted", "Reasoning left out of output tokens", "The visible text alone fills the reported output tokens, so the recorded reasoning is on top: at ≈3.2 characters per token, at least " + pct0(h.text.uncounted) + " more output than reported, and so left out of the cost.")); if (h.uncosted) cost.append(trials("uncosted", "Trials without a recorded cost")); // Two columns that fill independently: findings on the left; what ran, the trace and the cost on the right. const grid = el("div", "panels"); const left = el("div", "pcol"), right = el("div", "pcol"); left.append(found); right.append(ran, trace, cost); grid.append(left, right); main.replaceChildren(head, stats, actions, stripWrap, grid); // #k= (from the home page's tables) opens that list of trials; #found etc. scroll to a panel. const want = new URLSearchParams(location.hash.slice(1)).get("k"); const target = want ? document.getElementById("k-" + want) : location.hash ? document.getElementById(location.hash.slice(1)) : null; if (target) { if (target.tagName === "DETAILS") { target.open = true; target.dispatchEvent(new Event("toggle")); } requestAnimationFrame(() => target.scrollIntoView({ block: "start" })); } })();