Spaces:
Running
Running
| /* Shared by the TB2.1 annotated pages: data access, links, chips and theme. */ | |
| (function () { | |
| "use strict"; | |
| const D = window.TB21; | |
| const el = (tag, cls, text) => { | |
| const n = document.createElement(tag); | |
| if (cls) n.className = cls; | |
| if (text !== undefined && text !== null) n.textContent = text; | |
| return n; | |
| }; | |
| const link = (href, text, cls) => { const a = el("a", cls, text); a.href = href; return a; }; | |
| const pct0 = x => (x === null || x === undefined ? "–" : Math.round(100 * x) + "%"); | |
| const fmt1 = x => (x === null || x === undefined ? "–" : Number(x).toFixed(1)); | |
| const money = x => (x === null || x === undefined ? "–" : "$" + Number(x).toLocaleString("en-US", { maximumFractionDigits: 0 })); | |
| const shortModel = m => m.replace(/^.*\//, ""); | |
| const rowPath = r => "rows/" + encodeURIComponent(r.id) + "/"; | |
| const rowPage = r => "row.html?r=" + encodeURIComponent(r.id); | |
| const title = r => r.model + " · " + r.effort; | |
| const byId = Object.fromEntries(D.rows.map(r => [r.id, r])); | |
| const OUTCOME = { "1": "rewarded", "0": "failed", t: "timed out", e: "errored" }; | |
| const PRIORITY = { c: "critical", h: "high", m: "medium", l: "low", i: "info", n: "none" }; | |
| /* Trajectory links: the full chronology where this site publishes it (trials with | |
| highlight excerpts), otherwise the findings-only viewer of every trial. */ | |
| const viewerHref = (r, vid, finding) => rowPath(r) + "viewer/index.html#trial=" + encodeURIComponent(vid) + | |
| (finding !== null && finding !== undefined ? "&finding=" + encodeURIComponent(finding) + "&loc=0" : ""); | |
| const fullHref = (r, vid) => rowPath(r) + "trajectories/index.html#trial=" + encodeURIComponent(vid); | |
| const cellHref = (r, c) => (c.h ? fullHref(r, c.v) : viewerHref(r, c.v)); | |
| const trialInfo = {}; | |
| D.rows.forEach(r => { | |
| const info = trialInfo[r.id] = {}; | |
| Object.entries(r.tasks).forEach(([task, cells]) => cells.forEach((c, i) => { | |
| if (c.v !== null && c.v !== undefined) info[c.v] = { task, attempt: i + 1, ...c }; | |
| })); | |
| }); | |
| /* Reasoning and visibility. */ | |
| /* Why output tokens have no text in the trace. */ | |
| const RZ = { withheld: "withheld by provider", notKept: "not kept by harness", lost: "lost in conversion", | |
| summary: "summary only", complete: "reasoning recorded", noReasoning: "no reasoning recorded", unknown: "no token counts" }; | |
| const RZ_HELP = { | |
| withheld: "the model's API returns no reasoning text (encrypted, or thinking blocks with empty text): generated and billed, never readable", | |
| notKept: "the provider returns reasoning text, but the harness didn't record it", | |
| lost: "the harness's own session files hold the reasoning; converting them to the trajectory dropped it", | |
| summary: "the provider returns a summary of the reasoning, far shorter than its tokens", | |
| complete: "reasoning text is in the trace; for most trials the recorded text accounts for the output tokens", | |
| noReasoning: "no reasoning text or reasoning tokens in the trace; for most trials the recorded text accounts for the output tokens", | |
| unknown: "the trials report no output tokens to account for", | |
| }; | |
| const visibleShare = h => (h.text.unseen === null ? null : 1 - h.text.unseen); | |
| const reasoningChip = h => { | |
| const chip = el("span", "rz rz--" + h.text.why, RZ[h.text.why]); | |
| chip.title = RZ_HELP[h.text.why]; | |
| return chip; | |
| }; | |
| /* Problems worth a chip on the leaderboard: only what's actually wrong. */ | |
| function problems(r) { | |
| const h = r.scan.holes, n = r.scan.trials, out = []; | |
| const m = h.models; | |
| if (m.mismatch) out.push({ level: "bad", text: "served " + shortModel(m.served[0].model), key: "models", | |
| help: "configured " + Object.keys(m.configured).join(", ") + "; the steps ran " + m.served.map(s => shortModel(s.model) + " " + pct0(s.share)).join(", ") }); | |
| else if (m.foreignTrials) out.push({ level: "warn", text: "fallback model · " + m.foreignTrials + " trials", key: "models", | |
| help: pct0(m.foreignStepShare) + " of steps ran " + m.served.filter(s => !s.configured).map(s => shortModel(s.model)).join(", ") }); | |
| const costOff = h.lbCost && Math.abs(h.costSum - h.lbCost) / h.lbCost > 0.02; | |
| if (h.text.uncounted) out.push({ level: "bad", text: "output tokens ≥" + pct0(h.text.uncounted) + " under", key: "cost", | |
| help: "recorded reasoning isn't in the reported output tokens, so the cost leaves it out" }); | |
| if (costOff || h.uncosted / n > 0.01) out.push({ level: "warn", text: costOff ? "trial costs ≠ total" : h.uncosted + " uncosted", key: "cost", | |
| help: "trial costs sum to " + money(h.costSum) + " against the leaderboard's " + money(h.lbCost) + "; " + h.uncosted + " trials record no cost" }); | |
| if (h.webTrials >= 10 && h.webUnrecorded / h.webTrials > 0.5) out.push({ level: "warn", text: "web results missing", key: "trace", | |
| help: h.webUnrecorded + " of " + h.webTrials + " trials with web activity hold no web results" }); | |
| if (h.compacted / n > 0.02) out.push({ level: "warn", text: "compacted · " + h.compacted, key: "trace", help: "history compacted into a summary" }); | |
| if (h.noTrajectory / n > 0.02) out.push({ level: "warn", text: h.noTrajectory + " without trajectory", key: "trace", help: "listed trials whose trajectory is missing" }); | |
| return out; | |
| } | |
| const problemChip = p => { const c = el("span", "fl fl--" + (p.level === "bad" ? "bad" : "warn"), p.text); c.title = p.help; return c; }; | |
| /* The attempt strip: one column per task, five squares each. */ | |
| function strip(r, opts = {}) { | |
| const s = el("div", "strip" + (opts.large ? " strip--lg" : "")); | |
| D.tasks.forEach(task => { | |
| const col = el("div", "col"); | |
| (r.tasks[task] || []).forEach((c, i) => { | |
| const marked = opts.leadsOnly ? (c.c === "1" && (c.s === "c" || c.s === "h")) : "chm".includes(c.s); | |
| const cls = "sq sq--" + c.c + (marked ? " sq--" + c.s : "") + (c.d ? " sq--disc" : "") + (c.f ? " sq--fb" : "") + (c.n ? " sq--nt" : ""); | |
| const sq = el(c.v !== null && c.v !== undefined ? "a" : "span", cls); | |
| if (c.v !== null && c.v !== undefined) sq.href = cellHref(r, c); | |
| sq.dataset.tip = [task, "attempt " + (i + 1) + " · " + OUTCOME[c.c] + (c.d ? " · not counted in our estimate" : ""), | |
| "highest finding: " + PRIORITY[c.s], ...(c.f ? ["steps by a fallback model"] : []), ...(c.n ? ["no trajectory"] : []), | |
| c.h ? "opens the full trajectory" : "opens the findings viewer"].join("\n"); | |
| col.append(sq); | |
| }); | |
| s.append(col); | |
| }); | |
| return s; | |
| } | |
| /* One tooltip for every [data-tip], positioned at the pointer and hidden on anything else. */ | |
| function tooltips() { | |
| const tip = el("div", "tip"); | |
| tip.hidden = true; | |
| document.body.append(tip); | |
| const hide = () => { tip.hidden = true; }; | |
| document.addEventListener("pointerover", e => { | |
| const t = e.target.closest && e.target.closest("[data-tip]"); | |
| if (!t) { hide(); return; } | |
| const [head, ...rest] = t.dataset.tip.split("\n"); | |
| tip.replaceChildren(el("strong", null, head), ...rest.map(l => el("span", null, l))); | |
| tip.hidden = false; | |
| const x = Math.min(e.clientX + 14, innerWidth - tip.offsetWidth - 8); | |
| const y = e.clientY + 16 + tip.offsetHeight > innerHeight ? e.clientY - tip.offsetHeight - 10 : e.clientY + 16; | |
| tip.style.left = x + "px"; tip.style.top = y + "px"; | |
| }); | |
| ["scroll", "wheel", "touchstart", "blur", "pointerleave"].forEach(ev => addEventListener(ev, hide, { passive: true })); | |
| document.addEventListener("pointerleave", hide); | |
| } | |
| function theme() { | |
| const btn = document.getElementById("theme"); | |
| const set = t => { | |
| document.documentElement.dataset.theme = t; | |
| btn.textContent = t === "dark" ? "Light" : "Dark"; | |
| try { localStorage.setItem("tb21-theme", t); } catch (e) { /* private mode */ } | |
| }; | |
| let saved = null; | |
| try { saved = localStorage.getItem("tb21-theme"); } catch (e) { /* private mode */ } | |
| set(saved || (matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light")); | |
| btn.addEventListener("click", () => set(document.documentElement.dataset.theme === "dark" ? "light" : "dark")); | |
| } | |
| const sc = D.scanner; | |
| const scannerLabel = "atif-scan " + (sc.tag || sc.describe || sc.versions.join(", ")) + " (" + sc.commit.slice(0, 7) + ")"; | |
| /* What the output tokens went on: tool-call arguments, messages, reasoning the trace shows, | |
| reasoning it doesn't. Full width: shares, not sizes. */ | |
| const MIX = [["tool", "tool calls"], ["message", "messages"], ["reasoning", "reasoning"], ["hidden", "no text in trace"]]; | |
| function mixBar(t, large) { | |
| const wrap = el("span", "mix" + (large ? " mix--lg" : "")); | |
| if (!t.mix) { wrap.append(el("span", "z", "no token counts")); return wrap; } | |
| const bar = el("span", "mix-bar"); | |
| MIX.forEach(([k]) => { const i = el("i", "mx mx--" + k); i.style.width = (100 * t.mix[k]) + "%"; bar.append(i); }); | |
| const lab = el("span", "mix-lab"); | |
| lab.append(el("b", null, pct0(1 - t.mix.hidden)), document.createTextNode(" in trace")); | |
| wrap.append(bar, lab); | |
| wrap.title = MIX.map(([k, name]) => name + " " + pct0(t.mix[k])).join(" · ") + " of output tokens" + | |
| (t.mix.hidden ? " (no text: " + (t.measured ? "counted from reported reasoning tokens" : "inferred from the recorded text") + ")" : ""); | |
| return wrap; | |
| } | |
| const mixKey = () => { | |
| const k = el("span", "mix-key"); | |
| MIX.forEach(([key, name]) => k.append(el("i", "mx mx--" + key), document.createTextNode(name + " "))); | |
| return k; | |
| }; | |
| window.TB = { D, el, link, pct0, fmt1, money, shortModel, rowPath, rowPage, title, byId, OUTCOME, PRIORITY, | |
| viewerHref, fullHref, cellHref, trialInfo, RZ, RZ_HELP, visibleShare, reasoningChip, problems, problemChip, | |
| strip, tooltips, theme, scannerLabel, mixBar, mixKey }; | |
| document.addEventListener("DOMContentLoaded", () => { | |
| theme(); tooltips(); | |
| document.querySelectorAll("[data-scanner]").forEach(n => { n.textContent = scannerLabel + " · built " + D.generated; }); | |
| }); | |
| })(); | |