/* Shared by the TB2.1 annotated pages: data access, links, chips and theme. */ (function () { "use strict"; const D = window.TB21; const el = (tag, cls, text) => { const n = document.createElement(tag); if (cls) n.className = cls; if (text !== undefined && text !== null) n.textContent = text; return n; }; const link = (href, text, cls) => { const a = el("a", cls, text); a.href = href; return a; }; const pct0 = x => (x === null || x === undefined ? "–" : Math.round(100 * x) + "%"); const fmt1 = x => (x === null || x === undefined ? "–" : Number(x).toFixed(1)); const money = x => (x === null || x === undefined ? "–" : "$" + Number(x).toLocaleString("en-US", { maximumFractionDigits: 0 })); const shortModel = m => m.replace(/^.*\//, ""); const rowPath = r => "rows/" + encodeURIComponent(r.id) + "/"; const rowPage = r => "row.html?r=" + encodeURIComponent(r.id); const title = r => r.model + " · " + r.effort; const byId = Object.fromEntries(D.rows.map(r => [r.id, r])); const OUTCOME = { "1": "rewarded", "0": "failed", t: "timed out", e: "errored" }; const PRIORITY = { c: "critical", h: "high", m: "medium", l: "low", i: "info", n: "none" }; /* Trajectory links: the full chronology where this site publishes it (trials with highlight excerpts), otherwise the findings-only viewer of every trial. */ const viewerHref = (r, vid, finding) => rowPath(r) + "viewer/index.html#trial=" + encodeURIComponent(vid) + (finding !== null && finding !== undefined ? "&finding=" + encodeURIComponent(finding) + "&loc=0" : ""); const fullHref = (r, vid) => rowPath(r) + "trajectories/index.html#trial=" + encodeURIComponent(vid); const cellHref = (r, c) => (c.h ? fullHref(r, c.v) : viewerHref(r, c.v)); const trialInfo = {}; D.rows.forEach(r => { const info = trialInfo[r.id] = {}; Object.entries(r.tasks).forEach(([task, cells]) => cells.forEach((c, i) => { if (c.v !== null && c.v !== undefined) info[c.v] = { task, attempt: i + 1, ...c }; })); }); /* Reasoning and visibility. */ /* Why output tokens have no text in the trace. */ const RZ = { withheld: "withheld by provider", notKept: "not kept by harness", lost: "lost in conversion", summary: "summary only", complete: "reasoning recorded", noReasoning: "no reasoning recorded", unknown: "no token counts" }; const RZ_HELP = { withheld: "the model's API returns no reasoning text (encrypted, or thinking blocks with empty text): generated and billed, never readable", notKept: "the provider returns reasoning text, but the harness didn't record it", lost: "the harness's own session files hold the reasoning; converting them to the trajectory dropped it", summary: "the provider returns a summary of the reasoning, far shorter than its tokens", complete: "reasoning text is in the trace; for most trials the recorded text accounts for the output tokens", noReasoning: "no reasoning text or reasoning tokens in the trace; for most trials the recorded text accounts for the output tokens", unknown: "the trials report no output tokens to account for", }; const visibleShare = h => (h.text.unseen === null ? null : 1 - h.text.unseen); const reasoningChip = h => { const chip = el("span", "rz rz--" + h.text.why, RZ[h.text.why]); chip.title = RZ_HELP[h.text.why]; return chip; }; /* Problems worth a chip on the leaderboard: only what's actually wrong. */ function problems(r) { const h = r.scan.holes, n = r.scan.trials, out = []; const m = h.models; if (m.mismatch) out.push({ level: "bad", text: "served " + shortModel(m.served[0].model), key: "models", help: "configured " + Object.keys(m.configured).join(", ") + "; the steps ran " + m.served.map(s => shortModel(s.model) + " " + pct0(s.share)).join(", ") }); else if (m.foreignTrials) out.push({ level: "warn", text: "fallback model · " + m.foreignTrials + " trials", key: "models", help: pct0(m.foreignStepShare) + " of steps ran " + m.served.filter(s => !s.configured).map(s => shortModel(s.model)).join(", ") }); const costOff = h.lbCost && Math.abs(h.costSum - h.lbCost) / h.lbCost > 0.02; if (h.text.uncounted) out.push({ level: "bad", text: "output tokens ≥" + pct0(h.text.uncounted) + " under", key: "cost", help: "recorded reasoning isn't in the reported output tokens, so the cost leaves it out" }); if (costOff || h.uncosted / n > 0.01) out.push({ level: "warn", text: costOff ? "trial costs ≠ total" : h.uncosted + " uncosted", key: "cost", help: "trial costs sum to " + money(h.costSum) + " against the leaderboard's " + money(h.lbCost) + "; " + h.uncosted + " trials record no cost" }); if (h.webTrials >= 10 && h.webUnrecorded / h.webTrials > 0.5) out.push({ level: "warn", text: "web results missing", key: "trace", help: h.webUnrecorded + " of " + h.webTrials + " trials with web activity hold no web results" }); if (h.compacted / n > 0.02) out.push({ level: "warn", text: "compacted · " + h.compacted, key: "trace", help: "history compacted into a summary" }); if (h.noTrajectory / n > 0.02) out.push({ level: "warn", text: h.noTrajectory + " without trajectory", key: "trace", help: "listed trials whose trajectory is missing" }); return out; } const problemChip = p => { const c = el("span", "fl fl--" + (p.level === "bad" ? "bad" : "warn"), p.text); c.title = p.help; return c; }; /* The attempt strip: one column per task, five squares each. */ function strip(r, opts = {}) { const s = el("div", "strip" + (opts.large ? " strip--lg" : "")); D.tasks.forEach(task => { const col = el("div", "col"); (r.tasks[task] || []).forEach((c, i) => { const marked = opts.leadsOnly ? (c.c === "1" && (c.s === "c" || c.s === "h")) : "chm".includes(c.s); const cls = "sq sq--" + c.c + (marked ? " sq--" + c.s : "") + (c.d ? " sq--disc" : "") + (c.f ? " sq--fb" : "") + (c.n ? " sq--nt" : ""); const sq = el(c.v !== null && c.v !== undefined ? "a" : "span", cls); if (c.v !== null && c.v !== undefined) sq.href = cellHref(r, c); sq.dataset.tip = [task, "attempt " + (i + 1) + " · " + OUTCOME[c.c] + (c.d ? " · not counted in our estimate" : ""), "highest finding: " + PRIORITY[c.s], ...(c.f ? ["steps by a fallback model"] : []), ...(c.n ? ["no trajectory"] : []), c.h ? "opens the full trajectory" : "opens the findings viewer"].join("\n"); col.append(sq); }); s.append(col); }); return s; } /* One tooltip for every [data-tip], positioned at the pointer and hidden on anything else. */ function tooltips() { const tip = el("div", "tip"); tip.hidden = true; document.body.append(tip); const hide = () => { tip.hidden = true; }; document.addEventListener("pointerover", e => { const t = e.target.closest && e.target.closest("[data-tip]"); if (!t) { hide(); return; } const [head, ...rest] = t.dataset.tip.split("\n"); tip.replaceChildren(el("strong", null, head), ...rest.map(l => el("span", null, l))); tip.hidden = false; const x = Math.min(e.clientX + 14, innerWidth - tip.offsetWidth - 8); const y = e.clientY + 16 + tip.offsetHeight > innerHeight ? e.clientY - tip.offsetHeight - 10 : e.clientY + 16; tip.style.left = x + "px"; tip.style.top = y + "px"; }); ["scroll", "wheel", "touchstart", "blur", "pointerleave"].forEach(ev => addEventListener(ev, hide, { passive: true })); document.addEventListener("pointerleave", hide); } function theme() { const btn = document.getElementById("theme"); const set = t => { document.documentElement.dataset.theme = t; btn.textContent = t === "dark" ? "Light" : "Dark"; try { localStorage.setItem("tb21-theme", t); } catch (e) { /* private mode */ } }; let saved = null; try { saved = localStorage.getItem("tb21-theme"); } catch (e) { /* private mode */ } set(saved || (matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light")); btn.addEventListener("click", () => set(document.documentElement.dataset.theme === "dark" ? "light" : "dark")); } const sc = D.scanner; const scannerLabel = "atif-scan " + (sc.tag || sc.describe || sc.versions.join(", ")) + " (" + sc.commit.slice(0, 7) + ")"; /* What the output tokens went on: tool-call arguments, messages, reasoning the trace shows, reasoning it doesn't. Full width: shares, not sizes. */ const MIX = [["tool", "tool calls"], ["message", "messages"], ["reasoning", "reasoning"], ["hidden", "no text in trace"]]; function mixBar(t, large) { const wrap = el("span", "mix" + (large ? " mix--lg" : "")); if (!t.mix) { wrap.append(el("span", "z", "no token counts")); return wrap; } const bar = el("span", "mix-bar"); MIX.forEach(([k]) => { const i = el("i", "mx mx--" + k); i.style.width = (100 * t.mix[k]) + "%"; bar.append(i); }); const lab = el("span", "mix-lab"); lab.append(el("b", null, pct0(1 - t.mix.hidden)), document.createTextNode(" in trace")); wrap.append(bar, lab); wrap.title = MIX.map(([k, name]) => name + " " + pct0(t.mix[k])).join(" · ") + " of output tokens" + (t.mix.hidden ? " (no text: " + (t.measured ? "counted from reported reasoning tokens" : "inferred from the recorded text") + ")" : ""); return wrap; } const mixKey = () => { const k = el("span", "mix-key"); MIX.forEach(([key, name]) => k.append(el("i", "mx mx--" + key), document.createTextNode(name + " "))); return k; }; window.TB = { D, el, link, pct0, fmt1, money, shortModel, rowPath, rowPage, title, byId, OUTCOME, PRIORITY, viewerHref, fullHref, cellHref, trialInfo, RZ, RZ_HELP, visibleShare, reasoningChip, problems, problemChip, strip, tooltips, theme, scannerLabel, mixBar, mixKey }; document.addEventListener("DOMContentLoaded", () => { theme(); tooltips(); document.querySelectorAll("[data-scanner]").forEach(n => { n.textContent = scannerLabel + " · built " + D.generated; }); }); })();