Blame for row.js

/* A row's page: the hub between the leaderboard and atif-scan's highlights and trajectories. */
(function () {
  "use strict";
  const T = window.TB;
  const { D, el, link, pct0, fmt1, money, shortModel, rowPath, title, byId, OUTCOME, trialInfo, viewerHref, fullHref,
    reasoningChip, RZ, RZ_HELP, visibleShare, problems, problemChip, strip } = T;
  const DETAILS = window.TB21_DETAILS || {};
  const compactN = n => (n >= 1e6 ? (n / 1e6).toFixed(1) + "M" : n >= 1e3 ? Math.round(n / 1e3) + "k" : String(n));
  const main = document.getElementById("row");
  const r = byId[new URLSearchParams(location.search).get("r")];
  if (!r) { main.replaceChildren(el("p", null, "Unknown row. "), link("index.html", "Back to the leaderboard")); return; }
  const hubScript = document.createElement("script");
  hubScript.src = rowPath(r) + "hub.js";
  document.head.append(hubScript);
  document.title = "#" + r.rank + " " + title(r) + " · " + r.agent + " · " + (r.pr ? "TB2.1 submissions" : "TB2.1 annotated");
  const h = r.scan.holes, n = r.scan.trials;
  const full = Object.values(trialInfo[r.id]).filter(t => t.h).length;

  /* A count that opens the trials behind it. */
  function trials(key, label, note) {
    const items = (DETAILS[r.id] || {})[key] || [];
    const d = el("details", "trials");
    d.id = "k-" + key;
    const s = el("summary");
    s.append(el("span", "label", label), el("span", "count", items.length + (items.length === 1 ? " trial" : " trials")));
    d.append(s);
    d.addEventListener("toggle", () => {
      if (!d.open || d.querySelector("ol")) return;
      if (note) d.append(el("p", "muted small", note));
      const ol = el("ol", "trial-list");
      items.map(([vid, fid]) => ({ vid, fid, t: trialInfo[r.id][vid] || {} }))
        .sort((a, b) => (a.t.task || "").localeCompare(b.t.task || "") || (a.t.attempt || 0) - (b.t.attempt || 0))
        .forEach(({ vid, fid, t }) => {
          const li = el("li");
          li.append(el("i", "sq sq--" + (t.c || "0") + ("chm".includes(t.s) ? " sq--" + t.s : "")),
            link(viewerHref(r, vid, fid), (t.task || "trial " + vid) + " · " + (t.attempt || "?")),
            el("small", null, OUTCOME[t.c] || ""));
          if (t.h) li.append(link(fullHref(r, vid), "full ↗", "full"));
          const hub = (window.TB21_HUB || {})[vid];
          if (hub) { const a = link(hub, "Hub ↗", "full"); a.target = "_blank"; a.rel = "noopener noreferrer"; li.append(a); }
          ol.append(li);
        });
      d.append(ol);
    });
    return d;
  }
  function panel(id, heading, lede) {
    const p = el("section", "panel");
    p.id = id;
    p.append(el("h2", null, heading));
    if (lede) p.append(el("p", "lede-sm", lede));
    return p;
  }
  function stat(label, value, note, cls) {
    const s = el("div", "stat" + (cls ? " " + cls : ""));
    s.append(el("span", "stat-label", label));
    if (value instanceof Node) s.append(value); else s.append(el("b", null, value));
    if (note) s.append(el("small", null, note));
    return s;
  }

  /* Header. */
  const head = el("header", "row-head");
  head.append(el("p", "kicker", (r.pr ? "Terminal-Bench 2.1 submission · PR #" + r.pr + " (" + r.prState.toLowerCase() + ")"
    : "#" + r.rank + " on the Terminal-Bench 2.1 leaderboard") + " · " + (r.date || "")));
  head.append(el("h1", null, title(r)));
  const meta = el("p", "row-meta");
  meta.append(el("span", "harn", r.agent), el("span", null, [r.modelOrg].filter(Boolean).join(" · ")));
  meta.append(link(r.source, r.pr ? "PR #" + r.pr + " ↗" : "Submission ↗", "ext"));
  r.jobs.forEach(j => meta.append(link("https://hub.harborframework.com/jobs/" + j, "Hub job ↗", "ext")));
  head.append(meta);
  if (r.why) head.append(el("p", "why-line", r.why));
  const probs = problems(r);
  if (probs.length) { const c = el("p", "flags"); probs.forEach(p => c.append(problemChip(p))); head.append(c); }

  const adj = r.scan.adjusted, tk = h.tokens, ce = h.cost;
  const recount = r.lb.source === "recount";
  const stats = el("div", "stats");
  stats.append(
    stat(recount ? "Score" : "Leaderboard score", fmt1(r.lb.accuracy) + "%", "± " + fmt1(r.lb.ci95) + " · " + r.scan.rewarded + " of " + n + " trials rewarded"),
    stat("Our estimate", fmt1(adj.accuracy) + "%", adj.rewarded + " of " + n + (adj.leads ? " · less " + adj.leads + (adj.leads === 1 ? " review lead" : " review leads") : "") + (adj.fallback ? " · less " + adj.fallback + (adj.fallback === 1 ? " fallback trial" : " fallback trials") : ""), adj.accuracy < r.lb.accuracy - 1 ? "stat--hot" : ""),
    stat("Trace completeness", tk.mix ? pct0(1 - tk.mix.hidden) : "–", tk.mix ? "of output tokens have text in the trace · " + RZ[h.text.why] : "no token counts"),
    stat("Cost, our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate), (!recount && r.lb.cost !== null ? "leaderboard " + money(r.lb.cost) + " · " : "") + "recorded " + money(ce.recorded)));
  const actions = el("div", "actions");
  actions.append(link(rowPath(r) + "highlights/index.html", "Highlights", "btn btn--primary"),
    link(rowPath(r) + "trajectories/index.html", "Full trajectories · " + full + " trials", "btn"),
    link(rowPath(r) + "viewer/index.html", "All " + n + " trials · findings view", "btn"));
  const stripWrap = el("section", "strip-wrap");
  stripWrap.append(el("p", "muted small", "Every attempt: a column per task, easiest on the left. Click one to open its trajectory."),
    strip(r, { large: true }));

  /* What atif-scan found. */
  const found = panel("found", "What atif-scan found", "Medium-or-higher findings not explained by an allowance. Each opens its excerpts.");
  const hits = Object.entries(D.matrix[r.id] || {});
  const families = Object.keys(D.categories).concat([null]);
  if (!hits.length) found.append(el("p", "muted", "No medium-or-higher findings."));
  families.forEach(cat => {
    const rows = hits.filter(([id]) => D.checks[id].category === cat).sort((a, b) => b[1].trials - a[1].trials);
    if (!rows.length) return;
    found.append(el("h3", null, cat ? D.categories[cat] : "Other"));
    const ul = el("ul", "checks");
    rows.forEach(([id, v]) => {
      const c = D.checks[id], li = el("li");
      li.append(el("span", "sev sev--" + c.severity, c.severity[0].toUpperCase()),
        link(rowPath(r) + "highlights/index.html#check=" + encodeURIComponent(id), c.title),
        el("span", "count", v.trials + (v.rewarded ? " · " + v.rewarded + " rewarded" : "")));
      ul.append(li);
    });
    found.append(ul);
  });
  const quiet = Object.entries(r.scan.categories).filter(([k]) => !hits.some(([id]) => D.checks[id].category === k));
  if (quiet.length) found.append(el("p", "muted small", "At low priority only: " +
    quiet.map(([k, v]) => D.categories[k] + " " + v.trials).join(" · ") + " trials."));

  /* What ran. */
  const ran = panel("ran", "What ran", "Models by share of the trajectories' steps.");
  const m = h.models;
  const models = el("ul", "models");
  m.served.filter(x => x.share >= 0.001).forEach(x => {
    const li = el("li", x.configured ? "" : "off");
    const bar = el("span", "bar"); const fill = el("i"); fill.style.width = Math.max(1, Math.round(100 * x.share)) + "%"; bar.append(fill);
    li.append(el("code", null, shortModel(x.model)), bar, el("span", "count", pct0(x.share) + " · " + x.trials + " trials"));
    models.append(li);
  });
  ran.append(models, el("p", "muted small", "Configured: " + Object.keys(m.configured).join(", ")));
  if (m.foreignTrials) ran.append(trials("foreign", m.mismatch ? "Served by another model" : "Trials with fallback steps"));

  /* What the trace holds. */
  const trace = panel("trace", "What the trace holds");
  const rz = el("p", "rz-line");
  rz.append(reasoningChip(h), document.createTextNode(": " + RZ_HELP[h.text.why] + "."));
  trace.append(rz);
  const tstats = el("div", "stats stats--sm");
  tstats.append(
    stat("Output tokens, median trial", tk.outputMedian ? compactN(tk.outputMedian) : "–", tk.outputTotal ? compactN(tk.outputTotal) + " over the row" : ""),
    stat("No text in trace", tk.mix ? pct0(tk.mix.hidden) : "–",
      tk.measured ? "counted: reasoning tokens reported, no text" : "inferred: recorded text at ≈3.2 chars/token"),
    stat("Text per output token", h.text.allCpt === null ? "–" : String(h.text.allCpt), "reasoning text in " + pct0(h.text.withText) + " of trials"));
  trace.append(tstats);
  if (tk.mix) { const mx = el("div", "mix-row"); mx.append(T.mixBar(tk, true), T.mixKey()); trace.append(mx); }
  if (h.sourceNote) { const q = el("p", "source-note"); q.append(el("b", null, "Source session files: "), document.createTextNode(h.sourceNote)); trace.append(q); }
  const HOLES = [
    ["unseen", "Most output has no text in the trace", "Trials whose recorded text covers under 80% of their output tokens."],
    ["compacted", "History compacted"], ["webUnrecorded", "Web results not in the trace"],
    ["webUnresolved", "Web provenance unresolved"], ["unpaired", "Tool results not paired to calls"],
    ["callsMissing", "Model calls missing from the steps"], ["endsOnCall", "Ends on an unanswered tool call"],
    ["noTrajectory", "No trajectory"], ["timestamps", "Timestamps missing or out of order"],
  ];
  const holeList = el("div", "holes");
  HOLES.forEach(([key, label, note]) => { if (h[key]) holeList.append(trials(key, label, note)); });
  trace.append(holeList);
  trace.append(el("p", "muted small", h.evidenceIncomplete + " of " + n + " trials have at least one check atif-scan couldn't decide: unknown, never clean."));

  /* What it cost. */
  const cost = panel("cost", "What it cost");
  const cstats = el("div", "stats stats--sm");
  if (!recount && r.lb.cost !== null) cstats.append(stat("Leaderboard", money(r.lb.cost)));
  cstats.append(stat("Recorded, sum of trials", money(ce.recorded), h.uncosted ? h.uncosted + " trials uncosted" : "every trial costed"),
    stat("Our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate),
      [ce.imputedTrials ? "+" + money(ce.imputed) + " for " + ce.imputedTrials + " uncosted trials, from their tokens" : "",
       ce.unbilled ? "+" + money(ce.unbilled) + " undercounted reasoning" : "",
       ce.unpriced ? ce.unpriced + " trials unpriced: no token usage" + (ce.rates ? "" : ", or no rates fit the recorded costs") : ""].filter(Boolean).join(" · ") || "nothing to add"));
  cost.append(cstats);
  if (ce.rates) cost.append(el("p", "muted small", "Rates fitted to this row's own trial costs and tokens: $" + ce.rates.uncached + " / $" + ce.rates.cached +
    " / $" + ce.rates.output + " per million uncached input / cached input / output tokens (R² " + ce.rates.r2 + ")."));
  if (h.text.uncounted) cost.append(trials("uncounted", "Reasoning left out of output tokens",
    "The visible text alone fills the reported output tokens, so the recorded reasoning is on top: at ≈3.2 characters per token, at least " + pct0(h.text.uncounted) + " more output than reported, and so left out of the cost."));
  if (h.uncosted) cost.append(trials("uncosted", "Trials without a recorded cost"));

  // Two columns that fill independently: findings on the left; what ran, the trace and the cost on the right.
  const grid = el("div", "panels");
  const left = el("div", "pcol"), right = el("div", "pcol");
  left.append(found);
  right.append(ran, trace, cost);
  grid.append(left, right);
  main.replaceChildren(head, stats, actions, stripWrap, grid);

  // #k=<gap> (from the home page's tables) opens that list of trials; #found etc. scroll to a panel.
  const want = new URLSearchParams(location.hash.slice(1)).get("k");
  const target = want ? document.getElementById("k-" + want) : location.hash ? document.getElementById(location.hash.slice(1)) : null;
  if (target) {
    if (target.tagName === "DETAILS") { target.open = true; target.dispatchEvent(new Event("toggle")); }
    requestAnimationFrame(() => target.scrollIntoView({ block: "start" }));
  }
})();