Trace completeness: share of output tokens in the trace; why not (withheld by provider, not kept by harness, lost in conversion, summary only); counted/inferred basis; undercounted

common.js

@@ -36,19 +36,22 @@
   });
 
   /* Reasoning and visibility. */
-  const RZ = { full: "full text", summary: "summary", hidden: "hidden", unrecorded: "unrecorded", dropped: "dropped", none: "none" };
+  /* Why output tokens have no text in the trace. */
+  const RZ = { withheld: "withheld by provider", notKept: "not kept by harness", lost: "lost in conversion",
+    summary: "summary only", complete: "reasoning recorded", noReasoning: "no reasoning recorded", unknown: "no token counts" };
   const RZ_HELP = {
-    full: "reasoning text is recorded and covers the output tokens",
-    summary: "reasoning text is recorded, but far less of it than output tokens: a summary",
-    hidden: "reasoning tokens are reported, but no reasoning text (encrypted)",
-    unrecorded: "no reasoning text or reasoning tokens, yet most output has no text in the trace",
-    dropped: "the harness's own session files hold reasoning that the trajectories lost in conversion",
-    none: "no reasoning recorded, and the recorded text accounts for the output",
+    withheld: "the model's API returns no reasoning text (encrypted, or thinking blocks with empty text): generated and billed, never readable",
+    notKept: "the provider returns reasoning text, but the harness didn't record it",
+    lost: "the harness's own session files hold the reasoning; converting them to the trajectory dropped it",
+    summary: "the provider returns a summary of the reasoning, far shorter than its tokens",
+    complete: "reasoning text is in the trace; for most trials the recorded text accounts for the output tokens",
+    noReasoning: "no reasoning text or reasoning tokens in the trace; for most trials the recorded text accounts for the output tokens",
+    unknown: "the trials report no output tokens to account for",
   };
   const visibleShare = h => (h.text.unseen === null ? null : 1 - h.text.unseen);
   const reasoningChip = h => {
-    const chip = el("span", "rz rz--" + h.text.form, RZ[h.text.form]);
-    chip.title = RZ_HELP[h.text.form];
+    const chip = el("span", "rz rz--" + h.text.why, RZ[h.text.why]);
+    chip.title = RZ_HELP[h.text.why];
     return chip;
   };
 
@@ -131,17 +134,17 @@
 
   /* What the output tokens went on: tool-call arguments, messages, reasoning the trace shows,
      reasoning it doesn't. Full width: shares, not sizes. */
-  const MIX = [["tool", "tool calls"], ["message", "messages"], ["reasoning", "reasoning shown"], ["hidden", "reasoning hidden"]];
+  const MIX = [["tool", "tool calls"], ["message", "messages"], ["reasoning", "reasoning"], ["hidden", "no text in trace"]];
   function mixBar(t, large) {
     const wrap = el("span", "mix" + (large ? " mix--lg" : ""));
     if (!t.mix) { wrap.append(el("span", "z", "no token counts")); return wrap; }
     const bar = el("span", "mix-bar");
     MIX.forEach(([k]) => { const i = el("i", "mx mx--" + k); i.style.width = (100 * t.mix[k]) + "%"; bar.append(i); });
     const lab = el("span", "mix-lab");
-    lab.append(el("b", null, pct0(t.mix.hidden)), document.createTextNode(" hidden"));
+    lab.append(el("b", null, pct0(1 - t.mix.hidden)), document.createTextNode(" in trace"));
     wrap.append(bar, lab);
     wrap.title = MIX.map(([k, name]) => name + " " + pct0(t.mix[k])).join(" · ") + " of output tokens" +
-      (t.mix.hidden ? " (hidden " + (t.measured ? "measured from reported reasoning tokens" : "estimated from the recorded text") + ")" : "");
+      (t.mix.hidden ? " (no text: " + (t.measured ? "counted from reported reasoning tokens" : "inferred from the recorded text") + ")" : "");
     return wrap;
   }
   const mixKey = () => {

home.js

@@ -48,7 +48,7 @@
     const c = h.cost;
     if (c.imputedTrials) out.push(["warn", c.imputedTrials + " trials priced from tokens"]);
     if (c.unpriced / r.scan.trials > 0.02) out.push(["note", c.unpriced + " trials unpriced"]);
-    if (c.unbilled) out.push(["bad", "reasoning unbilled"]);
+    if (c.unbilled) out.push(["bad", "output undercounted"]);
     if (r.lb.disqualified) out.push(["bad", r.lb.disqualified + " disqualified by the PR pipeline"]);
     return out;
   }
@@ -71,8 +71,8 @@
       { name: hasLb ? "Leader­board" : "Score", sort: "score", help: hasLb ? "published accuracy ± 95% half-width" : "rewarded trials ± 95% half-width over the five attempt rounds" },
       { name: "Our estimate", sort: "adj", help: ESTIMATE_HELP },
       { name: "Leads", sort: "leads", help: "rewarded trials with an unexcused high or critical finding" },
-      { name: "Output tokens went on", sort: "hidden", cls: "l", help: "share of the row's output tokens: tool calls, messages, reasoning the trace shows, and (orange) reasoning it doesn't" },
-      { name: "Reasoning", help: "why the reasoning isn't in the trace" },
+      { name: "Output tokens · trace completeness", sort: "hidden", cls: "l", help: "the row's output tokens: tool calls, messages and reasoning with text in the trace, and (orange) output with no text in it. The figure is the share the trace holds." },
+      { name: "Why not in trace", help: "why some output tokens have no text in the trace" },
       { name: "Cost", sort: "cost", help: "our estimate: recorded trial costs, uncosted trials priced from their tokens at the row's own fitted rates, and any reasoning left out of the output tokens" },
       { name: "Notes", cls: "l" },
     ];
@@ -96,7 +96,7 @@
       else ld.append(el("span", "z", "0"));
       tr.append(ld);
       const oc = el("td", "l"); oc.append(mixBar(h.tokens)); tr.append(oc);
-      const rz = el("td", "rz rz--" + h.text.form, RZ[h.text.form]); rz.title = RZ_HELP[h.text.form]; tr.append(rz);
+      const rz = el("td", "rz rz--" + h.text.why, RZ[h.text.why]); rz.title = RZ_HELP[h.text.why]; tr.append(rz);
       const c = h.cost, cc = el("td");
       cc.append(num(c.estimate === null ? "–" : (c.lowerBound ? "≥" : "") + money(c.estimate), "n"));
       const rec = hasLb && r.lb.cost !== null ? "LB " + money(r.lb.cost) : c.imputed || c.unbilled ? "rec. " + money(c.recorded) : "";
@@ -122,11 +122,13 @@
   function gaps() {
     const groups = [
       ["Models", [["Served, by share of steps", "l", "per-step model names; red: another model than the configured one"]]],
-      ["Output tokens per trial", [
-        ["Median", "", "reported output tokens of the median trial"],
-        ["Hidden", "", "share of the row's output tokens with no text in the trace: reasoning the trace doesn't show"],
-        ["How known", "l", "measured: the harness reports reasoning tokens; estimated: from recorded characters at ≈3.2 per token"],
-        ["Unbilled", "", "recorded reasoning missing from the output tokens (and so the cost), as a share of reported output"]]],
+      ["Output tokens · trace completeness", [
+        ["Median trial", "", "reported output tokens of the median trial"],
+        ["In trace", "", "share of the row's output tokens whose text the trace holds"],
+        ["No text", "", "share with no text in the trace: generated and billed, not readable"],
+        ["Why", "l", "why that output has no text in the trace"],
+        ["Basis", "l", "counted: the harness reports reasoning tokens; inferred: from the recorded text at ≈3.2 characters per token"],
+        ["Under­counted", "", "the reverse: recorded reasoning text the reported output tokens leave out (and so the cost), as a share of reported output"]]],
       ["Web", [
         ["Trials using it", "", "trials that searched or opened web pages"],
         ["Results not in trace", "", "of those, trials where the search or fetch results aren't in the trace (only the query or URL is)"]]],
@@ -135,7 +137,7 @@
         ["No trajectory", "", "listed trials without a trajectory"]]],
       ["Cost", [
         ["Recorded", "", "sum of the trials' recorded costs"],
-        ["Our estimate", "", "recorded + uncosted trials priced from their tokens + unbilled reasoning, at the row's fitted rates"],
+        ["Our estimate", "", "recorded + uncosted trials priced from their tokens + undercounted reasoning, at the row's fitted rates"],
         ["Fitted $/M in · cached · out", "", "rates fitted to the row's own trial costs and tokens (R² shown on hover)"]]],
       ["Evidence", [["Undecided", "", "trials with a behavioural check atif-scan couldn't decide"]]],
     ];
@@ -163,15 +165,15 @@
       tr.append(start(md));
 
       tr.append(start(el("td", "n", tk.outputMedian ? compactN(tk.outputMedian) : "–")));
+      tr.append(el("td", "n", tk.mix ? pct0(1 - tk.mix.hidden) : "–"));
       const hid = el("td", "n");
-      if (tk.outputMedian) {
+      if (tk.mix) {
         heat(hid, tk.mix.hidden, 1, "warm");
         hid.append(link(holeHref(r, "unseen"), pct0(tk.mix.hidden)));
       } else hid.append(el("span", "z", "–"));
       tr.append(hid);
-      const how = el("td", "l small");
-      how.append(el("span", "rz rz--" + h.text.form, RZ[h.text.form]), document.createTextNode(tk.outputMedian ? (tk.measured ? " · measured" : " · estimated") : ""));
-      tr.append(how);
+      const why = el("td", "l rz rz--" + h.text.why, RZ[h.text.why]); why.title = RZ_HELP[h.text.why]; tr.append(why);
+      tr.append(el("td", "l small", tk.mix ? (tk.measured ? "counted" : "inferred") : "–"));
       const ub = el("td", "c");
       if (h.text.uncounted) ub.append(link(holeHref(r, "uncounted"), "≥" + pct0(h.text.uncounted), "bad"));
       else ub.append(el("span", "z", "·"));
@@ -189,7 +191,7 @@
       if (c.imputedTrials || c.unbilled || c.unpriced) {
         const parts = [];
         if (c.imputedTrials) parts.push("+" + money(c.imputed) + " " + c.imputedTrials + " priced");
-        if (c.unbilled) parts.push("+" + money(c.unbilled) + " unbilled");
+        if (c.unbilled) parts.push("+" + money(c.unbilled) + " undercounted");
         if (c.unpriced) parts.push(c.unpriced + " unpriced");
         est.append(el("small", null, parts.join(" · ")));
       }

index.html

@@ -53,18 +53,20 @@
         corner: highest finding <i class="sq sq--1 sq--c"></i>critical <i class="sq sq--1 sq--h"></i>high <i class="sq sq--1 sq--m"></i>medium<br>
         <b>Our estimate</b> leaves out rewarded trials with an unexcused high or critical finding (review leads) and trials partly run by a
         fallback model (a Gemini preview name served as another Gemini preview is an alias, not a fallback).
-        <b>Output tokens went on</b>: <i class="mx mx--tool"></i>tool calls <i class="mx mx--message"></i>messages
-        <i class="mx mx--reasoning"></i>reasoning the trace shows <i class="mx mx--hidden"></i>reasoning it doesn't (generated and
-        billed, but no text in the trace). <b>Cost</b>: recorded trial costs, uncosted trials priced from their tokens at the row's own
+        <b>Trace completeness</b>: the row's output tokens as <i class="mx mx--tool"></i>tool calls <i class="mx mx--message"></i>messages
+        <i class="mx mx--reasoning"></i>reasoning, all with text in the trace, and <i class="mx mx--hidden"></i>no text in the trace
+        (generated and billed, not readable); the figure is the share the trace holds. <b>Cost</b>: recorded trial costs, uncosted trials priced from their tokens at the row's own
         rates, and reasoning left out of the output tokens; ≥ where some trials can't be priced (no usage, or no rates that fit the recorded costs).</p>
     </section>
 
     <section id="gaps" class="block">
       <header class="block-head"><h2>Gaps &amp; accounting</h2>
         <span class="muted small">trials per row unless marked · shading is the share · click a figure for its trials</span></header>
-      <p class="explain-line"><b>Hidden</b> is output the model generated whose text isn't in the trace: reasoning, never web
-        results (those are input). <b>Measured</b> where the harness reports reasoning tokens; otherwise estimated from the recorded
-        characters (≈3.2 per token when the trace holds everything). <b>Web</b>: of the trials that searched or fetched, how many
+      <p class="explain-line">Every output token is accounted for: text in the trace (<b>In trace</b>) or not (<b>No text</b>: in
+        practice reasoning, never web results, which are input). <b>Why</b>: withheld by the model's provider, not kept by the
+        harness, lost converting its session files, or only a summary returned. <b>Basis</b>: counted where the harness reports
+        reasoning tokens, otherwise inferred from the recorded characters (≈3.2 per token when the trace holds everything).
+        <b>Undercounted</b> is the reverse: reasoning text the reported output tokens, and so the cost, leave out. <b>Web</b>: of the trials that searched or fetched, how many
         hold only the query or URL, not what came back. Tool-result pairing, call records missing from the steps, unanswered final
         calls and timestamps are on each row's page: recording-format quirks that limit some checks (counted under Undecided) but
         don't hide what the agent did.</p>
@@ -100,8 +102,8 @@
           <p>Recorded agent text (reasoning, messages, tool arguments) over reported output tokens. ≈3.2 when the trace holds all
             output (measured on visible text over visible tokens where reasoning tokens are reported). <b>In trace</b> = that
             ratio over 3.2. With the visible text alone filling the reported tokens, recorded reasoning is missing from the token
-            count and the cost (<b>output under</b>). Tokenizers differ: small deviations mean nothing.</p>
-          <h3>Reasoning</h3>
+            count and the cost (<b>undercounted</b>). Tokenizers differ: small deviations mean nothing.</p>
+          <h3>Why output has no text in the trace</h3>
           <dl class="defs" id="rz-defs"></dl>
         </div>
         <div>

row.js

@@ -3,7 +3,7 @@
   "use strict";
   const T = window.TB;
   const { D, el, link, pct0, fmt1, money, shortModel, rowPath, title, byId, OUTCOME, trialInfo, viewerHref, fullHref,
-    reasoningChip, RZ_HELP, visibleShare, problems, problemChip, strip } = T;
+    reasoningChip, RZ, RZ_HELP, visibleShare, problems, problemChip, strip } = T;
   const DETAILS = window.TB21_DETAILS || {};
   const compactN = n => (n >= 1e6 ? (n / 1e6).toFixed(1) + "M" : n >= 1e3 ? Math.round(n / 1e3) + "k" : String(n));
   const main = document.getElementById("row");
@@ -79,7 +79,7 @@
   stats.append(
     stat(recount ? "Score" : "Leaderboard score", fmt1(r.lb.accuracy) + "%", "± " + fmt1(r.lb.ci95) + " · " + r.scan.rewarded + " of " + n + " trials rewarded"),
     stat("Our estimate", fmt1(adj.accuracy) + "%", adj.rewarded + " of " + n + (adj.leads ? " · less " + adj.leads + (adj.leads === 1 ? " review lead" : " review leads") : "") + (adj.fallback ? " · less " + adj.fallback + (adj.fallback === 1 ? " fallback trial" : " fallback trials") : ""), adj.accuracy < r.lb.accuracy - 1 ? "stat--hot" : ""),
-    stat("Hidden reasoning", tk.mix ? pct0(tk.mix.hidden) : "–", tk.mix ? "of output tokens · " + (tk.measured ? "measured" : "estimated") + " · median trial " + compactN(tk.outputMedian) : "no token counts"),
+    stat("Trace completeness", tk.mix ? pct0(1 - tk.mix.hidden) : "–", tk.mix ? "of output tokens have text in the trace · " + RZ[h.text.why] : "no token counts"),
     stat("Cost, our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate), (!recount && r.lb.cost !== null ? "leaderboard " + money(r.lb.cost) + " · " : "") + "recorded " + money(ce.recorded)));
   const actions = el("div", "actions");
   actions.append(link(rowPath(r) + "highlights/index.html", "Highlights", "btn btn--primary"),
@@ -128,13 +128,13 @@
   /* What the trace holds. */
   const trace = panel("trace", "What the trace holds");
   const rz = el("p", "rz-line");
-  rz.append(reasoningChip(h), document.createTextNode(" " + RZ_HELP[h.text.form] + "."));
+  rz.append(reasoningChip(h), document.createTextNode(": " + RZ_HELP[h.text.why] + "."));
   trace.append(rz);
   const tstats = el("div", "stats stats--sm");
   tstats.append(
     stat("Output tokens, median trial", tk.outputMedian ? compactN(tk.outputMedian) : "–", tk.outputTotal ? compactN(tk.outputTotal) + " over the row" : ""),
-    stat("Hidden reasoning", tk.mix ? pct0(tk.mix.hidden) : "–",
-      tk.measured ? "measured: reasoning tokens reported, no text" : "estimated: recorded text at ≈3.2 chars/token"),
+    stat("No text in trace", tk.mix ? pct0(tk.mix.hidden) : "–",
+      tk.measured ? "counted: reasoning tokens reported, no text" : "inferred: recorded text at ≈3.2 chars/token"),
     stat("Text per output token", h.text.allCpt === null ? "–" : String(h.text.allCpt), "reasoning text in " + pct0(h.text.withText) + " of trials"));
   trace.append(tstats);
   if (tk.mix) { const mx = el("div", "mix-row"); mx.append(T.mixBar(tk, true), T.mixKey()); trace.append(mx); }
@@ -158,13 +158,13 @@
   cstats.append(stat("Recorded, sum of trials", money(ce.recorded), h.uncosted ? h.uncosted + " trials uncosted" : "every trial costed"),
     stat("Our estimate", ce.estimate === null ? "–" : (ce.lowerBound ? "≥" : "") + money(ce.estimate),
       [ce.imputedTrials ? "+" + money(ce.imputed) + " for " + ce.imputedTrials + " uncosted trials, from their tokens" : "",
-       ce.unbilled ? "+" + money(ce.unbilled) + " unbilled reasoning" : "",
+       ce.unbilled ? "+" + money(ce.unbilled) + " undercounted reasoning" : "",
        ce.unpriced ? ce.unpriced + " trials unpriced: no token usage" + (ce.rates ? "" : ", or no rates fit the recorded costs") : ""].filter(Boolean).join(" · ") || "nothing to add"));
   cost.append(cstats);
   if (ce.rates) cost.append(el("p", "muted small", "Rates fitted to this row's own trial costs and tokens: $" + ce.rates.uncached + " / $" + ce.rates.cached +
     " / $" + ce.rates.output + " per million uncached input / cached input / output tokens (R² " + ce.rates.r2 + ")."));
   if (h.text.uncounted) cost.append(trials("uncounted", "Reasoning left out of output tokens",
-    "The visible text alone fills the reported output tokens, so the recorded reasoning is on top: at ≈3.2 characters per token, at least " + pct0(h.text.uncounted) + " more output than reported, and so unbilled."));
+    "The visible text alone fills the reported output tokens, so the recorded reasoning is on top: at ≈3.2 characters per token, at least " + pct0(h.text.uncounted) + " more output than reported, and so left out of the cost."));
   if (h.uncosted) cost.append(trials("uncosted", "Trials without a recorded cost"));
 
   // Two columns that fill independently: findings on the left; what ran, the trace and the cost on the right.

style.css

@@ -155,9 +155,9 @@ details.trials summary .label { font-weight: 650; }
 .flags { margin: 6px 0 0; font-size: 13px; }
 /* Reasoning form: a word, coloured by how much of the reasoning the trace lacks. */
 .rz { font-size: 12px; font-weight: 650; text-align: left !important; }
-.rz--full { color: var(--ok); } .rz--none { color: var(--muted); }
+.rz--complete { color: var(--ok); } .rz--unknown, .rz--noReasoning { color: var(--muted); }
 .rz--summary { color: var(--accent-ink); }
-.rz--hidden, .rz--unrecorded, .rz--dropped { color: var(--danger); }
+.rz--withheld, .rz--notKept, .rz--lost { color: var(--danger); }
 /* Heat: one translucent hue per meaning; numbers stay ink. */
 .heat { background: color-mix(in srgb, var(--hue) calc(var(--a, 0) * 42%), transparent); }
 .heat--warm { --hue: var(--hl-h); } .heat--cool { --hue: var(--hl-m); } .heat--amber { --hue: var(--accent); }