Buckets:
| <title>Decision Reasoner</title> | |
| <meta name="claude:viewport-fit-sides" content="cover"> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600&family=Playfair+Display:ital,wght@0,500;0,600;1,500&family=JetBrains+Mono:wght@400;500&display=swap"> | |
| <style> | |
| /* Layout: sticky run bar, then sections top to bottom (summary before detail); charts in a 12-column grid that stacks on phones. | |
| Mercity tokens: indigo = identity, teal = measurement, amber = caution/live; neutrals at hue 240. */ | |
| :root{ | |
| --indigo-50:hsl(243 41% 97%);--indigo-100:hsl(243 47% 94%);--indigo-200:hsl(243 54% 87%);--indigo-400:hsl(243 69% 68%); | |
| --indigo-500:hsl(243 75% 59%);--indigo-600:hsl(243 69% 50%);--indigo-700:hsl(243 66% 41%);--indigo-900:hsl(243 56% 23%); | |
| --teal-50:hsl(178 34% 96%);--teal-100:hsl(178 38% 92%);--teal-400:hsl(178 57% 55%);--teal-500:hsl(178 62% 42%); | |
| --teal-600:hsl(178 66% 34%);--teal-700:hsl(178 70% 27%);--teal-800:hsl(178 72% 21%); | |
| --amber-50:hsl(34 78% 96%);--amber-100:hsl(34 82% 92%);--amber-300:hsl(34 88% 73%);--amber-500:hsl(34 84% 53%);--amber-700:hsl(32 74% 36%);--amber-800:hsl(30 70% 28%); | |
| --n-0:#ffffff;--n-50:hsl(240 24% 99%);--n-100:hsl(240 18% 96%);--n-200:hsl(240 13% 91%);--n-300:hsl(240 10% 83%); | |
| --n-400:hsl(240 8% 65%);--n-500:hsl(240 7% 49%);--n-600:hsl(240 8% 37%);--n-700:hsl(240 10% 27%);--n-800:hsl(240 12% 17%);--n-900:hsl(240 14% 10%); | |
| --ground:var(--n-50);--surface:var(--n-0);--sunk:var(--n-100);--ink:var(--n-900);--ink-2:var(--n-700);--ink-3:var(--n-500); | |
| --line:var(--n-200);--line-2:var(--n-300); | |
| --brand:var(--indigo-600);--brand-soft:var(--indigo-50);--brand-ink:var(--indigo-700); | |
| --measure:var(--teal-700);--measure-soft:var(--teal-50); | |
| --caution:var(--amber-700);--caution-soft:var(--amber-50);--caution-line:var(--amber-300); | |
| --good:var(--teal-700);--bad:hsl(354 62% 44%); | |
| /* chart series (validated: light surface #fff) */ | |
| --s-joint:#29aea9;--s-gold:#3028d7;--s-votes:#7b75e6;--s-base:var(--n-400); | |
| --grid:var(--n-200);--axis:var(--n-500); | |
| --sans:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif; | |
| --serif:'Playfair Display',Georgia,serif; | |
| --mono:'JetBrains Mono',ui-monospace,Menlo,monospace; | |
| --ease:cubic-bezier(.22,1,.36,1); | |
| --r:10px; | |
| } | |
| @media (prefers-color-scheme: dark){ | |
| :root:not([data-theme="light"]){ | |
| --ground:var(--n-900);--surface:var(--n-800);--sunk:hsl(240 12% 13%);--ink:var(--n-50);--ink-2:var(--n-300);--ink-3:var(--n-400); | |
| --line:var(--n-700);--line-2:var(--n-600); | |
| --brand:var(--indigo-400);--brand-soft:hsl(243 30% 22%);--brand-ink:var(--indigo-200); | |
| --measure:var(--teal-400);--measure-soft:hsl(178 30% 16%); | |
| --caution:var(--amber-300);--caution-soft:hsl(32 40% 18%);--caution-line:hsl(32 50% 35%); | |
| --good:var(--teal-400);--bad:hsl(354 70% 66%); | |
| --s-joint:#29a39f;--s-gold:#8580ef;--s-votes:#4841d2;--s-base:var(--n-500); | |
| --grid:var(--n-700);--axis:var(--n-400); | |
| color-scheme:dark; | |
| } | |
| } | |
| :root[data-theme="dark"]{ | |
| --ground:var(--n-900);--surface:var(--n-800);--sunk:hsl(240 12% 13%);--ink:var(--n-50);--ink-2:var(--n-300);--ink-3:var(--n-400); | |
| --line:var(--n-700);--line-2:var(--n-600); | |
| --brand:var(--indigo-400);--brand-soft:hsl(243 30% 22%);--brand-ink:var(--indigo-200); | |
| --measure:var(--teal-400);--measure-soft:hsl(178 30% 16%); | |
| --caution:var(--amber-300);--caution-soft:hsl(32 40% 18%);--caution-line:hsl(32 50% 35%); | |
| --good:var(--teal-400);--bad:hsl(354 70% 66%); | |
| --s-joint:#29a39f;--s-gold:#8580ef;--s-votes:#4841d2;--s-base:var(--n-500); | |
| --grid:var(--n-700);--axis:var(--n-400); | |
| color-scheme:dark; | |
| } | |
| *{box-sizing:border-box} | |
| body{background:var(--ground);color:var(--ink-2);font-family:var(--sans);font-size:15.5px;line-height:1.6;font-weight:400; | |
| padding-left:max(16px,env(safe-area-inset-left,0px));padding-right:max(16px,env(safe-area-inset-right,0px))} | |
| body::before{content:"";position:fixed;inset:0 0 auto;height:env(safe-area-inset-top,0px);z-index:10;background:var(--surface)} | |
| a{color:var(--brand)} | |
| :focus-visible{outline:2px solid var(--brand);outline-offset:2px} | |
| .wrap{max-width:1320px;margin:0 auto} | |
| .mono{font-family:var(--mono)} | |
| .num{font-variant-numeric:tabular-nums} | |
| h1,h2,h3{color:var(--ink);text-wrap:balance;margin:0} | |
| h2{font-family:var(--sans);font-weight:600;font-size:1.25rem;letter-spacing:-.01em} | |
| h3{font-weight:600;font-size:.98rem} | |
| .eyebrow{font-family:var(--mono);font-size:.72rem;letter-spacing:.08em;text-transform:uppercase;color:var(--ink-3)} | |
| p{margin:0;max-width:72ch} | |
| /* run bar */ | |
| .bar{position:sticky;top:env(safe-area-inset-top,0px);z-index:20;background:var(--surface);border-bottom:1px solid var(--line); | |
| margin-left:calc(-1 * max(16px,env(safe-area-inset-left,0px)));margin-right:calc(-1 * max(16px,env(safe-area-inset-right,0px))); | |
| padding-left:max(16px,env(safe-area-inset-left,0px));padding-right:max(16px,env(safe-area-inset-right,0px))} | |
| .bar .wrap{display:flex;flex-wrap:wrap;align-items:center;gap:8px 18px;padding-block:10px} | |
| .mark{font-family:var(--mono);font-size:.78rem;color:var(--ink-3);letter-spacing:.04em} | |
| .mark b{color:var(--ink);font-weight:500} | |
| .nav{display:flex;flex-wrap:wrap;gap:2px;margin-left:auto} | |
| .nav a{font-size:.85rem;color:var(--ink-3);text-decoration:none;padding:4px 9px;border-radius:6px} | |
| .nav a:hover{color:var(--ink);background:var(--sunk)} | |
| .pill{display:inline-flex;align-items:center;gap:7px;font-family:var(--mono);font-size:.74rem;padding:3px 10px;border-radius:999px;border:1px solid var(--line);white-space:nowrap} | |
| .pill i{width:7px;height:7px;border-radius:50%;background:currentColor;display:inline-block} | |
| .pill.live{color:var(--caution);background:var(--caution-soft);border-color:var(--caution-line)} | |
| .pill.live i{animation:pulse 1.6s var(--ease) infinite} | |
| .pill.done{color:var(--measure);background:var(--measure-soft);border-color:transparent} | |
| .pill.neutral{color:var(--ink-3);background:var(--sunk)} | |
| @keyframes pulse{50%{opacity:.25}} | |
| @media (prefers-reduced-motion:reduce){.pill.live i{animation:none}} | |
| /* hero */ | |
| .hero{display:grid;grid-template-columns:minmax(0,1.4fr) minmax(0,1fr);gap:28px;padding-block:34px 22px;align-items:end} | |
| .hero h1{font-family:var(--serif);font-weight:500;font-size:clamp(2.2rem,4.6vw,3.5rem);line-height:1.04;letter-spacing:-.015em} | |
| .hero h1 em{font-style:italic;color:var(--brand)} | |
| .hero p{margin-top:12px;color:var(--ink-2)} | |
| .runbox{background:var(--surface);border:1px solid var(--line);border-radius:var(--r);padding:16px 18px;display:grid;gap:10px} | |
| .runbox .row{display:flex;justify-content:space-between;gap:12px;font-size:.88rem} | |
| .runbox .row span:first-child{color:var(--ink-3)} | |
| .prog{height:6px;background:var(--sunk);border-radius:99px;overflow:hidden} | |
| .prog div{height:100%;background:var(--caution);border-radius:99px;transition:width .6s var(--ease)} | |
| @media (max-width:860px){.hero{grid-template-columns:1fr}} | |
| section{padding-block:26px;border-top:1px solid var(--line)} | |
| section > .head{display:flex;flex-wrap:wrap;align-items:baseline;justify-content:space-between;gap:6px 18px;margin-bottom:16px} | |
| section > .head p{color:var(--ink-3);font-size:.9rem} | |
| /* stat tiles */ | |
| .tiles{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:12px} | |
| .tile{background:var(--surface);border:1px solid var(--line);border-radius:var(--r);padding:14px 16px;display:grid;gap:4px;min-width:0} | |
| .tile .k{font-size:.8rem;color:var(--ink-3)} | |
| .tile .v{font-family:var(--mono);font-size:1.55rem;font-weight:500;color:var(--ink);font-variant-numeric:tabular-nums;line-height:1.2} | |
| .tile .v .arrow{color:var(--ink-3);font-weight:400;margin-inline:4px} | |
| .tile .v .to{color:var(--measure)} | |
| .tile .d{font-size:.78rem;color:var(--ink-3)} | |
| .delta{font-family:var(--mono);font-size:.78rem;padding:1px 7px;border-radius:999px;background:var(--measure-soft);color:var(--measure);justify-self:start} | |
| .delta.worse{background:var(--caution-soft);color:var(--caution)} | |
| /* hypotheses */ | |
| .hyps{display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:12px;margin-top:14px} | |
| .hyp{border:1px solid var(--line);border-radius:var(--r);padding:12px 14px;background:var(--surface);display:grid;gap:6px} | |
| .hyp .top{display:flex;justify-content:space-between;align-items:center;gap:10px} | |
| .hyp .id{font-family:var(--mono);font-weight:500;color:var(--ink)} | |
| .hyp p{font-size:.88rem} | |
| .hyp .n{font-family:var(--mono);font-size:.82rem;color:var(--ink-3)} | |
| /* grid of cards */ | |
| .grid{display:grid;grid-template-columns:repeat(12,minmax(0,1fr));gap:14px} | |
| .c6{grid-column:span 6}.c4{grid-column:span 4}.c8{grid-column:span 8}.c12{grid-column:span 12}.c5{grid-column:span 5}.c7{grid-column:span 7} | |
| @media (max-width:1000px){.c4,.c5,.c6,.c7,.c8{grid-column:span 12}} | |
| .card{background:var(--surface);border:1px solid var(--line);border-radius:var(--r);padding:14px 16px 12px;min-width:0} | |
| .cmpbar{display:flex;flex-wrap:wrap;align-items:center;gap:8px 12px} | |
| .cmpbar .lab{font-weight:500;font-size:.88rem} | |
| .cmpbar .note{font-size:.8rem;color:var(--ink-3)} | |
| .card h3{margin-bottom:2px} | |
| .card .sub{font-size:.8rem;color:var(--ink-3);margin-bottom:10px} | |
| .card .foot{font-size:.8rem;color:var(--ink-3);margin-top:8px} | |
| /* charts */ | |
| .chart{position:relative;width:100%} | |
| .chart svg{display:block;width:100%;height:auto;overflow:visible} | |
| .chart text{font-family:var(--mono);font-size:10.5px;fill:var(--axis)} | |
| .chart .gl{stroke:var(--grid);stroke-width:1} | |
| .chart .bl{stroke:var(--axis);stroke-width:1} | |
| .legend{display:flex;flex-wrap:wrap;gap:4px 14px;font-size:.78rem;color:var(--ink-2);margin-bottom:6px} | |
| .legend span{display:inline-flex;align-items:center;gap:6px} | |
| .legend i{width:14px;height:3px;border-radius:2px;display:inline-block} | |
| .legend i.box{width:10px;height:10px;border-radius:2px} | |
| .legend i.dash{background:repeating-linear-gradient(90deg,var(--s-base) 0 4px,transparent 4px 7px)} | |
| .tip{position:absolute;pointer-events:none;z-index:5;background:var(--surface);border:1px solid var(--line-2);border-radius:8px; | |
| padding:7px 9px;font-size:.78rem;color:var(--ink);box-shadow:0 6px 24px hsl(240 14% 10% / .12);min-width:120px;max-width:260px;display:none} | |
| .tip .t{font-family:var(--mono);color:var(--ink-3);font-size:.72rem;margin-bottom:3px} | |
| .tip .r{display:flex;justify-content:space-between;gap:12px;font-family:var(--mono)} | |
| .tip .r i{width:8px;height:8px;border-radius:2px;display:inline-block;margin-right:6px} | |
| .empty{font-size:.85rem;color:var(--ink-3);padding:26px 0;text-align:center;border:1px dashed var(--line-2);border-radius:8px} | |
| /* tables */ | |
| .tw{overflow-x:auto;border:1px solid var(--line);border-radius:8px} | |
| table{border-collapse:collapse;width:100%;font-size:.86rem} | |
| th,td{padding:7px 10px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top} | |
| th{font-weight:500;color:var(--ink-3);background:var(--sunk);font-size:.78rem;white-space:nowrap} | |
| td.n,th.n{text-align:right;font-family:var(--mono);font-variant-numeric:tabular-nums;white-space:nowrap} | |
| td.n.m{color:var(--measure)} | |
| tbody tr:last-child td{border-bottom:0} | |
| .better{color:var(--measure);font-weight:500} | |
| .worse{color:var(--caution)} | |
| /* samples */ | |
| .samples{display:grid;gap:14px} | |
| .sample{background:var(--surface);border:1px solid var(--line);border-radius:var(--r);padding:16px 18px;display:grid;gap:12px;min-width:0} | |
| .sample .meta{display:flex;flex-wrap:wrap;gap:6px} | |
| .chip{font-family:var(--mono);font-size:.72rem;padding:2px 8px;border-radius:6px;background:var(--sunk);color:var(--ink-3);white-space:nowrap} | |
| .chip.gold{background:var(--measure-soft);color:var(--measure)} | |
| .chip.warn{background:var(--caution-soft);color:var(--caution)} | |
| .q{color:var(--ink);font-weight:500} | |
| .ev{font-family:var(--mono);font-size:.76rem;white-space:pre-wrap;word-break:break-word;background:var(--sunk);border-radius:8px;padding:10px 12px;max-height:180px;overflow:auto;color:var(--ink-2)} | |
| .opts{display:grid;gap:4px} | |
| .opt{display:grid;grid-template-columns:22px minmax(0,1fr) 150px;gap:10px;align-items:center;font-size:.84rem} | |
| .opt .L{font-family:var(--mono);font-weight:500;color:var(--ink-3)} | |
| .opt.g .L{color:var(--measure)} | |
| .opt .txt{min-width:0;overflow-wrap:anywhere} | |
| .opt.g .txt{color:var(--ink);font-weight:500} | |
| .pb{display:grid;gap:2px} | |
| .pb div{display:grid;grid-template-columns:minmax(0,1fr) 38px;align-items:center;gap:6px;font-family:var(--mono);font-size:.68rem;color:var(--ink-3)} | |
| .pb span.b{height:5px;border-radius:3px;background:var(--sunk);position:relative;overflow:hidden;display:block} | |
| .pb span.b em{position:absolute;inset:0 auto 0 0;border-radius:3px} | |
| .chains{display:grid;grid-template-columns:repeat(auto-fit,minmax(300px,1fr));gap:10px} | |
| details.chain{border:1px solid var(--line);border-radius:8px;background:var(--surface)} | |
| details.chain summary{cursor:pointer;padding:8px 12px;font-size:.84rem;display:flex;justify-content:space-between;gap:10px;flex-wrap:wrap} | |
| details.chain summary b{font-weight:500;color:var(--ink)} | |
| details.chain pre{margin:0;padding:10px 12px;border-top:1px solid var(--line);font-family:var(--mono);font-size:.74rem;line-height:1.55;white-space:pre-wrap;word-break:break-word;max-height:420px;overflow:auto;color:var(--ink-2)} | |
| .why{font-size:.86rem;color:var(--ink-3)} | |
| /* browse */ | |
| .filters{display:flex;flex-wrap:wrap;gap:8px;margin-bottom:10px;align-items:center} | |
| .seg{display:inline-flex;border:1px solid var(--line);border-radius:8px;overflow:hidden} | |
| .seg button{font:inherit;font-size:.82rem;border:0;background:var(--surface);color:var(--ink-2);padding:6px 11px;cursor:pointer} | |
| .seg button+button{border-left:1px solid var(--line)} | |
| .seg button[aria-pressed="true"]{background:var(--brand-soft);color:var(--brand-ink);font-weight:500} | |
| tr.click{cursor:pointer} | |
| tr.click:hover td{background:var(--sunk)} | |
| tr.detail td{background:var(--sunk);padding:12px} | |
| .ok{color:var(--measure)}.no{color:var(--caution)} | |
| /* logs */ | |
| .tabs{display:flex;flex-wrap:wrap;gap:4px;margin-bottom:12px} | |
| .tabs button{font:inherit;font-size:.85rem;border:1px solid var(--line);background:var(--surface);color:var(--ink-2);padding:6px 12px;border-radius:8px;cursor:pointer} | |
| .tabs button[aria-selected="true"]{background:var(--brand-soft);color:var(--brand-ink);border-color:transparent;font-weight:500} | |
| .md{background:var(--surface);border:1px solid var(--line);border-radius:var(--r);padding:16px 20px;font-size:.9rem;overflow-x:auto} | |
| .md h1,.md h2,.md h3{margin:14px 0 8px}.md h1{font-size:1.15rem}.md h2{font-size:1.02rem} | |
| .md p{margin:8px 0}.md ul{margin:6px 0;padding-left:20px}.md li{margin:3px 0} | |
| .md code{font-family:var(--mono);font-size:.82em;background:var(--sunk);padding:1px 5px;border-radius:4px} | |
| .md table{margin:10px 0;font-size:.82rem}.md td,.md th{border:1px solid var(--line)} | |
| pre.log{margin:0;font-family:var(--mono);font-size:.72rem;line-height:1.5;background:var(--surface);border:1px solid var(--line);border-radius:var(--r);padding:12px 14px;max-height:460px;overflow:auto;white-space:pre-wrap;word-break:break-all;color:var(--ink-2)} | |
| footer{padding-block:28px 40px;color:var(--ink-3);font-size:.82rem;border-top:1px solid var(--line)} | |
| </style> | |
| <header class="bar"> | |
| <div class="wrap"> | |
| <span class="mark"><b>Mercity</b> · decision-reasoner · v1 + V2</span> | |
| <span class="pill neutral" id="statePill"><i></i><span>loading</span></span> | |
| <span class="mark" id="updated"></span> | |
| <nav class="nav" aria-label="Sections"> | |
| <a href="#summary">Summary</a><a href="#v2">V2</a><a href="#rl">RL</a><a href="#jevals">Jevals</a><a href="#twob">2B</a><a href="#s1mb">S1MB</a><a href="#pipeline">Pipeline</a><a href="#results">Results</a><a href="#system">System</a> | |
| <a href="#training">Training</a><a href="#generation">Generation</a><a href="#samples">Samples</a><a href="#browse">Browse</a><a href="#logs">Logs</a> | |
| </nav> | |
| </div> | |
| </header> | |
| <main class="wrap"> | |
| <div class="hero"> | |
| <div> | |
| <div class="eyebrow">Qwen3.5-4B · one A100 80GB · LoRA rank 32</div> | |
| <h1>Decision Reasoner, <em>fast when it can, thinking when it must</em></h1> | |
| <p>A Jev-style typed decision model: one forward pass reads a probability for every option; a thinking mode writes its reasoning first and decides from the letter right after the chain. The fast answer is trained toward what the model's own reasoning concludes.</p> | |
| </div> | |
| <div class="runbox" id="runbox"></div> | |
| </div> | |
| <section id="summary"> | |
| <div class="head"><h2>Summary</h2><p id="sumNote">Untrained base vs the jointly trained model, on the same eval items.</p></div> | |
| <div class="tiles" id="tiles"></div> | |
| <div class="hyps" id="hyps"></div> | |
| </section> | |
| <section id="v2"> | |
| <div class="head"><h2>V2 run</h2><p>About 42K readout questions, chains from the 4B itself plus an API teacher on the tail, a speed benchmark and a short learning-rate sweep before the joint run, then two small ablations and the full evaluation.</p></div> | |
| <div class="tiles" id="v2Tiles" style="margin-bottom:14px"></div> | |
| <div class="grid"> | |
| <div class="card c12"><h3>V2 stages</h3><div class="sub">UTC with IST beside it · decisions the pipeline made are listed after the stages</div><div class="tw"><table id="v2Stages"></table></div></div> | |
| <div class="c12 cmpbar"><span class="lab">Compare with v1</span><div class="seg" id="v1Seg"></div><span class="note">v1 joint drawn in gray on the training charts and added to the step table · x axis: tokens trained, so the two runs line up by how much they have seen</span></div> | |
| <div class="card c12"><h3>V2 vs v1 joint</h3><div class="sub" id="v2CmpSub"></div><div class="tw"><table id="v2Cmp"></table></div></div> | |
| <div class="card c12"><h3>Held-out dev, every V2 run</h3><div class="sub">fast readout on dev questions never trained on (999; the sweep runs used 500 of them) · 61 of the 999 repeat a training question exactly (procedural generators), so each accuracy is at most ~1.2 points high</div><div class="tw"><table id="v2Dev"></table></div></div> | |
| <div class="card c6"><h3>V2 joint: chain and post-chain letter loss</h3><div class="sub">chain: mean token cross-entropy · letter: soft target 0.5·own answer + 0.5·votes · 10-step means</div><div class="chart" id="v2Chain"></div></div> | |
| <div class="card c6"><h3>V2 joint: readout loss</h3><div class="sub">soft cross-entropy over option letters · the target's own entropy is the floor · 10-step means</div><div class="chart" id="v2Ro"></div></div> | |
| <div class="card c4"><h3>Learning rate</h3><div class="sub">cosine with 3% warmup</div><div class="chart" id="v2Lr"></div></div> | |
| <div class="card c4"><h3>Gradient norm before clipping</h3><div class="sub">clipped to 1.0</div><div class="chart" id="v2Gn"></div></div> | |
| <div class="card c4"><h3>Training throughput</h3><div class="sub">tokens per second, cumulative</div><div class="chart" id="v2Tps"></div></div> | |
| <div class="card c12"><h3>V2 joint: one optimizer step</h3><div class="sub" id="v2StepSub"></div><div class="tw"><table id="v2Step"></table></div></div> | |
| <div class="card c6"><h3>Learning-rate sweep</h3><div class="sub">1,500 questions, readout only · readout loss, 5-step means · pick: lowest rate within 0.01 dev NLL of the best</div><div class="chart" id="v2Sweep"></div><div class="tw"><table id="v2SweepT"></table></div></div> | |
| <div class="card c6"><h3>Speed benchmark</h3><div class="sub">same 260K tokens per setup · errors against an fp32 reference; a setup passes at most 1.5× the current setup's error</div><div class="tw"><table id="v2Bench"></table></div></div> | |
| <div class="card c12"><h3>Where the chains come from</h3><div class="sub">4 chains per question · share of questions by how many of the 4 match gold</div><div class="tw"><table id="v2Src"></table></div></div> | |
| <div class="card c12"><h3>Fast mode on the eval sets, first look</h3><div class="sub" id="v2FirstSub"></div><div class="tw"><table id="v2First"></table></div></div> | |
| <div class="card c12"><h3>V2 results</h3><div class="sub">eval set as in v1 · temperatures fitted on the calibration items only · reasoning on the items every reasoner answered</div><div class="tw"><table id="v2Res"></table></div></div> | |
| </div> | |
| </section> | |
| <section id="rl"> | |
| <div class="head"><h2>RL on V2 (Phase B)</h2><p>Steps 1-40 at lr 1e-5; from step 41, 20 more at 2.5e-5 from the step-40 adapter (fresh optimizer, questions not yet used). GRPO on the V2 joint model: 8 thinking chains per question, reward +1 right / -1 wrong (annotator share for soft labels), -1.5 for no answer by 8,192 tokens, a small penalty past 4,096. 480 questions: 288 never trained on (192 from reasoning-heavy sources), 192 in-domain. OpenDecisions and LocalLLaMA stay untouched.</p></div> | |
| <div class="tiles" id="rlTiles" style="margin-bottom:14px"></div> | |
| <div class="grid"> | |
| <div class="card c12"><h3>Result: thinking mode on the same test questions</h3><div class="sub" id="rlResSub"></div><div class="tw"><table id="rlRes"></table></div></div> | |
| <div class="card c6"><h3>Reward per generation round</h3><div class="sub">mean over the round's 128 chains · +1 all right, -1 all wrong, -1.5 all unfinished</div><div class="chart" id="rlReward"></div></div> | |
| <div class="card c6"><h3>Chains that hit the 8,192-token cap</h3><div class="sub">the loop problem V2 showed in its eval · share of the round's chains with no finished answer by the cap</div><div class="chart" id="rlClip"></div></div> | |
| <div class="card c4"><h3>Chain length</h3><div class="sub">mean completion tokens per round</div><div class="chart" id="rlLen"></div></div> | |
| <div class="card c4"><h3>Questions with no learning signal</h3><div class="sub">share of questions whose 8 chains all got the same reward</div><div class="chart" id="rlZero"></div></div> | |
| <div class="card c4"><h3>Gradient norm</h3><div class="sub">per optimizer step</div><div class="chart" id="rlGn"></div></div> | |
| <div class="card c6"><h3>Second-guessing in finished chains</h3><div class="sub">"Wait" per finished chain, per round · the loops are endless "Wait, I'll consider ..." cycles, so fewer of them in chains that do finish is the habit RL is meant to change · rounds draw different questions</div><div class="chart" id="rlWait"></div></div> | |
| <div class="card c6"><h3>Finished chains that are right</h3><div class="sub">share of the round's finished chains with the right answer · rounds draw different questions, so read the trend, not single rounds</div><div class="chart" id="rlFin"></div></div> | |
| </div> | |
| </section> | |
| <section id="s1mb"> | |
| <div class="head"><h2>S1MB</h2><p>System One Mosaic Benchmark (hotchpotch), English v1: 137 typed benchmarks (59 yes/no, 57 choice, 21 score), 26,269 judgments, dataset S1MB-english-v1 @ 10bcb8c6. Scored with our reimplementation of its SCORING.md (baseline-adjusted skill clipped per benchmark; Task Avg = mean of the three task scores; Borda = rank points over all 137 benchmarks across every complete published model plus ours), checked against the published scores of re-scored models. Our fast readout, plus routed rows: the least certain 30% of judgments (smallest readout margin) answered by thinking mode, the rest by the fast readout. Temperatures from our own calibration items, never S1MB.</p></div> | |
| <div class="grid"><div class="card c12"><h3>Leaderboard: top 25, every published 0.8B-4B model, and ours</h3><div class="sub" id="s1Sub"></div><div class="tw"><table id="s1Tab"></table></div></div></div> | |
| </section> | |
| <section id="twob"> | |
| <div class="head"><h2>2B model</h2><p>The V2 recipe on a ~2B model (overnight, on the user's instruction). Two candidates probed the way the 4B was: fast readout on the v1 eval set, 4 thinking chains on 128 pool items. Pre-registered pick: mean of readout accuracy and first-chain accuracy, parse rate at least 90%, Qwen wins within 0.02. It then trains on V2's data (chains written by the 4B and the 35B teacher).</p></div> | |
| <div class="tiles" id="tbTiles" style="margin-bottom:14px"></div> | |
| <div class="grid"><div class="card c12"><h3>Candidate probes</h3><div class="sub" id="tbSub"></div><div class="tw"><table id="tbProbe"></table></div></div> | |
| <div class="card c6"><h3>2B training: readout loss</h3><div class="sub">10-step means · gray: the 4B V2 run at the same tokens trained</div><div class="chart" id="tbRo"></div></div> | |
| <div class="card c6"><h3>2B training: chain loss</h3><div class="sub">the 4B's and the teacher's chains, as distillation targets · gray: the 4B V2 run</div><div class="chart" id="tbCh"></div></div></div> | |
| </section> | |
| <section id="jevals"> | |
| <div class="head"><h2>Jevals</h2><p>jevals.com suite 0.1.0, the same 300 items per board rebuilt from the source datasets and matched to every published state hash. Decision Score = 100 × (1 − loss / loss of the label prior): Brier for yes/no and choice, ranked probability score for the 5-level score board. Board rows are Jevals' release 2026-09-18; ours are scored with their rules (95% item bootstrap, our seed). Our training pool holds 40 Banking77 and 40 HelpSteer2-helpfulness train-split items (exact text overlap with these 300: 0 and 2); PubMedQA is unseen. Banking77 (77 options) runs in two stages: chunked readout, then the top 24 together (gold in the shortlist for 98.7% of items).</p></div> | |
| <div class="grid" id="jvGrid"></div> | |
| </section> | |
| <section id="pipeline"> | |
| <div class="head"><h2>Pipeline</h2><p>Every stage of the run, in UTC with IST beside it. Restarts are marked where a stage was stopped and resumed.</p><div class="seg" id="pipeSeg"></div></div> | |
| <div class="card" style="margin-bottom:12px"><h3>Up next</h3><div class="sub" id="upSub"></div><div class="tw"><table id="upTable"></table></div></div> | |
| <div class="card"><div class="chart" id="gantt"></div></div> | |
| <div class="tw" style="margin-top:12px"><table id="stageTable"></table></div> | |
| </section> | |
| <section id="results"> | |
| <div class="head"><h2>Results</h2><p>Readout accuracy uses per-type temperatures fitted on the calibration set only. OpenDecisions and LocalLLaMA were never trained on.</p></div> | |
| <div class="grid"> | |
| <div class="card c7"><h3>Fast readout accuracy by eval set</h3><div class="sub">argmax vs gold · 5,442 eval items</div><div class="chart" id="cReadout"></div></div> | |
| <div class="card c5"><h3>Readout agrees with its own reasoning</h3><div class="sub">same items, thinking call never sees the readout's answer</div><div class="chart" id="cAgree"></div></div> | |
| <div class="card c6"><h3>Reasoning accuracy, same items</h3><div class="sub">one thinking sample per item</div><div class="chart" id="cReason"></div></div> | |
| <div class="card c6"><h3>Median chain length, same items</h3><div class="sub">generated tokens in thinking mode</div><div class="chart" id="cTokens"></div></div> | |
| <div class="card c6"><h3>OpenDecisions by difficulty tier: readout</h3><div class="sub">accuracy · easy → hard</div><div class="chart" id="cTierR"></div></div> | |
| <div class="card c6"><h3>Procedural eval by difficulty level: readout</h3><div class="sub">accuracy · level 0 (easy) → 4 (hard)</div><div class="chart" id="cLevelR"></div></div> | |
| <div class="card c12"><h3>Comparison table</h3><div class="sub">same-item numbers for reasoning; readout over all eval items</div><div class="tw"><table id="cmpTable"></table></div></div> | |
| </div> | |
| </section> | |
| <section id="system"> | |
| <div class="head"><h2>The deployable system</h2><p>Readout by default; the least confident share of items goes to thinking mode, whose probabilities are the readout after the chain.</p></div> | |
| <div class="grid"> | |
| <div class="card c6"><h3>OpenDecisions Capability vs share routed to thinking</h3><div class="sub">official scorer · margin router</div><div class="chart" id="cRouteCap"></div></div> | |
| <div class="card c6"><h3>Accuracy over all eval sets vs share routed</h3><div class="sub">margin router against random routing at the same share</div><div class="chart" id="cRouteAcc"></div></div> | |
| <div class="card c12"><div class="tw"><table id="sysTable"></table></div><div class="foot" id="latNote"></div></div> | |
| </div> | |
| </section> | |
| <section id="training"> | |
| <div class="head"><h2>Training</h2><p>One LoRA adapter, joint loss: the model's shortest correct chain plus a readout target q = 0.5·gold + 0.5·votes. Ablations train the readout alone.</p></div> | |
| <div class="tiles" id="trainTiles" style="margin-bottom:14px"></div> | |
| <div class="grid"> | |
| <div class="card c6"><h3>Joint: chain loss per step</h3><div class="sub">mean token cross-entropy on the chain, letter and end token · line = 10-step mean</div><div class="chart" id="cChain"></div></div> | |
| <div class="card c6"><h3>Readout loss per step</h3><div class="sub">soft cross-entropy over option letters · 10-step means</div><div class="chart" id="cRoLoss"></div></div> | |
| <div class="card c6"><h3>Learning rate</h3><div class="sub">cosine with 3% warmup</div><div class="chart" id="cLr"></div></div> | |
| <div class="card c6"><h3>Gradient norm before clipping</h3><div class="sub">clipped to 1.0</div><div class="chart" id="cGn"></div></div> | |
| <div class="card c12"><h3>Training throughput</h3><div class="sub">tokens per second, cumulative per run</div><div class="chart" id="cTps"></div></div> | |
| </div> | |
| </section> | |
| <section id="generation"> | |
| <div class="head"><h2>Generation and serving</h2><p>vLLM's own engine stats (logged each minute from 03:09 UTC) and the stream of finished items.</p></div> | |
| <div class="tiles" id="genTiles" style="margin-bottom:14px"></div> | |
| <div class="grid"> | |
| <div class="card c6"><h3>Decode throughput</h3><div class="sub">generated tokens/s reported by vLLM</div><div class="chart" id="cEngGen"></div></div> | |
| <div class="card c6"><h3>KV cache in use</h3><div class="sub">% of the 1.7M-token cache</div><div class="chart" id="cEngKv"></div></div> | |
| <div class="card c6"><h3>Items finished over time</h3><div class="sub">each stage's own clock (minutes since it started)</div><div class="chart" id="cProg"></div></div> | |
| <div class="card c6"><h3>Chain length, generation</h3><div class="sub">all 4 samples vs the shortest correct chain per item · log2 bins</div><div class="chart" id="cHist"></div></div> | |
| <div class="card c7"><h3>Reasoner accuracy by procedural family</h3><div class="sub">share of 4 samples that are correct, training pool</div><div class="chart" id="cFam"></div></div> | |
| <div class="card c5"><h3>How the 4 votes fall</h3><div class="sub">share of generated items</div><div class="chart" id="cVotes"></div></div> | |
| </div> | |
| </section> | |
| <section id="samples"> | |
| <div class="head"><h2>Samples</h2><p>Real items picked by rule from the predictions, with full chains. Option bars show the readout's probabilities.</p></div> | |
| <div class="samples" id="sampleList"></div> | |
| </section> | |
| <section id="browse"> | |
| <div class="head"><h2>Browse eval items</h2><p>30 random items per eval set. Click a row for the item, both readouts and both chains.</p></div> | |
| <div class="filters"><div class="seg" id="browseSeg" role="group" aria-label="Eval set"></div><span class="mark" id="browseCount"></span></div> | |
| <div class="tw"><table id="browseTable"></table></div> | |
| </section> | |
| <section id="logs"> | |
| <div class="head"><h2>Logs</h2><p>The lab notebook, every deviation from the plan, the to-do list, the stage timeline and the tail of each stage's log.</p></div> | |
| <div class="tabs" id="logTabs" role="tablist"></div> | |
| <div id="logBody"></div> | |
| </section> | |
| </main> | |
| <footer><div class="wrap" id="foot"></div></footer> | |
| <script type="application/json" id="dash-data">__DATA__</script> | |
| <script> | |
| (() => { | |
| ; | |
| const D = JSON.parse(document.getElementById('dash-data').textContent); | |
| const $ = s => document.querySelector(s); | |
| const SV = 'http://www.w3.org/2000/svg'; | |
| function h(tag, attrs, ...kids){ const e = document.createElement(tag); if (attrs) for (const k in attrs){ const v = attrs[k]; if (v == null || v === false) continue; | |
| if (k === 'class') e.className = v; else if (k === 'style') e.style.cssText = v; else if (k.startsWith('on')) e.addEventListener(k.slice(2), v); else e.setAttribute(k, v === true ? '' : v); } | |
| for (const c of kids.flat(Infinity)) if (c != null && c !== false) e.append(c instanceof Node ? c : document.createTextNode(String(c))); return e; } | |
| function s(tag, attrs, ...kids){ const e = document.createElementNS(SV, tag); if (attrs) for (const k in attrs) if (attrs[k] != null) e.setAttribute(k, attrs[k]); | |
| for (const c of kids.flat(Infinity)) if (c != null) e.append(c instanceof Node ? c : document.createTextNode(String(c))); return e; } | |
| const pct = (v, d=1) => v == null ? '–' : (100*v).toFixed(d) + '%'; | |
| const f1 = v => v == null ? '–' : (+v).toFixed(1); | |
| const f1d = v => v == null ? '–' : (+v).toFixed(3); | |
| const int = v => v == null ? '–' : Math.round(v).toLocaleString('en-US'); | |
| const SET = {eval_proc:'Procedural', eval_ts:'tasksource', opendecisions:'OpenDecisions', localllama:'LocalLLaMA', all:'All sets'}; | |
| const SETS = ['eval_proc','eval_ts','opendecisions','localllama']; | |
| const COL = {base:'var(--s-base)', gold_only:'var(--s-gold)', votes_only:'var(--s-votes)', joint:'var(--s-joint)', v2_joint:'var(--caution)', v2_abl_gold:'var(--s-gold)', v2_abl_mix:'var(--s-votes)'}; | |
| const NAME = {base:'base (untrained)', gold_only:'gold-only', votes_only:'votes-only', joint:'joint', v2_joint:'V2 joint', v2_abl_gold:'V2 ablation gold', v2_abl_mix:'V2 ablation mix'}; | |
| const ist = iso => { if (!iso) return '–'; const d = new Date(iso); const u = d.toISOString().slice(11,16); const i = new Date(d.getTime() + 330*60000).toISOString().slice(11,16); return `${u} UTC · ${i} IST`; }; | |
| const hm = iso => iso ? new Date(iso).toISOString().slice(11,16) : '–'; | |
| const E = D.eval || {}; const RO = E.readout || {}; const RS = E.reasoning || {}; const PR = E.pairs || {}; | |
| /* ------------------------------------------------------------ run state */ | |
| const V = D.v2 || {}; | |
| const v2run = (V.stages || []).filter(x => x.how === 'running'); | |
| const running = (D.stages || []).filter(x => x.how === 'running').concat(v2run); | |
| const allDone = (D.events || []).some(e => e.event === 'finished') && !v2run.length && (!(V.stages || []).length || (V.decisions || []).some(e => e.event === 'finished')); | |
| const pill = $('#statePill'); | |
| if (allDone) { pill.className = 'pill done'; pill.lastChild.textContent = 'run complete'; } | |
| else if (running.length) { pill.className = 'pill live'; pill.lastChild.textContent = 'live · ' + running.map(x => x.stage.replace('_', ' ')).join(', '); } | |
| else { pill.className = 'pill neutral'; pill.lastChild.textContent = 'between stages'; } | |
| $('#updated').textContent = 'snapshot ' + ist(D.generated_at); | |
| (function runbox(){ | |
| const el = $('#runbox'); const st = D.run_start; | |
| const el_h = st ? ((new Date(D.generated_at) - new Date(st)) / 3.6e6) : null; | |
| const jr = RS.joint; const prog = jr ? jr.n_done / jr.n_total : null; | |
| const rows = [['Run started', ist(st)], ['Elapsed', el_h == null ? '–' : el_h.toFixed(1) + ' h'], | |
| ['Current stage', running.length ? running.map(x => x.stage).join(', ') : (allDone ? 'done' : '–')], | |
| ['Generated items (4 chains each)', int(D.gen_view && D.gen_view.items)], | |
| ['Joint reasoning eval', jr ? `${int(jr.n_done)} / ${int(jr.n_total)} items` : '–']]; | |
| el.append(h('div', {class:'eyebrow'}, 'Run status')); | |
| for (const [k, v] of rows) el.append(h('div', {class:'row'}, h('span', null, k), h('span', {class:'mono num'}, v))); | |
| if (prog != null && prog < 1) el.append(h('div', {class:'prog', role:'progressbar', 'aria-valuenow': Math.round(100*prog), 'aria-valuemin':0, 'aria-valuemax':100}, h('div', {style:`width:${(100*prog).toFixed(1)}%`}))); | |
| })(); | |
| /* ------------------------------------------------------------ tooltip + chart helpers */ | |
| function mkTip(box){ const t = h('div', {class:'tip'}); box.append(t); return t; } | |
| function showTip(tip, box, ev, title, rows){ | |
| tip.replaceChildren(h('div', {class:'t'}, title), ...rows.map(([c, k, v]) => h('div', {class:'r'}, h('span', null, c ? h('i', {style:`background:${c}`}) : null, k), h('b', null, v)))); | |
| tip.style.display = 'block'; const r = box.getBoundingClientRect(); let x = ev.clientX - r.left + 14, y = ev.clientY - r.top + 10; | |
| const w = tip.offsetWidth, hh = tip.offsetHeight; if (x + w > r.width) x = ev.clientX - r.left - w - 14; if (x < 0) x = 4; if (y + hh > r.height + 40) y = ev.clientY - r.top - hh - 10; tip.style.left = x + 'px'; tip.style.top = y + 'px'; } | |
| function hideTip(tip){ tip.style.display = 'none'; } | |
| function ticks(lo, hi, n=4){ if (hi === lo) { hi = lo + 1; } const raw = (hi - lo) / n, mag = Math.pow(10, Math.floor(Math.log10(raw))), nr = raw / mag; const st = (nr < 1.5 ? 1 : nr < 3 ? 2 : nr < 7 ? 5 : 10) * mag; const out = []; for (let v = Math.ceil(lo / st) * st; v <= hi + st * 1e-9; v += st) out.push(+v.toPrecision(10)); return out; } | |
| function legend(items){ return h('div', {class:'legend'}, items.map(it => h('span', null, h('i', {class:(it.box ? 'box ' : '') + (it.dash ? 'dash' : ''), style:`background:${it.color}`}), it.name))); } | |
| const registry = []; | |
| function chart(id, draw){ const el = document.getElementById(id); if (!el) return; registry.push(() => { el.replaceChildren(); draw(el, Math.max(280, el.clientWidth || 600)); }); } | |
| function lineChart(el, W, {series, xType='num', yFmt=v=>v, xFmt=v=>v, yMin, yMax, H=210, xLabel, refLines=[]}){ | |
| const pts = series.flatMap(s => s.pts); if (!pts.length) { el.append(h('div', {class:'empty'}, 'No data yet for this chart.')); return; } | |
| if (series.length > 1) el.append(legend(series.map(s => ({name:s.name, color:s.color, dash:s.dash})))); | |
| const m = {l:52, r:14, t:10, b: xLabel ? 36 : 24}; const iw = W - m.l - m.r, ih = H - m.t - m.b; | |
| let x0 = Math.min(...pts.map(p => p[0])), x1 = Math.max(...pts.map(p => p[0])); if (x0 === x1) x1 = x0 + 1; | |
| let y0 = yMin ?? Math.min(...pts.map(p => p[1])), y1 = yMax ?? Math.max(...pts.map(p => p[1])); if (y0 === y1) { y0 -= 1; y1 += 1; } | |
| const yt = ticks(y0, y1, 4); y0 = Math.min(y0, yt[0]); y1 = Math.max(y1, yt[yt.length-1]); | |
| const X = v => m.l + (v - x0) / (x1 - x0) * iw, Y = v => m.t + (1 - (v - y0) / (y1 - y0)) * ih; | |
| const svg = s('svg', {viewBox:`0 0 ${W} ${H}`, role:'img'}); | |
| for (const t of yt) svg.append(s('line', {x1:m.l, x2:W-m.r, y1:Y(t), y2:Y(t), class:'gl'}), s('text', {x:m.l-8, y:Y(t)+3.5, 'text-anchor':'end'}, yFmt(t))); | |
| const xt = xType === 'time' ? (() => { const span = x1 - x0; const st = span > 6*3.6e6 ? 3.6e6 : span > 2*3.6e6 ? 1.8e6 : span > 3.6e6 ? 9e5 : 6e5; const out = []; for (let v = Math.ceil(x0/st)*st; v <= x1; v += st) out.push(v); return out; })() : ticks(x0, x1, Math.max(3, Math.floor(iw / 90))); | |
| for (const t of xt) svg.append(s('text', {x:X(t), y:H-m.b+15, 'text-anchor':'middle'}, xFmt(t))); | |
| if (xLabel) svg.append(s('text', {x:m.l+iw/2, y:H-3, 'text-anchor':'middle'}, xLabel)); | |
| svg.append(s('line', {x1:m.l, x2:W-m.r, y1:Y(y0), y2:Y(y0), class:'bl'})); | |
| for (const r of refLines) svg.append(s('line', {x1:m.l, x2:W-m.r, y1:Y(r.y), y2:Y(r.y), style:`stroke:${r.color};stroke-width:1;stroke-dasharray:4 4`}), s('text', {x:W-m.r, y:Y(r.y)-4, 'text-anchor':'end'}, r.label)); | |
| for (const sr of series) { | |
| if (sr.dots) { for (const p of sr.pts) svg.append(s('circle', {cx:X(p[0]), cy:Y(p[1]), r:sr.r || 1.6, style:`fill:${sr.color};opacity:${sr.opacity ?? .35}`})); continue; } | |
| let d = '', pen = false; for (const p of sr.pts) { if (p[1] == null) { pen = false; continue; } d += (pen ? 'L' : 'M') + X(p[0]).toFixed(1) + ',' + Y(p[1]).toFixed(1); pen = true; } | |
| if (sr.area && sr.pts.length > 1) { const a = sr.pts.filter(p => p[1] != null); svg.append(s('path', {d:`M${X(a[0][0])},${Y(y0)}` + a.map(p => `L${X(p[0]).toFixed(1)},${Y(p[1]).toFixed(1)}`).join('') + `L${X(a[a.length-1][0])},${Y(y0)}Z`, style:`fill:${sr.color};opacity:.10`})); } | |
| svg.append(s('path', {d, style:`fill:none;stroke:${sr.color};stroke-width:${sr.w || 2};stroke-linejoin:round;stroke-linecap:round;${sr.dash ? 'stroke-dasharray:5 4;' : ''}`})); | |
| const last = sr.pts.filter(p => p[1] != null).slice(-1)[0]; if (last && !sr.noEnd) svg.append(s('circle', {cx:X(last[0]), cy:Y(last[1]), r:3.5, style:`fill:${sr.color};stroke:var(--surface);stroke-width:2`})); | |
| } | |
| const cross = s('line', {y1:m.t, y2:m.t+ih, style:'stroke:var(--line-2);stroke-width:1;display:none'}); svg.append(cross); | |
| const ov = s('rect', {x:m.l, y:m.t, width:iw, height:ih, style:'fill:transparent'}); svg.append(ov); | |
| el.append(svg); const tip = mkTip(el); | |
| ov.addEventListener('pointermove', ev => { const r = svg.getBoundingClientRect(); const xv = x0 + ((ev.clientX - r.left) * W / r.width - m.l) / iw * (x1 - x0); | |
| const rows = []; for (const sr of series) { if (sr.noTip) continue; let best = null; for (const p of sr.pts) if (p[1] != null && (best == null || Math.abs(p[0] - xv) < Math.abs(best[0] - xv))) best = p; if (best) rows.push([sr.color, sr.name, yFmt(best[1])]); } | |
| cross.setAttribute('x1', X(xv)); cross.setAttribute('x2', X(xv)); cross.style.display = 'block'; showTip(tip, el, ev, xFmt(xv), rows); }); | |
| ov.addEventListener('pointerleave', () => { cross.style.display = 'none'; hideTip(tip); }); | |
| } | |
| function barChart(el, W, {cats, series, yFmt=v=>pct(v,0), yMin=0, yMax, H=220, horizontal=false, catFmt=c=>c}){ | |
| const vals = series.flatMap(s => s.vals).filter(v => v != null); if (!vals.length) { el.append(h('div', {class:'empty'}, 'No data yet for this chart.')); return; } | |
| el.append(legend(series.map(s => ({name:s.name, color:s.color, box:true})))); | |
| const hi = yMax ?? Math.max(...vals) * 1.08; | |
| const yt = ticks(yMin, hi, 4); const top = Math.max(hi, yt[yt.length-1]); | |
| const tip = mkTip(el); | |
| if (horizontal) { | |
| const rowH = Math.max(16, 9 * series.length + 6), m = {l:Math.min(210, W*0.42), r:16, t:4, b:22}; const HH = m.t + m.b + cats.length * (rowH + 6); const iw = W - m.l - m.r; | |
| const X = v => m.l + (v - yMin) / (top - yMin) * iw; const svg = s('svg', {viewBox:`0 0 ${W} ${HH}`, role:'img'}); | |
| for (const t of yt) svg.append(s('line', {x1:X(t), x2:X(t), y1:m.t, y2:HH-m.b, class:'gl'}), s('text', {x:X(t), y:HH-6, 'text-anchor':'middle'}, yFmt(t))); | |
| cats.forEach((c, i) => { const y = m.t + i * (rowH + 6); svg.append(s('text', {x:m.l-8, y:y+rowH/2+3.5, 'text-anchor':'end', style:'fill:var(--ink-2)'}, catFmt(c))); | |
| const bh = (rowH - 2 * (series.length - 1)) / series.length; | |
| series.forEach((sr, j) => { const v = sr.vals[i]; if (v == null) return; const r = s('rect', {x:X(yMin), y:y + j * (bh + 2), width:Math.max(1, X(v) - X(yMin)), height:bh, rx:2, style:`fill:${sr.color}`}); | |
| r.addEventListener('pointermove', ev => showTip(tip, el, ev, catFmt(c), series.map(q => [q.color, q.name, q.vals[i] == null ? '–' : yFmt(q.vals[i])]))); r.addEventListener('pointerleave', () => hideTip(tip)); svg.append(r); }); }); | |
| el.append(svg); return; | |
| } | |
| const m = {l:44, r:10, t:8, b:30}; const iw = W - m.l - m.r, ih = H - m.t - m.b; const Y = v => m.t + (1 - (v - yMin) / (top - yMin)) * ih; | |
| const svg = s('svg', {viewBox:`0 0 ${W} ${H}`, role:'img'}); | |
| for (const t of yt) svg.append(s('line', {x1:m.l, x2:W-m.r, y1:Y(t), y2:Y(t), class:'gl'}), s('text', {x:m.l-7, y:Y(t)+3.5, 'text-anchor':'end'}, yFmt(t))); | |
| const band = iw / cats.length, gw = band * 0.74, bw = (gw - 2 * (series.length - 1)) / series.length; | |
| cats.forEach((c, i) => { const gx = m.l + i * band + (band - gw) / 2; | |
| series.forEach((sr, j) => { const v = sr.vals[i]; if (v == null) return; const x = gx + j * (bw + 2); | |
| const r = s('rect', {x, y:Y(v), width:bw, height:Math.max(0, Y(yMin) - Y(v)), rx:2, style:`fill:${sr.color}`}); | |
| r.addEventListener('pointermove', ev => showTip(tip, el, ev, catFmt(c), series.map(q => [q.color, q.name, q.vals[i] == null ? '–' : yFmt(q.vals[i])]))); r.addEventListener('pointerleave', () => hideTip(tip)); svg.append(r); }); | |
| svg.append(s('text', {x:m.l + i * band + band / 2, y:H - m.b + 16, 'text-anchor':'middle', style:'fill:var(--ink-2)'}, catFmt(c))); }); | |
| svg.append(s('line', {x1:m.l, x2:W-m.r, y1:Y(yMin), y2:Y(yMin), class:'bl'})); | |
| el.append(svg); | |
| } | |
| /* ------------------------------------------------------------ summary tiles */ | |
| (function tiles(){ | |
| const T = $('#tiles'); const all = PR.all || {}; const rb = RO.base, rj = RO.joint; | |
| const tile = (k, a, b, d, fmt, better='up') => { const dv = (a != null && b != null) ? b - a : null; const good = dv == null ? null : (better === 'up' ? dv >= 0 : dv <= 0); | |
| return h('div', {class:'tile'}, h('div', {class:'k'}, k), h('div', {class:'v'}, a == null ? '–' : fmt(a), h('span', {class:'arrow'}, '→'), h('span', {class:'to'}, b == null ? '–' : fmt(b))), | |
| dv == null ? null : h('span', {class:'delta' + (good ? '' : ' worse')}, (dv >= 0 ? '+' : '') + (fmt === pct ? (100*dv).toFixed(1) + ' pts' : fmt === int ? int(dv) : f1(dv))), h('div', {class:'d'}, d)); }; | |
| T.append(tile('Fast readout accuracy, all eval sets', rb && rb.acc.all, rj && rj.acc.all, 'one forward pass · 5,442 items', pct)); | |
| T.append(tile('OpenDecisions Capability (readout)', rb && rb.od.capability, rj && rj.od.capability, 'official scorer · never trained on', f1)); | |
| T.append(tile('Readout = its own reasoning', all.agree_base, all.agree_joint, `same ${int(all.n)} items · faithfulness check`, pct)); | |
| T.append(tile('Reasoning accuracy', all.reason_base, all.reason_joint, `one thinking sample · same ${int(all.n)} items`, pct)); | |
| T.append(tile('Median chain length', all.tok_base, all.tok_joint, 'generated tokens · same items', int, 'down')); | |
| T.append(tile('Chains that loop to 32k', all.loops_base, all.loops_joint, `of ${int(all.n)} chains each`, int, 'down')); | |
| const H = (D.metrics && D.metrics.hypotheses) || {}; | |
| const hyp = [['H1', 'The joint readout is more accurate than the gold-only readout', H.H1 && `${(100*H.H1.joint_minus_gold_only_acc).toFixed(1)} pts [${(100*H.H1.ci95[0]).toFixed(1)}, ${(100*H.H1.ci95[1]).toFixed(1)}]`, H.H1 && H.H1.confirmed], | |
| ['H2', 'The joint readout knows when reasoning would change its answer (disagreement AUROC +0.03)', H.H2 && Object.entries(H.H2).filter(([k]) => k !== 'confirmed').map(([k, v]) => `vs ${k.split('_minus_')[1]} ${v.delta >= 0 ? '+' : ''}${v.delta.toFixed(3)}`).join(' · '), H.H2 && H.H2.confirmed], | |
| ['H3', 'Joint training does not hurt the reasoner (at most 1 point below base)', H.H3 && `${(100*H.H3.joint_minus_base_reasoning_acc).toFixed(1)} pts`, H.H3 && H.H3.confirmed], | |
| ['H4', 'Routing 30% to thinking recovers at least 80% of the readout-to-reasoning gap', H.H4 && `${(100*H.H4['gap_recovered_f0.3']).toFixed(0)}% recovered`, H.H4 && H.H4.confirmed]]; | |
| const HY = $('#hyps'); | |
| for (const [id, text, nums, ok] of hyp) HY.append(h('div', {class:'hyp'}, h('div', {class:'top'}, h('span', {class:'id'}, id), | |
| ok == null ? h('span', {class:'pill neutral'}, h('i'), 'pending') : ok ? h('span', {class:'pill done'}, h('i'), 'confirmed') : h('span', {class:'pill live', style:'animation:none'}, h('i', {style:'animation:none'}), 'not confirmed')), | |
| h('p', null, text), h('div', {class:'n'}, nums || 'scored when the joint eval and both ablation readouts finish'))); | |
| })(); | |
| /* ------------------------------------------------------------ pipeline */ | |
| // which run the pipeline view shows: V2 + RL (default), v1, or both; remembered per viewer | |
| let PV = 'v2'; try { PV = localStorage.getItem('pipe_view') || 'v2'; } catch (e) {} | |
| const pipeStages = () => { const a = (D.stages || []).map(x => ({...x, stage:'v1 · ' + x.stage})); const b = (V.stages || []).map(x => ({...x, stage:x.stage.replace(/^v2_/, '').replace(/_/g, ' ')})); | |
| return PV === 'v1' ? a : PV === 'all' ? a.concat(b) : b; }; | |
| (function pipeSeg(){ const seg = $('#pipeSeg'); | |
| for (const [k, lab] of [['v2', 'V2 + RL'], ['v1', 'v1'], ['all', 'Both']]) seg.append(h('button', {type:'button', 'aria-pressed':String(k === PV), onclick:() => { PV = k; | |
| try { localStorage.setItem('pipe_view', k); } catch (e) {} | |
| seg.querySelectorAll('button').forEach(b => b.setAttribute('aria-pressed', String(b.textContent === lab))); registry.forEach(fn => fn()); }}, lab)); })(); | |
| (function upNext(){ const U = V.upcoming || {}; const t = $('#upTable'); | |
| $('#upSub').append(U.updated ? `planned order and estimated finish times (UTC, IST beside), as of ${ist(U.updated)} · "auto" steps start on their own` : 'no plan written yet'); | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Next'), h('th', null, 'Done by'), h('th', null, 'Starts'), h('th', null, 'Note')))); | |
| const tb = h('tbody'); const istOf = hhmm => { const m = /^(\d\d):(\d\d)$/.exec(hhmm || ''); if (!m) return hhmm || '–'; let mins = (+m[1] * 60 + +m[2] + 330) % 1440; return `${hhmm} UTC · ${String(Math.floor(mins / 60)).padStart(2, '0')}:${String(mins % 60).padStart(2, '0')} IST`; }; | |
| for (const x of U.items || []) tb.append(h('tr', null, h('td', null, x.what), h('td', {class:'mono'}, istOf(x.eta_utc)), h('td', null, x.auto ? 'auto' : 'by hand'), h('td', null, x.note || ''))); | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:4}, 'Nothing planned.'))); | |
| t.append(tb); })(); | |
| chart('gantt', (el, W) => { | |
| const st = pipeStages().filter(x => x.start); if (!st.length) { el.append(h('div', {class:'empty'}, 'No stages yet.')); return; } | |
| const now = new Date(D.generated_at).getTime(); const t0 = new Date(st[0].start).getTime(); const t1 = Math.max(now, ...st.map(x => x.end ? new Date(x.end).getTime() : now)); | |
| const rowsN = []; const lanes = {}; for (const x of st) if (!(x.stage in lanes)) { lanes[x.stage] = rowsN.length; rowsN.push(x.stage); } | |
| const m = {l:140, r:12, t:6, b:24}, rh = 20, H = m.t + m.b + rowsN.length * (rh + 6), iw = W - m.l - m.r; const X = t => m.l + (t - t0) / (t1 - t0) * iw; | |
| const svg = s('svg', {viewBox:`0 0 ${W} ${H}`, role:'img'}); const tip = mkTip(el); | |
| const step = (t1 - t0) > 6*3.6e6 ? 3.6e6 : 1.8e6; for (let t = Math.ceil(t0/step)*step; t <= t1; t += step) { svg.append(s('line', {x1:X(t), x2:X(t), y1:m.t, y2:H-m.b, class:'gl'}), s('text', {x:X(t), y:H-8, 'text-anchor':'middle'}, new Date(t).toISOString().slice(11,16))); } | |
| rowsN.forEach((r, i) => svg.append(s('text', {x:m.l-8, y:m.t + i*(rh+6) + rh/2 + 3.5, 'text-anchor':'end', style:'fill:var(--ink-2)'}, r))); | |
| for (const x of st) { const a = new Date(x.start).getTime(), b = x.end ? new Date(x.end).getTime() : now; const y = m.t + lanes[x.stage] * (rh + 6); | |
| const col = x.how === 'running' ? 'var(--caution)' : x.how === 'manual_stop' ? 'var(--s-votes)' : /^(v1 · )?(train|merge|sweep|rl)/.test(x.stage) ? 'var(--s-gold)' : 'var(--s-joint)'; | |
| const r = s('rect', {x:X(a), y, width:Math.max(2, X(b) - X(a)), height:rh, rx:3, style:`fill:${col}`}); | |
| r.addEventListener('pointermove', ev => showTip(tip, el, ev, x.stage, [[null, 'start', hm(x.start) + ' UTC'], [null, 'end', x.end ? hm(x.end) + ' UTC' : 'running'], [null, 'minutes', ((b - a)/6e4).toFixed(0)], [null, 'how', x.how]])); | |
| r.addEventListener('pointerleave', () => hideTip(tip)); svg.append(r); } | |
| svg.append(s('line', {x1:X(now), x2:X(now), y1:m.t, y2:H-m.b, style:'stroke:var(--caution);stroke-width:1.5;stroke-dasharray:3 3'})); | |
| el.append(legend([{name:'evaluation / generation', color:'var(--s-joint)', box:true}, {name:'training / merge', color:'var(--s-gold)', box:true}, {name:'stopped and resumed', color:'var(--s-votes)', box:true}, {name:'running now', color:'var(--caution)', box:true}])); | |
| el.append(svg); | |
| }); | |
| registry.push(function stageTable(){ | |
| const t = $('#stageTable'); t.replaceChildren(); t.append(h('thead', null, h('tr', null, h('th', null, 'Stage'), h('th', null, 'Start'), h('th', null, 'End'), h('th', {class:'n'}, 'Minutes'), h('th', null, 'Note')))); | |
| const tb = h('tbody'); const now = new Date(D.generated_at).getTime(); | |
| for (const x of pipeStages()) { const a = new Date(x.start).getTime(), b = x.end ? new Date(x.end).getTime() : now; | |
| tb.append(h('tr', null, h('td', {class:'mono'}, x.stage), h('td', {class:'mono'}, ist(x.start)), h('td', {class:'mono'}, x.end ? ist(x.end) : 'running'), h('td', {class:'n'}, ((b-a)/6e4).toFixed(0)), | |
| h('td', null, x.how === 'manual_stop' ? (x.note || 'stopped and resumed') : x.how === 'running' ? h('span', {class:'pill live'}, h('i'), 'running') : ''))); } | |
| t.append(tb); | |
| }); | |
| /* ------------------------------------------------------------ results */ | |
| const readNames = ['base', 'gold_only', 'votes_only', 'joint'].filter(n => RO[n]); | |
| chart('cReadout', (el, W) => barChart(el, W, {cats:SETS.concat(['all']), catFmt:c => SET[c], yMax:1, series:readNames.map(n => ({name:NAME[n], color:COL[n], vals:SETS.concat(['all']).map(sx => RO[n].acc[sx])}))})); | |
| const pairSeries = (kb, kj, fmt) => [{name:NAME.base, color:COL.base, vals:SETS.concat(['all']).map(sx => PR[sx] ? PR[sx][kb] : null)}, {name:NAME.joint, color:COL.joint, vals:SETS.concat(['all']).map(sx => PR[sx] ? PR[sx][kj] : null)}]; | |
| chart('cAgree', (el, W) => barChart(el, W, {cats:SETS.concat(['all']), catFmt:c => SET[c].replace('OpenDecisions','OpenDec.').replace('LocalLLaMA','LocalL.'), yMax:1, series:pairSeries('agree_base', 'agree_joint')})); | |
| chart('cReason', (el, W) => barChart(el, W, {cats:SETS.concat(['all']), catFmt:c => SET[c], yMax:1, series:pairSeries('reason_base', 'reason_joint')})); | |
| chart('cTokens', (el, W) => barChart(el, W, {cats:SETS.concat(['all']), catFmt:c => SET[c], yFmt:v => int(v), series:pairSeries('tok_base', 'tok_joint')})); | |
| chart('cTierR', (el, W) => barChart(el, W, {cats:['easy','standard','judge','hard'], yMax:1, series:readNames.map(n => ({name:NAME[n], color:COL[n], vals:['easy','standard','judge','hard'].map(t => RO[n].od_tier[t])}))})); | |
| chart('cLevelR', (el, W) => barChart(el, W, {cats:['0','1','2','3','4'], catFmt:c => 'level ' + c, yMax:1, series:readNames.map(n => ({name:NAME[n], color:COL[n], vals:['0','1','2','3','4'].map(l => RO[n].proc_level[l])}))})); | |
| (function cmpTable(){ | |
| const t = $('#cmpTable'); const cols = ['Set', 'Readout base', 'Readout joint', 'Reasoning base', 'Reasoning joint', 'Agree base', 'Agree joint', 'Tokens base', 'Tokens joint', 'Loops base', 'Loops joint', 'Items (same)']; | |
| t.append(h('thead', null, h('tr', null, cols.map((c, i) => h('th', {class: i ? 'n' : null}, c))))); | |
| const tb = h('tbody'); | |
| for (const sx of SETS.concat(['all'])) { const p = PR[sx] || {}; const b = RO.base, j = RO.joint; | |
| const cmp = (a, c, up=true) => (a == null || c == null) ? '' : (up ? c > a : c < a) ? 'better' : c === a ? '' : 'worse'; | |
| tb.append(h('tr', null, h('td', null, SET[sx]), | |
| h('td', {class:'n'}, pct(b && b.acc[sx])), h('td', {class:'n ' + cmp(b && b.acc[sx], j && j.acc[sx])}, pct(j && j.acc[sx])), | |
| h('td', {class:'n'}, pct(p.reason_base)), h('td', {class:'n ' + cmp(p.reason_base, p.reason_joint)}, pct(p.reason_joint)), | |
| h('td', {class:'n'}, pct(p.agree_base)), h('td', {class:'n ' + cmp(p.agree_base, p.agree_joint)}, pct(p.agree_joint)), | |
| h('td', {class:'n'}, int(p.tok_base)), h('td', {class:'n ' + cmp(p.tok_base, p.tok_joint, false)}, int(p.tok_joint)), | |
| h('td', {class:'n'}, int(p.loops_base)), h('td', {class:'n ' + cmp(p.loops_base, p.loops_joint, false)}, int(p.loops_joint)), h('td', {class:'n'}, int(p.n)))); } | |
| t.append(tb); | |
| })(); | |
| /* ------------------------------------------------------------ system */ | |
| function sysCurve(m){ if (!m || !m.system) return {}; const out = {}; for (const [name, b] of Object.entries(m.system)) { const keys = [['readout_only', 0], ['routed_margin_f0.1', .1], ['routed_margin_f0.2', .2], ['routed_margin_f0.3', .3], ['routed_margin_f0.5', .5], ['thinking_always', 1]]; | |
| out[name] = keys.filter(([k]) => b[k]).map(([k, f]) => ({f, cap:b[k].od_capability.capability, acc:b[k].acc_all, tok:b[k].mean_tokens})); } return out; } | |
| const SYS = Object.assign({}, sysCurve(D.metrics_base), sysCurve(D.metrics)); | |
| chart('cRouteCap', (el, W) => lineChart(el, W, {series:Object.entries(SYS).map(([n, c]) => ({name:NAME[n], color:COL[n], dash:n === 'base', pts:c.map(r => [r.f, r.cap])})), yFmt:v => v.toFixed(0), xFmt:v => pct(v, 0), xLabel:'share of items sent to thinking', | |
| refLines:[{y:80.1, label:'Jev 80.1', color:'var(--ink-3)'}, {y:77.1, label:'best 4B (Wald) 77.1', color:'var(--ink-3)'}]})); | |
| chart('cRouteAcc', (el, W) => { const rt = (D.metrics_base && D.metrics_base.routing && D.metrics_base.routing.base) || null; const series = Object.entries(SYS).map(([n, c]) => ({name:NAME[n] + ', margin router', color:COL[n], dash:n === 'base', pts:c.map(r => [r.f, r.acc])})); | |
| if (rt) series.push({name:'base, random routing', color:'var(--ink-3)', w:1, dash:true, noEnd:true, pts:rt.margin.map(r => [r.f, r.random_accuracy])}); | |
| if (D.metrics && D.metrics.routing && D.metrics.routing.joint) series.push({name:'joint, random routing', color:'var(--s-joint)', w:1, dash:true, noEnd:true, pts:D.metrics.routing.joint.margin.map(r => [r.f, r.random_accuracy])}); | |
| lineChart(el, W, {series, yFmt:v => pct(v, 0), xFmt:v => pct(v, 0), xLabel:'share of items sent to thinking'}); }); | |
| (function sysTable(){ | |
| const t = $('#sysTable'); t.append(h('thead', null, h('tr', null, ['Model', 'Routed', 'All-sets accuracy', 'OD Capability', 'Mean generated tokens'].map((c, i) => h('th', {class:i ? 'n' : null}, c))))); | |
| const tb = h('tbody'); for (const [n, c] of Object.entries(SYS)) for (const r of c) tb.append(h('tr', null, h('td', null, NAME[n]), h('td', {class:'n'}, pct(r.f, 0)), h('td', {class:'n m'}, pct(r.acc)), h('td', {class:'n m'}, f1(r.cap)), h('td', {class:'n'}, int(r.tok)))); | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:5}, 'System numbers appear after metrics run.'))); t.append(tb); | |
| const L = D.latency || {}; const parts = []; for (const n of ['base', 'joint']) if (L[n]) parts.push(`${NAME[n]}: readout median ${L[n].readout_ms.median.toFixed(0)} ms (p90 ${L[n].readout_ms.p90.toFixed(0)}), reasoning median ${(L[n].reasoning_ms.median/1000).toFixed(1)} s (p90 ${(L[n].reasoning_ms.p90/1000).toFixed(1)})`); | |
| $('#latNote').textContent = parts.length ? 'Latency at batch size 1 · ' + parts.join(' · ') : ''; | |
| })(); | |
| /* ------------------------------------------------------------ training */ | |
| const TR = D.training || {}; | |
| (function trainTiles(){ const T = $('#trainTiles'); for (const n of ['joint', 'gold_only', 'votes_only']) { const m = TR[n] && TR[n].meta; if (!m) continue; | |
| T.append(h('div', {class:'tile'}, h('div', {class:'k'}, NAME[n] + (n === 'joint' ? ' · reasoner + readout' : ' · readout only, α = ' + m.alpha)), | |
| h('div', {class:'v'}, `${m.steps_done}`, h('span', {class:'arrow'}, '/'), `${m.steps_planned} steps`), h('div', {class:'d'}, `${int(m.reasoner_examples)} reasoner + ${int(m.readout_examples)} readout examples · ${(m.train_tokens/1e6).toFixed(2)}M tokens · ${(m.train_seconds/60).toFixed(0)} min at ${int(m.tok_per_s)} tok/s`))); } })(); | |
| const roll = (arr, k=10) => arr.map((_, i) => { const w = arr.slice(Math.max(0, i - k + 1), i + 1).filter(v => v != null); return w.length ? w.reduce((a, b) => a + b, 0) / w.length : null; }); | |
| chart('cChain', (el, W) => { const st = (TR.joint || {}).steps || []; const raw = st.map(r => [r.step, r.chain_loss]).filter(p => p[1] != null); | |
| lineChart(el, W, {series:[{name:'per step', color:'var(--s-joint)', dots:true, noTip:false, pts:raw}, {name:'10-step mean', color:'var(--s-joint)', pts:raw.map((p, i) => [p[0], roll(raw.map(q => q[1]))[i]])}], yFmt:v => v.toFixed(2), xLabel:'optimizer step', yMin:0}); }); | |
| chart('cRoLoss', (el, W) => { const series = []; for (const n of ['joint', 'gold_only', 'votes_only']) { const st = (TR[n] || {}).steps || []; if (!st.length) continue; | |
| const raw = st.map(r => [r.step / st[st.length-1].of, r.readout_loss]).filter(p => p[1] != null); series.push({name:NAME[n], color:COL[n], pts:raw.map((p, i) => [p[0], roll(raw.map(q => q[1]))[i]])}); } | |
| lineChart(el, W, {series, yFmt:v => v.toFixed(2), xFmt:v => pct(v, 0), xLabel:'share of the run (runs have 207 and 37 steps)', yMin:0}); }); | |
| chart('cLr', (el, W) => lineChart(el, W, {series:['joint', 'gold_only', 'votes_only'].filter(n => TR[n]).map(n => ({name:NAME[n], color:COL[n], pts:TR[n].steps.map(r => [r.step / r.of, r.lr])})), yFmt:v => v.toExponential(0), xFmt:v => pct(v, 0), xLabel:'share of the run', yMin:0})); | |
| chart('cGn', (el, W) => lineChart(el, W, {series:['joint', 'gold_only', 'votes_only'].filter(n => TR[n]).map(n => ({name:NAME[n], color:COL[n], pts:TR[n].steps.map(r => [r.step / r.of, r.grad_norm])})), yFmt:v => v.toFixed(0), xFmt:v => pct(v, 0), xLabel:'share of the run', yMin:0, refLines:[{y:1, label:'clip 1.0', color:'var(--ink-3)'}]})); | |
| chart('cTps', (el, W) => lineChart(el, W, {H:180, series:['joint', 'gold_only', 'votes_only'].filter(n => TR[n]).map(n => ({name:NAME[n], color:COL[n], pts:TR[n].steps.map(r => [r.step / r.of, r.tok_per_s])})), yFmt:v => int(v), xFmt:v => pct(v, 0), xLabel:'share of the run', yMin:0})); | |
| /* ------------------------------------------------------------ S1MB */ | |
| (function s1mb(){ const S = V.s1mb; const t = $('#s1Tab'); if (!S) { t.append(h('tbody', null, h('tr', null, h('td', null, 'No S1MB run yet.')))); return; } | |
| const chk = (S.rescore_check || []).map(c => c.max_abs_diff).filter(x => x != null); | |
| $('#s1Sub').append(`cohort ${int(S.cohort)} models · scorer check: max |difference| vs published scores ${chk.length ? Math.max(...chk).toExponential(0) : '–'} · shaded rows are ours`); | |
| t.append(h('thead', null, h('tr', null, h('th', {class:'n'}, '#'), h('th', null, 'Model'), h('th', {class:'n'}, 'Borda'), h('th', {class:'n'}, 'Task Avg'), h('th', {class:'n'}, 'Yes/no'), h('th', {class:'n'}, 'Choice'), h('th', {class:'n'}, 'Score')))); | |
| const tb = h('tbody'); const f = v => v == null ? '–' : v.toFixed(1); | |
| const S1N = {rl60:'Ours: V2 + RL, fast', v1_joint:'Ours: v1, fast', base:'Ours: Qwen3.5-4B untrained, fast', v2_2b_ro:'Ours: 2B readout-only, fast', base_2b:'Ours: MiniCPM5-2B untrained, fast', rl60_routed30:'Ours: V2 + RL, 30% routed to thinking', v2_2b_hybrid30:'Ours: 2B hybrid, 30% routed to untrained thinking'}; | |
| for (const r of S.rows) tb.append(h('tr', r.model.startsWith('OURS') ? {style:'background:var(--brand-soft)'} : null, h('td', {class:'n mono'}, r.rank), | |
| h('td', null, r.model.startsWith('OURS:') ? (S1N[r.model.slice(5)] || 'Ours: ' + r.model.slice(5)) : r.model.replace('__', '/')), h('td', {class:'n mono'}, f(r.borda)), h('td', {class:'n mono'}, f(r.task_avg)), h('td', {class:'n mono'}, f(r.noul)), h('td', {class:'n mono'}, f(r.choice)), h('td', {class:'n mono'}, f(r.score)))); | |
| t.append(tb); })(); | |
| /* ------------------------------------------------------------ 2B */ | |
| (function twob(){ const T = V.twob || {}; const P = T.probes || {}; const t = $('#tbProbe'); const pick = (T.pick || {}).picked; | |
| $('#tbSub').append(pick ? `picked: ${pick}` : 'pick after both probes'); | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Model'), ...SETS.map(sx => h('th', {class:'n'}, 'Readout · ' + SET[sx])), h('th', {class:'n'}, 'Readout mean'), h('th', {class:'n'}, 'Think, 1st chain'), h('th', {class:'n'}, 'Think, majority of 4'), h('th', {class:'n'}, 'Parsed'), h('th', {class:'n'}, 'Looped'), h('th', {class:'n'}, 'Median tokens')))); | |
| const tb = h('tbody'); | |
| for (const [m, r] of Object.entries(P)) tb.append(h('tr', m === pick ? {style:'background:var(--brand-soft)'} : null, h('td', null, m + (m.endsWith('4B') ? ' (reference)' : '') + (m === pick ? ' · picked' : '')), | |
| ...SETS.map(sx => h('td', {class:'n mono'}, pct(r.readout[sx]))), h('td', {class:'n mono'}, pct(r.readout_mean)), h('td', {class:'n mono'}, pct(r.think1)), h('td', {class:'n mono'}, pct(r.maj4)), | |
| h('td', {class:'n mono'}, pct(r.parse)), h('td', {class:'n mono'}, `${int(r.looped)} / ${int(r.samples)}`), h('td', {class:'n mono'}, int(r.tokens_median)))); | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:11}, 'No probe yet.'))); | |
| t.append(tb); })(); | |
| /* ------------------------------------------------------------ Jevals */ | |
| (function jevals(){ const J = V.jevals; const G = $('#jvGrid'); if (!J) { G.append(h('div', {class:'card c12'}, 'No Jevals run yet.')); return; } | |
| const BN = {pubmedqa:'PubMedQA · yes/no (P(yes))', banking77:'Banking77 · choice of 77 intents', helpsteer2:'HelpSteer2 helpfulness · 5-level score'}; | |
| const ON = {base:'Ours: base 4B', v1_joint:'Ours: v1 joint', v2_joint:'Ours: V2 joint', rl60:'Ours: V2 + RL', rl40:'Ours: V2 + RL (step 40)', base_2b:'Ours: 2B base', v2_2b:'Ours: 2B, V2 recipe', v2_2b_ro:'Ours: 2B, readout only'}; | |
| for (const b of ['pubmedqa', 'banking77', 'helpsteer2']) { | |
| const rows = J.board.filter(r => r.board === b).map(r => ({name:r.name, ds:r.ds, ci:r.ci, acc:r.accuracy, ece:r.ece, ours:false})); | |
| for (const [k, o] of Object.entries(J.ours)) { const think = k.endsWith('_think'); const base = think ? k.slice(0, -6) : k; const s = (o.calibrated || o.raw)[b]; | |
| if (s) rows.push({name:(ON[base] || base) + (think ? ', thinking' : ', fast'), ds:s.ds, ci:s.ci, acc:s.accuracy, ece:s.ece_10_bin, ours:true}); } | |
| rows.sort((a, c) => c.ds - a.ds); | |
| const t = h('table'); t.append(h('thead', null, h('tr', null, h('th', {class:'n'}, '#'), h('th', null, 'System'), h('th', {class:'n'}, 'Decision Score'), h('th', {class:'n'}, '95% interval'), h('th', {class:'n'}, 'Accuracy'), h('th', {class:'n'}, 'ECE')))); | |
| const tb = h('tbody'); rows.forEach((r, i) => tb.append(h('tr', r.ours ? {style:'background:var(--brand-soft)'} : null, h('td', {class:'n mono'}, i + 1), h('td', null, r.name), | |
| h('td', {class:'n mono'}, r.ds.toFixed(1)), h('td', {class:'n mono'}, `${r.ci[0].toFixed(1)} – ${r.ci[1].toFixed(1)}`), h('td', {class:'n mono'}, pct(r.acc)), h('td', {class:'n mono'}, (r.ece * 100).toFixed(1))))); | |
| t.append(tb); | |
| G.append(h('div', {class:'card c12'}, h('h3', null, BN[b]), h('div', {class:'sub'}, 'shaded rows are ours · ours use temperatures fitted on our own calibration items (post-chain ones for thinking) · ECE in points'), h('div', {class:'tw'}, t))); | |
| } })(); | |
| /* ------------------------------------------------------------ RL */ | |
| const RL = V.rl || {}; const RLS = RL.steps || []; const RLG = RLS.filter(r => r.reward != null); | |
| (function rlTiles(){ const T = $('#rlTiles'); const t = (k, v, d) => T.append(h('div', {class:'tile'}, h('div', {class:'k'}, k), h('div', {class:'v'}, v), h('div', {class:'d'}, d))); | |
| const last = RLS[RLS.length - 1]; const g0 = RLG[0], g1 = RLG[RLG.length - 1]; | |
| const el = RL.started ? (new Date(D.generated_at) - new Date(RL.started)) / 6e4 : null; | |
| const st = RLS.filter(r => r.step_time != null).slice(-8).map(r => r.step_time); const per = st.length >= 4 ? st.reduce((a, b) => a + b, 0) / st.length / 60 : null; // minutes per step, recent rounds | |
| t('RL steps', last ? `${int(last.step)} / ${int(RL.planned)}` : (RL.started ? '0 / ' + int(RL.planned) : '–'), RL.error ? 'error: ' + RL.error.slice(0, 80) : (RL.started ? `started ${ist(RL.started)} · ${el ? int(el) + ' min in' : ''}${per ? ` · ~${int(per * (RL.planned - last.step))} min left` : (last ? ' · time left after the first rounds' : ' · first round generating')}` : 'starts after Phase A')); | |
| t('Reward', g1 ? g1.reward.toFixed(2) : '–', g0 && g1 && g1 !== g0 ? `first round ${g0.reward.toFixed(2)} · ${RLG.length} rounds` : 'per generation round'); | |
| t('Unfinished at the cap', g1 && g1.c_clipped_ratio != null ? pct(g1.c_clipped_ratio) : '–', g0 && g1 && g1 !== g0 ? `first round ${pct(g0.c_clipped_ratio)}` : 'share of chains'); | |
| t('Chain length', g1 && g1.c_mean_length != null ? int(g1.c_mean_length) : '–', g0 && g1 && g1 !== g0 ? `first round ${int(g0.c_mean_length)} tokens` : 'mean tokens per chain'); | |
| })(); | |
| (function rlRes(){ const C = V.reason_cmp; const t = $('#rlRes'); | |
| if (!C) { t.append(h('tbody', null, h('tr', null, h('td', null, 'No thinking-mode results yet.')))); return; } | |
| $('#rlResSub').append(`one thinking sample per question, the ${int(C.n)} test questions every model below answered · "looped" = ran to the 32K-token cap with no answer (counts as wrong) · tokens = generated per question`); | |
| const NM = {base:'Base 4B', v1_joint:'v1 joint', v2_joint:'V2 joint', rl60:'V2 + RL (step 60)', rl40:'V2 + RL (step 40, 1e-5 only)', base_2b:'2B base (MiniCPM5-2B)', v2_2b:'2B, V2 recipe', v2_2b_ro:'2B, readout only'}; | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Model'), ...SETS.map(sx => h('th', {class:'n'}, SET[sx])), h('th', {class:'n'}, 'All'), h('th', {class:'n'}, 'Looped'), h('th', {class:'n'}, 'Median tokens'), h('th', {class:'n'}, 'Mean tokens')))); | |
| const tb = h('tbody'); | |
| for (const [k, r] of Object.entries(C.rows)) tb.append(h('tr', null, h('td', null, NM[k] || k), ...SETS.map(sx => h('td', {class:'n mono'}, r[sx] ? pct(r[sx].acc) : '–')), | |
| h('td', {class:'n mono'}, pct(r.all.acc)), h('td', {class:'n mono'}, pct(r.all.looped)), h('td', {class:'n mono'}, int(r.all.tok_median)), h('td', {class:'n mono'}, int(r.all.tok_mean)))); | |
| t.append(tb); })(); | |
| const rlPts = key => RLG.filter(r => r[key] != null).map(r => [r.step, r[key]]); | |
| const LRREF = RLS.some(r => r.phase === 'b') ? [{y:0, label:'reward 0', color:'var(--ink-3)'}] : []; | |
| chart('rlReward', (el, W) => lineChart(el, W, {series:[{name:'lr 1e-5 (steps 1-40)', color:'var(--s-joint)', pts:RLG.filter(r => r.phase !== 'b').map(r => [r.step, r.reward])}, {name:'lr 2.5e-5 (from 41)', color:'var(--s-gold)', pts:RLG.filter(r => r.phase === 'b').map(r => [r.step, r.reward])}].filter(x => x.pts.length), yFmt:v => v.toFixed(2), xLabel:'optimizer step', refLines:LRREF})); | |
| chart('rlClip', (el, W) => lineChart(el, W, {series:[{name:'unfinished at cap', color:'var(--s-gold)', pts:rlPts('c_clipped_ratio')}], yFmt:v => pct(v, 0), xLabel:'optimizer step', yMin:0})); | |
| chart('rlLen', (el, W) => lineChart(el, W, {H:180, series:[{name:'mean tokens', color:'var(--s-joint)', pts:rlPts('c_mean_length')}], yFmt:v => int(v), xLabel:'optimizer step', yMin:0})); | |
| chart('rlZero', (el, W) => lineChart(el, W, {H:180, series:[{name:'no signal', color:'var(--s-votes)', pts:rlPts('frac_reward_zero_std')}], yFmt:v => pct(v, 0), xLabel:'optimizer step', yMin:0})); | |
| const RLR = (RL.rounds || []).filter(r => r.step != null); | |
| chart('rlWait', (el, W) => lineChart(el, W, {series:[{name:'"Wait" per finished chain', color:'var(--s-joint)', pts:RLR.filter(r => r.waits_per_fin != null).map(r => [r.step, r.waits_per_fin])}], yFmt:v => v.toFixed(1), xLabel:'optimizer step', yMin:0})); | |
| chart('rlFin', (el, W) => lineChart(el, W, {series:[{name:'share right among finished', color:'var(--s-gold)', pts:RLR.filter(r => r.right_among_fin != null).map(r => [r.step, r.right_among_fin])}], yFmt:v => pct(v, 0), xLabel:'optimizer step', yMin:0, yMax:1})); | |
| chart('rlGn', (el, W) => lineChart(el, W, {H:180, series:[{name:'gradient norm', color:'var(--s-joint)', pts:RLS.filter(r => r.grad_norm != null).map(r => [r.step, r.grad_norm])}], yFmt:v => v.toFixed(2), xLabel:'optimizer step', yMin:0})); | |
| /* ------------------------------------------------------------ V2 */ | |
| // the setup training actually uses: the last 'speed=' decision, matched to a benchmark config by its flags | |
| function usedSetup(){ const d = (V.decisions || []).filter(e => e.stage.startsWith('speed=')).pop(); const B = (V.bench || {}).configs; | |
| if (!d || !B) return null; const f = d.stage.slice(6).split(' (')[0]; const num = k => { const m = f.match(new RegExp(k + ' (\\d+)')); return m ? +m[1] : null; }; | |
| const want = {liger: f.includes('--liger'), micro: num('--micro-tokens') || 16384, ckpt_above: num('--ckpt-above') || 0}; | |
| const name = Object.keys(B).find(n => n !== 'R' && B[n].liger === want.liger && B[n].micro === want.micro && B[n].ckpt_above === want.ckpt_above); | |
| return name ? {name, how: name === V.bench.chosen ? 'the benchmark rule\'s pick' : 'user\'s call over the 72 GB line (deviation #26)'} : null; } | |
| (function v2Tiles(){ const T = $('#v2Tiles'); const t = (k, v, d) => T.append(h('div', {class:'tile'}, h('div', {class:'k'}, k), h('div', {class:'v'}, v), h('div', {class:'d'}, d))); | |
| const S = V.sources || {}; const sum = pre => Object.entries(S).filter(([k]) => k.startsWith(pre)).reduce((a, [, b]) => a + b.items, 0); | |
| const loc = sum('local:'), tea = sum('teacher:'); | |
| t('Questions with chains', int(loc + tea), `${int(loc)} from the 4B (incl. v1's) · ${int(tea)} from the API teacher`); | |
| const tg = V.teacher || {}; t('API teacher', tg.cost != null ? '$' + tg.cost.toFixed(2) : '–', tg.items_done ? `${int(tg.items_done)} questions × 4 · ${pct(tg.correct / tg.parsed)} correct · ${f1(tg.minutes)} min` : 'not run'); | |
| const b = V.bench || {}; const U = usedSetup(); | |
| if (U && U.name !== b.chosen) t('Speed setup', U.name, `${U.how} · ${((b.configs[U.name] || {}).tok_per_s / ((b.configs.A || {}).tok_per_s || 1)).toFixed(2)}× the current setup in the benchmark`); | |
| else t('Speed setup', b.chosen || '–', b.chosen ? `${b.flags || 'current setup kept'} · ${(b.speedup_vs_A || 1).toFixed(2)}× the current setup` : 'benchmark runs after generation'); | |
| const sw = ['1e-4', '2e-4', '3e-4'].filter(lr => TR['sweep_' + lr] && TR['sweep_' + lr].meta); const L = V.lr_sweep || {}; | |
| t('Learning rate', L.best || '–', L.best ? `dev NLL ${L[L.best].dev_nll.toFixed(3)} · dev acc ${pct(L[L.best].dev_acc)}` : (sw.length ? `sweep ${sw.length} / 3 done` : 'sweep runs after the benchmark')); | |
| const st = (TR.v2_joint || {}).steps || []; const last = st[st.length - 1]; | |
| const jm = (TR.v2_joint || {}).meta; | |
| if (jm && jm.dev_acc != null) t('V2 joint, held-out dev', pct(jm.dev_acc), `${int(jm.dev_n)} questions never trained on · NLL ${jm.dev_nll.toFixed(3)} · ECE ${jm.dev_ece15.toFixed(3)} · ${int(jm.train_seconds / 60)} min, ${int(jm.tok_per_s)} tok/s`); | |
| else t('V2 joint training', last ? `${int(last.step)} / ${int(last.of)}` : '–', last ? `${f1(last.elapsed_min)} min in · ETA ${f1(last.eta_min)} min · ${int(last.tok_per_s)} tok/s` : 'starts after the sweep'); | |
| })(); | |
| (function v2Stages(){ const t = $('#v2Stages'); | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Stage'), h('th', null, 'Start'), h('th', null, 'End'), h('th', {class:'n'}, 'Minutes'), h('th', null, 'State')))); | |
| const tb = h('tbody'); const now = new Date(D.generated_at); | |
| for (const x of V.stages || []) { const a = new Date(x.start), e = x.end ? new Date(x.end) : now; | |
| tb.append(h('tr', null, h('td', null, x.stage.replace(/^v2_/, '').replace(/_/g, ' ')), h('td', {class:'mono'}, ist(x.start)), h('td', {class:'mono'}, x.end ? ist(x.end) : '–'), | |
| h('td', {class:'n mono'}, ((e - a) / 6e4).toFixed(0)), h('td', null, x.how === 'running' ? 'running' : 'done'))); } | |
| for (const d of V.decisions || []) tb.append(h('tr', null, h('td', null, d.event === 'chosen' ? 'decision' : d.event), h('td', {class:'mono'}, ist(d.time)), h('td', null, '–'), h('td', {class:'n'}, '–'), h('td', {class:'mono'}, d.stage))); | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:5}, 'No V2 stage yet.'))); | |
| t.append(tb); })(); | |
| const V2T = (TR.v2_joint || {}).steps || []; | |
| const sm = (rows, key, k=10) => { const raw = rows.map(r => [r.step, r[key]]).filter(p => p[1] != null); const m = roll(raw.map(q => q[1]), k); return raw.map((p, i) => [p[0], m[i]]); }; | |
| // v1 joint beside V2, on a shared x axis of tokens trained; the switch is remembered per viewer | |
| const V1T = (TR.joint || {}).steps || [], V1M = (TR.joint || {}).meta || {}; | |
| let SHOW_V1 = true; try { SHOW_V1 = localStorage.getItem('v2_show_v1') !== '0'; } catch (e) {} | |
| const withTok = rows => { let c = 0; return rows.map(r => ({...r, tokM: (c += r.step_tokens || 0) / 1e6})); }; | |
| const V2X = withTok(V2T), V1X = withTok(V1T); | |
| const smT = (rows, key, k=10) => { const raw = rows.filter(r => r[key] != null).map(r => [r.tokM, r[key]]); const m = roll(raw.map(q => q[1]), k); return raw.map((p, i) => [p[0], m[i]]); }; | |
| const rawT = (rows, key) => rows.filter(r => r[key] != null).map(r => [r.tokM, r[key]]); | |
| const v1s = (name, pts) => SHOW_V1 && pts.length ? [{name:'v1 ' + name, color:'var(--s-base)', w:1.5, pts}] : []; | |
| const TOKX = {xLabel:'tokens trained (millions)', xFmt:v => (+v).toFixed(v < 10 ? 1 : 0) + 'M'}; | |
| const TBX = withTok((TR.v2b_joint || {}).steps || []); | |
| (function tbTiles(){ const T = $('#tbTiles'); const t = (k, v, d) => T.append(h('div', {class:'tile'}, h('div', {class:'k'}, k), h('div', {class:'v'}, v), h('div', {class:'d'}, d))); | |
| const W = V.twob || {}; const last = TBX[TBX.length - 1]; const m = (TR.v2b_joint || {}).meta; | |
| t('Model', (W.pick || {}).picked ? W.pick.picked.split('/')[1] : '–', 'picked by the probe rule'); | |
| t('Learning rate', (W.sweep || {}).best || '–', W.sweep && W.sweep.best ? `dev NLL ${W.sweep[W.sweep.best].dev_nll.toFixed(3)} · acc ${pct(W.sweep[W.sweep.best].dev_acc)}` : 'sweep first'); | |
| if (m && m.dev_acc != null) t('2B joint, held-out dev', pct(m.dev_acc), `${int(m.dev_n)} questions · NLL ${m.dev_nll.toFixed(3)} · ECE ${m.dev_ece15.toFixed(3)} (4B V2: 81.3%)`); | |
| else t('2B joint training', last ? `${int(last.step)} / ${int(last.of)}` : '–', last ? `${f1(last.elapsed_min)} min in · ETA ${f1(last.eta_min)} min · ${int(last.tok_per_s)} tok/s` : 'after the sweep'); | |
| })(); | |
| const TBR = withTok((TR.v2b_ro || {}).steps || []); | |
| chart('tbRo', (el, W) => lineChart(el, W, {series:[{name:'2B joint readout loss', color:'var(--s-joint)', pts:smT(TBX, 'readout_loss')}, {name:'2B readout-only', color:'var(--s-gold)', pts:smT(TBR, 'readout_loss')}, {name:'4B V2 readout loss', color:'var(--s-base)', w:1.5, pts:smT(V2X, 'readout_loss')}].filter(x => x.pts.length), yFmt:v => v.toFixed(2), yMin:0, ...TOKX})); | |
| chart('tbCh', (el, W) => lineChart(el, W, {series:[{name:'2B chain loss', color:'var(--s-joint)', pts:smT(TBX, 'chain_loss')}, {name:'4B V2 chain loss', color:'var(--s-base)', w:1.5, pts:smT(V2X, 'chain_loss')}].filter(x => x.pts.length), yFmt:v => v.toFixed(2), yMin:0, ...TOKX})); | |
| chart('v2Chain', (el, W) => lineChart(el, W, {series:[{name:'chain loss', color:'var(--s-joint)', pts:smT(V2X, 'chain_loss')}, {name:'post-chain letter loss', color:'var(--s-gold)', pts:smT(V2X, 'letter_loss')}, ...v1s('chain loss', smT(V1X, 'chain_loss'))], yFmt:v => v.toFixed(2), yMin:0, ...TOKX})); | |
| chart('v2Ro', (el, W) => lineChart(el, W, {series:[{name:'readout loss', color:'var(--s-joint)', pts:smT(V2X, 'readout_loss')}, {name:'target entropy (floor)', color:'var(--s-joint)', dash:true, w:1.5, noEnd:true, pts:smT(V2X, 'readout_target_entropy')}, ...v1s('readout loss', smT(V1X, 'readout_loss'))], yFmt:v => v.toFixed(2), yMin:0, ...TOKX})); | |
| chart('v2Lr', (el, W) => lineChart(el, W, {H:180, series:[{name:'learning rate', color:'var(--s-joint)', pts:rawT(V2X, 'lr')}, ...v1s('learning rate', rawT(V1X, 'lr'))], yFmt:v => v.toExponential(0), yMin:0, ...TOKX})); | |
| chart('v2Gn', (el, W) => lineChart(el, W, {H:180, series:[{name:'gradient norm', color:'var(--s-joint)', pts:rawT(V2X, 'grad_norm')}, ...v1s('gradient norm', rawT(V1X, 'grad_norm'))], yFmt:v => v.toFixed(1), yMin:0, refLines:[{y:1, label:'clip 1.0', color:'var(--ink-3)'}], ...TOKX})); | |
| chart('v2Tps', (el, W) => lineChart(el, W, {H:180, series:[{name:'tokens/s', color:'var(--s-joint)', pts:rawT(V2X, 'tok_per_s')}, ...v1s('tokens/s', rawT(V1X, 'tok_per_s'))], yFmt:v => int(v), yMin:0, ...TOKX})); | |
| (function v1Seg(){ const seg = $('#v1Seg'); | |
| for (const [on, lab] of [[false, 'Off'], [true, 'On']]) seg.append(h('button', {type:'button', 'aria-pressed':String(on === SHOW_V1), onclick:() => { SHOW_V1 = on; | |
| try { localStorage.setItem('v2_show_v1', on ? '1' : '0'); } catch (e) {} | |
| seg.querySelectorAll('button').forEach(b => b.setAttribute('aria-pressed', String(b.textContent === lab))); registry.forEach(fn => fn()); }}, lab)); })(); | |
| const qnt = (xs, p) => xs.slice().sort((a, b) => a - b)[Math.floor(p * (xs.length - 1))]; | |
| const mean = xs => xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null; | |
| const secs = rows => rows.slice(1).map((r, i) => (r.elapsed_min - rows[i].elapsed_min) * 60); | |
| const STEPROWS = rows => [['Seconds', secs(rows), f1], ['Examples', rows.map(r => r.examples), int], ['· readout questions', rows.map(r => r.readout_examples), int], | |
| ['· reasoning chains', rows.map(r => r.reasoner_examples), int], ['Tokens', rows.map(r => r.step_tokens), int], ['Micro-batches', rows.map(r => r.micro_batches), int]]; | |
| (function v2StepSub(){ const TL = V.train_log || {}; const U = usedSetup(); const lr = (V.lr_sweep || {}).best; | |
| const micro = U ? V.bench.configs[U.name].micro : 16384; const pk = V2T.length ? Math.max(...V2T.map(r => r.mem_gb || 0)) : null; | |
| $('#v2StepSub').append(`Batches are counted in tokens: each optimizer step takes examples until it holds ~32K tokens, split into micro-batches of up to ${micro / 1024}K padded tokens whose gradients add up (gradient accumulation), each example weighted equally. ` + | |
| `LoRA r32 α64 · AdamW ${lr ? 'lr ' + lr : ''}, cosine, 3% warmup · clipped at 1.0 · one pass over the data` + | |
| (TL.data ? ` · ${int(TL.data.examples)} examples, ${f1(TL.data.tokens / 1e6)}M tokens` : '') + | |
| (pk ? ` · peak memory ${f1(pk)} GB (${f1(pk * 1e9 / 2 ** 30)} GiB of 80)` : '') + ` · out-of-memory redos: ${TL.oom_retries || 0}`); })(); | |
| registry.push(function v2Step(){ const t = $('#v2Step'); t.replaceChildren(); const v1 = SHOW_V1 && V1T.length; | |
| const cols = v1 ? ['V2 median', 'V2 p10–p90', 'v1 median', 'v1 p10–p90'] : ['Mean', 'Median', 'p10', 'p90', 'Max']; | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Per step'), ...cols.map(x => h('th', {class:'n'}, x))))); | |
| const tb = h('tbody'); const R1 = STEPROWS(V1T); | |
| STEPROWS(V2T).forEach(([k, xs, fmt], i) => { if (!xs.length) return; const ys = R1[i][1]; | |
| const cells = v1 ? [fmt(qnt(xs, .5)), `${fmt(qnt(xs, .1))}–${fmt(qnt(xs, .9))}`, ys.length ? fmt(qnt(ys, .5)) : '–', ys.length ? `${fmt(qnt(ys, .1))}–${fmt(qnt(ys, .9))}` : '–'] | |
| : [mean(xs), qnt(xs, .5), qnt(xs, .1), qnt(xs, .9), Math.max(...xs)].map(fmt); | |
| tb.append(h('tr', null, h('td', null, k), ...cells.map(c => h('td', {class:'n mono'}, c)))); }); | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:6}, 'V2 joint has not started.'))); | |
| t.append(tb); }); | |
| (function v2Cmp(){ const t = $('#v2Cmp'); const TL = V.train_log || {}; const d = TL.data || {}; const U = usedSetup(); const last = V2X[V2X.length - 1]; | |
| const v1Tok = V1X.length ? V1X[V1X.length - 1].tokM : null, v2Tok = last ? last.tokM : null; | |
| const at = (rows, key, hi) => mean(rows.filter(r => r[key] != null && r.tokM > hi - 1 && r.tokM <= hi).map(r => r[key])); // last 1M tokens before `hi` | |
| const same = v1Tok != null && v2Tok != null ? Math.min(v1Tok, v2Tok) : null; | |
| $('#v2CmpSub').append('Same base model, LoRA and losses. V2 changes the data (every pool question as readout, half in the order the eval uses; stricter chain filters; ' + | |
| 'chains for 662 questions from the 35B teacher), the learning rate and the speed setup. Losses at equal tokens are a rough guide, not a score: the two runs train on different examples. The verdict is the eval table at the bottom.'); | |
| const v2ro = (d.readout_gold_only || 0) + (d.readout_from_votes || 0) + (d.readout_gold_fallback || 0); | |
| const rows = [ | |
| ['Readout questions', d.examples ? int(v2ro) : '–', int(V1M.readout_examples)], | |
| ['Reasoning chains', d.reasoner_examples != null ? int(d.reasoner_examples) : '–', int(V1M.reasoner_examples)], | |
| ['Tokens', d.tokens ? f1(d.tokens / 1e6) + 'M' : '–', V1M.train_tokens ? f1(V1M.train_tokens / 1e6) + 'M' : '–'], | |
| ['Optimizer steps', last ? int(last.of) : '–', int(V1M.steps_done)], | |
| ['Learning rate', (V.lr_sweep || {}).best || '–', V1M.lr != null ? V1M.lr.toExponential(0) : '–'], | |
| ['Micro-batch', U ? `${V.bench.configs[U.name].micro / 1024}K, setup ${U.name}` : '–', '16K, setup A'], | |
| ['Tokens per second', last ? int(last.tok_per_s) : '–', int(V1M.tok_per_s)], | |
| ['Training time', last ? `${int(last.elapsed_min)} min so far, ~${int(last.elapsed_min + last.eta_min)} total` : '–', V1M.train_seconds ? `${int(V1M.train_seconds / 60)} min` : '–'], | |
| [same != null ? `Readout loss at ${f1(same)}M tokens` : 'Readout loss at equal tokens', same != null ? f1d(at(V2X, 'readout_loss', same)) : '–', same != null ? f1d(at(V1X, 'readout_loss', same)) : '–'], | |
| [same != null ? `· its floor (target entropy) at ${f1(same)}M` : '· its floor (target entropy)', same != null ? f1d(at(V2X, 'readout_target_entropy', same)) : '–', same != null ? f1d(at(V1X, 'readout_target_entropy', same)) : '–'], | |
| [same != null ? `Readout top pick = target's top at ${f1(same)}M` : "Readout top pick = target's top", same != null ? pct(at(V2X, 'readout_argmax_matches_target', same)) : '–', same != null ? pct(at(V1X, 'readout_argmax_matches_target', same)) : '–'], | |
| [same != null ? `Chain loss at ${f1(same)}M tokens` : 'Chain loss at equal tokens', same != null ? f1d(at(V2X, 'chain_loss', same)) : '–', same != null ? f1d(at(V1X, 'chain_loss', same)) : '–'], | |
| ['Readout loss, end (last 1M tokens)', v2Tok != null ? f1d(at(V2X, 'readout_loss', v2Tok)) + (last.step < last.of ? ' (so far)' : '') : '–', v1Tok != null ? f1d(at(V1X, 'readout_loss', v1Tok)) : '–'], | |
| ['Chain loss, end (last 1M tokens)', v2Tok != null ? f1d(at(V2X, 'chain_loss', v2Tok)) + (last.step < last.of ? ' (so far)' : '') : '–', v1Tok != null ? f1d(at(V1X, 'chain_loss', v1Tok)) : '–']]; | |
| t.append(h('thead', null, h('tr', null, h('th', null, ''), h('th', {class:'n'}, 'V2 joint'), h('th', {class:'n'}, 'v1 joint')))); | |
| const tb = h('tbody'); for (const [k, a, b] of rows) tb.append(h('tr', null, h('td', null, k), h('td', {class:'n mono'}, a), h('td', {class:'n mono'}, b))); t.append(tb); })(); | |
| (function v2First(){ const t = $('#v2First'); const F = V.first_look || {}; const RW = F.rows || {}; | |
| $('#v2FirstSub').append(`Same 5,442 eval questions as v1, accuracy of the top pick · ECE and NLL after per-type temperatures fitted on the calibration items · "Procedural, no repeats" leaves out the ${F.n_dup ?? '–'} procedural items that exactly repeat a V2 training question · confidence intervals come with the full metrics`); | |
| const NM = {base:'Base model', v1_joint:'v1 joint', v2_abl_gold:'V2 ablation: correct answers only (12K)', v2_abl_mix:'V2 ablation: answers + votes (12K)', v2_joint:'V2 joint', | |
| rl60:'V2 + RL, step 60', rl40:'V2 + RL, step 40 (1e-5 only)', base_2b:'2B base model', v2_2b:'2B, V2 recipe', v2_2b_ro:'2B, readout only'}; | |
| const cols = [['eval_proc', 'Procedural'], ['proc_nodup', 'Procedural, no repeats'], ['eval_ts', 'tasksource'], ['opendecisions', 'OpenDecisions'], ['localllama', 'LocalLLaMA'], ['all', 'All']]; | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Readout'), ...cols.map(([, l]) => h('th', {class:'n'}, l)), h('th', {class:'n'}, 'ECE, all'), h('th', {class:'n'}, 'NLL, all')))); | |
| const tb = h('tbody'); const v1 = RW.v1_joint; | |
| for (const [k, r] of Object.entries(RW)) tb.append(h('tr', null, h('td', null, NM[k] || k), | |
| ...cols.map(([c]) => h('td', {class:'n mono'}, pct(r[c].acc) + (v1 && !['base', 'v1_joint', 'base_2b'].includes(k) ? ` (${r[c].acc >= v1[c].acc ? '+' : ''}${(100 * (r[c].acc - v1[c].acc)).toFixed(1)})` : ''))), | |
| h('td', {class:'n mono'}, r.all.ece.toFixed(3)), h('td', {class:'n mono'}, r.all.nll.toFixed(3)))); | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:9}, 'No V2 eval predictions yet.'))); | |
| t.append(tb); })(); | |
| (function v2Dev(){ const t = $('#v2Dev'); | |
| const RUNS = [['sweep_1e-4', 'Sweep, lr 1e-4', '1,500 questions, readout only, 50/50 where votes exist'], ['sweep_2e-4', 'Sweep, lr 2e-4', '1,500 questions, readout only, 50/50 where votes exist'], | |
| ['sweep_3e-4', 'Sweep, lr 3e-4', '1,500 questions, readout only, 50/50 where votes exist'], ['v2_abl_gold', 'Ablation: correct answers only', '12,000 questions, readout only, gold target'], | |
| ['v2_abl_mix', 'Ablation: answers + votes', 'same 12,000, readout only, 50/50 gold + votes where votes exist (7,320)'], ['v2_joint', 'V2 joint', '42,000 questions + 10,974 chains, 50/50 where votes exist']]; | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Run'), h('th', null, 'Trained on'), h('th', {class:'n'}, 'Dev n'), h('th', {class:'n'}, 'Accuracy'), h('th', {class:'n'}, 'NLL'), h('th', {class:'n'}, 'ECE')))); | |
| const tb = h('tbody'); | |
| for (const [k, name, what] of RUNS) { const m = (TR[k] || {}).meta; | |
| tb.append(h('tr', null, h('td', null, name), h('td', null, what), h('td', {class:'n mono'}, m ? int(m.dev_n) : '–'), h('td', {class:'n mono'}, m ? pct(m.dev_acc) : (TR[k] ? 'training' : '–')), | |
| h('td', {class:'n mono'}, m && m.dev_nll != null ? m.dev_nll.toFixed(3) : '–'), h('td', {class:'n mono'}, m && m.dev_ece15 != null ? m.dev_ece15.toFixed(3) : '–'))); } | |
| t.append(tb); })(); | |
| const LRC = {'1e-4':'var(--s-joint)', '2e-4':'var(--s-gold)', '3e-4':'var(--s-votes)'}; | |
| chart('v2Sweep', (el, W) => lineChart(el, W, {H:180, series:Object.keys(LRC).filter(lr => TR['sweep_' + lr]).map(lr => ({name:'lr ' + lr, color:LRC[lr], pts:sm(TR['sweep_' + lr].steps, 'readout_loss', 5)})), yFmt:v => v.toFixed(2), xLabel:'optimizer step', yMin:0})); | |
| (function v2SweepT(){ const t = $('#v2SweepT'); const L = V.lr_sweep || {}; | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Rate'), h('th', {class:'n'}, 'Steps'), h('th', {class:'n'}, 'Dev acc'), h('th', {class:'n'}, 'Dev NLL'), h('th', {class:'n'}, 'Dev ECE'), h('th', null, '')))); | |
| const tb = h('tbody'); | |
| for (const lr of Object.keys(LRC)) { const m = (TR['sweep_' + lr] || {}).meta; if (!m) continue; | |
| tb.append(h('tr', null, h('td', {class:'mono'}, lr), h('td', {class:'n mono'}, int(m.steps_done)), h('td', {class:'n mono'}, pct(m.dev_acc)), h('td', {class:'n mono'}, m.dev_nll == null ? '–' : m.dev_nll.toFixed(3)), | |
| h('td', {class:'n mono'}, m.dev_ece15 == null ? '–' : m.dev_ece15.toFixed(3)), h('td', null, L.best === lr ? 'chosen' : ''))); } | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:6}, 'Sweep not run yet.'))); | |
| t.append(tb); })(); | |
| (function v2Bench(){ const t = $('#v2Bench'); const B = (V.bench || {}).configs || {}; const A = B.A || {}; | |
| const desc = c => [c.liger ? 'Liger' : 'HF kernels', c.ckpt_above ? `no checkpoint ≤ ${c.ckpt_above / 1024}K` : 'checkpoint all', `${c.micro / 1024}K micro`].join(' · '); | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Setup'), h('th', null, 'What'), h('th', {class:'n'}, 'tok/s'), h('th', {class:'n'}, 'vs A'), h('th', {class:'n'}, 'Peak GB'), h('th', {class:'n'}, 'Grad err'), h('th', {class:'n'}, 'Loss err'), h('th', null, 'Result')))); | |
| const tb = h('tbody'); | |
| for (const [n, c] of Object.entries(B)) { if (n === 'R') continue; | |
| tb.append(h('tr', null, h('td', {class:'mono'}, n), h('td', null, desc(c)), h('td', {class:'n mono'}, c.failed ? '–' : int(c.tok_per_s)), h('td', {class:'n mono'}, c.failed || !A.tok_per_s ? '–' : (c.tok_per_s / A.tok_per_s).toFixed(2) + '×'), | |
| h('td', {class:'n mono'}, c.failed ? '–' : f1(c.peak_gb)), h('td', {class:'n mono'}, c.grad_err_vs_fp32 == null ? '–' : c.grad_err_vs_fp32.toExponential(1)), h('td', {class:'n mono'}, c.loss_err_vs_fp32 == null ? '–' : c.loss_err_vs_fp32.toExponential(1)), | |
| h('td', null, c.failed ? 'failed' : ((V.bench.chosen === n ? 'rule\'s pick · ' : '') + ((usedSetup() || {}).name === n ? 'in use · ' : '') + (c.equivalent ? 'passes' : 'fails equivalence') + (c.equivalent && c.peak_gb >= 72 ? ' · over 72 GB' : ''))))); } | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:8}, 'Benchmark not run yet.'))); | |
| t.append(tb); })(); | |
| (function v2Src(){ const t = $('#v2Src'); const S = V.sources || {}; | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Chains from'), h('th', null, 'Questions'), h('th', {class:'n'}, 'Count'), ...[0, 1, 2, 3, 4].map(k => h('th', {class:'n'}, `${k}/4`)), h('th', {class:'n'}, 'Median tokens'), h('th', {class:'n'}, 'p90 tokens'), h('th', {class:'n'}, 'Closed')))); | |
| const tb = h('tbody'); const NM = {local:'Qwen3.5-4B (local)', teacher:'Qwen3.5-35B-A3B (API)'}, ST = {proc:'procedural', ts:'tasksource'}; | |
| for (const [k, b] of Object.entries(S)) { const [src, set, v1] = k.split(':'); | |
| tb.append(h('tr', null, h('td', null, NM[src] || src), h('td', null, (ST[set] || set) + (v1 ? ' (v1 run)' : ' (V2 run)')), h('td', {class:'n mono'}, int(b.items)), ...b.correct_of_4.map(v => h('td', {class:'n mono'}, pct(v, 0))), | |
| h('td', {class:'n mono'}, int(b.tokens_median)), h('td', {class:'n mono'}, int(b.tokens_p90)), h('td', {class:'n mono'}, pct(b.closed)))); } | |
| if (!tb.children.length) tb.append(h('tr', null, h('td', {colspan:11}, 'No chains yet.'))); | |
| t.append(tb); })(); | |
| (function v2Res(){ const t = $('#v2Res'); const M = V.metrics; | |
| if (!M) { t.append(h('tbody', null, h('tr', null, h('td', null, 'Results arrive after the V2 evaluation.')))); return; } | |
| t.append(h('thead', null, h('tr', null, h('th', null, 'Readout'), ...SETS.map(x => h('th', {class:'n'}, SET[x])), h('th', {class:'n'}, 'All'), h('th', {class:'n'}, 'OD Capability'), h('th', {class:'n'}, 'ECE'), h('th', {class:'n'}, 'NLL'), h('th', {class:'n'}, 'Agree w/ own reasoning')))); | |
| const tb = h('tbody'); | |
| for (const [n, b] of Object.entries(M.readout || {})) { const pr = (M.pairs || {})[n]; | |
| tb.append(h('tr', null, h('td', null, n), ...SETS.map(x => h('td', {class:'n mono'}, pct(b.acc[x]))), h('td', {class:'n mono'}, pct(b.acc.all)), h('td', {class:'n mono'}, f1(b.od && b.od.capability)), | |
| h('td', {class:'n mono'}, b.ece15.toFixed(3)), h('td', {class:'n mono'}, b.nll.toFixed(3)), h('td', {class:'n mono'}, pr ? pct(pr.agree_all) : '–'))); } | |
| for (const [n, b] of Object.entries(M.reasoning || {})) tb.append(h('tr', null, h('td', null, 'reasoning · ' + n), ...SETS.map(x => h('td', {class:'n mono'}, pct(b.acc[x]))), h('td', {class:'n mono'}, pct(b.acc.all)), | |
| h('td', {class:'n mono'}, `${int(b.tokens_median)} tok`), h('td', {class:'n mono'}, b.post_ece15.toFixed(3)), h('td', {class:'n mono'}, `${int(b.loops)} loops`), h('td', null, ''))); | |
| t.append(tb); })(); | |
| /* ------------------------------------------------------------ generation and serving */ | |
| (function genTiles(){ const T = $('#genTiles'); const g = D.generation || {}; const p = D.probe || {}; | |
| const t = (k, v, d) => T.append(h('div', {class:'tile'}, h('div', {class:'k'}, k), h('div', {class:'v'}, v), h('div', {class:'d'}, d))); | |
| t('Items with 4 chains', int(g.items), `${int(g.samples)} chains · ${(g.tokens_total/1e6 || 0).toFixed(1)}M tokens`); | |
| t('Items with a correct chain', pct(g.items ? g.items_with_correct_chain / g.items : null), `${int(g.items_with_correct_chain)} reasoner examples available`); | |
| t('Majority vote correct', pct(g.majority_accuracy), 'training pool, vs gold'); | |
| t('Chain tokens median / p90', `${int(g.tokens_median)} / ${int(g.tokens_p90)}`, `max ${int(g.tokens_max)} · parse ${pct(g.parse_rate)}`); | |
| t('Chains looped to 32k', int(g.looped), pct(g.samples ? g.looped / g.samples : null, 2) + ' of chains'); | |
| })(); | |
| const STC = {probe:'var(--ink-3)', baseline:'var(--s-base)', generate:'var(--s-gold)', eval_joint:'var(--s-joint)', eval_gold:'var(--s-gold)', eval_votes:'var(--s-votes)', v2_generate:'var(--caution)', v2_eval_v2_joint:'var(--s-joint)'}; | |
| chart('cEngGen', (el, W) => lineChart(el, W, {xType:'time', series:Object.entries(D.engine || {}).map(([st, rows]) => ({name:st, color:STC[st] || 'var(--ink-3)', w:1.6, pts:rows.map(r => [new Date(r.t).getTime(), r.gen])})), yFmt:v => int(v), xFmt:v => new Date(v).toISOString().slice(11,16), yMin:0})); | |
| chart('cEngKv', (el, W) => lineChart(el, W, {xType:'time', series:Object.entries(D.engine || {}).map(([st, rows]) => ({name:st, color:STC[st] || 'var(--ink-3)', w:1.6, pts:rows.map(r => [new Date(r.t).getTime(), r.kv])})), yFmt:v => v.toFixed(0) + '%', xFmt:v => new Date(v).toISOString().slice(11,16), yMin:0, yMax:100, | |
| refLines:[{y:95, label:'preemption zone', color:'var(--caution)'}]})); | |
| chart('cProg', (el, W) => { const C = {'generation':'var(--s-gold)', 'baseline reasoning':'var(--s-base)', 'joint reasoning':'var(--s-joint)', 'V2 generation':'var(--caution)', 'V2 joint reasoning':'var(--s-votes)'}; | |
| lineChart(el, W, {series:Object.entries(D.progress || {}).map(([n, rows]) => { let off = 0, prev = null; const pts = []; for (const r of rows) { if (prev && r.finished < prev.finished) off += prev.finished; pts.push([pts.length, off + r.finished]); prev = r; } return {name:n + ' (segments joined)', color:C[n], pts}; }), | |
| yFmt:v => int(v), xLabel:'minutes of the stage (progress lines are one per minute)', yMin:0}); }); | |
| chart('cHist', (el, W) => { const g = D.gen_view || {}; const all = g.sample_tokens || [], sh = g.shortest_correct_tokens || []; if (!all.length) { el.append(h('div', {class:'empty'}, 'No generation yet.')); return; } | |
| const edges = []; for (let e = 64; e <= 65536; e *= 2) edges.push(e); const binOf = v => Math.min(edges.length - 1, Math.max(0, Math.floor(Math.log2(Math.max(v, 64))) - 6)); | |
| const cnt = arr => { const c = new Array(edges.length).fill(0); arr.forEach(v => c[binOf(v)]++); return c.map(x => x / arr.length); }; | |
| const ca = cnt(all), cs = cnt(sh); const keep = edges.map((e, i) => ca[i] > 0 || cs[i] > 0); | |
| barChart(el, W, {cats:edges.filter((_, i) => keep[i]).map(e => e), catFmt:c => c >= 1024 ? (c/1024) + 'k' : String(c), yFmt:v => pct(v, 0), | |
| series:[{name:'all 4 samples', color:'var(--s-votes)', vals:ca.filter((_, i) => keep[i])}, {name:'shortest correct (trained on)', color:'var(--s-joint)', vals:cs.filter((_, i) => keep[i])}]}); }); | |
| chart('cFam', (el, W) => { const g = D.gen_view || {}; const fam = Object.entries(g.family_acc || {}).sort((a, b) => a[1] - b[1]); | |
| barChart(el, W, {horizontal:true, cats:fam.map(f => f[0]), catFmt:c => c.replace(/_/g, ' '), yMin:0.6, yMax:1, yFmt:v => pct(v, 0), series:[{name:'share of samples correct', color:'var(--s-joint)', vals:fam.map(f => f[1])}]}); }); | |
| chart('cVotes', (el, W) => { const vs = (D.gen_view || {}).vote_share || {}; const keys = ['unanimous, = gold', 'split votes', 'unanimous, against gold', 'fewer than 2 parsed']; const sets = ['train_proc', 'train_ts'].filter(k => vs[k]); | |
| const tot = k => Object.values(vs[k]).reduce((a, b) => a + b, 0); const C = ['var(--s-joint)', 'var(--s-votes)', 'var(--s-gold)', 'var(--s-base)']; | |
| barChart(el, W, {cats:keys, catFmt:c => c, yMax:1, yFmt:v => pct(v, 0), series:sets.map((k, i) => ({name:k === 'train_proc' ? 'procedural' : 'tasksource', color:i ? 'var(--s-gold)' : 'var(--s-joint)', vals:keys.map(q => (vs[k][q] || 0) / tot(k))}))}); }); | |
| /* ------------------------------------------------------------ samples */ | |
| function probBars(item){ const rows = []; | |
| const pairs = [['readout_base', 'var(--s-base)', 'base'], ['readout_joint', 'var(--s-joint)', 'joint']].filter(([k]) => item[k]); | |
| return item.options.map((o, i) => h('div', {class:'opt' + (i === item.gold ? ' g' : '')}, h('span', {class:'L'}, o.letter), h('span', {class:'txt'}, o.text || '(no description)'), | |
| h('div', {class:'pb'}, pairs.map(([k, c, nm]) => h('div', {title:`${nm} readout`}, h('span', {class:'b'}, h('em', {style:`width:${(100*item[k][i]).toFixed(1)}%;background:${c}`})), h('span', null, (100*item[k][i]).toFixed(0) + '%')))))); | |
| } | |
| function chainBlock(label, r, gold, order){ if (!r) return null; const ans = r.answer; const ok = ans != null && (order ? order[ans] : ans) === gold; | |
| return h('details', {class:'chain'}, h('summary', null, h('b', null, label), h('span', {class:'mono'}, `${int(r.tokens)} tokens · `, r.looped ? h('span', {class:'no'}, 'looped, no answer') : h('span', {class:ok ? 'ok' : 'no'}, `answer ${ans == null ? '–' : String.fromCharCode(65 + ans)} ${ok ? '✓' : '✗'}`))), h('pre', null, r.text)); } | |
| function sampleCard(x){ const card = h('article', {class:'sample'}); | |
| card.append(h('div', {class:'eyebrow'}, x.title || ''), x.why ? h('p', {class:'why'}, x.why) : null); | |
| card.append(h('div', {class:'meta'}, h('span', {class:'chip'}, x.set), h('span', {class:'chip'}, x.family), x.level != null ? h('span', {class:'chip'}, 'level ' + x.level) : null, x.tier ? h('span', {class:'chip'}, x.tier + ' tier') : null, | |
| h('span', {class:'chip'}, x.qtype), h('span', {class:'chip gold'}, 'gold ' + String.fromCharCode(65 + x.gold)))); | |
| card.append(h('div', {class:'q'}, x.criterion), h('div', {class:'ev'}, x.evidence)); | |
| card.append(legend([x.readout_base ? {name:'base readout', color:'var(--s-base)', box:true} : null, x.readout_joint ? {name:'joint readout', color:'var(--s-joint)', box:true} : null].filter(Boolean)), h('div', {class:'opts'}, probBars(x))); | |
| const ch = h('div', {class:'chains'}, chainBlock('Base reasoning', x.reason_base, x.gold), chainBlock('Joint reasoning', x.reason_joint, x.gold), | |
| x.train_chain ? chainBlock('Training chain (shortest correct)', {answer:null, tokens:x.train_chain.tokens, looped:false, text:x.train_chain.text}, x.gold) : null); | |
| card.append(ch); return card; } | |
| (function(){ const L = $('#sampleList'); for (const x of (D.samples || [])) L.append(sampleCard(x)); if (!L.children.length) L.append(h('div', {class:'empty'}, 'Samples appear once predictions exist.')); })(); | |
| /* ------------------------------------------------------------ browse */ | |
| (function browse(){ | |
| const items = D.browse || []; const seg = $('#browseSeg'); let cur = 'all'; | |
| const opts = [['all', 'All']].concat(SETS.map(sx => [SET[sx], SET[sx]])); | |
| const draw = () => { const t = $('#browseTable'); t.replaceChildren(); const rows = items.filter(x => cur === 'all' || x.set === cur); $('#browseCount').textContent = `${rows.length} items`; | |
| t.append(h('thead', null, h('tr', null, ['Set', 'Question', 'Gold', 'Readout base', 'Readout joint', 'Reasoning base', 'Reasoning joint'].map((c, i) => h('th', {class:i > 1 ? 'n' : null}, c))))); | |
| const tb = h('tbody'); | |
| const ro = (x, k) => x[k] ? (() => { const a = x[k].indexOf(Math.max(...x[k])); return h('span', {class:a === x.gold ? 'ok' : 'no'}, `${String.fromCharCode(65 + a)} ${(100*x[k][a]).toFixed(0)}%`); })() : '–'; | |
| const rs = (x, k) => x[k] ? (x[k].looped ? h('span', {class:'no'}, 'looped') : h('span', {class:x[k].answer === x.gold ? 'ok' : 'no'}, `${x[k].answer == null ? '–' : String.fromCharCode(65 + x[k].answer)} · ${int(x[k].tokens)}`)) : '–'; | |
| rows.forEach((x, i) => { const tr = h('tr', {class:'click', tabindex:0, 'aria-expanded':'false'}, h('td', {class:'mono'}, x.set), h('td', null, x.criterion.length > 110 ? x.criterion.slice(0, 110) + '…' : x.criterion), | |
| h('td', {class:'n'}, String.fromCharCode(65 + x.gold)), h('td', {class:'n'}, ro(x, 'readout_base')), h('td', {class:'n'}, ro(x, 'readout_joint')), h('td', {class:'n'}, rs(x, 'reason_base')), h('td', {class:'n'}, rs(x, 'reason_joint'))); | |
| const toggle = () => { const nx = tr.nextElementSibling; if (nx && nx.classList.contains('detail')) { nx.remove(); tr.setAttribute('aria-expanded', 'false'); return; } | |
| tr.setAttribute('aria-expanded', 'true'); tr.after(h('tr', {class:'detail'}, h('td', {colspan:7}, sampleCard(x)))); }; | |
| tr.addEventListener('click', toggle); tr.addEventListener('keydown', e => { if (e.key === 'Enter') toggle(); }); tb.append(tr); }); | |
| t.append(tb); }; | |
| for (const [k, lab] of opts) seg.append(h('button', {type:'button', id:'b-' + k.replace(/\W/g, ''), 'aria-pressed': String(k === cur), onclick: () => { cur = k; seg.querySelectorAll('button').forEach(b => b.setAttribute('aria-pressed', String(b.textContent === lab))); draw(); }}, lab)); | |
| draw(); | |
| })(); | |
| /* ------------------------------------------------------------ logs */ | |
| function esc(t){ return t.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'); } | |
| function inline(t){ return esc(t).replace(/`([^`]+)`/g, '<code>$1</code>').replace(/\*\*([^*]+)\*\*/g, '<b>$1</b>'); } | |
| function md(src){ const out = h('div', {class:'md'}); const lines = src.split('\n'); let i = 0; | |
| while (i < lines.length) { const l = lines[i]; | |
| if (/^\s*\|/.test(l)) { const rows = []; while (i < lines.length && /^\s*\|/.test(lines[i])) { rows.push(lines[i]); i++; } const cells = r => r.trim().replace(/^\||\|$/g, '').split('|').map(c => c.trim()); | |
| const tw = h('div', {class:'tw'}); const tb = document.createElement('table'); const head = cells(rows[0]); const thead = document.createElement('thead'); const tr = document.createElement('tr'); head.forEach(c => { const th = document.createElement('th'); th.innerHTML = inline(c); tr.append(th); }); thead.append(tr); tb.append(thead); | |
| const body = document.createElement('tbody'); rows.slice(2).forEach(r => { const tr2 = document.createElement('tr'); cells(r).forEach(c => { const td = document.createElement('td'); td.innerHTML = inline(c); tr2.append(td); }); body.append(tr2); }); tb.append(body); tw.append(tb); out.append(tw); continue; } | |
| const hm2 = l.match(/^(#{1,3})\s+(.*)/); if (hm2) { const e = document.createElement('h' + hm2[1].length); e.innerHTML = inline(hm2[2]); out.append(e); i++; continue; } | |
| if (/^\s*[-*] /.test(l)) { const ul = document.createElement('ul'); while (i < lines.length && /^\s*([-*] |\s{2,}\S)/.test(lines[i]) && lines[i].trim()) { const li = document.createElement('li'); let txt = lines[i].replace(/^\s*[-*] /, ''); i++; while (i < lines.length && /^\s{2,}\S/.test(lines[i]) && !/^\s*[-*] /.test(lines[i])) { txt += ' ' + lines[i].trim(); i++; } li.innerHTML = inline(txt); ul.append(li); } out.append(ul); continue; } | |
| if (!l.trim()) { i++; continue; } | |
| let para = l; i++; while (i < lines.length && lines[i].trim() && !/^\s*(\||#|[-*] )/.test(lines[i])) { para += ' ' + lines[i].trim(); i++; } const p = document.createElement('p'); p.innerHTML = inline(para); out.append(p); } | |
| return out; } | |
| (function logs(){ | |
| const tabs = [['notebook', 'Lab notebook', () => md(D.notebook || '')], ['deviations', 'Deviations from the plan', () => md(D.deviations || '')], ['todo', 'To-do', () => md(D.todo || '')], | |
| ['timeline', 'Timeline events', () => { const tw = h('div', {class:'tw'}); const t = h('table'); t.append(h('thead', null, h('tr', null, h('th', null, 'Time'), h('th', null, 'Event'), h('th', null, 'Stage'), h('th', null, 'Note')))); | |
| const tb = h('tbody'); for (const e of (D.events || []).slice().reverse()) tb.append(h('tr', null, h('td', {class:'mono'}, ist(e.time)), h('td', null, e.event), h('td', {class:'mono'}, e.stage), h('td', null, e.note || ''))); t.append(tb); tw.append(t); return tw; }]] | |
| .concat(Object.keys(D.logs || {}).map(k => [k, k.replace('_', ' ') + ' log', () => h('pre', {class:'log'}, (D.logs[k] || []).join('\n'))])); | |
| const T = $('#logTabs'), B = $('#logBody'); let cur = 'notebook'; | |
| const show = k => { cur = k; T.querySelectorAll('button').forEach(b => b.setAttribute('aria-selected', String(b.dataset.k === k))); B.replaceChildren(tabs.find(t => t[0] === k)[2]()); }; | |
| for (const [k, lab] of tabs) T.append(h('button', {type:'button', role:'tab', id:'t-' + k, 'data-k':k, 'aria-selected':'false', onclick:() => show(k)}, lab)); | |
| show(cur); | |
| })(); | |
| $('#foot').textContent = `Snapshot of /workspace/decision-reasoner taken ${ist(D.generated_at)}. Files, logs and predictions sync to the Hugging Face bucket Mercity/Desicion-Model-Custom under run1/. This page is republished with a fresh snapshot while the run continues.`; | |
| /* draw all charts, redraw on resize */ | |
| const drawAll = () => registry.forEach(fn => fn()); drawAll(); | |
| let rt; new ResizeObserver(() => { clearTimeout(rt); rt = setTimeout(drawAll, 120); }).observe(document.querySelector('main')); | |
| })(); | |
| </script> | |
Xet Storage Details
- Size:
- 108 kB
- Xet hash:
- dd4d6fcfaac894b82f16c29d9e88f4cabd7431a429c5fcc20f7a3e4630a06fe1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.