Hugging Face's logo

Spaces:
aminamou
/
lm-playground-api
Sleeping

App Files Community
lm-playground-api / web /chapters /sampling.html
deploy
Deploy LM Playground API
f0a527a
Raw
History Blame Contribute Delete
33.7 kB
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>12. Build sampling — LM Playground</title>
<link rel="stylesheet" href="/platform/platform.css"/>
<script type="module" src="/platform/platform.js"></script>
<style>
:root{
 --bg:#0f1117; --panel:#181b24; --panel-2:#1f2330; --border:#2a2f3d;
 --text:#e6e8ee; --muted:#9aa3b2; --accent:#7c8cff; --accent-2:#5be0c0;
 --good:#5be08a; --warn:#ffc06b; --bad:#ff9090;
 --keep:#7c8cff; --cut:#3a3f4d;
 --mono:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace;
}
*{box-sizing:border-box}
html,body{margin:0;padding:0 20px 80px;background:radial-gradient(1200px 600px at 50% -10%,#1a1f2e 0%,var(--bg) 55%);color:var(--text);font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;line-height:1.5;-webkit-font-smoothing:antialiased}
header{max-width:980px;margin:0 auto;padding:34px 0 0}
.header-inner{display:flex;justify-content:space-between;align-items:flex-start;padding-bottom:12px}
h1{font-size:30px;margin:0 0 6px;letter-spacing:-.5px}
.sub{color:var(--muted);max-width:780px;margin:0;font-size:14.5px;line-height:1.6}
.sub b{color:var(--text);font-weight:600}
.page-tabs{display:flex;border-bottom:1px solid var(--border)}
.page-tab{background:none;border:none;color:var(--muted);padding:11px 22px 10px;font-size:14px;font-weight:600;cursor:pointer;border-bottom:2px solid transparent;margin-bottom:-1px;transition:.12s}
.page-tab:hover{color:var(--text)}.page-tab.active{color:var(--accent);border-bottom-color:var(--accent)}
.tab-panel{display:none}.tab-panel.active{display:block}
.guide{max-width:980px;margin:0 auto;padding:30px 0 80px;display:grid;grid-template-columns:1fr 220px;gap:0 48px}
.guide-toc{grid-column:2;grid-row:1/20;position:sticky;top:64px;align-self:start;background:var(--panel);border:1px solid var(--border);border-radius:12px;padding:16px;display:flex;flex-direction:column;gap:4px}
.toc-label{font-size:10px;text-transform:uppercase;letter-spacing:1px;color:var(--muted);font-weight:700;margin-bottom:6px}
.guide-toc a{color:var(--muted);text-decoration:none;font-size:12.5px;padding:5px 8px;border-radius:6px;transition:.1s}
.guide-toc a:hover{color:var(--text);background:var(--panel-2)}
.chapter{grid-column:1;padding:0 0 48px;border-bottom:1px solid var(--border);margin-bottom:48px}
.chapter:last-of-type{border-bottom:none}
.chapter h2{font-size:22px;margin:0 0 18px;display:flex;align-items:center;gap:12px;letter-spacing:-.3px}
.ch-num{display:inline-flex;align-items:center;justify-content:center;width:32px;height:32px;border-radius:99px;background:var(--accent-2);color:#0b0d14;font-size:14px;font-weight:800;flex-shrink:0}
.chapter h3{font-size:15px;margin:24px 0 8px;color:var(--accent-2);font-weight:700}
.chapter p{margin:0 0 14px;line-height:1.7;color:#ced3de;font-size:15px}
.chapter strong{color:var(--text)}.chapter em{font-style:italic}
code{background:var(--panel-2);border:1px solid var(--border);border-radius:5px;padding:1px 5px;font-family:var(--mono);font-size:12.5px;color:var(--accent-2)}
.callout{border-radius:10px;padding:14px 16px;margin:20px 0;font-size:14px;line-height:1.65;color:#ced3de}
.callout strong{display:block;margin-bottom:5px;font-size:12px;text-transform:uppercase;letter-spacing:.6px}
.callout.insight{background:#7c8cff18;border-left:3px solid var(--accent)}.callout.insight strong{color:var(--accent)}
.callout.warn{background:#ffc06b18;border-left:3px solid var(--warn)}.callout.warn strong{color:var(--warn)}
.formula-box{background:var(--panel);border:1px solid var(--border);border-radius:10px;padding:18px 16px;margin:16px 0;text-align:center}
.formula{font-family:var(--mono);font-size:15px;color:var(--text);line-height:1.7}
.formula .hl{color:var(--accent);font-weight:700}
.formula-note{font-size:12.5px;color:var(--muted);margin-top:10px;line-height:1.5}
.try-it{display:inline-block;margin-top:10px;padding:10px 18px;border-radius:9px;background:var(--panel-2);border:1px solid var(--border);color:var(--accent);font-size:13.5px;font-weight:600;cursor:pointer;transition:.14s;font-family:inherit}
.try-it:hover{background:#7c8cff22;border-color:var(--accent)}
.try-it.large{font-size:15px;padding:14px 28px;background:var(--accent-2);color:#0b0d14;border-color:var(--accent-2)}
.try-it.large:hover{filter:brightness(1.08)}
.guide-end{text-align:center;padding:18px 0 0}.guide-end p{color:var(--muted);margin-bottom:14px}
.panel-guide-item{display:flex;gap:12px;align-items:baseline;padding:9px 0;border-bottom:1px dashed var(--border)}
.panel-guide-item:last-child{border-bottom:none}
.pgi-label{font-size:13px;font-weight:800;color:var(--accent-2);min-width:20px;font-family:var(--mono)}
.panel-guide-item p{margin:0;font-size:13.5px;color:#ced3de;line-height:1.55}
.wrap{max-width:980px;margin:20px auto 0}
.panel{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:16px 18px;margin-bottom:16px}
.panel-head{display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;flex-wrap:wrap;gap:8px}
.panel-label{font-size:13px;font-weight:700;color:var(--text)}.panel-note{color:var(--muted);font-size:12px}
.ctrl-row{display:flex;flex-wrap:wrap;gap:16px;align-items:flex-end;margin:4px 0 6px}
.ctrl{display:flex;flex-direction:column;gap:6px}
.ctrl .lab{font-size:11px;text-transform:uppercase;letter-spacing:.7px;color:var(--muted);font-weight:600}
.ctrl .lab b{color:var(--text);font-family:var(--mono)}
input[type=range]{accent-color:var(--accent);width:150px}
.btn{border:1px solid var(--border);background:var(--panel-2);color:var(--text);font-size:13px;font-weight:600;padding:9px 16px;border-radius:9px;cursor:pointer;font-family:inherit}
.btn:hover{border-color:var(--accent)}.btn.primary{background:var(--accent-2);color:#0b0d14;border-color:var(--accent-2)}
.presets{display:flex;flex-wrap:wrap;gap:7px;margin-top:6px}
.presets button{font-size:12.5px;background:var(--panel-2);border:1px solid var(--border);color:var(--muted);border-radius:20px;padding:5px 12px;cursor:pointer;font-family:inherit}
.presets button:hover{color:var(--text);border-color:var(--accent-2)}
.presets button.on{border-color:var(--accent-2);color:var(--text)}
svg{display:block;width:100%;height:auto;overflow:visible}
.ctx{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin-bottom:8px}
.ctx .seg{display:inline-flex;background:var(--panel-2);border:1px solid var(--border);border-radius:8px;padding:3px;gap:2px}
.ctx .seg button{border:0;background:transparent;color:var(--muted);padding:5px 9px;border-radius:6px;cursor:pointer;font-family:var(--mono);font-size:12.5px}
.ctx .seg button.on{background:var(--accent);color:#0b0d14}
.statline{display:flex;flex-wrap:wrap;gap:20px;margin:8px 0 2px}
.stat .v{font-family:var(--mono);font-size:17px;font-weight:700}
.stat .l{font-size:10.5px;text-transform:uppercase;letter-spacing:.5px;color:var(--muted)}
.gen{font-family:var(--mono);font-size:13.5px;line-height:1.8;color:var(--accent-2);background:var(--bg);border:1px solid var(--border);border-radius:8px;padding:12px 14px;min-height:110px;white-space:pre-wrap;word-break:break-word}
.hint{color:var(--muted);font-size:12.5px;margin-top:10px;line-height:1.5}
.legend{display:flex;gap:16px;font-size:11.5px;color:var(--muted);margin-top:6px}
.legend i{display:inline-block;width:12px;height:12px;border-radius:3px;vertical-align:middle;margin-right:5px}
.fig{margin:22px 0;background:var(--panel);border:1px solid var(--border);border-radius:12px;padding:18px 16px 12px}
.fig svg{display:block;width:100%;height:auto;overflow:visible}
.fig figcaption{margin-top:10px;font-size:12px;color:var(--muted);text-align:center;line-height:1.55}
footer{max-width:980px;margin:30px auto 0;color:var(--muted);font-size:12px;text-align:center;line-height:1.6}
@media(max-width:760px){.guide{grid-template-columns:1fr}.guide-toc{grid-column:1;grid-row:auto;position:static;display:grid;grid-template-columns:1fr 1fr}.toc-label{grid-column:1/-1}}
</style>
</head>
<body data-chapter="sampling">
<header>
 <div class="header-inner">
 <div>
 <h1>Build sampling</h1>
 <p class="sub">
 A language model never outputs a word. It outputs a <b>probability for every word</b>,
 and someone has to choose. That choice — the decoding strategy — is where a model gets
 its voice: cautious and repetitive, or loose and surprising, from the very same
 weights. Here you turn the knobs on a real model's live distribution and watch the
 candidates widen, narrow, and loop.
 </p>
 </div>
 </div>
 <nav class="page-tabs" role="tablist">
 <button class="page-tab active" data-tab="guide">① Guide</button>
 <button class="page-tab" data-tab="playground">② Playground</button>
 </nav>
</header>
<!-- ═══════════════════════════════════════════════════════════════ GUIDE -->
<section id="guide-tab" class="tab-panel active">
<article class="guide">
<nav class="guide-toc">
 <span class="toc-label">Contents</span>
 <a href="#ch1">1 · Distribution to token</a>
 <a href="#ch2">2 · Why not greedy</a>
 <a href="#ch3">3 · Temperature</a>
 <a href="#ch4">4 · Top-k &amp; top-p</a>
 <a href="#ch5">5 · Taming repetition</a>
 <a href="#ch6">6 · Reading the playground</a>
 </nav>
<!-- 1 -->
 <section class="chapter" id="ch1">
 <h2><span class="ch-num">1</span> From distribution to token</h2>
 <p>
 Run a prompt through the model and the final layer hands you a <em>logit</em> for every
 token in the vocabulary — tens of thousands of raw scores. A softmax turns them into a
 probability distribution: token <code>the</code> at 12%, <code>a</code> at 8%,
 <code>quantum</code> at 0.001%, and so on across the whole vocabulary. The model's job
 ends here. It has stated its beliefs; it has not picked a word.
 </p>
 <figure class="fig"><svg viewBox="0 0 700 200" xmlns="http://www.w3.org/2000/svg" font-family="-apple-system,Segoe UI,Roboto,sans-serif"><defs><marker id="arsmp" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="var(--muted)"/></marker></defs>
<text x="150" y="24" text-anchor="middle" fill="var(--muted)" font-size="11" font-weight="700">model probabilities</text>
<rect x="70" y="129.0" width="15" height="21.0" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="89" y="134.6" width="15" height="15.4" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="108" y="138.8" width="15" height="11.2" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="127" y="142.3" width="15" height="7.7" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="146" y="144.4" width="15" height="5.6" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="165" y="146.5" width="15" height="3.5" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="184" y="147.2" width="15" height="2.8" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="203" y="147.9" width="15" height="2.1" rx="2" fill="var(--muted)" opacity="0.85"/><rect x="222" y="149.3" width="15" height="0.7" rx="2" fill="var(--muted)" opacity="0.85"/>
<line x1="64" y1="150" x2="300" y2="150" stroke="var(--border)"/>
<line x1="330" y1="105" x2="400" y2="105" stroke="var(--accent)" stroke-width="2" marker-end="url(#arsmp)"/>
<text x="365" y="96" text-anchor="middle" fill="var(--accent)" font-size="10">temp · top-k · top-p</text>
<text x="545" y="24" text-anchor="middle" fill="var(--accent-2)" font-size="11" font-weight="700">reshaped → sample</text>
<rect x="430" y="111.5" width="15" height="38.5" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="449" y="132.5" width="15" height="17.5" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="468" y="143.0" width="15" height="7.0" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="487" y="146.5" width="15" height="3.5" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="506" y="147.9" width="15" height="2.1" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="525" y="149.3" width="15" height="0.7" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="544" y="149.65" width="15" height="0.4" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="563" y="149.79" width="15" height="0.2" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="582" y="149.86" width="15" height="0.1" rx="2" fill="var(--accent-2)" opacity="0.85"/>
<line x1="424" y1="150" x2="660" y2="150" stroke="var(--border)"/>
<text x="150" y="172" text-anchor="middle" fill="var(--muted)" font-size="9.5">flat = creative / risky</text>
<text x="545" y="172" text-anchor="middle" fill="var(--muted)" font-size="9.5">peaked = safe / repetitive</text>
</svg><figcaption>Sampling turns the model's probability distribution into an actual next token. Temperature, top-k, and top-p reshape that distribution before you draw from it — flatten it for variety, sharpen it for focus. Same model, very different text depending on how you sample.</figcaption></figure>
 <p>
 <strong>Decoding</strong> is the separate step that turns that distribution into an
 actual token, and then repeats — append the token, run again, sample again. The same
 frozen weights can sound like a careful encyclopedia or a free-associating poet
 depending entirely on how you make that pick.
 </p>
 <div class="callout insight">
 <strong>The model proposes a distribution; decoding disposes a token.</strong>
 Every behaviour you associate with a model's "style" — its repetitiveness, its
 creativity, its determinism — is at least half a decoding choice, made after the weights
 have had their say.
 </div>
 <button class="try-it" data-action="open">▶ Reshape a real distribution in the Playground</button>
 </section>
<!-- 2 -->
 <section class="chapter" id="ch2">
 <h2><span class="ch-num">2</span> Why not just take the most likely?</h2>
 <p>
 The obvious strategy is <em>greedy</em>: at each step, take the single highest-probability
 token. It's deterministic and it sounds safe, but it has two real problems.
 </p>
 <p>
 First, the most likely token at every step does <strong>not</strong> add up to the most
 likely sentence — locally greedy choices paint you into globally bland corners. Second,
 greedy decoding loops. Once the model writes <em>"the best way to the best way to the
 best way…"</em>, each repetition makes the next repetition look even more probable, and
 it can't escape. Real text has variety that always-take-the-top can't produce.
 </p>
 <div class="callout warn">
 <strong>Greedy gets stuck.</strong>
 A high-probability loop is a trap with no exit under argmax: the very repetition that's
 gone wrong is what the model now scores highest. You need a pinch of randomness, or an
 explicit penalty, to break out — both of which this chapter builds.
 </div>
 </section>
<!-- 3 -->
 <section class="chapter" id="ch3">
 <h2><span class="ch-num">3</span> Temperature</h2>
 <p>
 Temperature is the master volume on randomness. Before the softmax, divide every logit
 by a number <code>T</code>:
 </p>
 <div class="formula-box">
 <div class="formula">p = softmax( <span class="hl">logits / T</span> )</div>
 <div class="formula-note">T &lt; 1 sharpens toward the top token; T &gt; 1 flattens toward uniform; T → 0 is greedy.</div>
 </div>
 <p>
 Low temperature (say 0.7) makes the peaks taller and the tails shorter — the model plays
 it safe and stays on-topic. High temperature (1.3+) levels the distribution, handing
 unlikely tokens a real chance and producing surprising, sometimes incoherent text. It's
 one knob that slides smoothly from rigid to unhinged, and most of a model's apparent
 personality lives on this dial.
 </p>
 <button class="try-it" data-action="temp">▶ Slide the temperature and watch the peaks move</button>
 </section>
<!-- 4 -->
 <section class="chapter" id="ch4">
 <h2><span class="ch-num">4</span> Top-k and top-p</h2>
 <p>
 Temperature reshapes the whole distribution but never closes the door on the junk in the
 far tail — at high T, a genuinely nonsensical token can still slip through. Truncation
 sampling slams that door by throwing the tail away before sampling.
 </p>
 <p>
 <strong>Top-k</strong> keeps only the <code>k</code> most probable tokens and zeroes the
 rest, then renormalizes and samples. Simple, but rigid: <code>k = 40</code> is too few
 when the model is genuinely unsure across hundreds of plausible tokens, and too many when
 it's confident about two. <strong>Top-p</strong> (nucleus sampling) fixes that by keeping
 the smallest set of tokens whose probabilities <em>sum to</em> <code>p</code> — say 0.9.
 When the model is certain, that nucleus is two or three tokens; when it's unsure, it's
 dozens. The candidate set breathes with the model's confidence.
 </p>
 <div class="callout insight">
 <strong>Top-p adapts; top-k doesn't.</strong>
 A fixed cutoff count can't tell a confident step from an uncertain one. Nucleus sampling
 sizes the candidate pool to the distribution in front of it, which is why <code>top-p ≈
 0.9</code> with a modest temperature is the workhorse default.
 </div>
 <button class="try-it" data-action="topp">▶ Cut the tail with top-k and top-p</button>
 </section>
<!-- 5 -->
 <section class="chapter" id="ch5">
 <h2><span class="ch-num">5</span> Taming repetition</h2>
 <p>
 Even with sampling, models drift into loops, especially at low temperature. The direct
 fix is a <strong>repetition penalty</strong>: before sampling, divide the logits of
 tokens that have appeared recently, making the model less keen to say them again.
 Relatives include <em>presence</em> and <em>frequency</em> penalties (used by the OpenAI
 API) and a <em>no-repeat n-gram</em> rule that simply forbids repeating any n-gram
 verbatim.
 </p>
 <p>
 These are blunt tools — push the penalty too hard and the model contorts itself to avoid
 common, necessary words like <em>the</em>, and the text turns stilted. As with every knob
 here, the goal isn't an extreme setting; it's the balance that reads as natural.
 </p>
 <button class="try-it" data-action="rep">▶ Break a loop with the repetition penalty</button>
 </section>
<!-- 6 -->
 <section class="chapter" id="ch6">
 <h2><span class="ch-num">6</span> Reading the playground</h2>
 <p>
 The distribution and the text here come from a <strong>real</strong> model — a small
 character-level network trained live in your browser. It's not a large LM, so it thinks
 in letters, not ideas; but the decoding mechanics are exactly the ones production systems
 use.
 </p>
 <div class="panel-guide-item"><span class="pgi-label">▮</span>
 <p>The live next-token distribution. Temperature reshapes the bars; top-k and top-p grey
 out the tail. Kept tokens are the only ones that can be sampled.</p></div>
 <div class="panel-guide-item"><span class="pgi-label">✎</span>
 <p>Generated text under your current settings. Flip to greedy and watch it loop; add a
 repetition penalty and watch the loop break.</p></div>
 <div class="panel-guide-item"><span class="pgi-label">◔</span>
 <p>Presets — Greedy, Balanced, Creative — to feel the span from rigid to wild in one click.</p></div>
 <div class="guide-end">
 <p>The reading is the setup. The playground is the point.</p>
 <button class="try-it large" data-action="open">▶ Open the Playground</button>
 </div>
 </section>
</article>
</section>
<!-- ═══════════════════════════════════════════════════════════════ PLAYGROUND -->
<section id="playground-tab" class="tab-panel">
<div class="wrap">
<div class="panel">
 <div class="panel-head"><span class="panel-label">Decoding knobs</span>
 <span class="panel-note" id="trainNote">real char-model · trained live</span></div>
 <div class="ctrl-row">
 <div class="ctrl"><span class="lab">Temperature: <b id="tVal">0.80</b></span>
 <input type="range" id="temp" min="5" max="200" value="80"></div>
 <div class="ctrl"><span class="lab">Top-k: <b id="kVal">off</b></span>
 <input type="range" id="topk" min="0" max="28" value="0"></div>
 <div class="ctrl"><span class="lab">Top-p: <b id="pVal">0.90</b></span>
 <input type="range" id="topp" min="10" max="100" value="90"></div>
 <div class="ctrl"><span class="lab">Repetition penalty: <b id="rVal">1.0</b></span>
 <input type="range" id="rep" min="100" max="200" value="100"></div>
 </div>
 <div class="presets" id="presets"></div>
 </div>
<div class="panel" id="panelDist">
 <div class="panel-head"><span class="panel-label">Step 1 · the next-token distribution</span>
 <span class="panel-note">after a given character — kept vs cut</span></div>
 <div class="ctx">
 <span style="font-size:12px;color:var(--muted)">context (last char):</span>
 <div class="seg" id="ctxSeg"></div>
 </div>
 <svg id="distPlot" viewBox="0 0 900 260"></svg>
 <div class="statline">
 <div class="stat"><span class="v" id="stCand" style="color:var(--keep)">—</span><span class="l">candidate tokens</span></div>
 <div class="stat"><span class="v" id="stCover">—</span><span class="l">prob. covered</span></div>
 <div class="stat"><span class="v" id="stTop">—</span><span class="l">top token prob.</span></div>
 </div>
 <div class="legend">
 <span><i style="background:var(--keep)"></i>kept (can be sampled)</span>
 <span><i style="background:var(--cut)"></i>cut by top-k / top-p</span>
 </div>
 </div>
<div class="panel" id="panelGen">
 <div class="panel-head"><span class="panel-label">Step 2 · generate</span>
 <span class="panel-note">these settings, rolled out</span></div>
 <div class="ctrl-row" style="margin-bottom:10px">
 <button class="btn primary" id="genBtn">↻ Generate</button>
 <label style="display:inline-flex;align-items:center;gap:8px;font-size:13px;color:var(--muted);cursor:pointer">
 <input type="checkbox" id="greedy"> greedy (argmax, no sampling)</label>
 </div>
 <div class="gen" id="genBox">…</div>
 <div class="statline">
 <div class="stat"><span class="v" id="stDiv" style="color:var(--accent-2)">—</span><span class="l">diversity (unique 4-grams)</span></div>
 <div class="stat"><span class="v" id="stLoop">—</span><span class="l">longest repeat</span></div>
 </div>
 <p class="hint">Greedy or very low temperature collapses diversity and the longest-repeat
 shoots up — the model loops. A repetition penalty or a little temperature pulls it back.</p>
 </div>
<footer>
 A real character-level model trained live in your browser. Temperature, top-k, top-p, and
 the repetition penalty are the exact decoding operations production systems run — applied
 here to a small model that thinks in letters.
 </footer>
</div>
</section>
<script>
"use strict";
/* ── tabs / toc / try-it ── */
function switchTab(name){document.querySelectorAll(".page-tab").forEach(b=>{const on=b.dataset.tab===name;b.classList.toggle("active",on);b.setAttribute("aria-selected",on);});document.querySelectorAll(".tab-panel").forEach(p=>p.classList.toggle("active",p.id===name+"-tab"));}
document.querySelectorAll(".page-tab").forEach(b=>b.addEventListener("click",()=>switchTab(b.dataset.tab)));
document.querySelectorAll(".guide-toc a").forEach(a=>a.addEventListener("click",e=>{e.preventDefault();document.querySelector(a.getAttribute("href"))?.scrollIntoView({behavior:"smooth",block:"start"});}));
document.querySelectorAll(".try-it[data-action]").forEach(b=>b.addEventListener("click",()=>{const a=b.dataset.action;switchTab("playground");window.scrollTo({top:0,behavior:"smooth"});const t=(a==="rep")?"panelGen":(a==="open"?null:"panelDist");if(t)setTimeout(()=>document.getElementById(t)?.scrollIntoView({behavior:"smooth",block:"start"}),350);}));
/* ════════════════ ENGINE — real tiny char model (reused from Ch.7) ════════════════ */
function mulberry32(a){return function(){a|=0;a=a+0x6D2B79F5|0;let t=Math.imul(a^a>>>15,1|a);t=t+Math.imul(t^t>>>7,61|t)^t;return((t^t>>>14)>>>0)/4294967296}}
const CORPUS=("the cat sat on the mat. the dog ran in the park. a small bird sang a song. she read the book by the window. the sun set over the calm sea. we walked along the road. the old man told a long story. the children played in the green field. rain fell on the roof. the river ran fast and clear. he wrote a note and left it on the table. the moon rose late and the stars came out over the hills.").toLowerCase();
const CHARS=[...new Set(CORPUS)].sort(), V=CHARS.length, D=8, H=24;
const c2i=new Map(CHARS.map((c,i)=>[c,i])), i2c=CHARS, SEQ=[...CORPUS].map(c=>c2i.get(c));
function gaussArr(seed,n){const r=mulberry32(seed),o=new Array(n);for(let i=0;i<n;i++){const u=r()||1e-9,v=r();o[i]=Math.sqrt(-2*Math.log(u))*Math.cos(2*Math.PI*v);}return o;}
let M;
function makeModel(seed){const r=mulberry32(seed),g=()=>{const u=r()||1e-9,v=r();return Math.sqrt(-2*Math.log(u))*Math.cos(2*Math.PI*v);};
 return {E:Array.from({length:V},()=>Array.from({length:D},()=>g()*0.4)),W1:Array.from({length:D},()=>Array.from({length:H},()=>g()*0.4)),b1:new Array(H).fill(0),W2:Array.from({length:H},()=>Array.from({length:V},()=>g()*0.4)),b2:new Array(V).fill(0)};}
function logitsFor(m,prev){const e=m.E[prev],z1=new Array(H);for(let h=0;h<H;h++){let s=m.b1[h];for(let d=0;d<D;d++)s+=e[d]*m.W1[d][h];z1[h]=Math.tanh(s);}const lo=new Array(V);for(let k=0;k<V;k++){let s=m.b2[k];for(let h=0;h<H;h++)s+=z1[h]*m.W2[h][k];lo[k]=s;}return {hid:z1,logits:lo};}
function trainStep(m,batch,lr){const gE=Array.from({length:V},()=>new Array(D).fill(0)),gW1=Array.from({length:D},()=>new Array(H).fill(0)),gb1=new Array(H).fill(0),gW2=Array.from({length:H},()=>new Array(V).fill(0)),gb2=new Array(V).fill(0);
 for(const i of batch){const prev=SEQ[i],tgt=SEQ[i+1],{hid,logits}=logitsFor(m,prev);let mx=-Infinity;for(const x of logits)if(x>mx)mx=x;let Z=0;const p=logits.map(x=>{const e=Math.exp(x-mx);Z+=e;return e;});for(let k=0;k<V;k++)p[k]/=Z;
 const dlog=p.slice();dlog[tgt]-=1;for(let k=0;k<V;k++){gb2[k]+=dlog[k];for(let h=0;h<H;h++)gW2[h][k]+=hid[h]*dlog[k];}
 const dh=new Array(H).fill(0);for(let h=0;h<H;h++){let s=0;for(let k=0;k<V;k++)s+=m.W2[h][k]*dlog[k];dh[h]=s*(1-hid[h]*hid[h]);}
 for(let h=0;h<H;h++){gb1[h]+=dh[h];for(let d=0;d<D;d++)gW1[d][h]+=m.E[prev][d]*dh[h];}
 const e=m.E[prev];for(let d=0;d<D;d++){let s=0;for(let h=0;h<H;h++)s+=m.W1[d][h]*dh[h];gE[prev][d]+=s;}}
 const B=batch.length,scl=1/B;const upd=(P,Gv)=>{for(let a=0;a<P.length;a++)for(let b=0;b<P[a].length;b++)P[a][b]-=lr*Gv[a][b]*scl;};const updV=(P,Gv)=>{for(let a=0;a<P.length;a++)P[a]-=lr*Gv[a]*scl;};
 upd(m.E,gE);upd(m.W1,gW1);updV(m.b1,gb1);upd(m.W2,gW2);updV(m.b2,gb2);}
function pretrain(steps){const r=mulberry32(99);for(let s=0;s<steps;s++){const batch=[];for(let b=0;b<16;b++)batch.push(Math.floor(r()*(SEQ.length-1)));trainStep(M,batch,0.3);}}
/* decoding ops */
function softmax(l){let m=-Infinity;for(const x of l)if(x>m)m=x;let Z=0;const e=l.map(x=>{const v=Math.exp(x-m);Z+=v;return v;});return e.map(x=>x/Z);}
function withTemp(l,T){const t=Math.max(T,1e-3);return l.map(x=>x/t);}
function keptSet(probs,k,p){
 const idx=[...probs.keys()].sort((a,b)=>probs[b]-probs[a]);
 let kept=idx;if(k>0)kept=kept.slice(0,k);
 if(p<1){const out=[];let c=0;for(const i of kept){out.push(i);c+=probs[i];if(c>=p)break;}kept=out;}
 return kept;
}
function renorm(probs,kept){const set=new Set(kept),s=kept.reduce((a,i)=>a+probs[i],0)||1,out=new Array(probs.length).fill(0);for(const i of kept)out[i]=probs[i]/s;return out;}
function sampleFrom(probs,rng){let x=rng(),c=0;for(let i=0;i<probs.length;i++){c+=probs[i];if(x<=c)return i;}for(let i=probs.length-1;i>=0;i--)if(probs[i]>0)return i;return 0;}
/* ════════════════ UI ════════════════ */
const $=id=>document.getElementById(id);
const css=v=>getComputedStyle(document.documentElement).getPropertyValue(v).trim();
const S={T:0.8,k:0,p:0.9,rep:1.0,ctx:c2i.get('t')??0,greedy:false};
function dispChar(c){return c===' '?'␣':c;}
function distFor(prev){
 let logits=logitsFor(M,prev).logits.slice();
 logits=withTemp(logits,S.T);
 const probs=softmax(logits); // distribution after temperature
 const kept=keptSet(probs,S.k,S.p);
 return {probs,kept};
}
function drawDist(){
 const {probs,kept}=distFor(S.ctx),keptS=new Set(kept);
 const idx=[...probs.keys()].sort((a,b)=>probs[b]-probs[a]).slice(0,18);
 const W=900,H2=260,padL=8,padR=8,padB=26,padT=10,bw=(W-padL-padR)/idx.length,mx=Math.max(...idx.map(i=>probs[i]))||1;
 let s="";
 idx.forEach((i,j)=>{const h=(probs[i]/mx)*(H2-padT-padB),x=padL+j*bw,on=keptS.has(i);
 s+='<rect x="'+(x+3)+'" y="'+(H2-padB-h)+'" width="'+(bw-6)+'" height="'+h.toFixed(1)+'" rx="3" fill="'+(on?css('--keep'):css('--cut'))+'"/>';
 s+='<text x="'+(x+bw/2)+'" y="'+(H2-padB+14)+'" font-size="11" text-anchor="middle" font-family="'+css('--mono')+'" fill="'+(on?css('--text'):css('--muted'))+'">'+dispChar(i2c[i])+'</text>';
 if(probs[i]/mx>0.08)s+='<text x="'+(x+bw/2)+'" y="'+(H2-padB-h-4)+'" font-size="8.5" text-anchor="middle" fill="'+css('--muted')+'">'+(probs[i]*100).toFixed(0)+'</text>';
 });
 $('distPlot').innerHTML=s;
 const cover=kept.reduce((a,i)=>a+probs[i],0);
 $('stCand').textContent=kept.length;
 $('stCover').textContent=Math.round(cover*100)+'%';
 const top=Math.max(...probs);$('stTop').textContent=Math.round(top*100)+'%';
}
function generate(){
 const rng=mulberry32((Date.now()&0xffff)^(S.greedy?1:7)^Math.floor(S.T*1000));
 let cur=c2i.get('t')??0,out="t",recent=[];
 for(let n=0;n<260;n++){
 let logits=logitsFor(M,cur).logits.slice();
 if(S.rep>1)recent.forEach(t=>{logits[t]=logits[t]>0?logits[t]/S.rep:logits[t]*S.rep;});
 logits=withTemp(logits,S.greedy?0.02:S.T);
 const probs=softmax(logits),kept=keptSet(probs,S.k,S.p),rp=renorm(probs,kept);
 let nx;
 if(S.greedy){nx=kept[0];let best=-1;for(const i of kept)if(rp[i]>best){best=rp[i];nx=i;}}
 else nx=sampleFrom(rp,rng);
 out+=i2c[nx];recent.push(nx);if(recent.length>16)recent.shift();cur=nx;
 }
 $('genBox').textContent=out;
 // diversity: unique 4-grams / total
 const g=new Set();let tot=0;for(let i=0;i+4<=out.length;i++){g.add(out.slice(i,i+4));tot++;}
 $('stDiv').textContent=tot?Math.round(100*g.size/tot)+'%':'—';
 // longest immediate repeated substring run (period 1..6)
 let longest=1;for(let p=1;p<=6;p++){let run=0;for(let i=p;i<out.length;i++){if(out[i]===out[i-p]){run++;longest=Math.max(longest,run/p+1);}else run=0;}}
 $('stLoop').textContent=Math.round(longest)+'×';
 $('stLoop').style.color=longest>6?css('--bad'):longest>3?css('--warn'):css('--text');
}
const PRESETS=[
 ['Greedy',{T:0.02,k:1,p:1.0,rep:1.0,greedy:true}],
 ['Balanced',{T:0.8,k:0,p:0.9,rep:1.1,greedy:false}],
 ['Creative',{T:1.3,k:0,p:0.98,rep:1.05,greedy:false}],
];
function applyPreset(p){
 S.T=p.T;S.k=p.k;S.p=p.p;S.rep=p.rep;S.greedy=p.greedy;
 $('temp').value=Math.round(p.T*100);$('tVal').textContent=p.T.toFixed(2);
 $('topk').value=p.k;$('kVal').textContent=p.k===0?'off':p.k;
 $('topp').value=Math.round(p.p*100);$('pVal').textContent=p.p.toFixed(2);
 $('rep').value=Math.round(p.rep*100);$('rVal').textContent=p.rep.toFixed(2);
 $('greedy').checked=p.greedy;
 drawDist();generate();
}
function renderCtx(){
 const opts=['t','e','a','o','h',' ','s','n'].filter(c=>c2i.has(c));
 $('ctxSeg').innerHTML=opts.map(c=>'<button data-c="'+c+'"'+(c2i.get(c)===S.ctx?' class="on"':'')+'>'+dispChar(c)+'</button>').join('');
 $('ctxSeg').querySelectorAll('button').forEach(b=>b.addEventListener('click',()=>{S.ctx=c2i.get(b.dataset.c);renderCtx();drawDist();}));
}
/* ── events ── */
$('temp').addEventListener('input',e=>{S.T=+e.target.value/100;$('tVal').textContent=S.T.toFixed(2);drawDist();});
$('topk').addEventListener('input',e=>{S.k=+e.target.value;$('kVal').textContent=S.k===0?'off':S.k;drawDist();});
$('topp').addEventListener('input',e=>{S.p=+e.target.value/100;$('pVal').textContent=S.p.toFixed(2);drawDist();});
$('rep').addEventListener('input',e=>{S.rep=+e.target.value/100;$('rVal').textContent=S.rep.toFixed(2);});
$('greedy').addEventListener('change',e=>{S.greedy=e.target.checked;generate();});
$('genBtn').addEventListener('click',generate);
$('presets').innerHTML=PRESETS.map((p,i)=>'<button data-i="'+i+'">'+p[0]+'</button>').join('');
$('presets').addEventListener('click',e=>{const b=e.target.closest('button');if(!b)return;$('presets').querySelectorAll('button').forEach(x=>x.classList.remove('on'));b.classList.add('on');applyPreset(PRESETS[+b.dataset.i][1]);});
/* ── init: train the real model, then render ── */
M=makeModel(7);
pretrain(600);
renderCtx();drawDist();generate();
</script>
</body>
</html>