Hugging Face's logo

Spaces:
aminamou
/
lm-playground-api
Sleeping

App Files Community
lm-playground-api / web /chapters /alignment.html
deploy
Deploy LM Playground API
f0a527a
Raw
History Blame Contribute Delete
29.8 kB
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>25. SFT / DPO / RLHF / GRPO β€” LM Playground</title>
<link rel="stylesheet" href="/platform/platform.css"/>
<script type="module" src="/platform/platform.js"></script>
<style>
:root{
 --bg:#0f1117; --panel:#181b24; --panel-2:#1f2330; --border:#2a2f3d;
 --text:#e6e8ee; --muted:#9aa3b2; --accent:#7c8cff; --accent-2:#5be0c0;
 --good:#5be08a; --warn:#ffc06b; --bad:#ff9090;
 --ref:#9aa3b2; --sft:#ffc06b; --dpo:#7c8cff; --grpo:#5be0c0;
 --mono:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace;
}
*{box-sizing:border-box}
html,body{margin:0;padding:0 20px 80px;background:radial-gradient(1200px 600px at 50% -10%,#1a1f2e 0%,var(--bg) 55%);color:var(--text);font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;line-height:1.5;-webkit-font-smoothing:antialiased}
header{max-width:980px;margin:0 auto;padding:34px 0 0}
.header-inner{display:flex;justify-content:space-between;align-items:flex-start;padding-bottom:12px}
h1{font-size:30px;margin:0 0 6px;letter-spacing:-.5px}
.sub{color:var(--muted);max-width:780px;margin:0;font-size:14.5px;line-height:1.6}
.sub b{color:var(--text);font-weight:600}
.page-tabs{display:flex;border-bottom:1px solid var(--border)}
.page-tab{background:none;border:none;color:var(--muted);padding:11px 22px 10px;font-size:14px;font-weight:600;cursor:pointer;border-bottom:2px solid transparent;margin-bottom:-1px;transition:.12s}
.page-tab:hover{color:var(--text)}.page-tab.active{color:var(--accent);border-bottom-color:var(--accent)}
.tab-panel{display:none}.tab-panel.active{display:block}
.guide{max-width:980px;margin:0 auto;padding:30px 0 80px;display:grid;grid-template-columns:1fr 220px;gap:0 48px}
.guide-toc{grid-column:2;grid-row:1/20;position:sticky;top:64px;align-self:start;background:var(--panel);border:1px solid var(--border);border-radius:12px;padding:16px;display:flex;flex-direction:column;gap:4px}
.toc-label{font-size:10px;text-transform:uppercase;letter-spacing:1px;color:var(--muted);font-weight:700;margin-bottom:6px}
.guide-toc a{color:var(--muted);text-decoration:none;font-size:12.5px;padding:5px 8px;border-radius:6px;transition:.1s}
.guide-toc a:hover{color:var(--text);background:var(--panel-2)}
.chapter{grid-column:1;padding:0 0 48px;border-bottom:1px solid var(--border);margin-bottom:48px}
.chapter:last-of-type{border-bottom:none}
.chapter h2{font-size:22px;margin:0 0 18px;display:flex;align-items:center;gap:12px;letter-spacing:-.3px}
.ch-num{display:inline-flex;align-items:center;justify-content:center;width:32px;height:32px;border-radius:99px;background:var(--accent-2);color:#0b0d14;font-size:14px;font-weight:800;flex-shrink:0}
.chapter h3{font-size:15px;margin:24px 0 8px;color:var(--accent-2);font-weight:700}
.chapter p{margin:0 0 14px;line-height:1.7;color:#ced3de;font-size:15px}
.chapter strong{color:var(--text)}.chapter em{font-style:italic}
code{background:var(--panel-2);border:1px solid var(--border);border-radius:5px;padding:1px 5px;font-family:var(--mono);font-size:12.5px;color:var(--accent-2)}
.callout{border-radius:10px;padding:14px 16px;margin:20px 0;font-size:14px;line-height:1.65;color:#ced3de}
.callout strong{display:block;margin-bottom:5px;font-size:12px;text-transform:uppercase;letter-spacing:.6px}
.callout.insight{background:#7c8cff18;border-left:3px solid var(--accent)}.callout.insight strong{color:var(--accent)}
.callout.warn{background:#ffc06b18;border-left:3px solid var(--warn)}.callout.warn strong{color:var(--warn)}
.try-it{display:inline-block;margin-top:10px;padding:10px 18px;border-radius:9px;background:var(--panel-2);border:1px solid var(--border);color:var(--accent);font-size:13.5px;font-weight:600;cursor:pointer;transition:.14s;font-family:inherit}
.try-it:hover{background:#7c8cff22;border-color:var(--accent)}
.try-it.large{font-size:15px;padding:14px 28px;background:var(--accent-2);color:#0b0d14;border-color:var(--accent-2)}
.try-it.large:hover{filter:brightness(1.08)}
.guide-end{text-align:center;padding:18px 0 0}.guide-end p{color:var(--muted);margin-bottom:14px}
.panel-guide-item{display:flex;gap:12px;align-items:baseline;padding:9px 0;border-bottom:1px dashed var(--border)}
.panel-guide-item:last-child{border-bottom:none}
.pgi-label{font-size:13px;font-weight:800;color:var(--accent-2);min-width:20px;font-family:var(--mono)}
.panel-guide-item p{margin:0;font-size:13.5px;color:#ced3de;line-height:1.55}
.wrap{max-width:980px;margin:20px auto 0}
.panel{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:16px 18px;margin-bottom:16px}
.panel-head{display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;flex-wrap:wrap;gap:8px}
.panel-label{font-size:13px;font-weight:700;color:var(--text)}.panel-note{color:var(--muted);font-size:12px}
.ctrl-row{display:flex;flex-wrap:wrap;gap:16px;align-items:flex-end;margin:4px 0 6px}
.ctrl{display:flex;flex-direction:column;gap:6px}
.ctrl .lab{font-size:11px;text-transform:uppercase;letter-spacing:.7px;color:var(--muted);font-weight:600}
.ctrl .lab b{color:var(--text);font-family:var(--mono)}
input[type=range]{accent-color:var(--accent);width:140px}
.btn{border:1px solid var(--border);background:var(--panel-2);color:var(--text);font-size:13px;font-weight:600;padding:9px 16px;border-radius:9px;cursor:pointer;font-family:inherit}
.btn:hover{border-color:var(--accent)}.btn.primary{background:var(--accent-2);color:#0b0d14;border-color:var(--accent-2)}
svg{display:block;width:100%;height:auto;overflow:visible}
.polrow{display:grid;grid-template-columns:repeat(4,1fr);gap:12px}
.polcard{background:var(--panel-2);border:1px solid var(--border);border-radius:11px;padding:12px}
.polcard h4{margin:0 0 6px;font-size:12.5px;display:flex;align-items:center;gap:6px}
.polcard .dot{width:9px;height:9px;border-radius:50%}
.polcard .note{font-size:10.5px;color:var(--muted);margin-top:6px;line-height:1.4}
.cmptable{width:100%;border-collapse:collapse;font-size:12px}
.cmptable th,.cmptable td{padding:7px 8px;text-align:left;border-bottom:1px solid var(--border);vertical-align:top}
.cmptable th{color:var(--muted);font-weight:600;font-size:11px;text-transform:uppercase;letter-spacing:.4px}
.cmptable .m{font-weight:700}
.hint{color:var(--muted);font-size:12.5px;margin-top:10px;line-height:1.5}
.verdict{font-size:12.5px;margin-top:10px;padding:9px 12px;border-radius:7px;font-weight:600;line-height:1.5;background:var(--panel-2);border:1px solid var(--border);color:#ced3de}
.fig{margin:22px 0;background:var(--panel);border:1px solid var(--border);border-radius:12px;padding:18px 16px 12px}
.fig svg{display:block;width:100%;height:auto;overflow:visible}
.fig figcaption{margin-top:10px;font-size:12px;color:var(--muted);text-align:center;line-height:1.55}
footer{max-width:980px;margin:30px auto 0;color:var(--muted);font-size:12px;text-align:center;line-height:1.6}
@media(max-width:760px){.guide{grid-template-columns:1fr}.guide-toc{grid-column:1;grid-row:auto;position:static;display:grid;grid-template-columns:1fr 1fr}.toc-label{grid-column:1/-1}.polrow{grid-template-columns:1fr 1fr}}
</style>
</head>
<body data-chapter="alignment">
<header>
 <div class="header-inner">
 <div>
 <h1>Alignment: SFT, RLHF, DPO &amp; GRPO</h1>
 <p class="sub">
 Pretraining gives a model knowledge and the knack for continuing text. It does not give it
 manners β€” the instinct to follow instructions, be helpful, refuse the harmful, and prefer the
 better answer over the merely plausible one. Post-training installs all of that, and there are
 a few ways to do it: imitate good answers, or learn from which answers people <b>preferred</b>.
 Same model, same preferences, surprisingly different results.
 </p>
 </div>
 </div>
 <nav class="page-tabs" role="tablist">
 <button class="page-tab active" data-tab="guide">β‘  Guide</button>
 <button class="page-tab" data-tab="playground">β‘‘ Playground</button>
 </nav>
</header>
<!-- ═══════════════════════════════════════════════════════════════ GUIDE -->
<section id="guide-tab" class="tab-panel active">
<article class="guide">
<nav class="guide-toc">
 <span class="toc-label">Contents</span>
 <a href="#ch1">1 Β· Knowledge isn't manners</a>
 <a href="#ch2">2 Β· SFT: imitate</a>
 <a href="#ch3">3 Β· RLHF: reward &amp; RL</a>
 <a href="#ch4">4 Β· DPO: skip the middle</a>
 <a href="#ch5">5 Β· GRPO: group-relative</a>
 <a href="#ch6">6 Β· Reading the playground</a>
 </nav>
<!-- 1 -->
 <section class="chapter" id="ch1">
 <h2><span class="ch-num">1</span> Knowledge isn't manners</h2>
 <p>
 A freshly pretrained model is a strange thing to talk to. It knows an enormous amount, but its
 only skill is continuing text, so it'll happily complete your question with more questions,
 ramble, or echo the worst of its training data. It has no notion that it should be helpful, that
 it should follow an instruction, or that one answer might be better than another.
 </p>
 <figure class="fig"><svg viewBox="0 0 700 200" xmlns="http://www.w3.org/2000/svg" font-family="-apple-system,Segoe UI,Roboto,sans-serif"><defs><marker id="aral" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="var(--muted)"/></marker></defs>
<rect x="30" y="74" width="96" height="46" rx="9" fill="var(--panel-2)" stroke="var(--border)"/>
<text x="78" y="94" text-anchor="middle" fill="var(--text)" font-size="11.5" font-weight="700">pretrained</text>
<text x="78" y="109" text-anchor="middle" fill="var(--muted)" font-size="9">knows language</text>
<line x1="126" y1="97" x2="166" y2="97" stroke="var(--border)" marker-end="url(#aral)"/>
<rect x="170" y="74" width="96" height="46" rx="9" fill="var(--panel-2)" stroke="var(--accent-2)"/>
<text x="218" y="94" text-anchor="middle" fill="var(--text)" font-size="11.5" font-weight="700">SFT</text>
<text x="218" y="109" text-anchor="middle" fill="var(--muted)" font-size="9">follows instructions</text>
<line x1="266" y1="97" x2="306" y2="97" stroke="var(--border)" marker-end="url(#aral)"/>
<rect x="310" y="44" width="150" height="34" rx="7" fill="var(--good)" opacity="0.18" stroke="var(--good)"/>
<text x="385" y="65" text-anchor="middle" fill="var(--good)" font-size="10.5" font-weight="700">βœ“ preferred answer</text>
<rect x="310" y="116" width="150" height="34" rx="7" fill="var(--bad)" opacity="0.15" stroke="var(--bad)"/>
<text x="385" y="137" text-anchor="middle" fill="var(--bad)" font-size="10.5" font-weight="700">βœ— rejected answer</text>
<line x1="266" y1="90" x2="306" y2="64" stroke="var(--border)" marker-end="url(#aral)"/>
<line x1="266" y1="104" x2="306" y2="130" stroke="var(--border)" marker-end="url(#aral)"/>
<line x1="460" y1="61" x2="520" y2="90" stroke="var(--good)" marker-end="url(#aral)"/>
<line x1="460" y1="133" x2="520" y2="104" stroke="var(--bad)" stroke-dasharray="3 3" marker-end="url(#aral)"/>
<rect x="524" y="74" width="146" height="46" rx="9" fill="var(--accent)" opacity="0.2" stroke="var(--accent)"/>
<text x="597" y="94" text-anchor="middle" fill="var(--text)" font-size="11.5" font-weight="700">aligned policy</text>
<text x="597" y="109" text-anchor="middle" fill="var(--muted)" font-size="9">DPO Β· RLHF Β· GRPO</text>
</svg><figcaption>Alignment turns a raw next-token predictor into a helpful assistant in stages. Supervised fine-tuning teaches it to follow instructions; preference methods (DPO, RLHF, GRPO) then push it toward answers people prefer and away from ones they reject β€” learning from comparisons, not just single correct labels.</figcaption></figure>
 <p>
 Post-training β€” alignment β€” is the stage that turns that raw model into something you'd want to
 use. The knowledge is already there; what's missing is a sense of what a <em>good response</em>
 looks like. And the cleanest signal for "good" turns out not to be a perfect example, but a
 comparison: this answer is better than that one. Most of modern alignment is about learning from
 those comparisons.
 </p>
 <div class="callout insight">
 <strong>Alignment shapes behaviour, not knowledge.</strong>
 Pretraining decides what the model knows; post-training decides how it acts on it. The same
 weights become a helpful assistant or stay an unruly text-completer depending entirely on this
 stage.
 </div>
 <button class="try-it" data-action="open">β–Ά Align the same policy three ways</button>
 </section>
<!-- 2 -->
 <section class="chapter" id="ch2">
 <h2><span class="ch-num">2</span> SFT: imitate the good answers</h2>
 <p>
 The first and simplest step is <strong>supervised fine-tuning</strong>. You collect a set of
 high-quality <em>(instruction, ideal response)</em> pairs β€” written by humans or distilled from a
 stronger model β€” and train the model to reproduce them, exactly the next-token objective from
 Chapter 8, now on curated data. This alone is transformative: it teaches the model the
 assistant format and basic helpfulness, and turns the text-completer into something that answers.
 </p>
 <p>
 But SFT can only imitate. It learns from <em>positive</em> examples β€” "produce this" β€” and never
 from "this is worse than that." If your demonstration is merely good and a better answer exists,
 SFT has no way to find it; it concentrates the model's probability on the example it was shown,
 whatever its quality. To go further, you need a signal about <em>relative</em> quality.
 </p>
 <div class="callout warn">
 <strong>SFT imitates; it can't compare.</strong>
 Trained only on demonstrations, the model's ceiling is the quality of those demonstrations. It
 will happily learn a mediocre answer if that's what it was shown, with no mechanism to prefer
 something better.
 </div>
 </section>
<!-- 3 -->
 <section class="chapter" id="ch3">
 <h2><span class="ch-num">3</span> RLHF: a reward model, then RL</h2>
 <p>
 <strong>Reinforcement learning from human feedback</strong> brings in the comparison signal. You
 show people two responses and ask which they prefer; from thousands of these pairs you train a
 <em>reward model</em> that scores any response with a number predicting human preference. Then you
 use reinforcement learning β€” classically PPO β€” to update the policy so it produces responses the
 reward model rates highly, with a leash (a KL penalty) keeping it from drifting too far from the
 SFT model.
 </p>
 <p>
 This is powerful: the model can now <em>exceed</em> its demonstrations, discovering high-reward
 responses no human wrote, because it's optimizing a score rather than copying examples. It's also
 a lot of moving parts β€” a separate reward model, an RL loop that's famously finicky to stabilize,
 and the risk of the policy learning to game the reward model rather than genuinely improve.
 </p>
 <div class="callout insight">
 <strong>RLHF optimizes a score, so it can beat the demos.</strong>
 By chasing a reward instead of imitating, the policy explores beyond what it was shown. The cost
 is complexity β€” a reward model plus a temperamental RL loop β€” and the ever-present danger of
 reward hacking.
 </div>
 </section>
<!-- 4 -->
 <section class="chapter" id="ch4">
 <h2><span class="ch-num">4</span> DPO: skip the reward model</h2>
 <p>
 <strong>Direct preference optimization</strong> is the insight that you don't actually need the
 reward model or the RL loop. The math of RLHF can be rearranged so that the optimal policy is
 expressed directly in terms of the preference data β€” which means you can train the policy on the
 preference pairs with a single, stable supervised-style loss. No reward model, no PPO, no
 sampling loop.
 </p>
 <p>
 The DPO loss simply pushes up the probability of each <em>chosen</em> response relative to its
 <em>rejected</em> partner, anchored to the reference model by a strength knob. It's far easier to
 implement and to stabilize than RLHF, which is why it became the default for preference tuning in
 the open-source world. The trade is that it learns only from the comparisons you give it β€” it
 can't explore for new high-reward responses the way an RL loop can.
 </p>
 <div class="callout insight">
 <strong>DPO is RLHF's objective without the machinery.</strong>
 Same goal β€” make preferred answers more likely β€” reached with one stable loss instead of a reward
 model and an RL loop. Simpler and steadier, at the cost of the RL family's ability to explore.
 </div>
 <button class="try-it" data-action="dpo">β–Ά Compare DPO and RLHF on the same preferences</button>
 </section>
<!-- 5 -->
 <section class="chapter" id="ch5">
 <h2><span class="ch-num">5</span> GRPO: group-relative, for reasoning</h2>
 <p>
 <strong>Group relative policy optimization</strong>, popularized by the DeepSeek reasoning models,
 is an RL method tuned for tasks where you can <em>check</em> the answer β€” math, code, anything
 with a verifier. For each prompt it samples a whole <em>group</em> of responses, scores them, and
 computes each one's advantage as its reward minus the group's average. Responses better than their
 peers are reinforced; worse-than-average ones are pushed down.
 </p>
 <p>
 The clever part is that the group's own average serves as the baseline, so GRPO needs no separate
 value model β€” a big simplification over PPO. Paired with a verifier that gives a clean reward
 (the code passes its tests, or it doesn't), it lets a model bootstrap strong reasoning by
 practising against itself, which is much of how the recent reasoning models were trained.
 </p>
 <div class="callout insight">
 <strong>Score a group, reinforce the above-average.</strong>
 Using the group mean as the baseline removes the value model and makes RL cheap and stable. With a
 verifiable reward, GRPO turns "generate and check" into a self-improvement loop β€” the engine of
 modern reasoning training.
 </div>
 <button class="try-it" data-action="grpo">β–Ά Watch GRPO find the best response</button>
 </section>
<!-- 6 -->
 <section class="chapter" id="ch6">
 <h2><span class="ch-num">6</span> Reading the playground</h2>
 <p>
 The same starting policy β€” a distribution over six candidate responses, each with a hidden true
 reward β€” is post-trained three ways with the <em>real</em> update rules: SFT's imitation
 gradient, the DPO preference loss, and GRPO's group-relative policy gradient. Watch where they
 end up.
 </p>
 <div class="panel-guide-item"><span class="pgi-label">β–¦</span>
 <p>Four distributions: the reference policy and the result of SFT, DPO, and GRPO β€” over the same
 six responses, with their true rewards shown beneath.</p></div>
 <div class="panel-guide-item"><span class="pgi-label">β–Ά</span>
 <p>Step through training and watch each method's probability mass move β€” SFT toward its
 demonstration, DPO toward the preferred pairs, GRPO toward the highest reward.</p></div>
 <div class="panel-guide-item"><span class="pgi-label">βš–</span>
 <p>The comparison table β€” what each method needs, and where they diverge.</p></div>
 <div class="guide-end">
 <p>The reading is the setup. The playground is the point.</p>
 <button class="try-it large" data-action="open">β–Ά Open the Playground</button>
 </div>
 </section>
</article>
</section>
<!-- ═══════════════════════════════════════════════════════════════ PLAYGROUND -->
<section id="playground-tab" class="tab-panel">
<div class="wrap">
<div class="panel">
 <div class="panel-head"><span class="panel-label">Post-train the same policy</span>
 <span class="panel-note">6 candidate responses Β· hidden true rewards</span></div>
 <div class="ctrl-row">
 <button class="btn primary" id="stepBtn">β–Ά Train 20 steps</button>
 <button class="btn" id="resetBtn">β†Ί Reset</button>
 <div class="ctrl"><span class="lab">Reference anchor Ξ²: <b id="betaVal">0.3</b></span>
 <input type="range" id="beta" min="0" max="100" value="30"></div>
 <div class="ctrl"><span class="lab">Steps trained: <b id="stepsVal">0</b></span></div>
 </div>
 </div>
<div class="panel" id="panelPolicies">
 <div class="panel-head"><span class="panel-label">Step 1 Β· where each method takes the policy</span>
 <span class="panel-note">probability over the six responses</span></div>
 <div class="polrow">
 <div class="polcard"><h4><span class="dot" style="background:var(--ref)"></span>Reference (pretrained)</h4><svg id="polRef" viewBox="0 0 200 130"></svg><div class="note">where we start β€” a default leaning on the safe response 0</div></div>
 <div class="polcard"><h4><span class="dot" style="background:var(--sft)"></span>SFT</h4><svg id="polSft" viewBox="0 0 200 130"></svg><div class="note" id="noteSft"></div></div>
 <div class="polcard"><h4><span class="dot" style="background:var(--dpo)"></span>DPO</h4><svg id="polDpo" viewBox="0 0 200 130"></svg><div class="note" id="noteDpo"></div></div>
 <div class="polcard"><h4><span class="dot" style="background:var(--grpo)"></span>GRPO</h4><svg id="polGrpo" viewBox="0 0 200 130"></svg><div class="note" id="noteGrpo"></div></div>
 </div>
 <div class="panel-head" style="margin-top:14px;margin-bottom:6px"><span class="panel-label" style="font-size:12px;color:var(--muted)">true reward of each response (hidden from SFT &amp; DPO)</span></div>
 <svg id="rewardBar" viewBox="0 0 900 70"></svg>
 <div class="verdict" id="alignVerdict"></div>
 </div>
<div class="panel" id="panelTable">
 <div class="panel-head"><span class="panel-label">Step 2 Β· what each method needs, and learns</span></div>
 <table class="cmptable">
 <thead><tr><th>method</th><th>signal it uses</th><th>extra machinery</th><th>can exceed demos?</th><th>stability</th></tr></thead>
 <tbody>
 <tr><td class="m" style="color:var(--sft)">SFT</td><td>good examples (positives only)</td><td>none</td><td>no β€” imitates</td><td>very stable</td></tr>
 <tr><td class="m" style="color:var(--accent)">RLHF</td><td>human preference pairs</td><td>reward model + PPO loop</td><td>yes</td><td>finicky</td></tr>
 <tr><td class="m" style="color:var(--dpo)">DPO</td><td>preference pairs</td><td>none (one loss)</td><td>limited to the pairs</td><td>stable</td></tr>
 <tr><td class="m" style="color:var(--grpo)">GRPO</td><td>a verifier / reward, per group</td><td>RL loop, no value model</td><td>yes β€” explores</td><td>stable-ish</td></tr>
 </tbody>
 </table>
 <p class="hint">SFT imitates a demonstration; DPO leans on which answers were preferred; the RL
 family (RLHF, GRPO) optimizes a reward and can discover the best response on its own.</p>
 </div>
<footer>
 A toy over six responses with real update rules β€” SFT's imitation gradient, the DPO preference
 loss (chosen vs rejected, anchored to the reference by Ξ²), and GRPO's group-relative policy
 gradient with a KL leash. Rewards are illustrative; the dynamics are the actual ones.
 </footer>
</div>
</section>
<script>
"use strict";
/* ── tabs / toc / try-it ── */
function switchTab(name){document.querySelectorAll(".page-tab").forEach(b=>{const on=b.dataset.tab===name;b.classList.toggle("active",on);b.setAttribute("aria-selected",on);});document.querySelectorAll(".tab-panel").forEach(p=>p.classList.toggle("active",p.id===name+"-tab"));}
document.querySelectorAll(".page-tab").forEach(b=>b.addEventListener("click",()=>switchTab(b.dataset.tab)));
document.querySelectorAll(".guide-toc a").forEach(a=>a.addEventListener("click",e=>{e.preventDefault();document.querySelector(a.getAttribute("href"))?.scrollIntoView({behavior:"smooth",block:"start"});}));
document.querySelectorAll(".try-it[data-action]").forEach(b=>b.addEventListener("click",()=>{switchTab("playground");window.scrollTo({top:0,behavior:"smooth"});}));
/* ════════════════ ENGINE β€” real policy updates over K responses ════════════════ */
const K=6;
// true rewards (hidden): response 3 is best, 0 is the "safe" default, 1 is the SFT demo
const REWARD=[0.35,0.55,0.70,0.95,0.45,0.25];
const DEMO=1; // SFT demonstration (a decent but not best answer)
const PREFS=[[3,0],[2,4],[1,5],[3,1],[2,0],[3,4]]; // (chosen, rejected) preference pairs
// reference logits: leaning on response 0 (safe default)
const THETA_REF=[1.2,0.3,0.1,0.0,0.2,0.4];
function softmax(th){let m=Math.max(...th);let Z=0;const e=th.map(x=>{const v=Math.exp(x-m);Z+=v;return v;});return e.map(x=>x/Z);}
function sigmoid(x){return 1/(1+Math.exp(-x));}
// SFT: maximize log Ο€(demo) β†’ grad ΞΈ = onehot(demo) - Ο€
function stepSFT(th,lr){const p=softmax(th);return th.map((v,i)=>v+lr*(((i===DEMO)?1:0)-p[i]));}
// DPO: for each pref (c,j), margin = Ξ²((ΞΈc-ΞΈref_c)-(ΞΈj-ΞΈref_j)); push ΞΈc up, ΞΈj down by lrΒ·Ξ²Β·(1-Οƒ(margin))
function stepDPO(th,lr,beta){const nt=th.slice();for(const [c,j] of PREFS){const margin=beta*((th[c]-THETA_REF[c])-(th[j]-THETA_REF[j]));const g=lr*beta*(1-sigmoid(margin));nt[c]+=g;nt[j]-=g;}return nt;}
// GRPO: policy gradient on reward with group-mean baseline, KL leash to reference.
// ΞΈi += lr*( Ο€_i*(r_i - E_Ο€[r]) - Ξ²*Ο€_i*((ΞΈi-ΞΈref_i) - mean) )
function stepGRPO(th,lr,beta){const p=softmax(th);const rbar=p.reduce((s,pi,i)=>s+pi*REWARD[i],0);
 const dev=th.map((v,i)=>v-THETA_REF[i]);const dbar=p.reduce((s,pi,i)=>s+pi*dev[i],0);
 return th.map((v,i)=>v+lr*(p[i]*(REWARD[i]-rbar) - beta*p[i]*(dev[i]-dbar))*6);}
/* ════════════════ UI ════════════════ */
const $=id=>document.getElementById(id);
const css=v=>getComputedStyle(document.documentElement).getPropertyValue(v).trim();
const S={steps:0,beta:0.3,sft:THETA_REF.slice(),dpo:THETA_REF.slice(),grpo:THETA_REF.slice()};
function reset(){S.steps=0;S.sft=THETA_REF.slice();S.dpo=THETA_REF.slice();S.grpo=THETA_REF.slice();render();}
function train(n){for(let i=0;i<n;i++){S.sft=stepSFT(S.sft,0.3);S.dpo=stepDPO(S.dpo,0.25,S.beta>0?S.beta:0.05);S.grpo=stepGRPO(S.grpo,0.3,S.beta);}S.steps+=n;render();}
function drawPol(svgId,th,col){const p=softmax(th),W=200,H=130,padB=20,padT=10,bw=(W-12)/K,mx=Math.max(...p,0.5);let s="";
 for(let i=0;i<K;i++){const h=(p[i]/mx)*(H-padT-padB),x=6+i*bw;s+='<rect x="'+(x+2)+'" y="'+(H-padB-h)+'" width="'+(bw-4)+'" height="'+h.toFixed(1)+'" rx="2" fill="'+col+'"/>';
 s+='<text x="'+(x+bw/2)+'" y="'+(H-padB+13)+'" font-size="8.5" text-anchor="middle" fill="'+css('--muted')+'">'+i+'</text>';
 if(p[i]>0.12)s+='<text x="'+(x+bw/2)+'" y="'+(H-padB-h-3)+'" font-size="8" text-anchor="middle" fill="'+css('--muted')+'">'+Math.round(p[i]*100)+'</text>';}
 $(svgId).innerHTML=s;}
function argmax(a){let bi=0;for(let i=1;i<a.length;i++)if(a[i]>a[bi])bi=i;return bi;}
function render(){
 $('stepsVal').textContent=S.steps;$('betaVal').textContent=S.beta.toFixed(2);
 drawPol('polRef',THETA_REF,css('--ref'));drawPol('polSft',S.sft,css('--sft'));drawPol('polDpo',S.dpo,css('--dpo'));drawPol('polGrpo',S.grpo,css('--grpo'));
 // reward bars
 const W=900,H=70,bw=(W-12)/K,mx=Math.max(...REWARD);let s="";
 for(let i=0;i<K;i++){const h=(REWARD[i]/mx)*40,x=6+i*bw,best=i===argmax(REWARD);
 s+='<rect x="'+(x+bw*0.3)+'" y="'+(50-h)+'" width="'+(bw*0.4)+'" height="'+h.toFixed(1)+'" rx="2" fill="'+(best?css('--good'):css('--muted'))+'"/>';
 s+='<text x="'+(x+bw/2)+'" y="'+62+'" font-size="9" text-anchor="middle" fill="'+(best?css('--good'):css('--muted'))+'">r'+i+'='+REWARD[i].toFixed(2)+(best?' β˜…':'')+'</text>';}
 $('rewardBar').innerHTML=s;
 const ps=softmax(S.sft),pd=softmax(S.dpo),pg=softmax(S.grpo);
 $('noteSft').textContent='mass β†’ response '+argmax(ps)+' (its demonstration), reward '+REWARD[argmax(ps)].toFixed(2);
 $('noteDpo').textContent='mass β†’ response '+argmax(pd)+' (most-preferred), reward '+REWARD[argmax(pd)].toFixed(2);
 $('noteGrpo').textContent='mass β†’ response '+argmax(pg)+' (highest reward), reward '+REWARD[argmax(pg)].toFixed(2);
 const v=$('alignVerdict');
 if(S.steps<10){v.textContent='Train a few steps to see the three methods pull the same policy in different directions.';}
 else v.innerHTML='After '+S.steps+' steps they diverge: <b style="color:var(--sft)">SFT</b> locked onto its demonstration (response '+argmax(ps)+', reward '+REWARD[argmax(ps)].toFixed(2)+'); <b style="color:var(--dpo)">DPO</b> moved to the preferred answer (response '+argmax(pd)+'); <b style="color:var(--grpo)">GRPO</b> optimized the reward and found the best response '+argmax(pg)+' ('+REWARD[argmax(pg)].toFixed(2)+') β€” even though no one demonstrated it.';
}
/* ── events ── */
$('stepBtn').addEventListener('click',()=>train(20));
$('resetBtn').addEventListener('click',reset);
$('beta').addEventListener('input',e=>{S.beta=+e.target.value/100;reset();});
reset();
</script>
</body>
</html>