Spaces:
Sleeping
Sleeping
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"/> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> | |
| <title>21. Diffusion language models — LM Playground</title> | |
| <link rel="stylesheet" href="/platform/platform.css"/> | |
| <script type="module" src="/platform/platform.js"></script> | |
| <style> | |
| :root{ | |
| --bg:#0f1117; --panel:#181b24; --panel-2:#1f2330; --border:#2a2f3d; | |
| --text:#e6e8ee; --muted:#9aa3b2; --accent:#7c8cff; --accent-2:#5be0c0; | |
| --good:#5be08a; --warn:#ffc06b; --bad:#ff9090; | |
| --ar:#7c8cff; --diff:#5be0c0; --mask:#3a3f4d; | |
| --mono:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace; | |
| } | |
| *{box-sizing:border-box} | |
| html,body{margin:0;padding:0 20px 80px;background:radial-gradient(1200px 600px at 50% -10%,#1a1f2e 0%,var(--bg) 55%);color:var(--text);font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;line-height:1.5;-webkit-font-smoothing:antialiased} | |
| header{max-width:980px;margin:0 auto;padding:34px 0 0} | |
| .header-inner{display:flex;justify-content:space-between;align-items:flex-start;padding-bottom:12px} | |
| h1{font-size:30px;margin:0 0 6px;letter-spacing:-.5px} | |
| .sub{color:var(--muted);max-width:780px;margin:0;font-size:14.5px;line-height:1.6} | |
| .sub b{color:var(--text);font-weight:600} | |
| .page-tabs{display:flex;border-bottom:1px solid var(--border)} | |
| .page-tab{background:none;border:none;color:var(--muted);padding:11px 22px 10px;font-size:14px;font-weight:600;cursor:pointer;border-bottom:2px solid transparent;margin-bottom:-1px;transition:.12s} | |
| .page-tab:hover{color:var(--text)}.page-tab.active{color:var(--accent);border-bottom-color:var(--accent)} | |
| .tab-panel{display:none}.tab-panel.active{display:block} | |
| .guide{max-width:980px;margin:0 auto;padding:30px 0 80px;display:grid;grid-template-columns:1fr 220px;gap:0 48px} | |
| .guide-toc{grid-column:2;grid-row:1/20;position:sticky;top:64px;align-self:start;background:var(--panel);border:1px solid var(--border);border-radius:12px;padding:16px;display:flex;flex-direction:column;gap:4px} | |
| .toc-label{font-size:10px;text-transform:uppercase;letter-spacing:1px;color:var(--muted);font-weight:700;margin-bottom:6px} | |
| .guide-toc a{color:var(--muted);text-decoration:none;font-size:12.5px;padding:5px 8px;border-radius:6px;transition:.1s} | |
| .guide-toc a:hover{color:var(--text);background:var(--panel-2)} | |
| .chapter{grid-column:1;padding:0 0 48px;border-bottom:1px solid var(--border);margin-bottom:48px} | |
| .chapter:last-of-type{border-bottom:none} | |
| .chapter h2{font-size:22px;margin:0 0 18px;display:flex;align-items:center;gap:12px;letter-spacing:-.3px} | |
| .ch-num{display:inline-flex;align-items:center;justify-content:center;width:32px;height:32px;border-radius:99px;background:var(--accent-2);color:#0b0d14;font-size:14px;font-weight:800;flex-shrink:0} | |
| .chapter h3{font-size:15px;margin:24px 0 8px;color:var(--accent-2);font-weight:700} | |
| .chapter p{margin:0 0 14px;line-height:1.7;color:#ced3de;font-size:15px} | |
| .chapter strong{color:var(--text)}.chapter em{font-style:italic} | |
| code{background:var(--panel-2);border:1px solid var(--border);border-radius:5px;padding:1px 5px;font-family:var(--mono);font-size:12.5px;color:var(--accent-2)} | |
| .callout{border-radius:10px;padding:14px 16px;margin:20px 0;font-size:14px;line-height:1.65;color:#ced3de} | |
| .callout strong{display:block;margin-bottom:5px;font-size:12px;text-transform:uppercase;letter-spacing:.6px} | |
| .callout.insight{background:#7c8cff18;border-left:3px solid var(--accent)}.callout.insight strong{color:var(--accent)} | |
| .callout.warn{background:#ffc06b18;border-left:3px solid var(--warn)}.callout.warn strong{color:var(--warn)} | |
| .try-it{display:inline-block;margin-top:10px;padding:10px 18px;border-radius:9px;background:var(--panel-2);border:1px solid var(--border);color:var(--accent);font-size:13.5px;font-weight:600;cursor:pointer;transition:.14s;font-family:inherit} | |
| .try-it:hover{background:#7c8cff22;border-color:var(--accent)} | |
| .try-it.large{font-size:15px;padding:14px 28px;background:var(--accent-2);color:#0b0d14;border-color:var(--accent-2)} | |
| .try-it.large:hover{filter:brightness(1.08)} | |
| .guide-end{text-align:center;padding:18px 0 0}.guide-end p{color:var(--muted);margin-bottom:14px} | |
| .panel-guide-item{display:flex;gap:12px;align-items:baseline;padding:9px 0;border-bottom:1px dashed var(--border)} | |
| .panel-guide-item:last-child{border-bottom:none} | |
| .pgi-label{font-size:13px;font-weight:800;color:var(--accent-2);min-width:20px;font-family:var(--mono)} | |
| .panel-guide-item p{margin:0;font-size:13.5px;color:#ced3de;line-height:1.55} | |
| .wrap{max-width:980px;margin:20px auto 0} | |
| .panel{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:16px 18px;margin-bottom:16px} | |
| .panel-head{display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;flex-wrap:wrap;gap:8px} | |
| .panel-label{font-size:13px;font-weight:700;color:var(--text)}.panel-note{color:var(--muted);font-size:12px} | |
| .ctrl-row{display:flex;flex-wrap:wrap;gap:16px;align-items:flex-end;margin:4px 0 6px} | |
| .ctrl{display:flex;flex-direction:column;gap:6px} | |
| .ctrl .lab{font-size:11px;text-transform:uppercase;letter-spacing:.7px;color:var(--muted);font-weight:600} | |
| .ctrl .lab b{color:var(--text);font-family:var(--mono)} | |
| input[type=range]{accent-color:var(--accent);width:140px} | |
| .btn{border:1px solid var(--border);background:var(--panel-2);color:var(--text);font-size:13px;font-weight:600;padding:9px 16px;border-radius:9px;cursor:pointer;font-family:inherit} | |
| .btn:hover{border-color:var(--accent)}.btn.primary{background:var(--accent-2);color:#0b0d14;border-color:var(--accent-2)} | |
| svg{display:block;width:100%;height:auto;overflow:visible} | |
| .gencol{background:var(--panel-2);border:1px solid var(--border);border-radius:11px;padding:14px} | |
| .gencol.ar{border-top:2px solid var(--ar)}.gencol.diff{border-top:2px solid var(--diff)} | |
| .gencol h4{margin:0 0 2px;font-size:13px}.gencol h4.ar{color:var(--ar)}.gencol h4.diff{color:var(--diff)} | |
| .gencol .sub{font-size:11px;color:var(--muted);margin:0 0 10px} | |
| .seq{display:flex;flex-wrap:wrap;gap:5px;min-height:120px;align-content:flex-start} | |
| .tok{font-family:var(--mono);font-size:13px;padding:5px 8px;border-radius:6px;border:1px solid var(--border);background:var(--bg);transition:.18s} | |
| .tok.mask{color:var(--muted);background:var(--mask);border-color:#4a4f5d} | |
| .tok.ar{color:var(--ar);border-color:var(--ar);background:#7c8cff15} | |
| .tok.diff{color:var(--diff);border-color:var(--diff);background:#5be0c015} | |
| .tok.just{background:#5be0c035;transform:scale(1.06)} | |
| .stepline{font-family:var(--mono);font-size:12px;color:var(--muted);margin-top:8px} | |
| .stepline b{color:var(--text)} | |
| .grid2{display:grid;grid-template-columns:1fr 1fr;gap:16px;align-items:start} | |
| .card{background:var(--panel-2);border:1px solid var(--border);border-radius:11px;padding:14px} | |
| .card h4{margin:0 0 4px;font-size:13px}.card .cap{color:var(--muted);font-size:11.5px;margin:0 0 8px;line-height:1.5} | |
| .cmptable{width:100%;border-collapse:collapse;font-size:12.5px} | |
| .cmptable td{padding:7px 8px;border-bottom:1px solid var(--border);vertical-align:top} | |
| .cmptable td:first-child{color:var(--muted);width:34%} | |
| .cmptable .ar{color:var(--ar)}.cmptable .diff{color:var(--diff)} | |
| .hint{color:var(--muted);font-size:12.5px;margin-top:10px;line-height:1.5} | |
| .fig{margin:22px 0;background:var(--panel);border:1px solid var(--border);border-radius:12px;padding:18px 16px 12px} | |
| .fig svg{display:block;width:100%;height:auto;overflow:visible} | |
| .fig figcaption{margin-top:10px;font-size:12px;color:var(--muted);text-align:center;line-height:1.55} | |
| footer{max-width:980px;margin:30px auto 0;color:var(--muted);font-size:12px;text-align:center;line-height:1.6} | |
| @media(max-width:760px){.guide{grid-template-columns:1fr}.guide-toc{grid-column:1;grid-row:auto;position:static;display:grid;grid-template-columns:1fr 1fr}.toc-label{grid-column:1/-1}.grid2{grid-template-columns:1fr}} | |
| </style> | |
| </head> | |
| <body data-chapter="diffusion-lm"> | |
| <header> | |
| <div class="header-inner"> | |
| <div> | |
| <h1>Diffusion language models</h1> | |
| <p class="sub"> | |
| Every model so far writes the same way: one token at a time, strictly left to right. But | |
| that's a choice, not a law. Image generators don't paint pixel by pixel — they start from | |
| noise and refine the <b>whole canvas at once</b>, over a handful of steps. Diffusion | |
| language models ask whether text can be written the same way: all positions in parallel, | |
| sharpened step by step from a blank, fully-masked sequence. | |
| </p> | |
| </div> | |
| </div> | |
| <nav class="page-tabs" role="tablist"> | |
| <button class="page-tab active" data-tab="guide">① Guide</button> | |
| <button class="page-tab" data-tab="playground">② Playground</button> | |
| </nav> | |
| </header> | |
| <!-- ═══════════════════════════════════════════════════════════════ GUIDE --> | |
| <section id="guide-tab" class="tab-panel active"> | |
| <article class="guide"> | |
| <nav class="guide-toc"> | |
| <span class="toc-label">Contents</span> | |
| <a href="#ch1">1 · The left-to-right habit</a> | |
| <a href="#ch2">2 · Noise, for discrete text</a> | |
| <a href="#ch3">3 · Refine in parallel</a> | |
| <a href="#ch4">4 · vs autoregressive</a> | |
| <a href="#ch5">5 · Why AR still rules</a> | |
| <a href="#ch6">6 · Reading the playground</a> | |
| </nav> | |
| <!-- 1 --> | |
| <section class="chapter" id="ch1"> | |
| <h2><span class="ch-num">1</span> The left-to-right habit</h2> | |
| <p> | |
| Autoregression — predict the next token, append, repeat — has been the only game in this | |
| course, and for good reason: it's simple, it trains on an exact likelihood, and it matches | |
| the way text seems to unfold. But it bakes in two constraints. Generation is strictly | |
| sequential, one token per forward pass, so a thousand-token output takes a thousand passes. | |
| And it only ever looks left — a token, once written, can never be revised in light of what | |
| comes after. | |
| </p> | |
| <figure class="fig"><svg viewBox="0 0 700 180" xmlns="http://www.w3.org/2000/svg" font-family="-apple-system,Segoe UI,Roboto,sans-serif"><defs><marker id="ardl" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="var(--muted)"/></marker></defs><rect x="40" y="50" width="108" height="56" rx="8" fill="var(--panel-2)" stroke="var(--border)"/><rect x="46" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="66" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="86" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="106" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="126" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="46" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="66" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="86" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="106" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="126" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="46" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="66" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="86" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="106" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="126" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><text x="94" y="128" text-anchor="middle" fill="var(--muted)" font-size="9.5">all noise</text><line x1="148" y1="78" x2="172" y2="78" stroke="var(--accent)" stroke-width="2" marker-end="url(#ardl)"/><rect x="172" y="50" width="108" height="56" rx="8" fill="var(--panel-2)" stroke="var(--border)"/><rect x="178" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="198" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="218" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="238" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="258" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="178" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="198" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="218" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="238" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="258" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="178" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="198" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="218" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="238" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="258" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><text x="226" y="128" text-anchor="middle" fill="var(--muted)" font-size="9.5"></text><line x1="280" y1="78" x2="304" y2="78" stroke="var(--accent)" stroke-width="2" marker-end="url(#ardl)"/><rect x="304" y="50" width="108" height="56" rx="8" fill="var(--panel-2)" stroke="var(--border)"/><rect x="310" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="330" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="350" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="370" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="390" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="310" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="330" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="350" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="370" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="390" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="310" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="330" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="350" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="370" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="390" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><text x="358" y="128" text-anchor="middle" fill="var(--muted)" font-size="9.5"></text><line x1="412" y1="78" x2="436" y2="78" stroke="var(--accent)" stroke-width="2" marker-end="url(#ardl)"/><rect x="436" y="50" width="108" height="56" rx="8" fill="var(--panel-2)" stroke="var(--border)"/><rect x="442" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="462" y="56" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="482" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="502" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="522" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="442" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="462" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="482" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="502" y="72" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><rect x="522" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="442" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="462" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="482" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="502" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="522" y="88" width="17" height="13" rx="2" fill="var(--accent-2)" opacity="0.85"/><text x="490" y="128" text-anchor="middle" fill="var(--muted)" font-size="9.5"></text><line x1="544" y1="78" x2="568" y2="78" stroke="var(--accent)" stroke-width="2" marker-end="url(#ardl)"/><rect x="568" y="50" width="108" height="56" rx="8" fill="var(--panel-2)" stroke="var(--border)"/><rect x="574" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="594" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="614" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="634" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="654" y="56" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="574" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="594" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="614" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="634" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="654" y="72" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="574" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="594" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="614" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="634" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><rect x="654" y="88" width="17" height="13" rx="2" fill="var(--muted)" opacity="0.3"/><text x="622" y="128" text-anchor="middle" fill="var(--muted)" font-size="9.5">clean text</text> | |
| <text x="350" y="26" text-anchor="middle" fill="var(--accent)" font-size="11.5" font-weight="700">iterative denoising — refine the whole sequence in parallel</text> | |
| </svg><figcaption>Diffusion language models generate differently from left-to-right transformers: they start from pure noise over the whole sequence and denoise it in a handful of parallel steps, sharpening every position at once. It trades the autoregressive token-by-token loop for a small number of full-sequence refinement passes.</figcaption></figure> | |
| <p> | |
| Other domains long ago abandoned this. Diffusion models — the technology behind modern image, | |
| audio, and video generators — don't produce their output in order at all. They start from | |
| pure noise and run a small, fixed number of refinement steps, each one improving the entire | |
| output simultaneously. The question this chapter asks is whether the same recipe works for | |
| language. | |
| </p> | |
| <div class="callout insight"> | |
| <strong>Autoregression is a default, not a requirement.</strong> | |
| Sequential, left-to-right generation is one way to turn a model into text — the one that | |
| won — but diffusion shows there's a fundamentally different shape the same goal can take. | |
| </div> | |
| <button class="try-it" data-action="open">▶ Watch both ways write the same sentence</button> | |
| </section> | |
| <!-- 2 --> | |
| <section class="chapter" id="ch2"> | |
| <h2><span class="ch-num">2</span> What "noise" means for discrete text</h2> | |
| <p> | |
| Image diffusion has it easy: pixels are continuous, so you can add a little Gaussian noise, | |
| then a little more, until the picture dissolves into static — and train a model to reverse | |
| each step. Text is discrete. There's no "slightly noisy" version of the word | |
| <em>cat</em>. So discrete diffusion uses a different corruption: <strong>masking</strong>. | |
| </p> | |
| <p> | |
| The forward process progressively replaces tokens with a special <code>[MASK]</code> symbol — | |
| a few at first, then more, until the whole sequence is masked. The model is trained to run | |
| that backwards: given a partly-masked sequence, predict what the masked tokens should be. | |
| Generation then starts from an all-masked sequence and unmasks its way to text. | |
| </p> | |
| <div class="callout insight"> | |
| <strong>For text, the noise is the mask.</strong> | |
| Corrupting toward a fully-masked sequence and learning to reverse it is the discrete analogue | |
| of dissolving an image into static and denoising it back. You met this exact masking objective | |
| in Chapter 8 — diffusion turns it into a multi-step generator. | |
| </div> | |
| </section> | |
| <!-- 3 --> | |
| <section class="chapter" id="ch3"> | |
| <h2><span class="ch-num">3</span> Refine the whole sequence in parallel</h2> | |
| <p> | |
| Here's the part that makes it interesting. Each denoising step looks at the entire sequence | |
| at once — every position, masked or not — and predicts all the masked tokens together. It | |
| then <em>commits</em> the predictions it's most confident about, leaves the uncertain | |
| positions masked, and moves to the next step. Over a handful of steps, the sequence sharpens | |
| from all-mask to finished text. | |
| </p> | |
| <p> | |
| Two things fall out of this. The number of steps is <strong>fixed and small</strong> — a few | |
| dozen, not one per token — so a long sequence doesn't cost proportionally more steps. And | |
| every step has <strong>bidirectional</strong> context: a token can be filled in based on | |
| words to its right that don't exist yet under autoregression. The model can lay down anchor | |
| words first and fill the gaps around them, in any order it likes. | |
| </p> | |
| <div class="callout insight"> | |
| <strong>Steps decouple from length, and context goes both ways.</strong> | |
| Diffusion trades "one pass per token, left to right" for "a few passes over everything, in | |
| any order." That parallelism and the ability to revise are its whole appeal. | |
| </div> | |
| <button class="try-it" data-action="steps">▶ Change the step count and re-denoise</button> | |
| </section> | |
| <!-- 4 --> | |
| <section class="chapter" id="ch4"> | |
| <h2><span class="ch-num">4</span> Diffusion vs autoregressive</h2> | |
| <p> | |
| Line them up and the trade is clear. Autoregression takes <code>N</code> sequential steps for | |
| <code>N</code> tokens, sees only the left context, can't revise, but trains on an exact | |
| likelihood and is, today, the strongest approach for text. Diffusion takes a fixed | |
| <code>T</code> steps regardless of length, sees the whole sequence, can revise earlier tokens | |
| as later ones firm up — but each of those <code>T</code> steps is a <em>full</em> forward pass | |
| over the entire sequence, and its training objective is a looser bound, not the exact | |
| likelihood. | |
| </p> | |
| <p> | |
| So "fewer steps" doesn't automatically mean "faster" — a diffusion step costs more than an | |
| autoregressive one, and you need enough steps for quality. The honest framing is that they | |
| occupy different points on the speed/quality/flexibility surface, and which wins depends on | |
| the sequence length, the hardware, and how much revision the task rewards. | |
| </p> | |
| <button class="try-it" data-action="cost">▶ Compare the step counts vs length</button> | |
| </section> | |
| <!-- 5 --> | |
| <section class="chapter" id="ch5"> | |
| <h2><span class="ch-num">5</span> Why autoregression still rules text</h2> | |
| <p> | |
| Despite the appeal, the frontier of language modelling is still overwhelmingly autoregressive, | |
| and there are real reasons. Left-to-right factorization matches the causal grain of language | |
| and gives an exact, easy-to-optimize training loss; the KV cache makes AR generation cheap per | |
| step; and decades of tooling and scaling know-how are built around it. Diffusion's looser | |
| objective and full-sequence steps have, so far, left it a step behind on raw quality. | |
| </p> | |
| <p> | |
| But it's an active, fast-moving frontier — recent diffusion and masked-generation language | |
| models have closed much of the gap, and the parallelism is genuinely attractive for long | |
| outputs and controllable, infilling-style generation. Diffusion already owns image, audio, and | |
| video; whether it takes a real share of text is one of the open questions of the field. | |
| </p> | |
| <div class="callout warn"> | |
| <strong>Don't mistake "less common" for "settled."</strong> | |
| Autoregression leads text today on the strength of likelihood, caching, and momentum — not | |
| because diffusion can't work. This is one of the places the architecture story is still being | |
| written. | |
| </div> | |
| </section> | |
| <!-- 6 --> | |
| <section class="chapter" id="ch6"> | |
| <h2><span class="ch-num">6</span> Reading the playground</h2> | |
| <p> | |
| The two columns illustrate the generation <em>processes</em> on the same target sentence: | |
| autoregressive fills left to right, one token per step; diffusion starts fully masked and | |
| unmasks confident tokens across the whole sequence over a few steps. It shows the mechanism | |
| and the step counts, not a trained model's samples. | |
| </p> | |
| <div class="panel-guide-item"><span class="pgi-label">▶</span> | |
| <p>Step or play both generators side by side: autoregressive marches left to right; diffusion | |
| lays down anchor words anywhere and fills the gaps.</p></div> | |
| <div class="panel-guide-item"><span class="pgi-label">◷</span> | |
| <p>Set the number of diffusion steps and re-run — fewer steps, coarser passes; more steps, a | |
| gentler reveal.</p></div> | |
| <div class="panel-guide-item"><span class="pgi-label">∿</span> | |
| <p>The steps-versus-length comparison — autoregressive grows with the sequence, diffusion stays | |
| flat — and the full trade-off table.</p></div> | |
| <div class="guide-end"> | |
| <p>The reading is the setup. The playground is the point.</p> | |
| <button class="try-it large" data-action="open">▶ Open the Playground</button> | |
| </div> | |
| </section> | |
| </article> | |
| </section> | |
| <!-- ═══════════════════════════════════════════════════════════════ PLAYGROUND --> | |
| <section id="playground-tab" class="tab-panel"> | |
| <div class="wrap"> | |
| <div class="panel"> | |
| <div class="panel-head"><span class="panel-label">Two ways to write a sentence</span> | |
| <span class="panel-note">same target, two generation processes</span></div> | |
| <div class="ctrl-row"> | |
| <button class="btn primary" id="stepBtn">▶ Step</button> | |
| <button class="btn" id="playBtn">▶▶ Play</button> | |
| <button class="btn" id="resetBtn">↺ Reset</button> | |
| <div class="ctrl"><span class="lab">Diffusion steps: <b id="tVal">6</b></span> | |
| <input type="range" id="tSteps" min="2" max="12" value="6"></div> | |
| </div> | |
| <div class="grid2"> | |
| <div class="gencol ar"> | |
| <h4 class="ar">Autoregressive</h4> | |
| <p class="sub">one token per step, strictly left → right</p> | |
| <div class="seq" id="arSeq"></div> | |
| <div class="stepline" id="arLine"></div> | |
| </div> | |
| <div class="gencol diff"> | |
| <h4 class="diff">Diffusion</h4> | |
| <p class="sub">all positions at once, masked → unmasked over a few steps</p> | |
| <div class="seq" id="diffSeq"></div> | |
| <div class="stepline" id="diffLine"></div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="panel" id="panelCost"> | |
| <div class="panel-head"><span class="panel-label">Steps, length & the trade-off</span> | |
| <span class="panel-note">generation steps vs sequence length</span></div> | |
| <div class="grid2"> | |
| <div class="card"> | |
| <h4>Generation steps vs length</h4> | |
| <p class="cap">Autoregressive needs one step per token (linear). Diffusion uses a fixed step | |
| budget regardless of length (flat) — though each step is a heavier pass.</p> | |
| <svg id="costPlot" viewBox="0 0 420 210"></svg> | |
| </div> | |
| <div class="card"> | |
| <h4>Side by side</h4> | |
| <table class="cmptable"> | |
| <tr><td>order</td><td class="ar">left → right</td><td class="diff">any order</td></tr> | |
| <tr><td>steps</td><td class="ar">N (one per token)</td><td class="diff">fixed T</td></tr> | |
| <tr><td>context</td><td class="ar">left only</td><td class="diff">bidirectional</td></tr> | |
| <tr><td>revise?</td><td class="ar">never</td><td class="diff">yes, until committed</td></tr> | |
| <tr><td>per-step cost</td><td class="ar">cheap (KV cache)</td><td class="diff">full pass</td></tr> | |
| <tr><td>training loss</td><td class="ar">exact likelihood</td><td class="diff">looser bound</td></tr> | |
| <tr><td>text quality today</td><td class="ar">leads</td><td class="diff">catching up</td></tr> | |
| </table> | |
| </div> | |
| </div> | |
| </div> | |
| <footer> | |
| An illustration of the two generation processes on a fixed target sentence — autoregressive | |
| left-to-right vs diffusion's parallel masked refinement. It shows the mechanism and step counts, | |
| not samples from a trained model. | |
| </footer> | |
| </div> | |
| </section> | |
| <script> | |
| ; | |
| /* ── tabs / toc / try-it ── */ | |
| function switchTab(name){document.querySelectorAll(".page-tab").forEach(b=>{const on=b.dataset.tab===name;b.classList.toggle("active",on);b.setAttribute("aria-selected",on);});document.querySelectorAll(".tab-panel").forEach(p=>p.classList.toggle("active",p.id===name+"-tab"));} | |
| document.querySelectorAll(".page-tab").forEach(b=>b.addEventListener("click",()=>switchTab(b.dataset.tab))); | |
| document.querySelectorAll(".guide-toc a").forEach(a=>a.addEventListener("click",e=>{e.preventDefault();document.querySelector(a.getAttribute("href"))?.scrollIntoView({behavior:"smooth",block:"start"});})); | |
| document.querySelectorAll(".try-it[data-action]").forEach(b=>b.addEventListener("click",()=>{const a=b.dataset.action;switchTab("playground");window.scrollTo({top:0,behavior:"smooth"});const t=(a==="cost")?"panelCost":(a==="open"?null:null);if(t)setTimeout(()=>document.getElementById(t)?.scrollIntoView({behavior:"smooth",block:"start"}),350);})); | |
| /* ════════════════ ENGINE ════════════════ */ | |
| const TARGET="the quiet river flowed past the old stone bridge at dawn".split(" "); | |
| const N=TARGET.length; | |
| const STOP=new Set("the a an of to in on at by is was and or for".split(" ")); | |
| // diffusion unmask order: most "confident" first — short/common words as anchors, then the rest. | |
| function confidence(tok){return (STOP.has(tok)?2:0) + 1/tok.length;} | |
| const ORDER=[...TARGET.keys()].sort((a,b)=>confidence(TARGET[b])-confidence(TARGET[a])); | |
| // cumulative tokens revealed by diffusion step s (cosine-ish schedule) | |
| function revealedBy(s,T){if(s<=0)return 0;if(s>=T)return N;return Math.round(N*(1-Math.cos(Math.PI*s/T))/2);} | |
| /* ════════════════ UI ════════════════ */ | |
| const $=id=>document.getElementById(id); | |
| const css=v=>getComputedStyle(document.documentElement).getPropertyValue(v).trim(); | |
| const S={T:6,arStep:0,diffStep:0,playing:false,raf:0,last:0,justAR:-1,justDiff:[]}; | |
| function reset(){S.arStep=0;S.diffStep=0;S.justAR=-1;S.justDiff=[];stop();render();} | |
| function done(){return S.arStep>=N&&S.diffStep>=S.T;} | |
| function step(){ | |
| if(S.arStep<N){S.justAR=S.arStep;S.arStep++;}else S.justAR=-1; | |
| if(S.diffStep<S.T){const before=revealedBy(S.diffStep,S.T);S.diffStep++;const after=revealedBy(S.diffStep,S.T); | |
| S.justDiff=ORDER.slice(before,after);}else S.justDiff=[]; | |
| render(); | |
| } | |
| function render(){ | |
| // AR: positions 0..arStep-1 revealed, rest masked | |
| let ar="";for(let i=0;i<N;i++){const shown=i<S.arStep;const cls="tok "+(shown?(i===S.justAR?"ar just":"ar"):"mask"); | |
| ar+='<span class="'+cls+'">'+(shown?TARGET[i]:'▒')+'</span>';} | |
| $('arSeq').innerHTML=ar; | |
| $('arLine').innerHTML='step <b>'+S.arStep+' / '+N+'</b> · '+(S.arStep>=N?'done':'next: position '+S.arStep+' (left→right)'); | |
| // diffusion: ORDER[0..revealed-1] revealed | |
| const rev=revealedBy(S.diffStep,S.T),revealedSet=new Set(ORDER.slice(0,rev)),justSet=new Set(S.justDiff); | |
| let df="";for(let i=0;i<N;i++){const shown=revealedSet.has(i);const cls="tok "+(shown?(justSet.has(i)?"diff just":"diff"):"mask"); | |
| df+='<span class="'+cls+'">'+(shown?TARGET[i]:'[mask]')+'</span>';} | |
| $('diffSeq').innerHTML=df; | |
| $('diffLine').innerHTML='step <b>'+S.diffStep+' / '+S.T+'</b> · '+rev+' / '+N+' tokens committed'+(S.diffStep>=S.T?' · done':''); | |
| drawCost(); | |
| } | |
| function drawCost(){ | |
| const W=420,H=210,padL=40,padR=14,padT=14,padB=30,maxN=64; | |
| const X=n=>padL+(n/maxN)*(W-padL-padR),Y=v=>H-padB-(v/maxN)*(H-padT-padB); | |
| let s=""; | |
| s+='<line x1="'+padL+'" y1="'+(H-padB)+'" x2="'+(W-padR)+'" y2="'+(H-padB)+'" stroke="'+css('--border')+'"/>'; | |
| // AR: steps = n | |
| s+='<line x1="'+X(0)+'" y1="'+Y(0)+'" x2="'+X(maxN)+'" y2="'+Y(maxN)+'" stroke="'+css('--ar')+'" stroke-width="2"/>'; | |
| // diffusion: flat at T | |
| s+='<line x1="'+X(0)+'" y1="'+Y(S.T)+'" x2="'+X(maxN)+'" y2="'+Y(S.T)+'" stroke="'+css('--diff')+'" stroke-width="2"/>'; | |
| s+='<text x="'+(W-padR)+'" y="'+(Y(maxN)+12)+'" font-size="9" text-anchor="end" fill="'+css('--ar')+'">autoregressive · N steps</text>'; | |
| s+='<text x="'+(W-padR)+'" y="'+(Y(S.T)-4)+'" font-size="9" text-anchor="end" fill="'+css('--diff')+'">diffusion · '+S.T+' steps (fixed)</text>'; | |
| // current N marker | |
| s+='<line x1="'+X(N)+'" y1="'+padT+'" x2="'+X(N)+'" y2="'+(H-padB)+'" stroke="'+css('--muted')+'" stroke-dasharray="3 3" opacity="0.4"/>'; | |
| s+='<text x="'+X(N)+'" y="'+(H-padB+13)+'" font-size="8.5" text-anchor="middle" fill="'+css('--muted')+'">N='+N+'</text>'; | |
| [0,16,32,48,64].forEach(t=>{if(Math.abs(t-N)>3)s+='<text x="'+X(t)+'" y="'+(H-padB+13)+'" font-size="8.5" text-anchor="middle" fill="'+css('--muted')+'">'+t+'</text>';}); | |
| s+='<text x="'+((padL+W-padR)/2)+'" y="'+(H-2)+'" font-size="9" text-anchor="middle" fill="'+css('--muted')+'">sequence length →</text>'; | |
| s+='<text x="8" y="'+(padT+30)+'" font-size="8.5" fill="'+css('--muted')+'" transform="rotate(-90 12 '+(padT+40)+')">generation steps</text>'; | |
| $('costPlot').innerHTML=s; | |
| } | |
| function stop(){S.playing=false;$('playBtn').textContent="▶▶ Play";cancelAnimationFrame(S.raf);} | |
| function play(){if(S.playing){stop();return;}if(done())reset();S.playing=true;$('playBtn').textContent="⏸ Pause";S.last=0; | |
| const loop=(ts)=>{if(!S.playing)return;if(ts-S.last>520){S.last=ts;step();}if(!done())S.raf=requestAnimationFrame(loop);else stop();}; | |
| S.raf=requestAnimationFrame(loop);} | |
| /* ── events ── */ | |
| $('stepBtn').addEventListener('click',()=>{stop();if(done())reset();step();}); | |
| $('playBtn').addEventListener('click',play); | |
| $('resetBtn').addEventListener('click',reset); | |
| $('tSteps').addEventListener('input',e=>{S.T=+e.target.value;$('tVal').textContent=S.T;reset();}); | |
| reset(); | |
| </script> | |
| </body> | |
| </html> | |