Spaces:
Paused
Paused
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>Nano-GLM — a from-scratch language model</title> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link href="https://fonts.googleapis.com/css2?family=Newsreader:ital,opsz,wght@0,6..72,400;0,6..72,500;0,6..72,600;1,6..72,400;1,6..72,500&family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet"> | |
| <style> | |
| :root{ | |
| --paper:#FAFAF7; | |
| --card:#FFFFFF; | |
| --ink:#17181B; | |
| --ink-soft:#5B5F66; | |
| --ink-faint:#9497A0; | |
| --line:#E4E1DA; | |
| --line-strong:#C9C6BE; | |
| --accent:#3F6355; | |
| --accent-ink:#2C4A3F; | |
| --accent-soft:#E9EFEB; | |
| } | |
| *{ box-sizing:border-box; } | |
| html{ scroll-behavior:smooth; } | |
| body{ | |
| margin:0; background:var(--paper); color:var(--ink); | |
| font-family:'Inter',sans-serif; font-size:16px; line-height:1.6; | |
| -webkit-font-smoothing:antialiased; | |
| } | |
| .serif{ font-family:'Newsreader',serif; } | |
| .mono{ font-family:'JetBrains Mono',monospace; } | |
| a{ color:inherit; } | |
| ::selection{ background:var(--accent-soft); color:var(--accent-ink); } | |
| /* ---------- shell ---------- */ | |
| .wrap{ max-width:720px; margin:0 auto; padding:0 24px; } | |
| .wrap-wide{ max-width:800px; margin:0 auto; padding:0 24px; } | |
| section{ padding:76px 0; } | |
| .hairline{ border:none; border-top:1px solid var(--line); margin:0; } | |
| /* ---------- nav ---------- */ | |
| nav{ | |
| display:flex; justify-content:space-between; align-items:center; | |
| padding:22px 24px; max-width:800px; margin:0 auto; | |
| } | |
| .nav-mark{ font-family:'Newsreader',serif; font-weight:600; font-size:19px; letter-spacing:-0.01em; } | |
| .nav-link{ | |
| font-size:13.5px; color:var(--ink-soft); text-decoration:none; | |
| border-bottom:1px solid transparent; padding-bottom:2px; transition:border-color .15s ease, color .15s ease; | |
| } | |
| .nav-link:hover{ color:var(--ink); border-color:var(--ink); } | |
| /* ---------- hero ---------- */ | |
| .hero{ padding:56px 0 0; text-align:left; } | |
| .eyebrow{ | |
| font-family:'JetBrains Mono',monospace; font-size:12px; letter-spacing:.06em; | |
| text-transform:uppercase; color:var(--accent-ink); margin-bottom:18px; | |
| display:flex; align-items:center; gap:8px; | |
| } | |
| .eyebrow .dot{ width:6px; height:6px; border-radius:50%; background:var(--accent); display:inline-block; } | |
| h1.title{ | |
| font-size:clamp(2.6rem, 6vw, 4rem); font-weight:500; letter-spacing:-0.02em; | |
| line-height:1.04; margin:0 0 20px; | |
| } | |
| .lede{ | |
| font-size:19px; color:var(--ink-soft); max-width:560px; margin:0 0 40px; line-height:1.55; | |
| } | |
| /* signature: architecture specimen strip */ | |
| .specimen{ | |
| font-family:'JetBrains Mono',monospace; font-size:12.5px; color:var(--ink-soft); | |
| border-top:1px solid var(--line); border-bottom:1px solid var(--line); | |
| padding:16px 0; margin-bottom:56px; overflow-x:auto; | |
| display:flex; align-items:center; gap:0; white-space:nowrap; | |
| } | |
| .specimen .node{ color:var(--ink); } | |
| .specimen .arrow{ color:var(--line-strong); margin:0 10px; } | |
| .specimen .tag{ | |
| font-size:10px; color:var(--ink-faint); display:block; margin-top:3px; font-family:'Inter',sans-serif; | |
| } | |
| .specimen-cell{ display:inline-flex; flex-direction:column; align-items:flex-start; } | |
| /* ---------- editorial sections ---------- */ | |
| .kicker{ | |
| font-family:'JetBrains Mono',monospace; font-size:11.5px; letter-spacing:.06em; text-transform:uppercase; | |
| color:var(--ink-faint); margin-bottom:10px; | |
| } | |
| h2.section-title{ | |
| font-family:'Newsreader',serif; font-weight:500; font-size:2rem; letter-spacing:-0.01em; | |
| margin:0 0 36px; | |
| } | |
| .technique{ padding:26px 0; border-top:1px solid var(--line); } | |
| .technique:last-child{ border-bottom:1px solid var(--line); } | |
| .technique-head{ display:flex; justify-content:space-between; align-items:baseline; gap:16px; margin-bottom:8px;} | |
| .technique-name{ font-family:'Newsreader',serif; font-size:1.3rem; font-weight:500; } | |
| .technique-from{ font-family:'JetBrains Mono',monospace; font-size:11.5px; color:var(--ink-faint); white-space:nowrap; } | |
| .technique p{ color:var(--ink-soft); margin:0; font-size:15.5px; max-width:600px; } | |
| /* stats */ | |
| .stats-grid{ | |
| display:grid; grid-template-columns:repeat(4,1fr); gap:0; | |
| border-top:1px solid var(--line); border-left:1px solid var(--line); | |
| } | |
| .stat{ | |
| border-right:1px solid var(--line); border-bottom:1px solid var(--line); | |
| padding:22px 18px; | |
| } | |
| .stat-num{ font-family:'Newsreader',serif; font-size:1.9rem; font-weight:500; letter-spacing:-0.01em; } | |
| .stat-label{ font-size:12.5px; color:var(--ink-soft); margin-top:4px; line-height:1.3; } | |
| .detail-list{ margin-top:40px; } | |
| .detail-row{ | |
| display:grid; grid-template-columns:180px 1fr; gap:20px; padding:14px 0; | |
| border-top:1px solid var(--line); font-size:14.5px; | |
| } | |
| .detail-row:last-child{ border-bottom:1px solid var(--line); } | |
| .detail-key{ font-family:'JetBrains Mono',monospace; font-size:12.5px; color:var(--ink-faint); padding-top:2px; } | |
| .detail-val{ color:var(--ink); } | |
| .detail-val b{ font-weight:600; } | |
| /* ---------- demo ---------- */ | |
| #demo{ background:var(--card); border-top:1px solid var(--line); border-bottom:1px solid var(--line); } | |
| .demo-card{ | |
| background:var(--card); border:1px solid var(--line); border-radius:10px; | |
| padding:32px; box-shadow:0 1px 2px rgba(23,24,27,0.04); | |
| } | |
| .presets-row{ display:flex; flex-wrap:wrap; gap:8px; margin-bottom:22px; } | |
| .preset-btn{ | |
| background:var(--paper); border:1px solid var(--line); color:var(--ink-soft); | |
| padding:7px 13px; border-radius:20px; font-size:13px; font-family:'Inter',sans-serif; | |
| cursor:pointer; transition:all .15s ease; | |
| } | |
| .preset-btn:hover{ border-color:var(--accent); color:var(--accent-ink); background:var(--accent-soft); } | |
| .field{ margin-bottom:20px; } | |
| .field label{ display:block; font-size:13px; font-weight:500; margin-bottom:8px; color:var(--ink); } | |
| textarea{ | |
| width:100%; height:96px; background:var(--paper); border:1px solid var(--line); border-radius:8px; | |
| padding:13px 14px; color:var(--ink); font-family:'Inter',sans-serif; font-size:15px; resize:vertical; | |
| transition:border-color .15s ease; | |
| } | |
| textarea:focus{ outline:none; border-color:var(--accent); box-shadow:0 0 0 3px var(--accent-soft); } | |
| textarea:focus-visible{ outline:2px solid var(--accent); outline-offset:1px; } | |
| .controls{ | |
| display:grid; grid-template-columns:repeat(auto-fit,minmax(160px,1fr)); gap:22px; | |
| padding:20px 0; margin-bottom:24px; border-top:1px solid var(--line); border-bottom:1px solid var(--line); | |
| } | |
| .control-head{ display:flex; justify-content:space-between; margin-bottom:8px; } | |
| .control-head label{ font-size:13px; color:var(--ink-soft); } | |
| .control-val{ font-family:'JetBrains Mono',monospace; font-size:12.5px; color:var(--accent-ink); font-weight:500; } | |
| input[type="range"]{ width:100%; accent-color:var(--accent); cursor:pointer; } | |
| input[type="range"]:focus-visible{ outline:2px solid var(--accent); outline-offset:3px; } | |
| .submit-btn{ | |
| width:100%; padding:13px; background:var(--ink); border:none; border-radius:8px; | |
| color:#fff; font-weight:500; font-size:15px; cursor:pointer; | |
| display:flex; justify-content:center; align-items:center; gap:9px; | |
| transition:background .15s ease; | |
| } | |
| .submit-btn:hover{ background:var(--accent-ink); } | |
| .submit-btn:focus-visible{ outline:2px solid var(--accent); outline-offset:2px; } | |
| .submit-btn:disabled{ opacity:.6; cursor:not-allowed; } | |
| .out{ margin-top:28px; display:none; } | |
| .out-head{ display:flex; justify-content:space-between; align-items:center; margin-bottom:10px; } | |
| .out-stats{ display:flex; gap:8px; } | |
| .out-badge{ | |
| font-family:'JetBrains Mono',monospace; font-size:11.5px; color:var(--ink-soft); | |
| background:var(--paper); border:1px solid var(--line); padding:3px 9px; border-radius:5px; | |
| } | |
| .copy-btn{ | |
| background:transparent; border:1px solid var(--line); color:var(--ink-soft); | |
| padding:4px 10px; border-radius:5px; font-size:12px; cursor:pointer; transition:all .15s ease; | |
| } | |
| .copy-btn:hover{ border-color:var(--accent); color:var(--accent-ink); } | |
| .out-box{ | |
| background:var(--paper); border:1px solid var(--line); border-radius:8px; padding:18px; | |
| font-family:'JetBrains Mono',monospace; font-size:14px; line-height:1.65; white-space:pre-wrap; | |
| min-height:110px; color:var(--ink); | |
| } | |
| .spinner{ | |
| width:15px; height:15px; border:2px solid rgba(255,255,255,.35); border-radius:50%; | |
| border-top-color:#fff; animation:spin .8s linear infinite; | |
| } | |
| @keyframes spin{ to{ transform:rotate(360deg); } } | |
| @media (prefers-reduced-motion:reduce){ .spinner{ animation:none; } html{ scroll-behavior:auto; } } | |
| footer{ padding:44px 0 60px; text-align:center; } | |
| footer p{ font-size:13px; color:var(--ink-faint); } | |
| @media (max-width:640px){ | |
| section{ padding:52px 0; } | |
| .stats-grid{ grid-template-columns:repeat(2,1fr); } | |
| .detail-row{ grid-template-columns:1fr; gap:4px; } | |
| nav{ flex-direction:column; gap:10px; align-items:flex-start; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <nav> | |
| <span class="nav-mark">Nano-GLM</span> | |
| <a class="nav-link" href="#demo">Try the model →</a> | |
| </nav> | |
| <div class="wrap"> | |
| <div class="hero"> | |
| <div class="eyebrow"><span class="dot"></span> 120M parameters · trained from scratch · single GPU</div> | |
| <h1 class="title">A small language model, built the way the frontier ones are.</h1> | |
| <p class="lede">Nano-GLM re-implements the architecture behind modern GLM and DeepSeek-class models at a scale that trains on one consumer GPU — multi-latent attention, mixture-of-experts routing, and sparse attention, none of it simplified.</p> | |
| </div> | |
| <div class="specimen"> | |
| <span class="specimen-cell"><span class="node">input</span><span class="tag">tokens</span></span> | |
| <span class="arrow">→</span> | |
| <span class="specimen-cell"><span class="node">MLA attention</span><span class="tag">compressed KV cache</span></span> | |
| <span class="arrow">→</span> | |
| <span class="specimen-cell"><span class="node">sparse router</span><span class="tag">DSA indexer</span></span> | |
| <span class="arrow">→</span> | |
| <span class="specimen-cell"><span class="node">MoE experts</span><span class="tag">1 shared + top-k</span></span> | |
| <span class="arrow">→</span> | |
| <span class="specimen-cell"><span class="node">output</span><span class="tag">next token</span></span> | |
| </div> | |
| </div> | |
| <section id="architecture"> | |
| <div class="wrap"> | |
| <div class="kicker">Architecture</div> | |
| <h2 class="section-title">Three ideas borrowed from the frontier, scaled down honestly.</h2> | |
| <div class="technique"> | |
| <div class="technique-head"> | |
| <span class="technique-name">Multi-Latent Attention</span> | |
| <span class="technique-from mono">from DeepSeek-V3</span> | |
| </div> | |
| <p>Instead of caching a full key/value tensor per token, MLA compresses attention through a low-rank projection, closer to LoRA than to a standard cache. The memory saved is what makes training possible on 6GB of VRAM at all.</p> | |
| </div> | |
| <div class="technique"> | |
| <div class="technique-head"> | |
| <span class="technique-name">Mixture of Experts</span> | |
| <span class="technique-from mono">from GLM-5</span> | |
| </div> | |
| <p>Rather than one dense feed-forward block, the model routes each token to a small set of specialized experts plus one shared expert. Capacity grows without every token paying for every parameter.</p> | |
| </div> | |
| <div class="technique"> | |
| <div class="technique-head"> | |
| <span class="technique-name">DeepSeek Sparse Attention</span> | |
| <span class="technique-from mono">from DeepSeek-V3.2</span> | |
| </div> | |
| <p>A learned indexer decides which previous tokens are actually worth attending to, and drops the rest — attention over a shortlist instead of the full context, without giving up long-range dependencies.</p> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="training"> | |
| <div class="wrap"> | |
| <div class="kicker">Training</div> | |
| <h2 class="section-title">The engineering that makes it fit on a laptop GPU.</h2> | |
| <div class="stats-grid"> | |
| <div class="stat"><div class="stat-num">120M</div><div class="stat-label">parameters</div></div> | |
| <div class="stat"><div class="stat-num">2.4B</div><div class="stat-label">training tokens</div></div> | |
| <div class="stat"><div class="stat-num">6GB</div><div class="stat-label">VRAM target</div></div> | |
| <div class="stat"><div class="stat-num">~4,900</div><div class="stat-label">tokens / sec</div></div> | |
| </div> | |
| <div class="detail-list"> | |
| <div class="detail-row"> | |
| <div class="detail-key">DATA</div> | |
| <div class="detail-val">2.0B tokens of FineWeb-Edu for the main run, <b>0.4B</b> mixed tokens for a final decay phase — a 20:1 Chinchilla-optimal ratio against model size.</div> | |
| </div> | |
| <div class="detail-row"> | |
| <div class="detail-key">SCHEDULE</div> | |
| <div class="detail-val">Warmup–Stable–Decay: peak learning rate of <b>6e-4</b> held stable for 2.0B tokens, then a cosine decay into the final phase.</div> | |
| </div> | |
| <div class="detail-row"> | |
| <div class="detail-key">PRECISION</div> | |
| <div class="detail-val">BF16 compute on NVIDIA TF32 tensor cores — roughly half the memory and twice the throughput of FP32.</div> | |
| </div> | |
| <div class="detail-row"> | |
| <div class="detail-key">MEMORY</div> | |
| <div class="detail-val">Gradient checkpointing recomputes activations during the backward pass instead of storing them, trading a little speed for about <b>40% less VRAM</b>.</div> | |
| </div> | |
| <div class="detail-row"> | |
| <div class="detail-key">BATCHING</div> | |
| <div class="detail-val">Gradient accumulation over 3 micro-batches simulates an effective batch of <b>9,216 tokens</b> per weight update.</div> | |
| </div> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="demo"> | |
| <div class="wrap-wide"> | |
| <div class="kicker">Live</div> | |
| <h2 class="section-title">Generate with it directly.</h2> | |
| <div class="demo-card"> | |
| <div class="presets-row"> | |
| <button class="preset-btn" type="button" onclick="setPreset('As per my last email,')">Corporate opener</button> | |
| <button class="preset-btn" type="button" onclick="setPreset('Artificial intelligence is defined as')">Encyclopedia lead</button> | |
| <button class="preset-btn" type="button" onclick="setPreset('1. The first rule of computer science is')">List starter</button> | |
| <button class="preset-btn" type="button" onclick="setPreset('The true nature of human consciousness is')">Open-ended</button> | |
| </div> | |
| <form id="generate-form" onsubmit="handleGenerate(event)"> | |
| <div class="field"> | |
| <label for="prompt">Prompt</label> | |
| <textarea id="prompt" required>Artificial intelligence is defined as</textarea> | |
| </div> | |
| <div class="controls"> | |
| <div class="control-item"> | |
| <div class="control-head"><label for="temp">Temperature</label><span class="control-val" id="temp-val">0.7</span></div> | |
| <input type="range" id="temp" min="0.1" max="2.0" step="0.1" value="0.7" oninput="document.getElementById('temp-val').innerText=this.value"> | |
| </div> | |
| <div class="control-item"> | |
| <div class="control-head"><label for="max-tokens">Max tokens</label><span class="control-val" id="tokens-val">100</span></div> | |
| <input type="range" id="max-tokens" min="10" max="300" step="10" value="100" oninput="document.getElementById('tokens-val').innerText=this.value"> | |
| </div> | |
| <div class="control-item"> | |
| <div class="control-head"><label for="top-k">Top-K</label><span class="control-val" id="topk-val">40</span></div> | |
| <input type="range" id="top-k" min="1" max="100" step="1" value="40" oninput="document.getElementById('topk-val').innerText=this.value"> | |
| </div> | |
| </div> | |
| <button type="submit" class="submit-btn" id="submit-btn"><span>Generate text</span></button> | |
| </form> | |
| <div class="out" id="output-container"> | |
| <div class="out-head"> | |
| <div class="out-stats"> | |
| <span class="out-badge" id="stat-tokens">0 tokens</span> | |
| <span class="out-badge" id="stat-time">0.00s</span> | |
| </div> | |
| <button class="copy-btn" type="button" onclick="copyOutput()">Copy text</button> | |
| </div> | |
| <div class="out-box" id="output-box"></div> | |
| </div> | |
| </div> | |
| </div> | |
| </section> | |
| <footer> | |
| <p>Nano-GLM — a from-scratch GLM/DeepSeek-style language model, built to prove the architecture at small scale.</p> | |
| </footer> | |
| <script> | |
| function setPreset(text){ | |
| document.getElementById('prompt').value = text; | |
| } | |
| async function handleGenerate(e){ | |
| e.preventDefault(); | |
| const btn = document.getElementById('submit-btn'); | |
| const outputContainer = document.getElementById('output-container'); | |
| const outputBox = document.getElementById('output-box'); | |
| const prompt = document.getElementById('prompt').value; | |
| const temperature = parseFloat(document.getElementById('temp').value); | |
| const max_new_tokens = parseInt(document.getElementById('max-tokens').value); | |
| const top_k = parseInt(document.getElementById('top-k').value); | |
| btn.disabled = true; | |
| btn.innerHTML = '<div class="spinner"></div><span>Generating…</span>'; | |
| outputContainer.style.display = 'block'; | |
| outputBox.innerText = 'Generating completion…'; | |
| try { | |
| const response = await fetch('/generate', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify({ prompt, temperature, max_new_tokens, top_k, seed: 42 }) | |
| }); | |
| if (!response.ok) throw new Error('API server returned error status'); | |
| const data = await response.json(); | |
| outputBox.innerText = data.generated_text; | |
| document.getElementById('stat-tokens').innerText = `${data.num_tokens} tokens`; | |
| document.getElementById('stat-time').innerText = `${data.inference_time_sec}s`; | |
| } catch (err) { | |
| outputBox.innerText = `Error: ${err.message}. Make sure the FastAPI backend is running.`; | |
| } finally { | |
| btn.disabled = false; | |
| btn.innerHTML = '<span>Generate text</span>'; | |
| } | |
| } | |
| function copyOutput(){ | |
| const text = document.getElementById('output-box').innerText; | |
| navigator.clipboard.writeText(text); | |
| } | |
| </script> | |
| </body> | |
| </html> | |