Spaces:
Running on Zero
Running on Zero
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>Sillage — a frozen LM that remembers what it reads</title> | |
| <style> | |
| :root { | |
| --bg:#0d1117; --panel:#161b22; --line:#26303d; --ink:#c9d1d9; | |
| --dim:#8b949e; --hl:#79c0ff; --warm:#ffc457; --ok:#3fb950; | |
| } | |
| * { box-sizing:border-box } | |
| body { | |
| margin:0; background:var(--bg); color:var(--ink); line-height:1.6; | |
| font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,Roboto, | |
| Helvetica,Arial,sans-serif; | |
| } | |
| .wrap { max-width:980px; margin:0 auto; padding:0 20px 80px } | |
| header { text-align:center; padding:56px 0 24px } | |
| h1 { font-size:2.6rem; margin:0 0 10px; letter-spacing:-.02em } | |
| .tag { font-size:1.15rem; color:var(--ink); max-width:640px; margin:0 auto } | |
| .etym { color:var(--dim); font-style:italic; font-size:.95rem; | |
| max-width:560px; margin:18px auto 0 } | |
| .badges { margin:22px 0 8px; display:flex; gap:8px; justify-content:center; | |
| flex-wrap:wrap } | |
| .badges img { height:20px } | |
| h2 { font-size:1.5rem; margin:52px 0 6px; letter-spacing:-.01em } | |
| h2 .num { color:var(--dim); font-weight:400; margin-right:8px } | |
| .lede { color:var(--dim); margin:0 0 20px } | |
| .gif { width:100%; border:1px solid var(--line); border-radius:10px; | |
| display:block; margin:26px 0 } | |
| .chips { display:flex; gap:8px; flex-wrap:wrap; margin-bottom:16px } | |
| .chip { | |
| background:var(--panel); border:1px solid var(--line); color:var(--ink); | |
| border-radius:999px; padding:7px 14px; cursor:pointer; font-size:.9rem; | |
| font-family:inherit; transition:border-color .15s, color .15s; | |
| } | |
| .chip:hover { border-color:var(--hl); color:var(--hl) } | |
| .chip[aria-pressed="true"] { border-color:var(--hl); color:var(--hl); | |
| background:#16273b } | |
| .cols { display:grid; grid-template-columns:1fr 1fr; gap:14px } | |
| @media (max-width:760px) { .cols { grid-template-columns:1fr } } | |
| .col { background:var(--panel); border:1px solid var(--line); | |
| border-radius:10px; padding:14px 16px; min-height:170px } | |
| .col h3 { margin:0 0 10px; font-size:.85rem; text-transform:uppercase; | |
| letter-spacing:.08em; color:var(--dim); font-weight:600 } | |
| .col.win h3 { color:var(--ok) } | |
| .out { font-family:"Cascadia Mono",Consolas,ui-monospace,monospace; | |
| font-size:.9rem; white-space:pre-wrap; word-break:break-word; margin:0 } | |
| .out .p { color:var(--dim) } | |
| .out .new { color:var(--warm) } | |
| .col.win .out .new { color:var(--hl) } | |
| .note { color:var(--dim); font-size:.9rem; margin:12px 0 0 } | |
| table { width:100%; border-collapse:collapse; margin:18px 0; font-size:.92rem } | |
| th,td { text-align:left; padding:9px 12px; border-bottom:1px solid var(--line) } | |
| th { color:var(--dim); font-weight:600; font-size:.82rem; | |
| text-transform:uppercase; letter-spacing:.06em } | |
| td.mono,th.mono { font-family:"Cascadia Mono",Consolas,ui-monospace,monospace; | |
| font-size:.86rem } | |
| tr.best td { color:var(--hl); font-weight:600 } | |
| .panel { background:var(--panel); border:1px solid var(--line); | |
| border-radius:10px; padding:18px 20px; margin:18px 0 } | |
| .panel p:first-child { margin-top:0 } | |
| .panel p:last-child { margin-bottom:0 } | |
| pre.sh { background:var(--panel); border:1px solid var(--line); | |
| border-radius:10px; padding:16px 18px; overflow-x:auto; | |
| font-family:"Cascadia Mono",Consolas,ui-monospace,monospace; | |
| font-size:.9rem; color:var(--ink) } | |
| pre.sh .c { color:var(--dim) } | |
| a { color:var(--hl) } | |
| footer { color:var(--dim); font-size:.9rem; text-align:center; | |
| margin-top:64px; border-top:1px solid var(--line); padding-top:24px } | |
| .kbd { font-family:"Cascadia Mono",Consolas,ui-monospace,monospace; | |
| background:#1c2432; border:1px solid var(--line); border-radius:5px; | |
| padding:1px 6px; font-size:.85em } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="wrap"> | |
| <header> | |
| <h1>Sillage</h1> | |
| <p class="tag">A frozen language model that <b>remembers what it reads</b> — | |
| 4.2 MB, no gradients, no fine-tuning, no vector database.</p> | |
| <p class="etym">sillage (n., French) — the trace left behind by something | |
| that has passed: a ship's wake, a scent in a room. What a model keeps of | |
| what it read.</p> | |
| <div class="badges"> | |
| <a href="https://pypi.org/project/sillage/"><img alt="PyPI" | |
| src="https://img.shields.io/pypi/v/sillage.svg"></a> | |
| <a href="https://github.com/riscoss63/sillage"><img alt="GitHub" | |
| src="https://img.shields.io/badge/code-GitHub-181717?logo=github"></a> | |
| <a href="https://doi.org/10.5281/zenodo.22079016"><img alt="DOI" | |
| src="https://zenodo.org/badge/DOI/10.5281/zenodo.22079016.svg"></a> | |
| <a href="https://github.com/riscoss63/sillage/blob/main/LICENSE"><img | |
| alt="MIT" src="https://img.shields.io/badge/License-MIT-blue.svg"></a> | |
| </div> | |
| </header> | |
| <img class="gif" src="demo.gif" alt="Two sessions a day apart: the model | |
| reads a draft on Monday and halves the perplexity of the next draft on | |
| Tuesday, then completes a sentence it can only know from what it read."> | |
| <h2><span class="num">1</span>It has already read a paper</h2> | |
| <p class="lede">The memory below has read | |
| <b id="read-what"></b> — the paper describing this very mechanism, which | |
| GPT-2 has never seen. Same prompt, same greedy decoding, one column with the | |
| memory and one without. Pick a beginning:</p> | |
| <div class="chips" id="chips"></div> | |
| <div class="cols"> | |
| <div class="col"><h3>GPT-2, frozen</h3><p class="out" id="out-frozen"></p></div> | |
| <div class="col win"><h3>GPT-2 + Sillage memory</h3> | |
| <p class="out" id="out-memory"></p></div> | |
| </div> | |
| <p class="note" id="note"></p> | |
| <h2><span class="num">2</span>What it does to a document it has never seen</h2> | |
| <p class="lede">An operations manual invented for this demo, so GPT-2 cannot | |
| have seen it. Read <b>once</b>, left to right: the first half builds the | |
| memory, the second half is measured with it — the papers' own dev/test split — | |
| and every token is scored <i>before</i> being written.</p> | |
| <div class="panel" id="manual-summary"></div> | |
| <h3 style="margin:26px 0 4px">Predictions the memory corrected</h3> | |
| <p class="lede">The frozen model had no way of knowing these. They are facts | |
| that exist only in that document.</p> | |
| <table id="fixes"><thead><tr> | |
| <th class="mono">context</th><th class="mono">what came next</th> | |
| <th class="mono">what frozen GPT-2 said</th> | |
| </tr></thead><tbody></tbody></table> | |
| <h2><span class="num">3</span>How much better, exactly</h2> | |
| <p class="lede">36k tokens of technical text the model had never seen, frozen | |
| GPT-2 124M, every system tuned identically on a held-out prefix, | |
| 95 % bootstrap confidence intervals.</p> | |
| <table> | |
| <thead><tr><th>system</th><th>perplexity</th><th>change</th> | |
| <th>memory used</th></tr></thead> | |
| <tbody> | |
| <tr><td>frozen GPT-2</td><td>31.2</td><td>—</td><td>0</td></tr> | |
| <tr><td>+ RAG-style retrieve & rescore</td><td>29.9</td><td>−4 %</td> | |
| <td>corpus + index</td></tr> | |
| <tr><td>+ kNN-LM, <b>unbounded</b> store</td><td>23.6</td><td>−24 %</td> | |
| <td>55 MB, grows forever</td></tr> | |
| <tr class="best"><td>+ this memory (fixed)</td><td>19.2</td><td>−38 %</td> | |
| <td>4.2 MB, constant</td></tr> | |
| <tr class="best"><td>+ memory and fast weights</td><td>16.6</td><td>−47 %</td> | |
| <td>7.4 MB, constant</td></tr> | |
| </tbody></table> | |
| <p class="note">Paired block bootstrap <b>P = 1.000</b> against the unbounded | |
| datastore, replicated over 5 random seeds and on a second model | |
| (Qwen3-0.6B).</p> | |
| <h2><span class="num">4</span>Where it does not work</h2> | |
| <div class="panel"> | |
| <p><b>On long, low-repetition narrative, an unbounded kNN-LM still wins</b> | |
| (+0.048 vs +0.007 nats). This memory captures verbatim recurrence, not | |
| paraphrase. The boundary is measured and published rather than hidden.</p> | |
| <p>Three results that did <i>not</i> work are published too. Hidden states | |
| make poor Hebbian keys — their geometry is too entangled. Surprise gating | |
| helps the memory and <i>hurts</i> the fast-weight adapter, because the delta | |
| rule already carries its own error term. And calibrating the readout on your | |
| own stream loses to a proper tuning (+0.109 against +0.120 nats), because the | |
| calibration window is read by a colder memory than the one it will govern.</p> | |
| <p>A fixed matrix also saturates at long horizons, near 0.5 writes per | |
| parameter. Forgetting recovers ×2.3 of the gain and a 4× larger matrix | |
| recovers ×3.4 — both are one flag away.</p> | |
| </div> | |
| <h2><span class="num">5</span>Run it yourself</h2> | |
| <p class="lede">Every output on this page came out of the command line below, | |
| on a laptop CPU. It works with any causal language model — a Hugging Face id | |
| or a local folder — not just the two the papers measured.</p> | |
| <pre class="sh">pip install sillage | |
| sillage index notes.md <span class="c"># instant: no model needed</span> | |
| sillage ask "what did the report say?" | |
| sillage read notes.md <span class="c"># memorise it</span> | |
| sillage complete "The report said" <span class="c"># generate WITH the memory</span> | |
| sillage status <span class="c"># what it knows, tier by tier</span> | |
| sillage read notes.md --model HuggingFaceTB/SmolLM2-135M <span class="c"># any LM</span></pre> | |
| <footer> | |
| <p>Four preprints with permanent DOIs, the full reproduction pipeline, every | |
| number as committed JSON and 24 tests: | |
| <a href="https://github.com/riscoss63/sillage">github.com/riscoss63/sillage</a> | |
| · <a href="https://pypi.org/project/sillage/">pypi.org/project/sillage</a> | |
| · <a href="https://doi.org/10.5281/zenodo.22079016">doi:10.5281/zenodo.22079016</a></p> | |
| <p id="provenance"></p> | |
| </footer> | |
| </div> | |
| <script> | |
| const DATA = {"model": "openai-community/gpt2", "read": "papers/sillage/sillage.tex (8969 tokens)", "state_mb": 7.4, "completions": [{"prompt": "On a 36k-token stream of novel technical text, the memory", "frozen": " of the original text is not lost.\n\nThe original text is not lost.\n\nThe", "memory": " improves GPT-2's test negative log-likelihood by +0.486 +/- 0.", "same": false}, {"prompt": "the memory improves GPT-2's test negative log-likelihood by", "frozen": " a factor of 1.5.\n\nThe results of the study are summarized in Table 1.", "memory": " +0.486 +/- 0.005 nats (perplexity 31.2 -> 19.", "same": false}, {"prompt": "At 500k tokens the fixed matrix", "frozen": " is a bit more complex.\n\nThe first thing to note is that the fixed matrix is not", "memory": " saturates (0.5 writes per parameter); leaky decay recovers 2.3k writes per", "same": false}, {"prompt": "surprise gating quadruples the gain of", "frozen": " the previous year.\n\nThe new study, published in the journal Nature Communications, found that the", "memory": " uniform writes at equal plasticity budget (+0.203 vs +0.050 for the count variant", "same": false}, {"prompt": "Every write is gated by the model's own", "frozen": " rules.\n\nThe model's rules are:\n\nThe model must be able to write a", "memory": " surprise\n\n\nThe model's surprise is the surprise of the augmented reality -- the surprise of the augmented", "same": false}, {"prompt": "a three-factor plasticity rule whose modulator is", "frozen": " a single-cell polyethylene (CPM) polyethylene (PPM) polyethyl", "memory": " free at inference. We call the system . On the other hand, the system is free at inference", "same": false}], "manual": {"summary": "**Read once, left to right: 652 tokens built the memory, the next 653 were measured with it.** That split is the papers' own protocol — the numbers below are on text the memory had not seen when it scored them, and every token was scored *before* being written.\n\nPerplexity on that second half: 11.16 frozen → 9.84 with the rank-16 adapter → **9.80** with the memory on top — **12% lower**. The memory spoke on 36% of those positions and kept quiet on the rest; abstaining when it has nothing to say is what keeps it from doing harm.\n\nState on disk: **7.4 MB**, and it would still be 7.4 MB after a million tokens. A kNN-LM datastore over the same text would already hold about 8.0 MB, and would keep growing.", "fixes": [["wick valves or the ninth cabinet is reported immediately", "','", "' to'"], ["incoming duty officer. The morning routine begins at", "' 05'", "' 19'"], [" holds the amber cipher. The rotation changes every eleven", "' days'", "'\\n'"], [" incident, and is recorded in the gallery log", "' beside'", "'.'"], [". Record the seal number in the gallery log.", "'\\n'", "' The'"], ["in manifold. Confirm the Kelbrin manifold", "' reads'", "' is'"], [". Confirm the Kelbrin manifold reads between", "' 4'", "'\\n'"], ["4.6 bar. Open the Fenwick valves", "' in'", "'.'"], ["6 bar. Open the Fenwick valves in the", "' order'", "' west'"], [" bar. Open the Fenwick valves in the order", "' three'", "' in'"], [" one, two. Inspect the condensate", "' trap'", "'.'"], [" the seal number. Confirm the technician", "' on'", "\"'s\""]], "perplexity": {"frozen GPT-2": 11.161853381368227, "+ fast weights": 9.837810800300112, "+ memory": 9.795396904604075}, "suggestion": "The Fenwick valves open three, one,", "completion": {"prompt": "The Fenwick valves open three, one,", "frozen": " two, three, four, five, six, seven,", "memory": " two. Inspect the\n\nfenwick valve and the"}}}; | |
| function split(prompt, text) { | |
| // the prompt is echoed back by the tool; colour only what was generated | |
| const rest = text.startsWith(prompt) ? text.slice(prompt.length) : text; | |
| const head = text.startsWith(prompt) ? prompt : ""; | |
| return [head, rest]; | |
| } | |
| function render(i) { | |
| const c = DATA.completions[i]; | |
| for (const [id, key] of [["out-frozen", "frozen"], ["out-memory", "memory"]]) { | |
| const [head, rest] = split(c.prompt, c.prompt + c[key]); | |
| document.getElementById(id).innerHTML = | |
| '<span class="p">' + esc(head) + '</span><span class="new">' | |
| + esc(rest) + '</span>'; | |
| } | |
| document.getElementById("note").textContent = c.same | |
| ? "Identical here — the memory abstains when it is not confident, which " | |
| + "is what keeps it from doing harm." | |
| : "Everything after the prompt on the right comes from the paper the " | |
| + "memory read, not from GPT-2's weights."; | |
| document.querySelectorAll(".chip").forEach((b, k) => | |
| b.setAttribute("aria-pressed", k === i ? "true" : "false")); | |
| } | |
| function esc(s) { | |
| return s.replace(/[&<>]/g, m => ({"&":"&","<":"<",">":">"}[m])); | |
| } | |
| const chips = document.getElementById("chips"); | |
| DATA.completions.forEach((c, i) => { | |
| const b = document.createElement("button"); | |
| b.className = "chip"; | |
| b.type = "button"; | |
| b.textContent = c.prompt.length > 52 ? c.prompt.slice(0, 52) + "…" : c.prompt; | |
| b.title = c.prompt; | |
| b.onclick = () => render(i); | |
| chips.appendChild(b); | |
| }); | |
| document.getElementById("read-what").textContent = DATA.read; | |
| document.getElementById("manual-summary").innerHTML = md(DATA.manual.summary); | |
| const body = document.querySelector("#fixes tbody"); | |
| DATA.manual.fixes.forEach(row => { | |
| const tr = document.createElement("tr"); | |
| row.forEach(cell => { | |
| const td = document.createElement("td"); | |
| td.className = "mono"; | |
| td.textContent = cell; | |
| tr.appendChild(td); | |
| }); | |
| body.appendChild(tr); | |
| }); | |
| document.getElementById("provenance").textContent = | |
| "The completions and corrections on this page are recorded runs of " | |
| + DATA.model + " on a laptop CPU, state " + DATA.state_mb | |
| + " MB — reproduce them with the commands above."; | |
| function md(s) { | |
| return "<p>" + esc(s) | |
| .replace(/\*\*(.+?)\*\*/g, "<b>$1</b>") | |
| .replace(/\*(.+?)\*/g, "<i>$1</i>") | |
| .replace(/\n\n/g, "</p><p>") + "</p>"; | |
| } | |
| render(0); | |
| </script> | |
| </body> | |
| </html> | |