Spaces:
Running on Zero
Running on Zero
| """Sillage — Hugging Face Space. | |
| Three screens, in the order that convinces: | |
| 1. "It has already read this paper." A memory built at deploy time is | |
| already loaded, so the very first thing a visitor sees is a frozen model | |
| and a remembering model completing the same sentence side by side. No | |
| waiting, no setup, no account. | |
| 2. "Give it your own text." Half a minute, then the same comparison on | |
| THEIR document, plus the list of predictions the memory actually | |
| corrected -- the part that turns a number into a fact. | |
| 3. "Where it does not work." The regime boundary, stated by the authors | |
| before anyone else has to point it out. | |
| Everything here runs on `pip install sillage`; the read loop below is the | |
| library's own, instrumented to report which tokens changed. | |
| On ZeroGPU the visitor pays quota only for time spent inside a @spaces.GPU | |
| function, so the split matters: the frozen forward passes are batched into | |
| one decorated call, the Hebbian mechanism stays in numpy on the CPU, and the | |
| suggested prompts of screen 1 are served from runs recorded at build time. | |
| Looking at the page costs nothing; a click costs a second or two. | |
| """ | |
| import collections | |
| import io | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import tempfile | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| from sillage import Sillage | |
| from sillage.core import CAP | |
| try: # ZeroGPU: a real GPU is attached only | |
| import spaces # inside a decorated function | |
| GPU = spaces.GPU | |
| except ImportError: # anywhere else the decorator is a no-op | |
| def GPU(*args, **kwargs): | |
| if args and callable(args[0]): | |
| return args[0] | |
| return lambda fn: fn | |
| import sample | |
| STATE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "state") | |
| MAX_TOKENS = 1600 # 160 MB of logits, ~30 s end to end | |
| MIN_TOKENS = 900 # below this, the split has nothing to say | |
| WINDOW, STRIDE = 1024, 512 | |
| PAPER = "Sillage (paper 1), 8 969 tokens" | |
| # Chosen by running them: each one makes the two columns visibly diverge, | |
| # and the right-hand column recites the paper rather than inventing. | |
| EXAMPLE_PROMPTS = [ | |
| "On a 36k-token stream of novel technical text, the memory", | |
| "At 500k tokens the fixed matrix", | |
| "surprise gating quadruples the gain of", | |
| "Every write is gated by the model's own", | |
| ] | |
| # --------------------------------------------------------------- backend --- | |
| DEVICE = "cuda" if torch.cuda.is_available() else "cpu" | |
| memory = Sillage(model="gpt2", state=STATE, device=DEVICE, quiet=True) | |
| frozen = Sillage(model="gpt2", state=tempfile.mkdtemp(), device=DEVICE, | |
| quiet=True) | |
| # Each visitor who reads a document leaves a 7.4 MB state behind. Keep the | |
| # most recent few so the follow-up completion still works, and delete the | |
| # rest: a public Space should not fill its disk with strangers' memories. | |
| SESSIONS = collections.deque(maxlen=8) | |
| def new_session_dir(): | |
| if len(SESSIONS) == SESSIONS.maxlen: | |
| shutil.rmtree(SESSIONS[0], ignore_errors=True) | |
| d = tempfile.mkdtemp(prefix="sillage-session-") | |
| SESSIONS.append(d) | |
| return d | |
| def models(): | |
| """Load GPT-2 once and let both assistants share the frozen weights.""" | |
| tok, model = memory.load_model() | |
| frozen._tok, frozen._model = tok, model | |
| frozen.device = memory.device | |
| return tok, model | |
| # ZeroGPU asks for the weights to be placed at startup rather than inside a | |
| # decorated call, where a cold load would eat into the visitor's quota and | |
| # could outlast the function's duration budget. | |
| models() | |
| # Screen 1's suggested prompts were run once, at build time, into data.json. | |
| # Greedy decoding over a fixed memory is deterministic, so a cached answer is | |
| # the same answer -- and serving it costs the visitor no GPU quota at all. | |
| # Anything a visitor types instead is computed live. | |
| CACHED = {} | |
| try: | |
| for _c in json.load(io.open(os.path.join( | |
| os.path.dirname(os.path.abspath(__file__)), "data.json"), | |
| encoding="utf-8"))["completions"]: | |
| CACHED[_c["prompt"].strip()] = ( | |
| _c["prompt"] + _c["frozen"], _c["prompt"] + _c["memory"], | |
| _c["same"]) | |
| except Exception: # no capture yet: everything runs live | |
| CACHED = {} | |
| def compare(prompt, n_tokens=24): | |
| """Screen 1. Recorded prompts are served from cache, the rest is live.""" | |
| hit = CACHED.get((prompt or "").strip()) | |
| if hit: | |
| return hit[0], hit[1], _verdict(hit[2]) | |
| return compare_live(prompt, n_tokens) | |
| def compare_live(prompt, n_tokens=24): | |
| """The decorated path: generation is the only part that needs the model. | |
| Twenty-four tokens twice is a second or two of GPU, which is what a | |
| visitor's daily ZeroGPU quota can afford many times over. | |
| """ | |
| return _compare(prompt, n_tokens) | |
| def _verdict(same): | |
| return ("identical here — the memory abstains when it is not confident, " | |
| "which is the point of the abstention threshold" if same else | |
| "the two columns diverge: everything after the prompt on the " | |
| "right comes from what was read, not from GPT-2's weights") | |
| def forward_all(ids): | |
| """Every frozen forward pass for a document, in one GPU visit. | |
| The Hebbian loop that follows is numpy on the CPU: keeping it outside | |
| this function is what stops a 30-second read from costing 30 seconds of | |
| the visitor's GPU quota. Returned in fp16 -- 1600 positions of GPT-2 | |
| logits are 160 MB, and the loop only ever reads them. | |
| """ | |
| tok, model = models() | |
| x = torch.tensor(ids, device=memory.device) | |
| n = len(ids) - 1 | |
| logits = np.empty((n + 1, model.config.vocab_size), dtype=np.float16) | |
| hidden = np.empty((n + 1, model.config.n_embd), dtype=np.float16) | |
| a = 0 | |
| while a < n: | |
| w = min(WINDOW, len(ids) - a) | |
| out = model(x[a:a + w].unsqueeze(0), output_hidden_states=True) | |
| lo = 0 if a == 0 else WINDOW - STRIDE | |
| logits[a + lo:a + w] = ( | |
| out.logits[0, lo:w].to(torch.float16).cpu().numpy()) | |
| hidden[a + lo:a + w] = ( | |
| out.hidden_states[-1][0, lo:w].to(torch.float16).cpu().numpy()) | |
| if a + w >= len(ids): | |
| break | |
| a += STRIDE | |
| return logits, hidden | |
| def _compare(prompt, n_tokens=24): | |
| """The same prompt, decoded greedily, with and without the memory.""" | |
| if not prompt.strip(): | |
| return "", "", "" | |
| models() | |
| a = frozen.complete(prompt, n=n_tokens) | |
| b = memory.complete(prompt, n=n_tokens) | |
| return prompt + a, prompt + b, _verdict(a.strip() == b.strip()) | |
| def suggest_prompt(text, tail): | |
| """Half of a sentence whose opening recurs — so the memory knows the rest. | |
| A prompt taken at random usually lands where the memory has nothing to | |
| say, and the demo falls flat through no fault of the mechanism. Picking a | |
| sentence whose first words already appeared earlier is the honest way to | |
| show the memory doing what it is for. | |
| """ | |
| best = "" | |
| for sentence in re.split(r"(?<=[.:])\s+|\n+", tail): | |
| words = sentence.split() | |
| if len(words) < 9: | |
| continue | |
| if text.count(" ".join(words[:5])) >= 2: | |
| cut = " ".join(words[:max(5, len(words) // 2)]) | |
| if len(cut) > len(best): | |
| best = cut | |
| return best or " ".join(tail.split()[:10]) | |
| def read_and_report(text, progress=gr.Progress()): | |
| """Stream a document through a fresh memory, reporting what it changed. | |
| This is `sillage.runtime.Sillage.read_text`, unrolled for two reasons: | |
| the demo collects the positions where the memory turned a wrong | |
| prediction into a right one, and the frozen forward passes are done in | |
| one batch beforehand (on the GPU when there is one) so that this loop -- | |
| plain numpy, no model -- costs the visitor nothing. Strictly prequential | |
| either way: every token is scored before it is written. | |
| """ | |
| text = (text or "").strip() | |
| tok, lm = models() | |
| state_dir = new_session_dir() | |
| fresh = Sillage(model="gpt2", state=state_dir, device=memory.device, | |
| quiet=True) | |
| fresh._tok, fresh._model = tok, lm | |
| mem = fresh.mem | |
| ids = np.array(tok.encode(text), dtype=np.int64)[:MAX_TOKENS] | |
| n = len(ids) - 1 | |
| if n < MIN_TOKENS: | |
| return (None, f"That is {max(n, 0)} tokens. Below about " | |
| f"{MIN_TOKENS} there is nothing to measure honestly: the " | |
| f"first half of a document is what builds the memory, and " | |
| f"the abstention threshold needs a few hundred observations " | |
| f"before it will let the memory speak at all. Paste a longer " | |
| f"one — or click an example below.", None, "", "") | |
| dev_end = n // 2 # first half: the memory reads and calibrates | |
| mem.new_stream() | |
| thrG = thrS = np.inf # silent until the dev half is behind us | |
| nll_b = nll_f = nll_m = 0.0 | |
| cnt = active = 0 | |
| fixes = [] | |
| logits, hidden = forward_all(ids) # the only GPU visit | |
| for j in range(n): | |
| if j == dev_end: # the papers' protocol: dev decides, test tells | |
| thrG, thrS = mem.thresholds() | |
| truth = int(ids[j + 1]) | |
| lb = logits[j].astype(np.float32) | |
| mx = lb.max() | |
| lpb = lb - (mx + np.log(np.exp(lb - mx).sum())) | |
| lp = float(lpb[truth]) | |
| la, phi = mem.adapt(lb, hidden[j].astype(np.float32)) | |
| m2 = la.max() | |
| p_ad = np.exp(la - m2) | |
| p_ad /= p_ad.sum() | |
| lp_f = float(np.log(max(p_ad[truth], 1e-30))) | |
| qG = mem.step_key(int(ids[j])) | |
| uG, sG = mem.scores(mem.M, qG) | |
| mem.res_G.append(float(sG.max())) | |
| pc = mem.cold_lookup(truth) | |
| p_true = mem.mix_true(np.exp(lp_f), sG, truth, None, pc, thrG, thrS) | |
| speaks = float(sG.max()) >= thrG | |
| if j >= dev_end: | |
| active += speaks | |
| nll_b += -lp | |
| nll_f += -lp_f | |
| nll_m += -np.log(max(p_true, 1e-30)) | |
| cnt += 1 | |
| if speaks and len(fixes) < 12: | |
| # mix_true takes the probability of one token, mix_full takes the | |
| # whole distribution -- so the cold tier is asked twice, once for | |
| # each shape | |
| full = mem.mix_full(p_ad.copy(), sG, None, mem.cold_lookup(), | |
| thrG, thrS) | |
| said, now = int(np.argmax(p_ad)), int(np.argmax(full)) | |
| if now == truth != said: | |
| ctx = tok.decode(ids[max(0, j - 9):j + 1]) | |
| fixes.append([" ".join(ctx.split())[-58:], | |
| repr(tok.decode([truth])), | |
| repr(tok.decode([said]))]) | |
| g = min(CAP, max(0.0, -lp)) | |
| mem.write_all(qG, uG, None, None, truth, g, phi, p_ad) | |
| if j % 128 == 0: | |
| progress(j / n, desc=f"reading {j}/{n} tokens") | |
| fresh.index.add(text, "your document") | |
| fresh.save() | |
| ppl = [float(np.exp(v / cnt)) for v in (nll_b, nll_f, nll_m)] | |
| plot = {"frozen GPT-2": ppl[0], "+ fast weights": ppl[1], | |
| "+ memory": ppl[2]} | |
| knn = n * 1536 * 4 / 1e6 # one 1536-float key+value per token | |
| summary = ( | |
| f"**Read once, left to right: {dev_end} tokens built the memory, the " | |
| f"next {cnt} were measured with it.** That split is the papers' own " | |
| f"protocol — the numbers below are on text the memory had not seen " | |
| f"when it scored them, and every token was scored *before* being " | |
| f"written.\n\n" | |
| f"Perplexity on that second half: {ppl[0]:.2f} frozen → " | |
| f"{ppl[1]:.2f} with the rank-16 adapter → **{ppl[2]:.2f}** with the " | |
| f"memory on top — **{100 * (1 - ppl[2] / ppl[0]):.0f}% lower**. The " | |
| f"memory spoke on {100 * active / cnt:.0f}% of those positions and " | |
| f"kept quiet on the rest; abstaining when it has nothing to say is " | |
| f"what keeps it from doing harm.\n\n" | |
| f"State on disk: **7.4 MB**, and it would still be 7.4 MB after a " | |
| f"million tokens. A kNN-LM datastore over the same text would " | |
| f"already hold about {knn:.1f} MB, and would keep growing.") | |
| return (plot, summary, fixes or None, state_dir, | |
| suggest_prompt(text, tok.decode(ids[dev_end:]))) | |
| def read_then_compare(text, progress=gr.Progress()): | |
| """Read, report, and hand back a prompt worth trying plus the new state.""" | |
| plot, summary, fixes, state_dir, suggestion = read_and_report(text, | |
| progress) | |
| return (plot, summary, fixes, state_dir, | |
| gr.update(value=suggestion, visible=bool(state_dir)), | |
| gr.update(visible=bool(state_dir))) | |
| def compare_user(prompt, state_dir, n_tokens=12): | |
| """The same comparison, against the memory the visitor just built.""" | |
| if not state_dir or not os.path.exists(os.path.join(state_dir, | |
| "state.npz")): | |
| return "", "", ("That session's memory has been cleared — read a " | |
| "document again.") | |
| if not (prompt or "").strip(): | |
| return "", "", "" | |
| tok, model = models() | |
| theirs = Sillage(model="gpt2", state=state_dir, device=memory.device, | |
| quiet=True) | |
| theirs._tok, theirs._model = tok, model | |
| a = frozen.complete(prompt, n=n_tokens) | |
| b = theirs.complete(prompt, n=n_tokens) | |
| note = ("identical here — try a phrase that recurs in your document, " | |
| "the memory only speaks where it is confident" | |
| if a.strip() == b.strip() else | |
| "the right-hand column comes from your document, not from " | |
| "GPT-2's weights") | |
| return prompt + a, prompt + b, note | |
| # ------------------------------------------------------------------ ui ----- | |
| CSS = """ | |
| .hero {text-align:center} | |
| .small {font-size:0.9em; opacity:0.75} | |
| footer {visibility:hidden} | |
| """ | |
| with gr.Blocks(title="Sillage: a frozen LM that remembers " | |
| "what it reads") as demo: | |
| gr.Markdown( | |
| "# Sillage\n" | |
| "### A frozen language model that remembers what it reads — " | |
| "4.2 MB, no gradients, no fine-tuning, no vector database.\n" | |
| "[]" | |
| "(https://pypi.org/project/sillage/) " | |
| "[]" | |
| "(https://github.com/riscoss63/sillage) " | |
| "[]" | |
| "(https://doi.org/10.5281/zenodo.22079016)", | |
| elem_classes="hero") | |
| gr.Markdown( | |
| "> This Space runs **GPT-2 124M on a free CPU**. Its prose is weak in " | |
| "absolute terms — that is not what is on display. What is on display " | |
| "is the **difference** between the two columns, and it comes from a " | |
| "4.2 MB matrix written while reading, with no gradient anywhere.", | |
| elem_classes="small") | |
| with gr.Tab("1 · It has already read a paper"): | |
| gr.Markdown( | |
| f"The memory loaded here has read **{PAPER}** — the paper that " | |
| "describes this very mechanism. GPT-2 has never seen that text. " | |
| "Complete a sentence from it and watch the right-hand column " | |
| "recall what the left one cannot know.") | |
| prompt = gr.Textbox(label="Beginning of a sentence", | |
| value=EXAMPLE_PROMPTS[0], lines=2) | |
| gr.Examples(examples=[[p] for p in EXAMPLE_PROMPTS], inputs=prompt, | |
| label="Try one of these") | |
| go = gr.Button("Complete both", variant="primary") | |
| with gr.Row(): | |
| out_frozen = gr.Textbox(label="GPT-2, frozen", lines=6) | |
| out_memory = gr.Textbox(label="GPT-2 + Sillage memory", lines=6) | |
| verdict = gr.Markdown(elem_classes="small") | |
| go.click(compare, prompt, [out_frozen, out_memory, verdict]) | |
| demo.load(compare, prompt, [out_frozen, out_memory, verdict]) | |
| with gr.Tab("2 · Give it your own text"): | |
| gr.Markdown( | |
| f"Paste something GPT-2 has never seen: your notes, an internal " | |
| f"document, a README, a specification. It is read **once**, left " | |
| f"to right, and every token is scored *before* it is memorised — " | |
| f"so the numbers below are honest online measurements, not a " | |
| f"replay. Capped at {MAX_TOKENS} tokens (about 90 seconds on this " | |
| f"free CPU).") | |
| user_text = gr.Textbox(label="Your document", lines=12, | |
| value=sample.MANUAL) | |
| gr.Examples( | |
| examples=[[sample.MANUAL], [sample.paper_excerpt()]], | |
| inputs=user_text, example_labels=[ | |
| "An invented operations manual (GPT-2 cannot know it)", | |
| "The opening of paper 1 (novel technical prose)"], | |
| label="Or start from one of these") | |
| read_btn = gr.Button("Read it", variant="primary") | |
| chart = gr.Label(label="Perplexity, lower is better") | |
| report = gr.Markdown() | |
| fixed = gr.Dataframe( | |
| headers=["context", "what came next", "what frozen GPT-2 said"], | |
| label="Predictions the memory corrected — the frozen model had " | |
| "no way of knowing these", wrap=True) | |
| gr.Markdown("### Now ask it to continue a sentence from *your* text") | |
| session = gr.State("") | |
| your_prompt = gr.Textbox(label="Beginning of a sentence", lines=2, | |
| visible=False) | |
| your_btn = gr.Button("Complete both", variant="primary", | |
| visible=False) | |
| with gr.Row(): | |
| your_frozen = gr.Textbox(label="GPT-2, frozen", lines=5) | |
| your_memory = gr.Textbox(label="GPT-2 + the memory you just " | |
| "built", lines=5) | |
| your_note = gr.Markdown(elem_classes="small") | |
| read_btn.click(read_then_compare, user_text, | |
| [chart, report, fixed, session, your_prompt, your_btn]) | |
| your_btn.click(compare_user, [your_prompt, session], | |
| [your_frozen, your_memory, your_note]) | |
| with gr.Tab("3 · Where it does not work"): | |
| gr.Markdown( | |
| "**The regime matters, and it is measured.**\n\n" | |
| "| system | perplexity | memory used |\n|---|---|---|\n" | |
| "| frozen GPT-2 | 31.2 | 0 |\n" | |
| "| + RAG-style retrieve & rescore | 29.9 | corpus + index |\n" | |
| "| + kNN-LM, unbounded store | 23.6 | 55 MB, grows forever |\n" | |
| "| + **this memory** (fixed) | **19.2** | **4.2 MB, constant** |\n" | |
| "| + memory and fast weights | **16.8** | 7.4 MB, constant |\n\n" | |
| "That is on 36k tokens of novel technical text. **On long, " | |
| "low-repetition narrative, an unbounded kNN-LM still wins** " | |
| "(+0.048 vs +0.007 nats): this memory captures verbatim " | |
| "recurrence, not paraphrase. The boundary is mapped in the " | |
| "papers rather than hidden.\n\n" | |
| "Three results that did *not* work are published too: hidden " | |
| "states make poor Hebbian keys; surprise gating helps the memory " | |
| "and *hurts* the fast-weight adapter; and calibrating the readout " | |
| "on your own stream loses to a proper tuning, because the " | |
| "calibration window is read by a colder memory than the one it " | |
| "will govern.\n\n" | |
| "---\n\n" | |
| "**Run it yourself, on any causal LM:**\n\n" | |
| "```bash\npip install sillage\nsillage read notes.md\n" | |
| "sillage ask \"what did the report say?\"\n" | |
| "sillage complete \"The report said\"\n```\n\n" | |
| "Four preprints with DOIs, the full reproduction pipeline and " | |
| "every number as committed JSON: " | |
| "[github.com/riscoss63/sillage](https://github.com/riscoss63/sillage)") | |
| if __name__ == "__main__": | |
| # one reading at a time: the whole point is a single CPU | |
| demo.queue(default_concurrency_limit=1).launch( | |
| theme=gr.themes.Soft(), css=CSS) | |