Instructions to use dappalumbo91/fluid-language-model with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use dappalumbo91/fluid-language-model with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="dappalumbo91/fluid-language-model", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("dappalumbo91/fluid-language-model", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use dappalumbo91/fluid-language-model with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "dappalumbo91/fluid-language-model" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "dappalumbo91/fluid-language-model", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/dappalumbo91/fluid-language-model
- SGLang
How to use dappalumbo91/fluid-language-model with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "dappalumbo91/fluid-language-model" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "dappalumbo91/fluid-language-model", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "dappalumbo91/fluid-language-model" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "dappalumbo91/fluid-language-model", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use dappalumbo91/fluid-language-model with Docker Model Runner:
docker model run hf.co/dappalumbo91/fluid-language-model
Download scratch.py from dappalumbo91/fluid-language-model: direct link, hf CLI and curl.
- Browser
- Download file 17.9 kB
-
https://huggingface.co/dappalumbo91/fluid-language-model/resolve/main/scratch.py
- Command line
-
hf download hf://dappalumbo91/fluid-language-model/scratch.py
-
curl -L -o scratch.py https://huggingface.co/dappalumbo91/fluid-language-model/resolve/main/scratch.py
17.9 kB
| """A small model trained from nothing, on the relation itself. | |
| Each number is a sign and four digits, least place first, so a place is a | |
| token. The slow stack learns the true sum, difference, product, exact | |
| quotient, and comparison. A second head says whether a finished equation | |
| holds. A short-term residual stays silent during that training. It moves | |
| only when the second head rejects a draft, and the draft it then prefers | |
| is one of its own alternatives. An outside answer is a score, not a target. | |
| This run writes ``weights/scratch/`` only. | |
| python scratch.py | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import operator | |
| from pathlib import Path | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| ROOT = Path(__file__).resolve().parent | |
| DEST = ROOT / "weights" / "scratch" | |
| SEED = 20261109 | |
| WIDTH = 4 | |
| D_MODEL = 128 | |
| LAYERS = 4 | |
| HEADS = 4 | |
| RANK = 8 | |
| EPOCHS = 100 | |
| BATCH = 256 | |
| LR = 1e-3 | |
| FAR_LO = 31 | |
| FAR_HI = 45 | |
| PLUS_SIGN = 2 | |
| MINUS_SIGN = 3 | |
| OP_ADD = 4 | |
| OP_SUB = 5 | |
| OP_MUL = 6 | |
| OP_DIV = 7 | |
| OP_LARGER = 8 | |
| OP_SMALLER = 9 | |
| EQ = 10 | |
| DIGIT0 = 11 | |
| VOCAB = DIGIT0 + 10 | |
| PROMPT_LEN = 12 | |
| ANSWER_LEN = 5 | |
| LENGTH = PROMPT_LEN + ANSWER_LEN | |
| YES = 0 | |
| NO = 1 | |
| OPS = (OP_ADD, OP_SUB, OP_MUL, OP_DIV, OP_LARGER, OP_SMALLER) | |
| STIFF = ((OP_ADD, -5, 3), (OP_LARGER, 3, 9), (OP_SMALLER, 9, 3)) | |
| def _truth(op: int, left: int, right: int) -> int | None: | |
| if op == OP_DIV: | |
| if right == 0 or left % right != 0: | |
| return None | |
| return operator.floordiv(left, right) | |
| return { | |
| OP_ADD: operator.add, | |
| OP_SUB: operator.sub, | |
| OP_MUL: operator.mul, | |
| OP_LARGER: max, | |
| OP_SMALLER: min, | |
| }[op](left, right) | |
| def _expect(op: int, left: int, right: int) -> int | None: | |
| if op == OP_ADD: | |
| return left + right | |
| if op == OP_SUB: | |
| return left - right | |
| if op == OP_MUL: | |
| return left * right | |
| if op == OP_DIV: | |
| if right == 0 or left % right != 0: | |
| return None | |
| return left // right | |
| if op == OP_LARGER: | |
| return left if left >= right else right | |
| if op == OP_SMALLER: | |
| return left if left <= right else right | |
| return None | |
| def _span_held(left: int, right: int) -> bool: | |
| """Scattered holdout. | |
| A mod-5 test collapses to the ones digits, because 10 ≡ 0 (mod 5). | |
| That deletes the whole digit sum, so a later place can never learn it. | |
| """ | |
| return (left * 3 + right * 5) % 7 == 0 | |
| def _holds(op: int, left: int, right: int) -> bool: | |
| return (op, left, right) in STIFF or _span_held(left, right) | |
| def _number(value: int) -> list[int]: | |
| sign = PLUS_SIGN if value >= 0 else MINUS_SIGN | |
| remain = abs(value) | |
| digits = [] | |
| for _ in range(WIDTH): | |
| digits.append(DIGIT0 + remain % 10) | |
| remain //= 10 | |
| if remain: | |
| raise RuntimeError(f"{value} does not fit in {WIDTH} digits") | |
| return [sign, *digits] | |
| def _read_number(tokens: list[int]) -> int | None: | |
| if len(tokens) != ANSWER_LEN or tokens[0] not in (PLUS_SIGN, MINUS_SIGN): | |
| return None | |
| if any(token < DIGIT0 or token >= DIGIT0 + 10 for token in tokens[1:]): | |
| return None | |
| value = 0 | |
| for place, token in enumerate(tokens[1:]): | |
| value += (token - DIGIT0) * 10**place | |
| return value if tokens[0] == PLUS_SIGN else -value | |
| def _sequence(op: int, left: int, right: int, value: int) -> list[int]: | |
| row = _number(left) + [op] + _number(right) + [EQ] + _number(value) | |
| if len(row) != LENGTH: | |
| raise RuntimeError("sequence length drifted") | |
| return row | |
| def _rows(ops: tuple[int, ...], lo: int, hi: int, keep) -> list[tuple[int, int, int, int]]: | |
| found = [] | |
| for op in ops: | |
| for left in range(lo, hi + 1): | |
| for right in range(lo, hi + 1): | |
| if not keep(op, left, right): | |
| continue | |
| value = _truth(op, left, right) | |
| if value is None or value != _expect(op, left, right): | |
| continue | |
| found.append((op, left, right, value)) | |
| return found | |
| def _banks() -> dict[str, list[tuple[int, int, int, int]]]: | |
| core = _rows(OPS, 0, 30, lambda op, left, right: not _holds(op, left, right)) | |
| signed = _rows((OP_ADD, OP_SUB), -12, 12, lambda op, left, right: (left < 0 or right < 0) and not _holds(op, left, right)) | |
| hold_in = _rows(OPS, 0, 30, lambda op, left, right: _span_held(left, right) and (op, left, right) not in STIFF) | |
| hold_far = _rows(OPS, FAR_LO, FAR_HI, lambda op, left, right: True) | |
| stiff = [] | |
| for op, left, right in STIFF: | |
| value = _truth(op, left, right) | |
| if value != _expect(op, left, right): | |
| raise RuntimeError(f"stiff label drifted {op} {left} {right}") | |
| stiff.append((op, left, right, value)) | |
| train_keys = {(op, left, right) for op, left, right, _value in core + signed} | |
| for name, rows in (("in", hold_in), ("far", hold_far), ("stiff", stiff)): | |
| leaked = [row for row in rows if (row[0], row[1], row[2]) in train_keys] | |
| if leaked: | |
| raise RuntimeError(f"{name} leaked: {leaked[:3]}") | |
| return {"train": core + signed, "in": hold_in, "far": hold_far, "stiff": stiff} | |
| def _audit(banks: dict[str, list[tuple[int, int, int, int]]]) -> None: | |
| for name, rows in banks.items(): | |
| if not rows: | |
| raise RuntimeError(f"{name} bank is empty") | |
| for op, left, right, value in rows: | |
| if _truth(op, left, right) != value or _expect(op, left, right) != value: | |
| raise RuntimeError(f"bad label in {name}: {op} {left} {right} {value}") | |
| if _read_number(_number(value)) != value: | |
| raise RuntimeError(f"digit roundtrip failed for {value}") | |
| print( | |
| f"audit train {len(banks['train'])} in {len(banks['in'])} far {len(banks['far'])} stiff {len(banks['stiff'])}", | |
| flush=True, | |
| ) | |
| class Scratch(nn.Module): | |
| """Slow stack plus a silent short-term residual and a yes/no head.""" | |
| def __init__(self) -> None: | |
| super().__init__() | |
| self.embed = nn.Embedding(VOCAB, D_MODEL) | |
| block = nn.TransformerEncoderLayer( | |
| d_model=D_MODEL, | |
| nhead=HEADS, | |
| dim_feedforward=D_MODEL * 4, | |
| dropout=0.0, | |
| activation="gelu", | |
| batch_first=True, | |
| norm_first=True, | |
| ) | |
| self.stack = nn.TransformerEncoder(block, num_layers=LAYERS, enable_nested_tensor=False) | |
| self.read = nn.Linear(D_MODEL, VOCAB) | |
| self.verdict = nn.Linear(D_MODEL, 2) | |
| self.fast_down = nn.Parameter(torch.empty(D_MODEL, RANK)) | |
| self.fast_up = nn.Parameter(torch.zeros(RANK, D_MODEL)) | |
| nn.init.normal_(self.fast_down, std=0.02) | |
| self.register_buffer("fast_down0", self.fast_down.detach().clone()) | |
| def reset_fast(self) -> None: | |
| self.fast_down.data.copy_(self.fast_down0) | |
| self.fast_up.data.zero_() | |
| def forward(self, tokens: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| hidden = self.embed(tokens) | |
| hidden = hidden + (hidden @ self.fast_down @ self.fast_up) | |
| mask = nn.Transformer.generate_square_subsequent_mask( | |
| hidden.shape[1], device=hidden.device, dtype=hidden.dtype | |
| ) | |
| hidden = self.stack(hidden, mask=mask, is_causal=True) | |
| return self.read(hidden), self.verdict(hidden[:, -1]) | |
| def _tensors(rows: list[tuple[int, int, int, int]], device: torch.device) -> torch.Tensor: | |
| data = [_sequence(*row) for row in rows] | |
| return torch.tensor(data, dtype=torch.long, device=device) | |
| def _corrupt(batch: torch.Tensor, generator: torch.Generator) -> torch.Tensor: | |
| forged = batch.clone() | |
| for row in forged: | |
| value = _read_number(row[PROMPT_LEN:].tolist()) | |
| if value is None: | |
| raise RuntimeError("corrupt saw a non-number") | |
| for _ in range(6): | |
| trial = row.clone() | |
| if value != 0 and int(torch.randint(0, 2, (1,), generator=generator)) == 0: | |
| trial[PROMPT_LEN] = MINUS_SIGN if trial[PROMPT_LEN] == PLUS_SIGN else PLUS_SIGN | |
| else: | |
| slot = PROMPT_LEN + 1 + int(torch.randint(0, WIDTH, (1,), generator=generator)) | |
| trial[slot] = DIGIT0 + (trial[slot] - DIGIT0 + 1 + int(torch.randint(0, 9, (1,), generator=generator))) % 10 | |
| if _read_number(trial[PROMPT_LEN:].tolist()) != value: | |
| row.copy_(trial) | |
| break | |
| else: | |
| raise RuntimeError("could not forge a wrong answer") | |
| return forged | |
| def _answer_loss(logits: torch.Tensor, batch: torch.Tensor) -> torch.Tensor: | |
| pred = logits[:, :-1, :] | |
| labels = batch[:, 1:].clone() | |
| labels[:, : PROMPT_LEN - 1] = -100 | |
| return F.cross_entropy(pred.reshape(-1, VOCAB), labels.reshape(-1), ignore_index=-100) | |
| def _slow_parameters(model: Scratch) -> list[nn.Parameter]: | |
| return [param for name, param in model.named_parameters() if not name.startswith("fast_")] | |
| def _greedy(model: Scratch, prompt: torch.Tensor) -> torch.Tensor: | |
| tokens = prompt.clone() | |
| for index in range(ANSWER_LEN): | |
| logits, _verdict = model(tokens) | |
| nxt = logits[:, -1, :] | |
| if index == 0: | |
| choice = nxt[:, PLUS_SIGN : MINUS_SIGN + 1].argmax(dim=-1) + PLUS_SIGN | |
| else: | |
| choice = nxt[:, DIGIT0 : DIGIT0 + 10].argmax(dim=-1) + DIGIT0 | |
| tokens = torch.cat((tokens, choice.unsqueeze(1)), dim=1) | |
| return tokens | |
| def _exact(model: Scratch, rows: list[tuple[int, int, int, int]], device: torch.device) -> tuple[int, int]: | |
| if not rows: | |
| return 0, 0 | |
| hits = 0 | |
| prompts = _tensors(rows, device)[:, :PROMPT_LEN] | |
| for start in range(0, len(rows), BATCH): | |
| drafted = _greedy(model, prompts[start : start + BATCH]) | |
| for row, draft in zip(rows[start : start + BATCH], drafted.tolist()): | |
| if _read_number(draft[PROMPT_LEN:]) == row[3]: | |
| hits += 1 | |
| return hits, len(rows) | |
| def _span(rows: list[tuple[int, int, int, int]], count: int = 512) -> list[tuple[int, int, int, int]]: | |
| if len(rows) <= count: | |
| return rows | |
| step = max(1, len(rows) // count) | |
| return rows[::step][:count] | |
| def _copy_left(rows: list[tuple[int, int, int, int]]) -> int: | |
| return sum(value == left for _op, left, _right, value in rows) | |
| def _verdict_hits(model: Scratch, rows: list[tuple[int, int, int, int]], device: torch.device) -> tuple[int, int]: | |
| hits = 0 | |
| prompts = _tensors(rows, device)[:, :PROMPT_LEN] | |
| for start in range(0, len(rows), BATCH): | |
| chunk = rows[start : start + BATCH] | |
| drafted = _greedy(model, prompts[start : start + BATCH]) | |
| _logits, verdict = model(drafted) | |
| called = verdict.argmax(dim=-1).tolist() | |
| for row, draft, call in zip(chunk, drafted.tolist(), called): | |
| right = _read_number(draft[PROMPT_LEN:]) == row[3] | |
| if (call == YES and right) or (call == NO and not right): | |
| hits += 1 | |
| return hits, len(rows) | |
| def _alternate(model: Scratch, draft: torch.Tensor) -> torch.Tensor: | |
| """Second token at the answer place where the margin is thinnest.""" | |
| tokens = draft[:PROMPT_LEN].clone().unsqueeze(0) | |
| margins = [] | |
| choices = [] | |
| for index in range(ANSWER_LEN): | |
| logits, _verdict = model(tokens) | |
| nxt = logits[0, -1] | |
| if index == 0: | |
| pool = nxt[PLUS_SIGN : MINUS_SIGN + 1] | |
| base = PLUS_SIGN | |
| else: | |
| pool = nxt[DIGIT0 : DIGIT0 + 10] | |
| base = DIGIT0 | |
| order = torch.topk(pool, k=2) | |
| margins.append(float(order.values[0] - order.values[1])) | |
| choices.append(base + int(order.indices[1])) | |
| taken = draft[PROMPT_LEN + index].view(1, 1) | |
| tokens = torch.cat((tokens, taken), dim=1) | |
| slot = min(range(ANSWER_LEN), key=margins.__getitem__) | |
| revised = draft.clone() | |
| revised[PROMPT_LEN + slot] = choices[slot] | |
| return revised | |
| def _revise(model: Scratch, rows: list[tuple[int, int, int, int]], device: torch.device) -> tuple[int, int, int]: | |
| """On a rejected draft, keep the alternative the verdict prefers.""" | |
| improved = changed = rejected = 0 | |
| was_right = 0 | |
| prompts = _tensors(rows, device) | |
| fast = [model.fast_down, model.fast_up] | |
| for row, full in zip(rows, prompts): | |
| model.reset_fast() | |
| draft = _greedy(model, full[:PROMPT_LEN].unsqueeze(0))[0] | |
| _logits, verdict = model(draft.unsqueeze(0)) | |
| if int(verdict.argmax()) == YES: | |
| continue | |
| rejected += 1 | |
| other = _alternate(model, draft) | |
| _logits, other_verdict = model(other.unsqueeze(0)) | |
| if float(other_verdict[0, YES]) <= float(verdict[0, YES]): | |
| continue | |
| optimizer = torch.optim.SGD(fast, lr=0.1) | |
| target = other[PROMPT_LEN:].unsqueeze(0) | |
| prompt = full[:PROMPT_LEN].unsqueeze(0) | |
| optimizer.zero_grad(set_to_none=True) | |
| made = _greedy_train(model, prompt, target) | |
| made.backward() | |
| optimizer.step() | |
| revised = _greedy(model, prompt)[0] | |
| before = _read_number(draft[PROMPT_LEN:].tolist()) == row[3] | |
| after = _read_number(revised[PROMPT_LEN:].tolist()) == row[3] | |
| if revised[PROMPT_LEN:].tolist() != draft[PROMPT_LEN:].tolist(): | |
| changed += 1 | |
| if after and not before: | |
| improved += 1 | |
| if before: | |
| was_right += 1 | |
| return rejected, changed, improved | |
| def _greedy_train(model: Scratch, prompt: torch.Tensor, target: torch.Tensor) -> torch.Tensor: | |
| tokens = prompt | |
| losses = [] | |
| for index in range(ANSWER_LEN): | |
| logits, _verdict = model(tokens) | |
| losses.append(F.cross_entropy(logits[:, -1, :], target[:, index])) | |
| tokens = torch.cat((tokens, target[:, index].unsqueeze(1)), dim=1) | |
| return torch.stack(losses).mean() | |
| def _fit(model: Scratch, train: list[tuple[int, int, int, int]], banks: dict[str, list[tuple[int, int, int, int]]], device: torch.device) -> None: | |
| generator = torch.Generator(device="cpu") | |
| generator.manual_seed(SEED) | |
| optimizer = torch.optim.AdamW(_slow_parameters(model), lr=LR, weight_decay=0.01) | |
| data = _tensors(train, device) | |
| best_far = -1 | |
| best_state = None | |
| for epoch in range(1, EPOCHS + 1): | |
| model.train() | |
| order = torch.randperm(len(train), generator=generator) | |
| total = 0.0 | |
| seen = 0 | |
| for start in range(0, len(train), BATCH): | |
| batch = data[order[start : start + BATCH]] | |
| forged = _corrupt(batch, generator) | |
| optimizer.zero_grad(set_to_none=True) | |
| logits, verdict = model(batch) | |
| _forged_logits, forged_verdict = model(forged) | |
| loss = _answer_loss(logits, batch) | |
| loss = loss + 0.5 * F.cross_entropy(verdict, torch.full((len(batch),), YES, device=device, dtype=torch.long)) | |
| loss = loss + 0.5 * F.cross_entropy(forged_verdict, torch.full((len(batch),), NO, device=device, dtype=torch.long)) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(_slow_parameters(model), 1.0) | |
| optimizer.step() | |
| model.reset_fast() | |
| total += float(loss.detach()) * len(batch) | |
| seen += len(batch) | |
| if epoch == 1 or epoch % 10 == 0 or epoch == EPOCHS: | |
| model.eval() | |
| model.reset_fast() | |
| train_hits, train_n = _exact(model, _span(train), device) | |
| far_hits, far_n = _exact(model, banks["far"], device) | |
| print( | |
| f"epoch {epoch} loss {total / seen:.4f} train {train_hits}/{train_n} far {far_hits}/{far_n} gpu_gib {torch.cuda.memory_allocated() / 1024**3:.2f}", | |
| flush=True, | |
| ) | |
| if far_hits > best_far: | |
| best_far = far_hits | |
| best_state = {key: value.detach().clone() for key, value in model.state_dict().items()} | |
| if best_state is None: | |
| raise RuntimeError("no epoch was scored") | |
| model.load_state_dict(best_state) | |
| print(f"kept far {best_far}/{len(banks['far'])}", flush=True) | |
| def main() -> None: | |
| if not torch.cuda.is_available(): | |
| raise SystemExit("CUDA is not available. Refusing to build the model in system RAM.") | |
| print(f"device {torch.cuda.get_device_name(0)}", flush=True) | |
| banks = _banks() | |
| _audit(banks) | |
| torch.manual_seed(SEED) | |
| device = torch.device("cuda") | |
| model = Scratch().to(device) | |
| if any(param.device.type != "cuda" for param in model.parameters()): | |
| raise SystemExit("a parameter is off the GPU") | |
| print(f"gpu_mib {torch.cuda.memory_allocated() / 1024**2:.1f} params {sum(param.numel() for param in model.parameters())}", flush=True) | |
| if float(model.fast_up.detach().norm()) != 0.0: | |
| raise SystemExit("short-term layer is not silent") | |
| _fit(model, banks["train"], banks, device) | |
| model.eval() | |
| model.reset_fast() | |
| report = {} | |
| for name in ("train", "in", "far", "stiff"): | |
| sample = banks[name] if name != "train" else _span(banks["train"]) | |
| hits, count = _exact(model, sample, device) | |
| copied = _copy_left(sample) | |
| verdict_hits, verdict_n = _verdict_hits(model, sample, device) | |
| report[name] = {"exact": hits, "count": count, "copy_left": copied, "verdict": verdict_hits, "verdict_n": verdict_n} | |
| print( | |
| f"{name} {hits}/{count} copy_left {copied}/{count} verdict {verdict_hits}/{verdict_n}", | |
| flush=True, | |
| ) | |
| rejected, changed, improved = _revise(model, banks["stiff"] + banks["far"][:64], device) | |
| print(f"revise rejected {rejected} changed {changed} improved {improved}", flush=True) | |
| DEST.mkdir(parents=True, exist_ok=True) | |
| torch.save(model.state_dict(), DEST / "model.pt") | |
| (DEST / "scores.json").write_text(json.dumps(report, indent=2), encoding="utf-8") | |
| print(f"wrote {DEST / 'model.pt'}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |