Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittlePotion-1BPW with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittlePotion-1BPW with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittlePotion-1BPW") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
Download scripts/potion_lb.py from MisasAI/LittlePotion-1BPW: direct link, hf CLI and curl.
- Browser
- Download file 9.78 kB
-
https://huggingface.co/MisasAI/LittlePotion-1BPW/resolve/main/scripts/potion_lb.py
- Command line
-
hf download hf://MisasAI/LittlePotion-1BPW/scripts/potion_lb.py
-
curl -L -o potion_lb.py https://huggingface.co/MisasAI/LittlePotion-1BPW/resolve/main/scripts/potion_lb.py
9.78 kB
| """LittleBit compression of Model2Vec potion-base-8M (static embeddings, 29528 x 256). | |
| python potion_lb.py --bpw 0.55 # train, eval on STS-B, write potion_0.55.lbit | |
| python potion_lb.py --bpw 0.55 --steps 0 # just Dual-SVID init, no retraining | |
| python potion_lb.py --expand potion_0.55.lbit out_dir # write a normal Model2Vec folder | |
| Sentence embedding (as Model2Vec): mean of token rows (no special tokens), L2-normalised. | |
| Retraining = distillation: the original table is the teacher; loss = 1 - cos(student, teacher) | |
| on sentence embeddings, plus a small row-level term so rare tokens are not ignored. | |
| """ | |
| import argparse, json, math, os, time, shutil, torch, torch.nn as nn, torch.nn.functional as F | |
| from huggingface_hub import hf_hub_download | |
| from safetensors.torch import load_file, save_file, safe_open | |
| from tokenizers import Tokenizer | |
| REPO = "minishlab/potion-base-8M" | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--bpw", type=float, default=0.55) | |
| p.add_argument("--steps", type=int, default=3000) | |
| p.add_argument("--bs", type=int, default=512) | |
| p.add_argument("--lr", type=float, default=3e-3) | |
| p.add_argument("--row_w", type=float, default=0.1, help="weight of the per-row MSE term") | |
| p.add_argument("--no_residual", action="store_true") | |
| p.add_argument("--loss", default="rel", choices=["rel", "cos"], help="rel = match pairwise similarities, cos = match each vector") | |
| p.add_argument("--expand", nargs=2, metavar=("LBIT", "OUTDIR")) | |
| args = p.parse_args() | |
| dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| torch.manual_seed(0) | |
| # ---------------- LittleBit core (same maths as littlebit.py) ---------------- | |
| class SmoothSign(torch.autograd.Function): | |
| def forward(ctx, x): | |
| ctx.save_for_backward(x); return torch.where(x >= 0, 1.0, -1.0).to(x.dtype) | |
| def backward(ctx, g): | |
| x, = ctx.saved_tensors; t = torch.tanh(100 * x); return g * 100 * (1 - t * t) | |
| def rank1(M): | |
| U, S, Vh = torch.linalg.svd(M, full_matrices=False) | |
| a, b = U[:, 0] * S[0].sqrt(), Vh[0] * S[0].sqrt() | |
| if a.sum() < 0: a, b = -a, -b | |
| return a.abs().clamp_min(1e-8), b.abs().clamp_min(1e-8) | |
| class Path(nn.Module): | |
| def __init__(self, W, r): | |
| super().__init__() | |
| U, S, Vh = torch.linalg.svd(W, full_matrices=False) | |
| s = S[:r].sqrt(); Up, Vp = U[:, :r] * s, Vh[:r].T * s | |
| h, lu = rank1(Up.abs()); g, lv = rank1(Vp.abs()) | |
| self.U, self.V = nn.Parameter(Up), nn.Parameter(Vp) | |
| self.h, self.g, self.l = nn.Parameter(h), nn.Parameter(g), nn.Parameter(lu * lv) | |
| def weight(self): | |
| return (self.h[:, None] * SmoothSign.apply(self.U) * self.l) @ (SmoothSign.apply(self.V) * self.g[:, None]).T | |
| class LBTable(nn.Module): | |
| def __init__(self, W, r, residual): | |
| super().__init__() | |
| self.pri = Path(W, r) | |
| self.res = None | |
| if residual: | |
| with torch.no_grad(): self.res = Path(W - self.pri.weight(), r) | |
| def weight(self): | |
| return self.pri.weight() + (self.res.weight() if self.res is not None else 0) | |
| def rank_for(bpw, n, d, npath): | |
| return max(1, int((bpw * n * d / npath - 16 * (n + d)) / (n + d + 16))) | |
| # ---------------- packing ---------------- | |
| SH = torch.tensor([7, 6, 5, 4, 3, 2, 1, 0], dtype=torch.uint8) | |
| def pack(x): | |
| rows, r = x.shape; b = (x >= 0).to(torch.uint8) | |
| if r % 8: b = torch.cat([b, torch.zeros(rows, (-r) % 8, dtype=torch.uint8)], 1) | |
| return (b.view(rows, -1, 8) << SH).sum(-1).to(torch.uint8) | |
| def unpack(pk, r): | |
| return (((pk.unsqueeze(-1) >> SH) & 1).view(pk.shape[0], -1)[:, :r].float() * 2 - 1) | |
| def save_lbit(table, path, r): | |
| out = {} | |
| for nm in ("pri", "res"): | |
| pth = getattr(table, nm) | |
| if pth is None: continue | |
| out[f"embeddings.{nm}.U_bits"] = pack(pth.U.detach().cpu()); out[f"embeddings.{nm}.V_bits"] = pack(pth.V.detach().cpu()) | |
| for s in "hgl": out[f"embeddings.{nm}.{s}"] = getattr(pth, s).detach().cpu().half() | |
| save_file(out, path, metadata={"format": "littlebit-model2vec-v1", "base": REPO, "rank": str(r)}) | |
| def load_lbit(path): | |
| with safe_open(path, "pt") as f: r = int(f.metadata()["rank"]) | |
| t = load_file(path); W = 0 | |
| for nm in ("pri", "res"): | |
| if f"embeddings.{nm}.U_bits" not in t: continue | |
| Us, Vs = unpack(t[f"embeddings.{nm}.U_bits"], r), unpack(t[f"embeddings.{nm}.V_bits"], r) | |
| h, g, l = (t[f"embeddings.{nm}.{s}"].float() for s in "hgl") | |
| W = W + (h[:, None] * Us * l) @ (Vs * g[:, None]).T | |
| return W | |
| if args.expand: | |
| src, outdir = args.expand | |
| os.makedirs(outdir, exist_ok=True) | |
| save_file({"embeddings": load_lbit(src).contiguous()}, os.path.join(outdir, "model.safetensors")) | |
| for f in ("config.json", "tokenizer.json", "modules.json", "special_tokens_map.json", "tokenizer_config.json"): | |
| shutil.copy(hf_hub_download(REPO, f), outdir) | |
| print(f"expanded {src} -> {outdir} (loadable with model2vec StaticModel.from_pretrained)"); raise SystemExit | |
| # ---------------- model + data ---------------- | |
| E = load_file(hf_hub_download(REPO, "model.safetensors"))["embeddings"].float().to(dev) | |
| tok = Tokenizer.from_file(hf_hub_download(REPO, "tokenizer.json")) | |
| N, D = E.shape | |
| def ids_of(texts): | |
| encs = tok.encode_batch(texts, add_special_tokens=False) | |
| flat, offs = [], [] | |
| for e in encs: | |
| offs.append(len(flat)); flat.extend(e.ids if e.ids else [0]) | |
| return torch.tensor(flat, device=dev), torch.tensor(offs, device=dev) | |
| def embed(W, flat, offs): | |
| return F.normalize(F.embedding_bag(flat, W, offs, mode="mean"), dim=-1) | |
| from datasets import load_dataset | |
| import random | |
| def sentence_stream(seed=0, buffer=50_000): | |
| """Streams WikiText-103 (no full download), splits paragraphs into sentence-ish chunks, | |
| and shuffles through a buffer. Restarts from the top if it runs out.""" | |
| rng = random.Random(seed); buf = [] | |
| while True: | |
| ds = load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train", streaming=True) | |
| for row in ds: | |
| l = row["text"].strip() | |
| if len(l) <= 20 or l.startswith("="): continue | |
| for s in l.split(". "): | |
| s = s.strip() | |
| if len(s) <= 10: continue | |
| if len(buf) < buffer: buf.append(s); continue | |
| i = rng.randrange(buffer); yield buf[i]; buf[i] = s | |
| stream = sentence_stream() | |
| # held-out sentences come from the validation split, never seen in training | |
| held = [s.strip() for l in load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="validation")["text"] | |
| for s in l.split(". ") if len(s.strip()) > 10 and not l.strip().startswith("=")][:5000] | |
| sts = load_dataset("sentence-transformers/stsb", split="test") | |
| def sts_spearman(W): | |
| a, b = ids_of(sts["sentence1"]), ids_of(sts["sentence2"]) | |
| with torch.no_grad(): | |
| cos = (embed(W, *a) * embed(W, *b)).sum(-1).cpu() | |
| gold = torch.tensor(sts["score"]) | |
| rk = lambda x: x.argsort().argsort().float() | |
| return torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0, 1].item() | |
| # held-out agreement: mean cosine between student and teacher sentence embeddings | |
| held_ids = ids_of(held) | |
| def agreement(W): | |
| with torch.no_grad(): return (embed(W, *held_ids) * embed(E, *held_ids)).sum(-1).mean().item() | |
| # ---------------- run ---------------- | |
| npath = 1 if args.no_residual else 2 | |
| r = rank_for(args.bpw, N, D, npath) | |
| bits = npath * (r * (N + D) + 16 * (N + D + r)) | |
| print(f"original: {N}x{D}, fp32 {N*D*4/1e6:.1f} MB / fp16 {N*D*2/1e6:.1f} MB", flush=True) | |
| print(f"LittleBit {args.bpw} BPW: rank {r}, {npath} path(s), {bits/8e6:.2f} MB ({bits/(N*D):.3f} BPW)", flush=True) | |
| print(f"teacher STS-B spearman {sts_spearman(E):.4f}", flush=True) | |
| table = LBTable(E, r, not args.no_residual).to(dev) | |
| with torch.no_grad(): W0 = table.weight() | |
| print(f"init STS-B spearman {sts_spearman(W0):.4f} | agreement {agreement(W0):.4f}", flush=True) | |
| opt = torch.optim.Adam(table.parameters(), lr=args.lr) | |
| sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=args.lr, total_steps=max(1, args.steps), pct_start=0.02) | |
| t0 = time.time() | |
| for step in range(args.steps): | |
| batch = [next(stream) for _ in range(args.bs)] | |
| flat, offs = ids_of(batch) | |
| W = table.weight() | |
| with torch.no_grad(): tgt = embed(E, flat, offs) | |
| s = embed(W, flat, offs) | |
| if args.loss == "cos": | |
| loss = 1 - (s * tgt).sum(-1).mean() | |
| else: # relational: match the batch's sentence-to-sentence similarity matrix | |
| loss = F.mse_loss(s @ s.T, tgt @ tgt.T) * 100 | |
| if args.row_w: # keep every row (incl. rare tokens) near the teacher, scaled per row | |
| rows = torch.randint(0, N, (4096,), device=dev) | |
| loss = loss + args.row_w * (1 - F.cosine_similarity(W[rows], E[rows], dim=-1).mean()) | |
| opt.zero_grad(); loss.backward(); opt.step(); sched.step() | |
| if step % 500 == 0 or step == args.steps - 1: | |
| print(f"step {step} loss {loss.item():.4f} {time.time()-t0:.0f}s", flush=True) | |
| with torch.no_grad(): Wf = table.weight() | |
| res = dict(bpw=args.bpw, rank=r, mb=bits / 8e6, residual=not args.no_residual, steps=args.steps, | |
| teacher_sts=sts_spearman(E), init_sts=sts_spearman(W0), final_sts=sts_spearman(Wf), | |
| agreement=agreement(Wf)) | |
| print(f"final STS-B spearman {res['final_sts']:.4f} | agreement {res['agreement']:.4f}", flush=True) | |
| out = f"potion_{args.bpw}{'_nores' if args.no_residual else ''}.lbit" | |
| save_lbit(table, out, r) | |
| Wl = load_lbit(out).to(dev) # round-trip check through the packed file | |
| res["file_mb"] = os.path.getsize(out) / 1e6 | |
| res["packed_sts"] = sts_spearman(Wl) | |
| print(f"wrote {out}: {res['file_mb']:.2f} MB, packed STS-B {res['packed_sts']:.4f}", flush=True) | |
| json.dump(res, open(f"result_{args.bpw}{'_nores' if args.no_residual else ''}.json", "w"), indent=1) | |