Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittlePotion-1BPW with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittlePotion-1BPW with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittlePotion-1BPW") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
File size: 9,776 Bytes
dd973c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 | """LittleBit compression of Model2Vec potion-base-8M (static embeddings, 29528 x 256).
python potion_lb.py --bpw 0.55 # train, eval on STS-B, write potion_0.55.lbit
python potion_lb.py --bpw 0.55 --steps 0 # just Dual-SVID init, no retraining
python potion_lb.py --expand potion_0.55.lbit out_dir # write a normal Model2Vec folder
Sentence embedding (as Model2Vec): mean of token rows (no special tokens), L2-normalised.
Retraining = distillation: the original table is the teacher; loss = 1 - cos(student, teacher)
on sentence embeddings, plus a small row-level term so rare tokens are not ignored.
"""
import argparse, json, math, os, time, shutil, torch, torch.nn as nn, torch.nn.functional as F
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file, save_file, safe_open
from tokenizers import Tokenizer
REPO = "minishlab/potion-base-8M"
p = argparse.ArgumentParser()
p.add_argument("--bpw", type=float, default=0.55)
p.add_argument("--steps", type=int, default=3000)
p.add_argument("--bs", type=int, default=512)
p.add_argument("--lr", type=float, default=3e-3)
p.add_argument("--row_w", type=float, default=0.1, help="weight of the per-row MSE term")
p.add_argument("--no_residual", action="store_true")
p.add_argument("--loss", default="rel", choices=["rel", "cos"], help="rel = match pairwise similarities, cos = match each vector")
p.add_argument("--expand", nargs=2, metavar=("LBIT", "OUTDIR"))
args = p.parse_args()
dev = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)
# ---------------- LittleBit core (same maths as littlebit.py) ----------------
class SmoothSign(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
ctx.save_for_backward(x); return torch.where(x >= 0, 1.0, -1.0).to(x.dtype)
@staticmethod
def backward(ctx, g):
x, = ctx.saved_tensors; t = torch.tanh(100 * x); return g * 100 * (1 - t * t)
def rank1(M):
U, S, Vh = torch.linalg.svd(M, full_matrices=False)
a, b = U[:, 0] * S[0].sqrt(), Vh[0] * S[0].sqrt()
if a.sum() < 0: a, b = -a, -b
return a.abs().clamp_min(1e-8), b.abs().clamp_min(1e-8)
class Path(nn.Module):
def __init__(self, W, r):
super().__init__()
U, S, Vh = torch.linalg.svd(W, full_matrices=False)
s = S[:r].sqrt(); Up, Vp = U[:, :r] * s, Vh[:r].T * s
h, lu = rank1(Up.abs()); g, lv = rank1(Vp.abs())
self.U, self.V = nn.Parameter(Up), nn.Parameter(Vp)
self.h, self.g, self.l = nn.Parameter(h), nn.Parameter(g), nn.Parameter(lu * lv)
def weight(self):
return (self.h[:, None] * SmoothSign.apply(self.U) * self.l) @ (SmoothSign.apply(self.V) * self.g[:, None]).T
class LBTable(nn.Module):
def __init__(self, W, r, residual):
super().__init__()
self.pri = Path(W, r)
self.res = None
if residual:
with torch.no_grad(): self.res = Path(W - self.pri.weight(), r)
def weight(self):
return self.pri.weight() + (self.res.weight() if self.res is not None else 0)
def rank_for(bpw, n, d, npath):
return max(1, int((bpw * n * d / npath - 16 * (n + d)) / (n + d + 16)))
# ---------------- packing ----------------
SH = torch.tensor([7, 6, 5, 4, 3, 2, 1, 0], dtype=torch.uint8)
def pack(x):
rows, r = x.shape; b = (x >= 0).to(torch.uint8)
if r % 8: b = torch.cat([b, torch.zeros(rows, (-r) % 8, dtype=torch.uint8)], 1)
return (b.view(rows, -1, 8) << SH).sum(-1).to(torch.uint8)
def unpack(pk, r):
return (((pk.unsqueeze(-1) >> SH) & 1).view(pk.shape[0], -1)[:, :r].float() * 2 - 1)
def save_lbit(table, path, r):
out = {}
for nm in ("pri", "res"):
pth = getattr(table, nm)
if pth is None: continue
out[f"embeddings.{nm}.U_bits"] = pack(pth.U.detach().cpu()); out[f"embeddings.{nm}.V_bits"] = pack(pth.V.detach().cpu())
for s in "hgl": out[f"embeddings.{nm}.{s}"] = getattr(pth, s).detach().cpu().half()
save_file(out, path, metadata={"format": "littlebit-model2vec-v1", "base": REPO, "rank": str(r)})
def load_lbit(path):
with safe_open(path, "pt") as f: r = int(f.metadata()["rank"])
t = load_file(path); W = 0
for nm in ("pri", "res"):
if f"embeddings.{nm}.U_bits" not in t: continue
Us, Vs = unpack(t[f"embeddings.{nm}.U_bits"], r), unpack(t[f"embeddings.{nm}.V_bits"], r)
h, g, l = (t[f"embeddings.{nm}.{s}"].float() for s in "hgl")
W = W + (h[:, None] * Us * l) @ (Vs * g[:, None]).T
return W
if args.expand:
src, outdir = args.expand
os.makedirs(outdir, exist_ok=True)
save_file({"embeddings": load_lbit(src).contiguous()}, os.path.join(outdir, "model.safetensors"))
for f in ("config.json", "tokenizer.json", "modules.json", "special_tokens_map.json", "tokenizer_config.json"):
shutil.copy(hf_hub_download(REPO, f), outdir)
print(f"expanded {src} -> {outdir} (loadable with model2vec StaticModel.from_pretrained)"); raise SystemExit
# ---------------- model + data ----------------
E = load_file(hf_hub_download(REPO, "model.safetensors"))["embeddings"].float().to(dev)
tok = Tokenizer.from_file(hf_hub_download(REPO, "tokenizer.json"))
N, D = E.shape
def ids_of(texts):
encs = tok.encode_batch(texts, add_special_tokens=False)
flat, offs = [], []
for e in encs:
offs.append(len(flat)); flat.extend(e.ids if e.ids else [0])
return torch.tensor(flat, device=dev), torch.tensor(offs, device=dev)
def embed(W, flat, offs):
return F.normalize(F.embedding_bag(flat, W, offs, mode="mean"), dim=-1)
from datasets import load_dataset
import random
def sentence_stream(seed=0, buffer=50_000):
"""Streams WikiText-103 (no full download), splits paragraphs into sentence-ish chunks,
and shuffles through a buffer. Restarts from the top if it runs out."""
rng = random.Random(seed); buf = []
while True:
ds = load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train", streaming=True)
for row in ds:
l = row["text"].strip()
if len(l) <= 20 or l.startswith("="): continue
for s in l.split(". "):
s = s.strip()
if len(s) <= 10: continue
if len(buf) < buffer: buf.append(s); continue
i = rng.randrange(buffer); yield buf[i]; buf[i] = s
stream = sentence_stream()
# held-out sentences come from the validation split, never seen in training
held = [s.strip() for l in load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="validation")["text"]
for s in l.split(". ") if len(s.strip()) > 10 and not l.strip().startswith("=")][:5000]
sts = load_dataset("sentence-transformers/stsb", split="test")
def sts_spearman(W):
a, b = ids_of(sts["sentence1"]), ids_of(sts["sentence2"])
with torch.no_grad():
cos = (embed(W, *a) * embed(W, *b)).sum(-1).cpu()
gold = torch.tensor(sts["score"])
rk = lambda x: x.argsort().argsort().float()
return torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0, 1].item()
# held-out agreement: mean cosine between student and teacher sentence embeddings
held_ids = ids_of(held)
def agreement(W):
with torch.no_grad(): return (embed(W, *held_ids) * embed(E, *held_ids)).sum(-1).mean().item()
# ---------------- run ----------------
npath = 1 if args.no_residual else 2
r = rank_for(args.bpw, N, D, npath)
bits = npath * (r * (N + D) + 16 * (N + D + r))
print(f"original: {N}x{D}, fp32 {N*D*4/1e6:.1f} MB / fp16 {N*D*2/1e6:.1f} MB", flush=True)
print(f"LittleBit {args.bpw} BPW: rank {r}, {npath} path(s), {bits/8e6:.2f} MB ({bits/(N*D):.3f} BPW)", flush=True)
print(f"teacher STS-B spearman {sts_spearman(E):.4f}", flush=True)
table = LBTable(E, r, not args.no_residual).to(dev)
with torch.no_grad(): W0 = table.weight()
print(f"init STS-B spearman {sts_spearman(W0):.4f} | agreement {agreement(W0):.4f}", flush=True)
opt = torch.optim.Adam(table.parameters(), lr=args.lr)
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=args.lr, total_steps=max(1, args.steps), pct_start=0.02)
t0 = time.time()
for step in range(args.steps):
batch = [next(stream) for _ in range(args.bs)]
flat, offs = ids_of(batch)
W = table.weight()
with torch.no_grad(): tgt = embed(E, flat, offs)
s = embed(W, flat, offs)
if args.loss == "cos":
loss = 1 - (s * tgt).sum(-1).mean()
else: # relational: match the batch's sentence-to-sentence similarity matrix
loss = F.mse_loss(s @ s.T, tgt @ tgt.T) * 100
if args.row_w: # keep every row (incl. rare tokens) near the teacher, scaled per row
rows = torch.randint(0, N, (4096,), device=dev)
loss = loss + args.row_w * (1 - F.cosine_similarity(W[rows], E[rows], dim=-1).mean())
opt.zero_grad(); loss.backward(); opt.step(); sched.step()
if step % 500 == 0 or step == args.steps - 1:
print(f"step {step} loss {loss.item():.4f} {time.time()-t0:.0f}s", flush=True)
with torch.no_grad(): Wf = table.weight()
res = dict(bpw=args.bpw, rank=r, mb=bits / 8e6, residual=not args.no_residual, steps=args.steps,
teacher_sts=sts_spearman(E), init_sts=sts_spearman(W0), final_sts=sts_spearman(Wf),
agreement=agreement(Wf))
print(f"final STS-B spearman {res['final_sts']:.4f} | agreement {res['agreement']:.4f}", flush=True)
out = f"potion_{args.bpw}{'_nores' if args.no_residual else ''}.lbit"
save_lbit(table, out, r)
Wl = load_lbit(out).to(dev) # round-trip check through the packed file
res["file_mb"] = os.path.getsize(out) / 1e6
res["packed_sts"] = sts_spearman(Wl)
print(f"wrote {out}: {res['file_mb']:.2f} MB, packed STS-B {res['packed_sts']:.4f}", flush=True)
json.dump(res, open(f"result_{args.bpw}{'_nores' if args.no_residual else ''}.json", "w"), indent=1)
|