Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittleLittlePotion-164kb with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittleLittlePotion-164kb with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittleLittlePotion-164kb") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
Download scripts/micro_lb.py from MisasAI/LittleLittlePotion-164kb: direct link, hf CLI and curl.
- Browser
- Download file 16.1 kB
-
https://huggingface.co/MisasAI/LittleLittlePotion-164kb/resolve/main/scripts/micro_lb.py
- Command line
-
hf download hf://MisasAI/LittleLittlePotion-164kb/scripts/micro_lb.py
-
curl -L -o micro_lb.py https://huggingface.co/MisasAI/LittleLittlePotion-164kb/resolve/main/scripts/micro_lb.py
16.1 kB
| """LittleBit compression of micro700kb (Model2Vec vocab-quantized static model). | |
| Original layout: embeddings int8 [K=2000, 256] (shared vectors), mapping int32 [V] (token -> vector), | |
| weights fp32 [V] (per-token scale). Token vector = weights[t] * embeddings[mapping[t]]. | |
| Compressed layout (.lbit): | |
| centroids.{pri,res}.{U_bits,V_bits,h,g,l} LittleBit factors of the 2000 x 256 table | |
| mapping_bits uint8 mapping packed at ceil(log2 K) bits per token (lossless) | |
| weights fp16 per-token scale (trainable during retraining) | |
| python micro_lb.py --bpw 1.0 --steps 6000 | |
| python micro_lb.py --baseline # just re-pack mapping/weights, no LittleBit | |
| """ | |
| import argparse, json, math, os, random, time, torch, torch.nn as nn, torch.nn.functional as F | |
| from safetensors import safe_open | |
| from safetensors.torch import load_file, save_file | |
| from tokenizers import Tokenizer | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--bpw", type=float, default=1.0, help="bits per weight for the 2000x256 centroid table") | |
| p.add_argument("--steps", type=int, default=6000) | |
| p.add_argument("--bs", type=int, default=512) | |
| p.add_argument("--lr", type=float, default=3e-3) | |
| p.add_argument("--no_residual", action="store_true") | |
| p.add_argument("--freeze_weights", action="store_true", help="keep per-token scales fixed") | |
| p.add_argument("--baseline", action="store_true") | |
| p.add_argument("--teacher", default="self", choices=["self", "bge", "bge-small"], | |
| help="self = copy the original micro model; bge / bge-small = learn from BAAI/bge-{base,small}-en-v1.5") | |
| p.add_argument("--alpha", type=float, default=1.0, | |
| help="teacher similarity = alpha * model + (1 - alpha) * BM25F. 1.0 = model only") | |
| p.add_argument("--title_w", type=float, default=2.0, help="BM25F weight of the section-title field (body = 1)") | |
| p.add_argument("--k1", type=float, default=1.2) | |
| p.add_argument("--b", type=float, default=0.75) | |
| p.add_argument("--idf_n", type=int, default=300_000, help="sentences used to estimate IDF / field lengths") | |
| p.add_argument("--no_retrieval", action="store_true", help="skip the SciFact retrieval eval") | |
| p.add_argument("--teacher_bs", type=int, default=128, help="sub-batch size for the bge forward pass") | |
| args = p.parse_args() | |
| dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| torch.manual_seed(0) | |
| # ---------------- original model ---------------- | |
| t = load_file(os.path.join(HERE, "orig", "model.safetensors")) | |
| C0 = t["embeddings"].float().to(dev) # global scale is irrelevant after L2-normalisation | |
| MAP = t["mapping"].long().to(dev) | |
| WT0 = t["weights"].float().to(dev) | |
| K, D = C0.shape; V = MAP.numel() | |
| tok = Tokenizer.from_file(os.path.join(HERE, "orig", "tokenizer.json")) | |
| def table(C, wt): return wt[:, None] * C[MAP] | |
| T0 = table(C0, WT0) | |
| # ---------------- LittleBit core ---------------- | |
| class SmoothSign(torch.autograd.Function): | |
| def forward(ctx, x): ctx.save_for_backward(x); return torch.where(x >= 0, 1.0, -1.0).to(x.dtype) | |
| def backward(ctx, g): x, = ctx.saved_tensors; th = torch.tanh(100 * x); return g * 100 * (1 - th * th) | |
| def rank1(M): | |
| U, S, Vh = torch.linalg.svd(M, full_matrices=False) | |
| a, b = U[:, 0] * S[0].sqrt(), Vh[0] * S[0].sqrt() | |
| if a.sum() < 0: a, b = -a, -b | |
| return a.abs().clamp_min(1e-8), b.abs().clamp_min(1e-8) | |
| class Path(nn.Module): | |
| def __init__(self, W, r): | |
| super().__init__() | |
| U, S, Vh = torch.linalg.svd(W, full_matrices=False) | |
| s = S[:r].sqrt(); Up, Vp = U[:, :r] * s, Vh[:r].T * s | |
| h, lu = rank1(Up.abs()); g, lv = rank1(Vp.abs()) | |
| self.U, self.V = nn.Parameter(Up), nn.Parameter(Vp) | |
| self.h, self.g, self.l = nn.Parameter(h), nn.Parameter(g), nn.Parameter(lu * lv) | |
| def weight(self): | |
| return (self.h[:, None] * SmoothSign.apply(self.U) * self.l) @ (SmoothSign.apply(self.V) * self.g[:, None]).T | |
| class Student(nn.Module): | |
| def __init__(self, r, residual): | |
| super().__init__() | |
| self.pri = Path(C0, r) | |
| self.res = None | |
| if residual: | |
| with torch.no_grad(): self.res = Path(C0 - self.pri.weight(), r) | |
| self.wt = nn.Parameter(WT0.clone(), requires_grad=not args.freeze_weights) | |
| def centroids(self): return self.pri.weight() + (self.res.weight() if self.res is not None else 0) | |
| def table(self): return table(self.centroids(), self.wt) | |
| def rank_for(bpw, n, d, npath): return max(1, int((bpw * n * d / npath - 16 * (n + d)) / (n + d + 16))) | |
| # ---------------- packing ---------------- | |
| SH = torch.tensor([7, 6, 5, 4, 3, 2, 1, 0], dtype=torch.uint8) | |
| def pack_signs(x): | |
| rows, r = x.shape; b = (x >= 0).to(torch.uint8) | |
| if r % 8: b = torch.cat([b, torch.zeros(rows, (-r) % 8, dtype=torch.uint8)], 1) | |
| return (b.view(rows, -1, 8) << SH).sum(-1).to(torch.uint8) | |
| def unpack_signs(pk, r): return ((pk.unsqueeze(-1) >> SH) & 1).view(pk.shape[0], -1)[:, :r].float() * 2 - 1 | |
| def pack_ints(x, nbits): | |
| """little-endian bit stream of non-negative ints, nbits each -> uint8""" | |
| x = x.cpu().long() | |
| bits = ((x.unsqueeze(1) >> torch.arange(nbits)) & 1).view(-1).to(torch.uint8) | |
| if bits.numel() % 8: bits = torch.cat([bits, torch.zeros((-bits.numel()) % 8, dtype=torch.uint8)]) | |
| return (bits.view(-1, 8) << torch.arange(8, dtype=torch.uint8)).sum(-1).to(torch.uint8) | |
| def unpack_ints(b, nbits, n): | |
| bits = ((b.long().unsqueeze(1) >> torch.arange(8)) & 1).view(-1)[: n * nbits].view(n, nbits) | |
| return (bits << torch.arange(nbits)).sum(-1) | |
| NB = max(1, math.ceil(math.log2(K))) | |
| def save(path, student=None, r=0): | |
| out = {"mapping_bits": pack_ints(MAP, NB)} | |
| if student is None: # baseline: original int8 centroids, re-packed mapping, fp16 weights | |
| out["centroids.int8"] = t["embeddings"]; out["weights"] = WT0.cpu().half() | |
| else: | |
| for nm in ("pri", "res"): | |
| pth = getattr(student, nm) | |
| if pth is None: continue | |
| out[f"centroids.{nm}.U_bits"] = pack_signs(pth.U.detach().cpu()) | |
| out[f"centroids.{nm}.V_bits"] = pack_signs(pth.V.detach().cpu()) | |
| for s in "hgl": out[f"centroids.{nm}.{s}"] = getattr(pth, s).detach().cpu().half() | |
| out["weights"] = student.wt.detach().cpu().half() | |
| save_file(out, path, metadata={"format": "littlebit-model2vec-vq-v1", "rank": str(r), "K": str(K), | |
| "vocab": str(V), "map_bits": str(NB)}) | |
| def load_table(path): | |
| with safe_open(path, "pt") as f: m = f.metadata() | |
| x = load_file(path); r, n, nb = int(m["rank"]), int(m["vocab"]), int(m["map_bits"]) | |
| mp = unpack_ints(x["mapping_bits"], nb, n) | |
| if "centroids.int8" in x: C = x["centroids.int8"].float() | |
| else: | |
| C = 0 | |
| for nm in ("pri", "res"): | |
| if f"centroids.{nm}.U_bits" not in x: continue | |
| Us, Vs = unpack_signs(x[f"centroids.{nm}.U_bits"], r), unpack_signs(x[f"centroids.{nm}.V_bits"], r) | |
| h, g, l = (x[f"centroids.{nm}.{s}"].float() for s in "hgl") | |
| C = C + (h[:, None] * Us * l) @ (Vs * g[:, None]).T | |
| return x["weights"].float()[:, None] * C[mp] | |
| # ---------------- data + eval ---------------- | |
| from datasets import load_dataset | |
| def ids_of(texts): | |
| flat, offs = [], [] | |
| for e in tok.encode_batch(texts, add_special_tokens=False): | |
| offs.append(len(flat)); flat.extend(e.ids if e.ids else [0]) | |
| return torch.tensor(flat, device=dev), torch.tensor(offs, device=dev) | |
| def embed(W, flat, offs): return F.normalize(F.embedding_bag(flat, W, offs, mode="mean"), dim=-1) | |
| sts = load_dataset("sentence-transformers/stsb", split="test") | |
| sa, sb = ids_of(sts["sentence1"]), ids_of(sts["sentence2"]) | |
| gold = torch.tensor(sts["score"]) | |
| def sts_spearman(W): | |
| with torch.no_grad(): cos = (embed(W.to(dev), *sa) * embed(W.to(dev), *sb)).sum(-1).cpu() | |
| rk = lambda x: x.argsort().argsort().float() | |
| return torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0, 1].item() | |
| def raw_pairs(): | |
| """(section title, sentence) from streamed WikiText-103, in order. Title = latest heading.""" | |
| title = "" | |
| for row in load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train", streaming=True): | |
| l = row["text"].strip() | |
| if l.startswith("=") and l.endswith("="): | |
| title = l.strip("= ").strip(); continue | |
| if len(l) <= 20: continue | |
| for s in l.split(". "): | |
| s = s.strip() | |
| if len(s) > 10: yield title, s | |
| def sentence_stream(seed=0, buffer=50_000): | |
| rng = random.Random(seed); buf = [] | |
| while True: | |
| for pair in raw_pairs(): | |
| if len(buf) < buffer: buf.append(pair); continue | |
| i = rng.randrange(buffer); yield buf[i]; buf[i] = pair | |
| # ---------------- BM25F (fields: title, body) over the model's own token ids ---------------- | |
| def bm25f_stats(): | |
| path = os.path.join(HERE, f"bm25_stats_{args.idf_n}.pt") | |
| if os.path.exists(path): return torch.load(path) | |
| df = torch.zeros(V); n = 0; len_b = len_t = 0.0; chunk = [] | |
| def flush(chunk): | |
| nonlocal n, len_b, len_t | |
| eb = tok.encode_batch([s for _, s in chunk], add_special_tokens=False) | |
| et = tok.encode_batch([t for t, _ in chunk], add_special_tokens=False) | |
| for b_, t_ in zip(eb, et): | |
| ids = set(b_.ids) | set(t_.ids) | |
| if ids: df[list(ids)] += 1 | |
| len_b += len(b_.ids); len_t += len(t_.ids); n += 1 | |
| for pair in raw_pairs(): | |
| chunk.append(pair) | |
| if len(chunk) == 10_000: flush(chunk); chunk = [] | |
| if n + len(chunk) >= args.idf_n: break | |
| if chunk: flush(chunk) | |
| st_ = dict(idf=torch.log(1 + (n - df + 0.5) / (df + 0.5)), avg_b=len_b / n, avg_t=len_t / n, n=n) | |
| torch.save(st_, path); return st_ | |
| def bm25f_sim(pairs, stats): | |
| """Symmetric, self-normalised BM25F similarity between all sentences in a batch -> [B, B] in ~[0, 1].""" | |
| B = len(pairs) | |
| eb = tok.encode_batch([s for _, s in pairs], add_special_tokens=False) | |
| et = tok.encode_batch([t for t, _ in pairs], add_special_tokens=False) | |
| def tf_matrix(encs, avg): | |
| rows = torch.cat([torch.full((len(e.ids),), i) for i, e in enumerate(encs)] + [torch.zeros(0, dtype=torch.long)]).long() | |
| cols = torch.tensor([j for e in encs for j in e.ids], dtype=torch.long) | |
| M = torch.zeros(B, V, device=dev) | |
| M.index_put_((rows.to(dev), cols.to(dev)), torch.ones(len(cols), device=dev), accumulate=True) | |
| lens = torch.tensor([len(e.ids) for e in encs], device=dev, dtype=torch.float) | |
| return M / (1 - args.b + args.b * lens[:, None] / avg) | |
| tf = tf_matrix(eb, stats["avg_b"]) + args.title_w * tf_matrix(et, stats["avg_t"]) # BM25F pseudo-tf | |
| sat = tf / (args.k1 + tf) # saturation | |
| q = (tf_matrix(eb, stats["avg_b"]) > 0).float() * stats["idf"].to(dev) # query = body terms, idf-weighted | |
| S = q @ sat.T # S[i, j] = BM25F(query i -> doc j) | |
| S = (S + S.T) / 2 | |
| d = S.diag().clamp_min(1e-6).sqrt() | |
| return (S / d[:, None] / d[None, :]).clamp(0, 1) | |
| # ---------------- retrieval eval: SciFact (BEIR), nDCG@10 ---------------- | |
| def load_scifact(): | |
| corpus = load_dataset("mteb/scifact", "corpus", split="corpus") | |
| queries = load_dataset("mteb/scifact", "queries", split="queries") | |
| qrels = load_dataset("mteb/scifact", "default", split="test") | |
| qrel = {} | |
| for r in qrels: | |
| if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"])) | |
| qs = [(str(q["_id"]), q["text"]) for q in queries if str(q["_id"]) in qrel] | |
| docs = [(str(c["_id"]), (c.get("title") or "") + " " + c["text"]) for c in corpus] | |
| return docs, qs, qrel | |
| SCIFACT = None if args.no_retrieval else load_scifact() | |
| def ndcg10(W): | |
| docs, qs, qrel = SCIFACT | |
| with torch.no_grad(): | |
| De = embed(W.to(dev), *ids_of([d for _, d in docs])); Qe = embed(W.to(dev), *ids_of([q for _, q in qs])) | |
| top = (Qe @ De.T).topk(10, dim=1).indices.cpu() | |
| tot = 0.0 | |
| for qi, (qid, _) in enumerate(qs): | |
| rel = qrel[qid] | |
| dcg = sum(1 / math.log2(r + 2) for r, di in enumerate(top[qi].tolist()) if docs[di][0] in rel) | |
| idcg = sum(1 / math.log2(r + 2) for r in range(min(len(rel), 10))) | |
| tot += dcg / idcg | |
| return tot / len(qs) | |
| def report(name, W): | |
| msg = f"{name:10s} STS-B {sts_spearman(W):.4f}" | |
| if SCIFACT: msg += f" | SciFact nDCG@10 {ndcg10(W):.4f}" | |
| print(msg, flush=True) | |
| orig_kb = os.path.getsize(os.path.join(HERE, "orig", "model.safetensors")) / 1e3 | |
| print(f"original micro700kb: {orig_kb:.0f} KB | K={K} centroids x {D}, vocab {V}", flush=True) | |
| print(f"original STS-B spearman {sts_spearman(T0):.4f}", flush=True) | |
| if args.baseline: | |
| save("micro_repacked.lbit") | |
| print(f"re-packed (int8 centroids, {NB}-bit mapping, fp16 weights): " | |
| f"{os.path.getsize('micro_repacked.lbit')/1e3:.0f} KB, STS-B {sts_spearman(load_table('micro_repacked.lbit')):.4f}") | |
| raise SystemExit | |
| npath = 1 if args.no_residual else 2 | |
| r = rank_for(args.bpw, K, D, npath) | |
| st = Student(r, not args.no_residual).to(dev) | |
| with torch.no_grad(): print(f"LittleBit {args.bpw} BPW centroids: rank {r}, {npath} path(s) | init STS-B {sts_spearman(st.table()):.4f}", flush=True) | |
| bge = None | |
| if args.teacher in ("bge", "bge-small"): | |
| from sentence_transformers import SentenceTransformer | |
| bge = SentenceTransformer("BAAI/bge-base-en-v1.5" if args.teacher == "bge" else "BAAI/bge-small-en-v1.5", device=dev) | |
| bge.half() | |
| with torch.no_grad(): | |
| a = bge.encode(list(sts["sentence1"]), batch_size=128, convert_to_tensor=True, normalize_embeddings=True) | |
| b = bge.encode(list(sts["sentence2"]), batch_size=128, convert_to_tensor=True, normalize_embeddings=True) | |
| cos = (a.float() * b.float()).sum(-1).cpu(); rk = lambda x: x.argsort().argsort().float() | |
| print(f"teacher {args.teacher} STS-B spearman {torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0,1].item():.4f} (ceiling)", flush=True) | |
| def teacher_emb(texts, flat, offs): | |
| if bge is None: | |
| return embed(T0, flat, offs) | |
| return bge.encode(texts, batch_size=args.teacher_bs, convert_to_tensor=True, | |
| normalize_embeddings=True, show_progress_bar=False).float() | |
| BM25 = bm25f_stats() if args.alpha < 1 else None | |
| if BM25: print(f"BM25F stats from {BM25['n']:,} sentences (avg body {BM25['avg_b']:.1f}, title {BM25['avg_t']:.1f} tokens)", flush=True) | |
| report("original", T0) | |
| opt = torch.optim.Adam([q for q in st.parameters() if q.requires_grad], lr=args.lr) | |
| sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=args.lr, total_steps=max(4, args.steps), | |
| pct_start=max(0.02, 2 / max(4, args.steps))) # short runs need >=2 warmup steps | |
| stream = sentence_stream(); t0 = time.time() | |
| for step in range(args.steps): | |
| pairs = [next(stream) for _ in range(args.bs)] | |
| texts = [s_ for _, s_ in pairs] | |
| flat, offs = ids_of(texts) | |
| with torch.no_grad(): | |
| tgt = teacher_emb(texts, flat, offs); T = tgt @ tgt.T | |
| if args.alpha < 1: T = args.alpha * T + (1 - args.alpha) * bm25f_sim(pairs, BM25) | |
| s = embed(st.table(), flat, offs) | |
| loss = F.mse_loss(s @ s.T, T) * 100 # relational distillation (what worked for potion) | |
| opt.zero_grad(); loss.backward(); opt.step(); sched.step() | |
| if step % 1000 == 0 or step == args.steps - 1: | |
| print(f"step {step} loss {loss.item():.4f} {time.time()-t0:.0f}s", flush=True) | |
| tag = f"{args.bpw}{'_nores' if args.no_residual else ''}{'_' + args.teacher if args.teacher != 'self' else ''}{f'_a{args.alpha}' if args.alpha < 1 else ''}" | |
| out = f"micro_lb_{tag}.lbit"; save(out, st, r) | |
| res = dict(bpw=args.bpw, rank=r, steps=args.steps, orig_kb=orig_kb, file_kb=os.path.getsize(out) / 1e3, | |
| orig_sts=sts_spearman(T0), final_sts=sts_spearman(load_table(out))) | |
| print(f"wrote {out}: {res['file_kb']:.0f} KB (from {orig_kb:.0f} KB) | packed STS-B {res['final_sts']:.4f}", flush=True) | |
| report("final", load_table(out)) | |
| json.dump(res, open(f"result_{tag}.json", "w"), indent=1) | |