Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittleLittlePotion-164kb with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittleLittlePotion-164kb with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittleLittlePotion-164kb") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
File size: 16,113 Bytes
92f6dce | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 | """LittleBit compression of micro700kb (Model2Vec vocab-quantized static model).
Original layout: embeddings int8 [K=2000, 256] (shared vectors), mapping int32 [V] (token -> vector),
weights fp32 [V] (per-token scale). Token vector = weights[t] * embeddings[mapping[t]].
Compressed layout (.lbit):
centroids.{pri,res}.{U_bits,V_bits,h,g,l} LittleBit factors of the 2000 x 256 table
mapping_bits uint8 mapping packed at ceil(log2 K) bits per token (lossless)
weights fp16 per-token scale (trainable during retraining)
python micro_lb.py --bpw 1.0 --steps 6000
python micro_lb.py --baseline # just re-pack mapping/weights, no LittleBit
"""
import argparse, json, math, os, random, time, torch, torch.nn as nn, torch.nn.functional as F
from safetensors import safe_open
from safetensors.torch import load_file, save_file
from tokenizers import Tokenizer
HERE = os.path.dirname(os.path.abspath(__file__))
p = argparse.ArgumentParser()
p.add_argument("--bpw", type=float, default=1.0, help="bits per weight for the 2000x256 centroid table")
p.add_argument("--steps", type=int, default=6000)
p.add_argument("--bs", type=int, default=512)
p.add_argument("--lr", type=float, default=3e-3)
p.add_argument("--no_residual", action="store_true")
p.add_argument("--freeze_weights", action="store_true", help="keep per-token scales fixed")
p.add_argument("--baseline", action="store_true")
p.add_argument("--teacher", default="self", choices=["self", "bge", "bge-small"],
help="self = copy the original micro model; bge / bge-small = learn from BAAI/bge-{base,small}-en-v1.5")
p.add_argument("--alpha", type=float, default=1.0,
help="teacher similarity = alpha * model + (1 - alpha) * BM25F. 1.0 = model only")
p.add_argument("--title_w", type=float, default=2.0, help="BM25F weight of the section-title field (body = 1)")
p.add_argument("--k1", type=float, default=1.2)
p.add_argument("--b", type=float, default=0.75)
p.add_argument("--idf_n", type=int, default=300_000, help="sentences used to estimate IDF / field lengths")
p.add_argument("--no_retrieval", action="store_true", help="skip the SciFact retrieval eval")
p.add_argument("--teacher_bs", type=int, default=128, help="sub-batch size for the bge forward pass")
args = p.parse_args()
dev = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)
# ---------------- original model ----------------
t = load_file(os.path.join(HERE, "orig", "model.safetensors"))
C0 = t["embeddings"].float().to(dev) # global scale is irrelevant after L2-normalisation
MAP = t["mapping"].long().to(dev)
WT0 = t["weights"].float().to(dev)
K, D = C0.shape; V = MAP.numel()
tok = Tokenizer.from_file(os.path.join(HERE, "orig", "tokenizer.json"))
def table(C, wt): return wt[:, None] * C[MAP]
T0 = table(C0, WT0)
# ---------------- LittleBit core ----------------
class SmoothSign(torch.autograd.Function):
@staticmethod
def forward(ctx, x): ctx.save_for_backward(x); return torch.where(x >= 0, 1.0, -1.0).to(x.dtype)
@staticmethod
def backward(ctx, g): x, = ctx.saved_tensors; th = torch.tanh(100 * x); return g * 100 * (1 - th * th)
def rank1(M):
U, S, Vh = torch.linalg.svd(M, full_matrices=False)
a, b = U[:, 0] * S[0].sqrt(), Vh[0] * S[0].sqrt()
if a.sum() < 0: a, b = -a, -b
return a.abs().clamp_min(1e-8), b.abs().clamp_min(1e-8)
class Path(nn.Module):
def __init__(self, W, r):
super().__init__()
U, S, Vh = torch.linalg.svd(W, full_matrices=False)
s = S[:r].sqrt(); Up, Vp = U[:, :r] * s, Vh[:r].T * s
h, lu = rank1(Up.abs()); g, lv = rank1(Vp.abs())
self.U, self.V = nn.Parameter(Up), nn.Parameter(Vp)
self.h, self.g, self.l = nn.Parameter(h), nn.Parameter(g), nn.Parameter(lu * lv)
def weight(self):
return (self.h[:, None] * SmoothSign.apply(self.U) * self.l) @ (SmoothSign.apply(self.V) * self.g[:, None]).T
class Student(nn.Module):
def __init__(self, r, residual):
super().__init__()
self.pri = Path(C0, r)
self.res = None
if residual:
with torch.no_grad(): self.res = Path(C0 - self.pri.weight(), r)
self.wt = nn.Parameter(WT0.clone(), requires_grad=not args.freeze_weights)
def centroids(self): return self.pri.weight() + (self.res.weight() if self.res is not None else 0)
def table(self): return table(self.centroids(), self.wt)
def rank_for(bpw, n, d, npath): return max(1, int((bpw * n * d / npath - 16 * (n + d)) / (n + d + 16)))
# ---------------- packing ----------------
SH = torch.tensor([7, 6, 5, 4, 3, 2, 1, 0], dtype=torch.uint8)
def pack_signs(x):
rows, r = x.shape; b = (x >= 0).to(torch.uint8)
if r % 8: b = torch.cat([b, torch.zeros(rows, (-r) % 8, dtype=torch.uint8)], 1)
return (b.view(rows, -1, 8) << SH).sum(-1).to(torch.uint8)
def unpack_signs(pk, r): return ((pk.unsqueeze(-1) >> SH) & 1).view(pk.shape[0], -1)[:, :r].float() * 2 - 1
def pack_ints(x, nbits):
"""little-endian bit stream of non-negative ints, nbits each -> uint8"""
x = x.cpu().long()
bits = ((x.unsqueeze(1) >> torch.arange(nbits)) & 1).view(-1).to(torch.uint8)
if bits.numel() % 8: bits = torch.cat([bits, torch.zeros((-bits.numel()) % 8, dtype=torch.uint8)])
return (bits.view(-1, 8) << torch.arange(8, dtype=torch.uint8)).sum(-1).to(torch.uint8)
def unpack_ints(b, nbits, n):
bits = ((b.long().unsqueeze(1) >> torch.arange(8)) & 1).view(-1)[: n * nbits].view(n, nbits)
return (bits << torch.arange(nbits)).sum(-1)
NB = max(1, math.ceil(math.log2(K)))
def save(path, student=None, r=0):
out = {"mapping_bits": pack_ints(MAP, NB)}
if student is None: # baseline: original int8 centroids, re-packed mapping, fp16 weights
out["centroids.int8"] = t["embeddings"]; out["weights"] = WT0.cpu().half()
else:
for nm in ("pri", "res"):
pth = getattr(student, nm)
if pth is None: continue
out[f"centroids.{nm}.U_bits"] = pack_signs(pth.U.detach().cpu())
out[f"centroids.{nm}.V_bits"] = pack_signs(pth.V.detach().cpu())
for s in "hgl": out[f"centroids.{nm}.{s}"] = getattr(pth, s).detach().cpu().half()
out["weights"] = student.wt.detach().cpu().half()
save_file(out, path, metadata={"format": "littlebit-model2vec-vq-v1", "rank": str(r), "K": str(K),
"vocab": str(V), "map_bits": str(NB)})
def load_table(path):
with safe_open(path, "pt") as f: m = f.metadata()
x = load_file(path); r, n, nb = int(m["rank"]), int(m["vocab"]), int(m["map_bits"])
mp = unpack_ints(x["mapping_bits"], nb, n)
if "centroids.int8" in x: C = x["centroids.int8"].float()
else:
C = 0
for nm in ("pri", "res"):
if f"centroids.{nm}.U_bits" not in x: continue
Us, Vs = unpack_signs(x[f"centroids.{nm}.U_bits"], r), unpack_signs(x[f"centroids.{nm}.V_bits"], r)
h, g, l = (x[f"centroids.{nm}.{s}"].float() for s in "hgl")
C = C + (h[:, None] * Us * l) @ (Vs * g[:, None]).T
return x["weights"].float()[:, None] * C[mp]
# ---------------- data + eval ----------------
from datasets import load_dataset
def ids_of(texts):
flat, offs = [], []
for e in tok.encode_batch(texts, add_special_tokens=False):
offs.append(len(flat)); flat.extend(e.ids if e.ids else [0])
return torch.tensor(flat, device=dev), torch.tensor(offs, device=dev)
def embed(W, flat, offs): return F.normalize(F.embedding_bag(flat, W, offs, mode="mean"), dim=-1)
sts = load_dataset("sentence-transformers/stsb", split="test")
sa, sb = ids_of(sts["sentence1"]), ids_of(sts["sentence2"])
gold = torch.tensor(sts["score"])
def sts_spearman(W):
with torch.no_grad(): cos = (embed(W.to(dev), *sa) * embed(W.to(dev), *sb)).sum(-1).cpu()
rk = lambda x: x.argsort().argsort().float()
return torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0, 1].item()
def raw_pairs():
"""(section title, sentence) from streamed WikiText-103, in order. Title = latest heading."""
title = ""
for row in load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train", streaming=True):
l = row["text"].strip()
if l.startswith("=") and l.endswith("="):
title = l.strip("= ").strip(); continue
if len(l) <= 20: continue
for s in l.split(". "):
s = s.strip()
if len(s) > 10: yield title, s
def sentence_stream(seed=0, buffer=50_000):
rng = random.Random(seed); buf = []
while True:
for pair in raw_pairs():
if len(buf) < buffer: buf.append(pair); continue
i = rng.randrange(buffer); yield buf[i]; buf[i] = pair
# ---------------- BM25F (fields: title, body) over the model's own token ids ----------------
def bm25f_stats():
path = os.path.join(HERE, f"bm25_stats_{args.idf_n}.pt")
if os.path.exists(path): return torch.load(path)
df = torch.zeros(V); n = 0; len_b = len_t = 0.0; chunk = []
def flush(chunk):
nonlocal n, len_b, len_t
eb = tok.encode_batch([s for _, s in chunk], add_special_tokens=False)
et = tok.encode_batch([t for t, _ in chunk], add_special_tokens=False)
for b_, t_ in zip(eb, et):
ids = set(b_.ids) | set(t_.ids)
if ids: df[list(ids)] += 1
len_b += len(b_.ids); len_t += len(t_.ids); n += 1
for pair in raw_pairs():
chunk.append(pair)
if len(chunk) == 10_000: flush(chunk); chunk = []
if n + len(chunk) >= args.idf_n: break
if chunk: flush(chunk)
st_ = dict(idf=torch.log(1 + (n - df + 0.5) / (df + 0.5)), avg_b=len_b / n, avg_t=len_t / n, n=n)
torch.save(st_, path); return st_
def bm25f_sim(pairs, stats):
"""Symmetric, self-normalised BM25F similarity between all sentences in a batch -> [B, B] in ~[0, 1]."""
B = len(pairs)
eb = tok.encode_batch([s for _, s in pairs], add_special_tokens=False)
et = tok.encode_batch([t for t, _ in pairs], add_special_tokens=False)
def tf_matrix(encs, avg):
rows = torch.cat([torch.full((len(e.ids),), i) for i, e in enumerate(encs)] + [torch.zeros(0, dtype=torch.long)]).long()
cols = torch.tensor([j for e in encs for j in e.ids], dtype=torch.long)
M = torch.zeros(B, V, device=dev)
M.index_put_((rows.to(dev), cols.to(dev)), torch.ones(len(cols), device=dev), accumulate=True)
lens = torch.tensor([len(e.ids) for e in encs], device=dev, dtype=torch.float)
return M / (1 - args.b + args.b * lens[:, None] / avg)
tf = tf_matrix(eb, stats["avg_b"]) + args.title_w * tf_matrix(et, stats["avg_t"]) # BM25F pseudo-tf
sat = tf / (args.k1 + tf) # saturation
q = (tf_matrix(eb, stats["avg_b"]) > 0).float() * stats["idf"].to(dev) # query = body terms, idf-weighted
S = q @ sat.T # S[i, j] = BM25F(query i -> doc j)
S = (S + S.T) / 2
d = S.diag().clamp_min(1e-6).sqrt()
return (S / d[:, None] / d[None, :]).clamp(0, 1)
# ---------------- retrieval eval: SciFact (BEIR), nDCG@10 ----------------
def load_scifact():
corpus = load_dataset("mteb/scifact", "corpus", split="corpus")
queries = load_dataset("mteb/scifact", "queries", split="queries")
qrels = load_dataset("mteb/scifact", "default", split="test")
qrel = {}
for r in qrels:
if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"]))
qs = [(str(q["_id"]), q["text"]) for q in queries if str(q["_id"]) in qrel]
docs = [(str(c["_id"]), (c.get("title") or "") + " " + c["text"]) for c in corpus]
return docs, qs, qrel
SCIFACT = None if args.no_retrieval else load_scifact()
def ndcg10(W):
docs, qs, qrel = SCIFACT
with torch.no_grad():
De = embed(W.to(dev), *ids_of([d for _, d in docs])); Qe = embed(W.to(dev), *ids_of([q for _, q in qs]))
top = (Qe @ De.T).topk(10, dim=1).indices.cpu()
tot = 0.0
for qi, (qid, _) in enumerate(qs):
rel = qrel[qid]
dcg = sum(1 / math.log2(r + 2) for r, di in enumerate(top[qi].tolist()) if docs[di][0] in rel)
idcg = sum(1 / math.log2(r + 2) for r in range(min(len(rel), 10)))
tot += dcg / idcg
return tot / len(qs)
def report(name, W):
msg = f"{name:10s} STS-B {sts_spearman(W):.4f}"
if SCIFACT: msg += f" | SciFact nDCG@10 {ndcg10(W):.4f}"
print(msg, flush=True)
orig_kb = os.path.getsize(os.path.join(HERE, "orig", "model.safetensors")) / 1e3
print(f"original micro700kb: {orig_kb:.0f} KB | K={K} centroids x {D}, vocab {V}", flush=True)
print(f"original STS-B spearman {sts_spearman(T0):.4f}", flush=True)
if args.baseline:
save("micro_repacked.lbit")
print(f"re-packed (int8 centroids, {NB}-bit mapping, fp16 weights): "
f"{os.path.getsize('micro_repacked.lbit')/1e3:.0f} KB, STS-B {sts_spearman(load_table('micro_repacked.lbit')):.4f}")
raise SystemExit
npath = 1 if args.no_residual else 2
r = rank_for(args.bpw, K, D, npath)
st = Student(r, not args.no_residual).to(dev)
with torch.no_grad(): print(f"LittleBit {args.bpw} BPW centroids: rank {r}, {npath} path(s) | init STS-B {sts_spearman(st.table()):.4f}", flush=True)
bge = None
if args.teacher in ("bge", "bge-small"):
from sentence_transformers import SentenceTransformer
bge = SentenceTransformer("BAAI/bge-base-en-v1.5" if args.teacher == "bge" else "BAAI/bge-small-en-v1.5", device=dev)
bge.half()
with torch.no_grad():
a = bge.encode(list(sts["sentence1"]), batch_size=128, convert_to_tensor=True, normalize_embeddings=True)
b = bge.encode(list(sts["sentence2"]), batch_size=128, convert_to_tensor=True, normalize_embeddings=True)
cos = (a.float() * b.float()).sum(-1).cpu(); rk = lambda x: x.argsort().argsort().float()
print(f"teacher {args.teacher} STS-B spearman {torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0,1].item():.4f} (ceiling)", flush=True)
def teacher_emb(texts, flat, offs):
if bge is None:
return embed(T0, flat, offs)
return bge.encode(texts, batch_size=args.teacher_bs, convert_to_tensor=True,
normalize_embeddings=True, show_progress_bar=False).float()
BM25 = bm25f_stats() if args.alpha < 1 else None
if BM25: print(f"BM25F stats from {BM25['n']:,} sentences (avg body {BM25['avg_b']:.1f}, title {BM25['avg_t']:.1f} tokens)", flush=True)
report("original", T0)
opt = torch.optim.Adam([q for q in st.parameters() if q.requires_grad], lr=args.lr)
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=args.lr, total_steps=max(4, args.steps),
pct_start=max(0.02, 2 / max(4, args.steps))) # short runs need >=2 warmup steps
stream = sentence_stream(); t0 = time.time()
for step in range(args.steps):
pairs = [next(stream) for _ in range(args.bs)]
texts = [s_ for _, s_ in pairs]
flat, offs = ids_of(texts)
with torch.no_grad():
tgt = teacher_emb(texts, flat, offs); T = tgt @ tgt.T
if args.alpha < 1: T = args.alpha * T + (1 - args.alpha) * bm25f_sim(pairs, BM25)
s = embed(st.table(), flat, offs)
loss = F.mse_loss(s @ s.T, T) * 100 # relational distillation (what worked for potion)
opt.zero_grad(); loss.backward(); opt.step(); sched.step()
if step % 1000 == 0 or step == args.steps - 1:
print(f"step {step} loss {loss.item():.4f} {time.time()-t0:.0f}s", flush=True)
tag = f"{args.bpw}{'_nores' if args.no_residual else ''}{'_' + args.teacher if args.teacher != 'self' else ''}{f'_a{args.alpha}' if args.alpha < 1 else ''}"
out = f"micro_lb_{tag}.lbit"; save(out, st, r)
res = dict(bpw=args.bpw, rank=r, steps=args.steps, orig_kb=orig_kb, file_kb=os.path.getsize(out) / 1e3,
orig_sts=sts_spearman(T0), final_sts=sts_spearman(load_table(out)))
print(f"wrote {out}: {res['file_kb']:.0f} KB (from {orig_kb:.0f} KB) | packed STS-B {res['final_sts']:.4f}", flush=True)
report("final", load_table(out))
json.dump(res, open(f"result_{tag}.json", "w"), indent=1)
|