Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittleLittlePotion-164kb with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittleLittlePotion-164kb with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittleLittlePotion-164kb") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
Download scripts/pipeline.py from MisasAI/LittleLittlePotion-164kb: direct link, hf CLI and curl.
- Browser
- Download file 7.56 kB
-
https://huggingface.co/MisasAI/LittleLittlePotion-164kb/resolve/main/scripts/pipeline.py
- Command line
-
hf download hf://MisasAI/LittleLittlePotion-164kb/scripts/pipeline.py
-
curl -L -o pipeline.py https://huggingface.co/MisasAI/LittleLittlePotion-164kb/resolve/main/scripts/pipeline.py
7.56 kB
| """3-stage retrieval on SciFact (BEIR test set), scored with nDCG@10: | |
| 1. BM25F (fields: title x2, body x1) picks the top --k1 candidates from the whole corpus | |
| 2. dense (bge-small, or a static model) re-scores those and keeps the top --k2 | |
| 3. rerank (Ettin cross-encoder) orders the final list | |
| python pipeline.py # bge-small + ettin-reranker-32m | |
| python pipeline.py --reranker 68m --k1 200 --k2 30 | |
| """ | |
| import argparse, math, os, re, time, numpy as np, torch, torch.nn.functional as F | |
| from collections import Counter | |
| from datasets import load_dataset | |
| from scipy.sparse import csr_matrix | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--k1", type=int, default=100, help="BM25F candidates") | |
| p.add_argument("--k2", type=int, default=20, help="kept after dense stage, sent to reranker") | |
| p.add_argument("--reranker", default="32m", choices=["17m", "32m", "68m", "150m"]) | |
| p.add_argument("--title_w", type=float, default=2.0) | |
| p.add_argument("--bm25_k1", type=float, default=1.2) | |
| p.add_argument("--bm25_b", type=float, default=0.75) | |
| args = p.parse_args() | |
| dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| # ---------------- data ---------------- | |
| corpus = load_dataset("mteb/scifact", "corpus", split="corpus") | |
| queries = load_dataset("mteb/scifact", "queries", split="queries") | |
| qrels = load_dataset("mteb/scifact", "default", split="test") | |
| qrel = {} | |
| for r in qrels: | |
| if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"])) | |
| qs = [(str(q["_id"]), q["text"]) for q in queries if str(q["_id"]) in qrel] | |
| doc_ids = [str(c["_id"]) for c in corpus] | |
| titles = [c.get("title") or "" for c in corpus] | |
| bodies = [c["text"] for c in corpus] | |
| full = [t + " " + b for t, b in zip(titles, bodies)] | |
| N = len(doc_ids) | |
| print(f"SciFact: {N} docs, {len(qs)} test queries", flush=True) | |
| def ndcg10(ranked): # ranked: list (per query) of doc indices, best first | |
| tot = 0.0 | |
| for (qid, _), lst in zip(qs, ranked): | |
| rel = qrel[qid] | |
| dcg = sum(1 / math.log2(r + 2) for r, di in enumerate(lst[:10]) if doc_ids[di] in rel) | |
| tot += dcg / sum(1 / math.log2(r + 2) for r in range(min(len(rel), 10))) | |
| return tot / len(qs) | |
| def recall_at(ranked, k): # share of relevant docs present in the candidate list | |
| hit = tot = 0 | |
| for (qid, _), lst in zip(qs, ranked): | |
| rel = qrel[qid]; got = {doc_ids[d] for d in lst[:k]} | |
| hit += len(rel & got); tot += len(rel) | |
| return hit / tot | |
| # ---------------- stage 1: BM25F ---------------- | |
| WORD = re.compile(r"[a-z0-9]+") | |
| STOP = set("a an and are as at be by for from has have in is it its of on or that the this to was were with".split()) | |
| def words(s): return [w for w in WORD.findall(s.lower()) if w not in STOP] | |
| vocab = {} | |
| def ids(ws): return [vocab.setdefault(w, len(vocab)) for w in ws] | |
| tw = [Counter(ids(words(t))) for t in titles] | |
| bw = [Counter(ids(words(b))) for b in bodies] | |
| V = len(vocab) | |
| def field_matrix(counts): | |
| rows, cols, vals = [], [], [] | |
| for i, c in enumerate(counts): | |
| for j, v in c.items(): rows.append(i); cols.append(j); vals.append(v) | |
| M = csr_matrix((vals, (rows, cols)), shape=(N, V), dtype=np.float32) | |
| lens = np.asarray(M.sum(1)).ravel() | |
| return M, lens | |
| Mt, lt = field_matrix(tw); Mb, lb = field_matrix(bw) | |
| def norm(M, lens): | |
| d = 1 - args.bm25_b + args.bm25_b * lens / max(lens.mean(), 1e-6) | |
| return csr_matrix(M.multiply(1 / d[:, None])) | |
| TF = args.title_w * norm(Mt, lt) + norm(Mb, lb) # BM25F pseudo term frequency | |
| TF.data = TF.data / (args.bm25_k1 + TF.data) # saturation, once over the combined fields | |
| df = np.bincount(TF.indices, minlength=V) | |
| idf = np.log(1 + (N - df + 0.5) / (df + 0.5)).astype(np.float32) | |
| def bm25f_search(q, k): | |
| qi = [vocab[w] for w in set(words(q)) if w in vocab] | |
| if not qi: return list(range(k)) | |
| s = np.asarray(TF[:, qi] @ idf[qi]).ravel() | |
| return list(np.argsort(-s)[:k]) | |
| t0 = time.time() | |
| stage1 = [bm25f_search(q, args.k1) for _, q in qs] | |
| print(f"\n[1] BM25F alone nDCG@10 {ndcg10(stage1):.4f} | recall@{args.k1} {recall_at(stage1, args.k1):.3f} ({time.time()-t0:.1f}s)", flush=True) | |
| # ---------------- dense encoders ---------------- | |
| from sentence_transformers import SentenceTransformer, CrossEncoder | |
| bge = SentenceTransformer("BAAI/bge-small-en-v1.5", device=dev).half() | |
| BGE_Q = "Represent this sentence for searching relevant passages: " # bge's recommended query prefix | |
| t0 = time.time() | |
| D_bge = bge.encode(full, batch_size=64, convert_to_tensor=True, normalize_embeddings=True).float() | |
| Q_bge = bge.encode([BGE_Q + q for _, q in qs], batch_size=64, convert_to_tensor=True, normalize_embeddings=True).float() | |
| print(f" (bge-small encoded corpus in {time.time()-t0:.0f}s)", flush=True) | |
| def static_encoder(lbit_or_orig): | |
| """Mean-pooled static table (micro700kb original or a LittleBit .lbit).""" | |
| from tokenizers import Tokenizer | |
| import importlib.util | |
| spec = importlib.util.spec_from_file_location("lbdec", os.path.join(HERE, "lbit_decode.py")) | |
| m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) | |
| W = torch.tensor(m.decode(lbit_or_orig)).to(dev) | |
| tok = Tokenizer.from_file(os.path.join(HERE, "orig", "tokenizer.json")) | |
| def enc(texts): | |
| flat, offs = [], [] | |
| for e in tok.encode_batch(texts, add_special_tokens=False): | |
| offs.append(len(flat)); flat.extend(e.ids if e.ids else [0]) | |
| return F.normalize(F.embedding_bag(torch.tensor(flat, device=dev), W, torch.tensor(offs, device=dev), mode="mean"), dim=-1) | |
| return enc | |
| def dense_full(Qe, De): # dense retrieval over the whole corpus (no BM25F) | |
| return (Qe @ De.T).topk(10, dim=1).indices.cpu().tolist() | |
| def dense_rerank(cands, Qe, De, k): | |
| out = [] | |
| for qi, c in enumerate(cands): | |
| c = torch.tensor(c, device=dev) | |
| s = De[c] @ Qe[qi] | |
| out.append(c[s.argsort(descending=True)[:k]].cpu().tolist()) | |
| return out | |
| ce = CrossEncoder(f"cross-encoder/ettin-reranker-{args.reranker}-v1", device=dev) | |
| def ce_rerank(cands): | |
| t0 = time.time(); out = [] | |
| for (_, q), c in zip(qs, cands): | |
| s = ce.predict([(q, full[d]) for d in c], batch_size=64, show_progress_bar=False) | |
| out.append([c[i] for i in np.argsort(-s)]) | |
| return out, time.time() - t0 | |
| print(f"[2] bge-small alone nDCG@10 {ndcg10(dense_full(Q_bge, D_bge)):.4f}", flush=True) | |
| s2 = dense_rerank(stage1, Q_bge, D_bge, args.k2) | |
| print(f"[3] BM25F -> bge-small nDCG@10 {ndcg10(s2):.4f} | recall@{args.k2} {recall_at(s2, args.k2):.3f}", flush=True) | |
| s3, sec = ce_rerank(s2) | |
| print(f"[4] BM25F -> bge-small -> ettin-{args.reranker} nDCG@10 {ndcg10(s3):.4f} ({sec:.0f}s for {len(qs)} queries)", flush=True) | |
| for name, src in [("micro700kb original", os.path.join(HERE, "orig", "model.safetensors")), | |
| ("micro LittleBit 164KB", os.path.join(HERE, "micro_lb_1.0.lbit"))]: | |
| enc = static_encoder(src) | |
| De, Qe = enc(full), enc([q for _, q in qs]) | |
| print(f"\n {name} alone nDCG@10 {ndcg10(dense_full(Qe, De)):.4f}", flush=True) | |
| s2 = dense_rerank(stage1, Qe, De, args.k2) | |
| print(f" BM25F -> {name} nDCG@10 {ndcg10(s2):.4f} | recall@{args.k2} {recall_at(s2, args.k2):.3f}", flush=True) | |
| s3, sec = ce_rerank(s2) | |
| print(f" BM25F -> {name} -> ettin-{args.reranker} nDCG@10 {ndcg10(s3):.4f}", flush=True) | |
| s3, sec = ce_rerank([c[:args.k2] for c in stage1]) | |
| print(f"\n BM25F -> ettin-{args.reranker} (no dense stage) nDCG@10 {ndcg10(s3):.4f}", flush=True) | |