Icodetools's picture
LittleLittlePotion-164kb: 1 BPW Model2Vec embeddings in 164 KB
92f6dce verified
Raw History Blame Contribute Delete
7.56 kB
"""3-stage retrieval on SciFact (BEIR test set), scored with nDCG@10:
1. BM25F (fields: title x2, body x1) picks the top --k1 candidates from the whole corpus
2. dense (bge-small, or a static model) re-scores those and keeps the top --k2
3. rerank (Ettin cross-encoder) orders the final list
python pipeline.py # bge-small + ettin-reranker-32m
python pipeline.py --reranker 68m --k1 200 --k2 30
"""
import argparse, math, os, re, time, numpy as np, torch, torch.nn.functional as F
from collections import Counter
from datasets import load_dataset
from scipy.sparse import csr_matrix
HERE = os.path.dirname(os.path.abspath(__file__))
p = argparse.ArgumentParser()
p.add_argument("--k1", type=int, default=100, help="BM25F candidates")
p.add_argument("--k2", type=int, default=20, help="kept after dense stage, sent to reranker")
p.add_argument("--reranker", default="32m", choices=["17m", "32m", "68m", "150m"])
p.add_argument("--title_w", type=float, default=2.0)
p.add_argument("--bm25_k1", type=float, default=1.2)
p.add_argument("--bm25_b", type=float, default=0.75)
args = p.parse_args()
dev = "cuda" if torch.cuda.is_available() else "cpu"
# ---------------- data ----------------
corpus = load_dataset("mteb/scifact", "corpus", split="corpus")
queries = load_dataset("mteb/scifact", "queries", split="queries")
qrels = load_dataset("mteb/scifact", "default", split="test")
qrel = {}
for r in qrels:
if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"]))
qs = [(str(q["_id"]), q["text"]) for q in queries if str(q["_id"]) in qrel]
doc_ids = [str(c["_id"]) for c in corpus]
titles = [c.get("title") or "" for c in corpus]
bodies = [c["text"] for c in corpus]
full = [t + " " + b for t, b in zip(titles, bodies)]
N = len(doc_ids)
print(f"SciFact: {N} docs, {len(qs)} test queries", flush=True)
def ndcg10(ranked): # ranked: list (per query) of doc indices, best first
tot = 0.0
for (qid, _), lst in zip(qs, ranked):
rel = qrel[qid]
dcg = sum(1 / math.log2(r + 2) for r, di in enumerate(lst[:10]) if doc_ids[di] in rel)
tot += dcg / sum(1 / math.log2(r + 2) for r in range(min(len(rel), 10)))
return tot / len(qs)
def recall_at(ranked, k): # share of relevant docs present in the candidate list
hit = tot = 0
for (qid, _), lst in zip(qs, ranked):
rel = qrel[qid]; got = {doc_ids[d] for d in lst[:k]}
hit += len(rel & got); tot += len(rel)
return hit / tot
# ---------------- stage 1: BM25F ----------------
WORD = re.compile(r"[a-z0-9]+")
STOP = set("a an and are as at be by for from has have in is it its of on or that the this to was were with".split())
def words(s): return [w for w in WORD.findall(s.lower()) if w not in STOP]
vocab = {}
def ids(ws): return [vocab.setdefault(w, len(vocab)) for w in ws]
tw = [Counter(ids(words(t))) for t in titles]
bw = [Counter(ids(words(b))) for b in bodies]
V = len(vocab)
def field_matrix(counts):
rows, cols, vals = [], [], []
for i, c in enumerate(counts):
for j, v in c.items(): rows.append(i); cols.append(j); vals.append(v)
M = csr_matrix((vals, (rows, cols)), shape=(N, V), dtype=np.float32)
lens = np.asarray(M.sum(1)).ravel()
return M, lens
Mt, lt = field_matrix(tw); Mb, lb = field_matrix(bw)
def norm(M, lens):
d = 1 - args.bm25_b + args.bm25_b * lens / max(lens.mean(), 1e-6)
return csr_matrix(M.multiply(1 / d[:, None]))
TF = args.title_w * norm(Mt, lt) + norm(Mb, lb) # BM25F pseudo term frequency
TF.data = TF.data / (args.bm25_k1 + TF.data) # saturation, once over the combined fields
df = np.bincount(TF.indices, minlength=V)
idf = np.log(1 + (N - df + 0.5) / (df + 0.5)).astype(np.float32)
def bm25f_search(q, k):
qi = [vocab[w] for w in set(words(q)) if w in vocab]
if not qi: return list(range(k))
s = np.asarray(TF[:, qi] @ idf[qi]).ravel()
return list(np.argsort(-s)[:k])
t0 = time.time()
stage1 = [bm25f_search(q, args.k1) for _, q in qs]
print(f"\n[1] BM25F alone nDCG@10 {ndcg10(stage1):.4f} | recall@{args.k1} {recall_at(stage1, args.k1):.3f} ({time.time()-t0:.1f}s)", flush=True)
# ---------------- dense encoders ----------------
from sentence_transformers import SentenceTransformer, CrossEncoder
bge = SentenceTransformer("BAAI/bge-small-en-v1.5", device=dev).half()
BGE_Q = "Represent this sentence for searching relevant passages: " # bge's recommended query prefix
t0 = time.time()
D_bge = bge.encode(full, batch_size=64, convert_to_tensor=True, normalize_embeddings=True).float()
Q_bge = bge.encode([BGE_Q + q for _, q in qs], batch_size=64, convert_to_tensor=True, normalize_embeddings=True).float()
print(f" (bge-small encoded corpus in {time.time()-t0:.0f}s)", flush=True)
def static_encoder(lbit_or_orig):
"""Mean-pooled static table (micro700kb original or a LittleBit .lbit)."""
from tokenizers import Tokenizer
import importlib.util
spec = importlib.util.spec_from_file_location("lbdec", os.path.join(HERE, "lbit_decode.py"))
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
W = torch.tensor(m.decode(lbit_or_orig)).to(dev)
tok = Tokenizer.from_file(os.path.join(HERE, "orig", "tokenizer.json"))
def enc(texts):
flat, offs = [], []
for e in tok.encode_batch(texts, add_special_tokens=False):
offs.append(len(flat)); flat.extend(e.ids if e.ids else [0])
return F.normalize(F.embedding_bag(torch.tensor(flat, device=dev), W, torch.tensor(offs, device=dev), mode="mean"), dim=-1)
return enc
def dense_full(Qe, De): # dense retrieval over the whole corpus (no BM25F)
return (Qe @ De.T).topk(10, dim=1).indices.cpu().tolist()
def dense_rerank(cands, Qe, De, k):
out = []
for qi, c in enumerate(cands):
c = torch.tensor(c, device=dev)
s = De[c] @ Qe[qi]
out.append(c[s.argsort(descending=True)[:k]].cpu().tolist())
return out
ce = CrossEncoder(f"cross-encoder/ettin-reranker-{args.reranker}-v1", device=dev)
def ce_rerank(cands):
t0 = time.time(); out = []
for (_, q), c in zip(qs, cands):
s = ce.predict([(q, full[d]) for d in c], batch_size=64, show_progress_bar=False)
out.append([c[i] for i in np.argsort(-s)])
return out, time.time() - t0
print(f"[2] bge-small alone nDCG@10 {ndcg10(dense_full(Q_bge, D_bge)):.4f}", flush=True)
s2 = dense_rerank(stage1, Q_bge, D_bge, args.k2)
print(f"[3] BM25F -> bge-small nDCG@10 {ndcg10(s2):.4f} | recall@{args.k2} {recall_at(s2, args.k2):.3f}", flush=True)
s3, sec = ce_rerank(s2)
print(f"[4] BM25F -> bge-small -> ettin-{args.reranker} nDCG@10 {ndcg10(s3):.4f} ({sec:.0f}s for {len(qs)} queries)", flush=True)
for name, src in [("micro700kb original", os.path.join(HERE, "orig", "model.safetensors")),
("micro LittleBit 164KB", os.path.join(HERE, "micro_lb_1.0.lbit"))]:
enc = static_encoder(src)
De, Qe = enc(full), enc([q for _, q in qs])
print(f"\n {name} alone nDCG@10 {ndcg10(dense_full(Qe, De)):.4f}", flush=True)
s2 = dense_rerank(stage1, Qe, De, args.k2)
print(f" BM25F -> {name} nDCG@10 {ndcg10(s2):.4f} | recall@{args.k2} {recall_at(s2, args.k2):.3f}", flush=True)
s3, sec = ce_rerank(s2)
print(f" BM25F -> {name} -> ettin-{args.reranker} nDCG@10 {ndcg10(s3):.4f}", flush=True)
s3, sec = ce_rerank([c[:args.k2] for c in stage1])
print(f"\n BM25F -> ettin-{args.reranker} (no dense stage) nDCG@10 {ndcg10(s3):.4f}", flush=True)