"""Benchmark LittlePotion. Downloads the test sets (and original potion-base-8M) from Hugging Face. python benchmark.py # both models python benchmark.py littlepotion-931kb.lbit # just one Tasks: STS-B and SICK-R test (sentence similarity, Spearman), SciFact test (search, nDCG@10). """ import math, os, sys, torch, torch.nn.functional as F from datasets import load_dataset from huggingface_hub import hf_hub_download from safetensors.torch import load_file from scipy.stats import spearmanr from tokenizers import Tokenizer HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) from littlepotion import decode, _DEFAULT_TOKENIZER import glob files = sys.argv[1:] or sorted(os.path.basename(f) for f in glob.glob(os.path.join(HERE, "*.lbit"))) stsb = load_dataset("sentence-transformers/stsb", split="test") sick = load_dataset("mteb/sickr-sts", split="test") corpus = load_dataset("mteb/scifact", "corpus", split="corpus") qtext = {str(x["_id"]): x["text"] for x in load_dataset("mteb/scifact", "queries", split="queries")} qrel = {} for r in load_dataset("mteb/scifact", "default", split="test"): if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"])) queries = [(k, qtext[k]) for k in qrel] docs = [(str(c["_id"]), (c.get("title") or "") + " " + c["text"]) for c in corpus] def scores(W, tok): def enc(texts): flat, offs = [], [] for e in tok.encode_batch(list(texts), add_special_tokens=False): offs.append(len(flat)); flat.extend(e.ids or [0]) return F.normalize(F.embedding_bag(torch.tensor(flat), W, torch.tensor(offs), mode="mean"), dim=-1) sts = lambda ds: spearmanr((enc(ds["sentence1"]) * enc(ds["sentence2"])).sum(1), ds["score"]).correlation top = (enc([q for _, q in queries]) @ enc([t for _, t in docs]).T).topk(10, dim=1).indices nd = sum(sum(1 / math.log2(k + 2) for k, di in enumerate(top[i].tolist()) if docs[di][0] in qrel[q]) / sum(1 / math.log2(k + 2) for k in range(min(len(qrel[q]), 10))) for i, (q, _) in enumerate(queries)) / len(queries) return sts(stsb), sts(sick), nd rows = [] from safetensors import safe_open def fmt_of(f): with safe_open(os.path.join(HERE, f), "pt") as s: return s.metadata()["format"] if any(fmt_of(f) == "littlebit-model2vec-v1" for f in files): # potion model: compare against the original W = load_file(hf_hub_download("minishlab/potion-base-8M", "model.safetensors"))["embeddings"].float() rows.append(("original potion-base-8M", scores(W, Tokenizer.from_file(os.path.join(HERE, "tokenizer.json"))))) for f in files: W, fmt = decode(os.path.join(HERE, f)) tp = os.path.join(HERE, _DEFAULT_TOKENIZER[fmt]) tok = Tokenizer.from_file(tp if os.path.exists(tp) else os.path.join(HERE, "tokenizer.json")) rows.append((f"{f} ({os.path.getsize(os.path.join(HERE, f)) / 1024:.0f} KB)", scores(torch.tensor(W), tok))) print(f"\n{'model':38s} {'STS-B':>7s} {'SICK-R':>7s} {'SciFact':>8s}") for name, r in rows: print(f"{name:38s} {r[0]:7.4f} {r[1]:7.4f} {r[2]:8.4f}")