LittlePotion-1BPW / benchmark.py
Icodetools's picture
931 KB model only; 164 KB moved to LittleLittlePotion-164kb
9a5f6d7 verified
Raw History Blame Contribute Delete
3.11 kB
"""Benchmark LittlePotion. Downloads the test sets (and original potion-base-8M) from Hugging Face.
python benchmark.py # both models
python benchmark.py littlepotion-931kb.lbit # just one
Tasks: STS-B and SICK-R test (sentence similarity, Spearman), SciFact test (search, nDCG@10).
"""
import math, os, sys, torch, torch.nn.functional as F
from datasets import load_dataset
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file
from scipy.stats import spearmanr
from tokenizers import Tokenizer
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from littlepotion import decode, _DEFAULT_TOKENIZER
import glob
files = sys.argv[1:] or sorted(os.path.basename(f) for f in glob.glob(os.path.join(HERE, "*.lbit")))
stsb = load_dataset("sentence-transformers/stsb", split="test")
sick = load_dataset("mteb/sickr-sts", split="test")
corpus = load_dataset("mteb/scifact", "corpus", split="corpus")
qtext = {str(x["_id"]): x["text"] for x in load_dataset("mteb/scifact", "queries", split="queries")}
qrel = {}
for r in load_dataset("mteb/scifact", "default", split="test"):
if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"]))
queries = [(k, qtext[k]) for k in qrel]
docs = [(str(c["_id"]), (c.get("title") or "") + " " + c["text"]) for c in corpus]
def scores(W, tok):
def enc(texts):
flat, offs = [], []
for e in tok.encode_batch(list(texts), add_special_tokens=False):
offs.append(len(flat)); flat.extend(e.ids or [0])
return F.normalize(F.embedding_bag(torch.tensor(flat), W, torch.tensor(offs), mode="mean"), dim=-1)
sts = lambda ds: spearmanr((enc(ds["sentence1"]) * enc(ds["sentence2"])).sum(1), ds["score"]).correlation
top = (enc([q for _, q in queries]) @ enc([t for _, t in docs]).T).topk(10, dim=1).indices
nd = sum(sum(1 / math.log2(k + 2) for k, di in enumerate(top[i].tolist()) if docs[di][0] in qrel[q]) /
sum(1 / math.log2(k + 2) for k in range(min(len(qrel[q]), 10))) for i, (q, _) in enumerate(queries)) / len(queries)
return sts(stsb), sts(sick), nd
rows = []
from safetensors import safe_open
def fmt_of(f):
with safe_open(os.path.join(HERE, f), "pt") as s: return s.metadata()["format"]
if any(fmt_of(f) == "littlebit-model2vec-v1" for f in files): # potion model: compare against the original
W = load_file(hf_hub_download("minishlab/potion-base-8M", "model.safetensors"))["embeddings"].float()
rows.append(("original potion-base-8M", scores(W, Tokenizer.from_file(os.path.join(HERE, "tokenizer.json")))))
for f in files:
W, fmt = decode(os.path.join(HERE, f))
tp = os.path.join(HERE, _DEFAULT_TOKENIZER[fmt])
tok = Tokenizer.from_file(tp if os.path.exists(tp) else os.path.join(HERE, "tokenizer.json"))
rows.append((f"{f} ({os.path.getsize(os.path.join(HERE, f)) / 1024:.0f} KB)", scores(torch.tensor(W), tok)))
print(f"\n{'model':38s} {'STS-B':>7s} {'SICK-R':>7s} {'SciFact':>8s}")
for name, r in rows:
print(f"{name:38s} {r[0]:7.4f} {r[1]:7.4f} {r[2]:8.4f}")