Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittlePotion-1BPW with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittlePotion-1BPW with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittlePotion-1BPW") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
Download benchmark.py from MisasAI/LittlePotion-1BPW: direct link, hf CLI and curl.
- Browser
- Download file 3.11 kB
-
https://huggingface.co/MisasAI/LittlePotion-1BPW/resolve/main/benchmark.py
- Command line
-
hf download hf://MisasAI/LittlePotion-1BPW/benchmark.py
-
curl -L -o benchmark.py https://huggingface.co/MisasAI/LittlePotion-1BPW/resolve/main/benchmark.py
3.11 kB
| """Benchmark LittlePotion. Downloads the test sets (and original potion-base-8M) from Hugging Face. | |
| python benchmark.py # both models | |
| python benchmark.py littlepotion-931kb.lbit # just one | |
| Tasks: STS-B and SICK-R test (sentence similarity, Spearman), SciFact test (search, nDCG@10). | |
| """ | |
| import math, os, sys, torch, torch.nn.functional as F | |
| from datasets import load_dataset | |
| from huggingface_hub import hf_hub_download | |
| from safetensors.torch import load_file | |
| from scipy.stats import spearmanr | |
| from tokenizers import Tokenizer | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| sys.path.insert(0, HERE) | |
| from littlepotion import decode, _DEFAULT_TOKENIZER | |
| import glob | |
| files = sys.argv[1:] or sorted(os.path.basename(f) for f in glob.glob(os.path.join(HERE, "*.lbit"))) | |
| stsb = load_dataset("sentence-transformers/stsb", split="test") | |
| sick = load_dataset("mteb/sickr-sts", split="test") | |
| corpus = load_dataset("mteb/scifact", "corpus", split="corpus") | |
| qtext = {str(x["_id"]): x["text"] for x in load_dataset("mteb/scifact", "queries", split="queries")} | |
| qrel = {} | |
| for r in load_dataset("mteb/scifact", "default", split="test"): | |
| if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"])) | |
| queries = [(k, qtext[k]) for k in qrel] | |
| docs = [(str(c["_id"]), (c.get("title") or "") + " " + c["text"]) for c in corpus] | |
| def scores(W, tok): | |
| def enc(texts): | |
| flat, offs = [], [] | |
| for e in tok.encode_batch(list(texts), add_special_tokens=False): | |
| offs.append(len(flat)); flat.extend(e.ids or [0]) | |
| return F.normalize(F.embedding_bag(torch.tensor(flat), W, torch.tensor(offs), mode="mean"), dim=-1) | |
| sts = lambda ds: spearmanr((enc(ds["sentence1"]) * enc(ds["sentence2"])).sum(1), ds["score"]).correlation | |
| top = (enc([q for _, q in queries]) @ enc([t for _, t in docs]).T).topk(10, dim=1).indices | |
| nd = sum(sum(1 / math.log2(k + 2) for k, di in enumerate(top[i].tolist()) if docs[di][0] in qrel[q]) / | |
| sum(1 / math.log2(k + 2) for k in range(min(len(qrel[q]), 10))) for i, (q, _) in enumerate(queries)) / len(queries) | |
| return sts(stsb), sts(sick), nd | |
| rows = [] | |
| from safetensors import safe_open | |
| def fmt_of(f): | |
| with safe_open(os.path.join(HERE, f), "pt") as s: return s.metadata()["format"] | |
| if any(fmt_of(f) == "littlebit-model2vec-v1" for f in files): # potion model: compare against the original | |
| W = load_file(hf_hub_download("minishlab/potion-base-8M", "model.safetensors"))["embeddings"].float() | |
| rows.append(("original potion-base-8M", scores(W, Tokenizer.from_file(os.path.join(HERE, "tokenizer.json"))))) | |
| for f in files: | |
| W, fmt = decode(os.path.join(HERE, f)) | |
| tp = os.path.join(HERE, _DEFAULT_TOKENIZER[fmt]) | |
| tok = Tokenizer.from_file(tp if os.path.exists(tp) else os.path.join(HERE, "tokenizer.json")) | |
| rows.append((f"{f} ({os.path.getsize(os.path.join(HERE, f)) / 1024:.0f} KB)", scores(torch.tensor(W), tok))) | |
| print(f"\n{'model':38s} {'STS-B':>7s} {'SICK-R':>7s} {'SciFact':>8s}") | |
| for name, r in rows: | |
| print(f"{name:38s} {r[0]:7.4f} {r[1]:7.4f} {r[2]:8.4f}") | |