Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittleLittlePotion-164kb with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittleLittlePotion-164kb with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittleLittlePotion-164kb") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
File size: 2,921 Bytes
92f6dce | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Convert a {query, positive, hard_negatives[], paraphrases[]} jsonl into the 4-file format
distill_stages.py reads (queries_/passages_/triplets_/paraphrases_*.jsonl with train/validation/test splits).
python convert_generated.py <input.jsonl> <out_dir>
"""
import json, os, random, sys
src, out = sys.argv[1], sys.argv[2]
os.makedirs(out, exist_ok=True)
def fix(s):
"""Repair UTF-8 text that was decoded as cp1252/latin-1 (e.g. '“' -> '“')."""
if "â" not in s and "Ã" not in s:
return s
for enc in ("cp1252", "latin-1"):
try:
t = s.encode(enc).decode("utf8")
if "â" not in t and "Ã" not in t:
return t
except (UnicodeEncodeError, UnicodeDecodeError):
pass
return s
rows = [json.loads(l) for l in open(src, encoding="utf8") if l.strip()]
fixed = 0
for r in rows:
for k in ("query", "positive"):
f = fix(r[k]); fixed += f != r[k]; r[k] = f
for k in ("hard_negatives", "paraphrases"):
new = [fix(x) for x in r.get(k, []) if isinstance(x, str) and x.strip()]
fixed += sum(a != b for a, b in zip(new, r.get(k, [])))
r[k] = new
random.Random(0).shuffle(rows)
n = len(rows); n_val = n_test = max(1, n // 20)
split_of = lambda i: "validation" if i < n_val else "test" if i < n_val + n_test else "train"
passages, queries, triplets, paras, pid_of = [], [], [], [], {}
def pid(text, split):
if text not in pid_of:
pid_of[text] = f"p{len(pid_of):06d}"
passages.append({"passage_id": pid_of[text], "text": text, "split": split})
return pid_of[text]
for i, r in enumerate(rows):
s, qid = split_of(i), f"q{i:05d}"
pos = pid(r["positive"], s)
negs = [pid(x, s) for x in r["hard_negatives"] if x != r["positive"]]
queries.append({"query_id": qid, "text": r["query"], "positive_passage_id": pos, "split": s,
"candidate_negative_passage_id": negs[0] if negs else None})
for ng in negs:
triplets.append({"query_id": qid, "anchor": r["query"], "positive_passage_id": pos,
"negative_passage_id": ng, "split": s})
for j, ptxt in enumerate(r["paraphrases"]):
paras.append({"pair_id": f"{qid}_{j}", "query_id": qid, "text_a": r["query"], "text_b": ptxt, "split": s})
for name, data in [("queries", queries), ("passages", passages), ("triplets", triplets), ("paraphrases", paras)]:
with open(os.path.join(out, f"{name}_{n}.jsonl"), "w", encoding="utf8") as f:
for x in data:
f.write(json.dumps(x, ensure_ascii=False) + "\n")
cnt = lambda xs, s: sum(x["split"] == s for x in xs)
print(f"{n} rows -> {len(queries)} queries (train {cnt(queries,'train')} / val {cnt(queries,'validation')} / test {cnt(queries,'test')}), "
f"{len(passages)} passages, {len(triplets)} triplets, {len(paras)} paraphrase pairs; repaired {fixed} mis-encoded strings -> {out}")
|