Sentence Similarity
Model2Vec
English
embeddings
static-embeddings
feature-extraction
quantization
1-bit
littlebit
Instructions to use MisasAI/LittleLittlePotion-164kb with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Model2Vec
How to use MisasAI/LittleLittlePotion-164kb with Model2Vec:
from model2vec import StaticModel model = StaticModel.from_pretrained("MisasAI/LittleLittlePotion-164kb") embeddings = model.encode(["It's dangerous to go alone!", "It's a secret to everybody."]) print(embeddings.shape) - Notebooks
- Google Colab
- Kaggle
Download scripts/convert_generated.py from MisasAI/LittleLittlePotion-164kb: direct link, hf CLI and curl.
- Browser
- Download file 2.92 kB
-
https://huggingface.co/MisasAI/LittleLittlePotion-164kb/resolve/main/scripts/convert_generated.py
- Command line
-
hf download hf://MisasAI/LittleLittlePotion-164kb/scripts/convert_generated.py
-
curl -L -o convert_generated.py https://huggingface.co/MisasAI/LittleLittlePotion-164kb/resolve/main/scripts/convert_generated.py
2.92 kB
| """Convert a {query, positive, hard_negatives[], paraphrases[]} jsonl into the 4-file format | |
| distill_stages.py reads (queries_/passages_/triplets_/paraphrases_*.jsonl with train/validation/test splits). | |
| python convert_generated.py <input.jsonl> <out_dir> | |
| """ | |
| import json, os, random, sys | |
| src, out = sys.argv[1], sys.argv[2] | |
| os.makedirs(out, exist_ok=True) | |
| def fix(s): | |
| """Repair UTF-8 text that was decoded as cp1252/latin-1 (e.g. '“' -> '“').""" | |
| if "â" not in s and "Ã" not in s: | |
| return s | |
| for enc in ("cp1252", "latin-1"): | |
| try: | |
| t = s.encode(enc).decode("utf8") | |
| if "â" not in t and "Ã" not in t: | |
| return t | |
| except (UnicodeEncodeError, UnicodeDecodeError): | |
| pass | |
| return s | |
| rows = [json.loads(l) for l in open(src, encoding="utf8") if l.strip()] | |
| fixed = 0 | |
| for r in rows: | |
| for k in ("query", "positive"): | |
| f = fix(r[k]); fixed += f != r[k]; r[k] = f | |
| for k in ("hard_negatives", "paraphrases"): | |
| new = [fix(x) for x in r.get(k, []) if isinstance(x, str) and x.strip()] | |
| fixed += sum(a != b for a, b in zip(new, r.get(k, []))) | |
| r[k] = new | |
| random.Random(0).shuffle(rows) | |
| n = len(rows); n_val = n_test = max(1, n // 20) | |
| split_of = lambda i: "validation" if i < n_val else "test" if i < n_val + n_test else "train" | |
| passages, queries, triplets, paras, pid_of = [], [], [], [], {} | |
| def pid(text, split): | |
| if text not in pid_of: | |
| pid_of[text] = f"p{len(pid_of):06d}" | |
| passages.append({"passage_id": pid_of[text], "text": text, "split": split}) | |
| return pid_of[text] | |
| for i, r in enumerate(rows): | |
| s, qid = split_of(i), f"q{i:05d}" | |
| pos = pid(r["positive"], s) | |
| negs = [pid(x, s) for x in r["hard_negatives"] if x != r["positive"]] | |
| queries.append({"query_id": qid, "text": r["query"], "positive_passage_id": pos, "split": s, | |
| "candidate_negative_passage_id": negs[0] if negs else None}) | |
| for ng in negs: | |
| triplets.append({"query_id": qid, "anchor": r["query"], "positive_passage_id": pos, | |
| "negative_passage_id": ng, "split": s}) | |
| for j, ptxt in enumerate(r["paraphrases"]): | |
| paras.append({"pair_id": f"{qid}_{j}", "query_id": qid, "text_a": r["query"], "text_b": ptxt, "split": s}) | |
| for name, data in [("queries", queries), ("passages", passages), ("triplets", triplets), ("paraphrases", paras)]: | |
| with open(os.path.join(out, f"{name}_{n}.jsonl"), "w", encoding="utf8") as f: | |
| for x in data: | |
| f.write(json.dumps(x, ensure_ascii=False) + "\n") | |
| cnt = lambda xs, s: sum(x["split"] == s for x in xs) | |
| print(f"{n} rows -> {len(queries)} queries (train {cnt(queries,'train')} / val {cnt(queries,'validation')} / test {cnt(queries,'test')}), " | |
| f"{len(passages)} passages, {len(triplets)} triplets, {len(paras)} paraphrase pairs; repaired {fixed} mis-encoded strings -> {out}") | |