LittleLittlePotion-164kb / scripts /convert_generated.py
Icodetools's picture
LittleLittlePotion-164kb: 1 BPW Model2Vec embeddings in 164 KB
92f6dce verified
Raw History Blame Contribute Delete
2.92 kB
"""Convert a {query, positive, hard_negatives[], paraphrases[]} jsonl into the 4-file format
distill_stages.py reads (queries_/passages_/triplets_/paraphrases_*.jsonl with train/validation/test splits).
python convert_generated.py <input.jsonl> <out_dir>
"""
import json, os, random, sys
src, out = sys.argv[1], sys.argv[2]
os.makedirs(out, exist_ok=True)
def fix(s):
"""Repair UTF-8 text that was decoded as cp1252/latin-1 (e.g. '“' -> '“')."""
if "â" not in s and "Ã" not in s:
return s
for enc in ("cp1252", "latin-1"):
try:
t = s.encode(enc).decode("utf8")
if "â" not in t and "Ã" not in t:
return t
except (UnicodeEncodeError, UnicodeDecodeError):
pass
return s
rows = [json.loads(l) for l in open(src, encoding="utf8") if l.strip()]
fixed = 0
for r in rows:
for k in ("query", "positive"):
f = fix(r[k]); fixed += f != r[k]; r[k] = f
for k in ("hard_negatives", "paraphrases"):
new = [fix(x) for x in r.get(k, []) if isinstance(x, str) and x.strip()]
fixed += sum(a != b for a, b in zip(new, r.get(k, [])))
r[k] = new
random.Random(0).shuffle(rows)
n = len(rows); n_val = n_test = max(1, n // 20)
split_of = lambda i: "validation" if i < n_val else "test" if i < n_val + n_test else "train"
passages, queries, triplets, paras, pid_of = [], [], [], [], {}
def pid(text, split):
if text not in pid_of:
pid_of[text] = f"p{len(pid_of):06d}"
passages.append({"passage_id": pid_of[text], "text": text, "split": split})
return pid_of[text]
for i, r in enumerate(rows):
s, qid = split_of(i), f"q{i:05d}"
pos = pid(r["positive"], s)
negs = [pid(x, s) for x in r["hard_negatives"] if x != r["positive"]]
queries.append({"query_id": qid, "text": r["query"], "positive_passage_id": pos, "split": s,
"candidate_negative_passage_id": negs[0] if negs else None})
for ng in negs:
triplets.append({"query_id": qid, "anchor": r["query"], "positive_passage_id": pos,
"negative_passage_id": ng, "split": s})
for j, ptxt in enumerate(r["paraphrases"]):
paras.append({"pair_id": f"{qid}_{j}", "query_id": qid, "text_a": r["query"], "text_b": ptxt, "split": s})
for name, data in [("queries", queries), ("passages", passages), ("triplets", triplets), ("paraphrases", paras)]:
with open(os.path.join(out, f"{name}_{n}.jsonl"), "w", encoding="utf8") as f:
for x in data:
f.write(json.dumps(x, ensure_ascii=False) + "\n")
cnt = lambda xs, s: sum(x["split"] == s for x in xs)
print(f"{n} rows -> {len(queries)} queries (train {cnt(queries,'train')} / val {cnt(queries,'validation')} / test {cnt(queries,'test')}), "
f"{len(passages)} passages, {len(triplets)} triplets, {len(paras)} paraphrase pairs; repaired {fixed} mis-encoded strings -> {out}")