"""Convert a {query, positive, hard_negatives[], paraphrases[]} jsonl into the 4-file format distill_stages.py reads (queries_/passages_/triplets_/paraphrases_*.jsonl with train/validation/test splits). python convert_generated.py """ import json, os, random, sys src, out = sys.argv[1], sys.argv[2] os.makedirs(out, exist_ok=True) def fix(s): """Repair UTF-8 text that was decoded as cp1252/latin-1 (e.g. '“' -> '“').""" if "â" not in s and "Ã" not in s: return s for enc in ("cp1252", "latin-1"): try: t = s.encode(enc).decode("utf8") if "â" not in t and "Ã" not in t: return t except (UnicodeEncodeError, UnicodeDecodeError): pass return s rows = [json.loads(l) for l in open(src, encoding="utf8") if l.strip()] fixed = 0 for r in rows: for k in ("query", "positive"): f = fix(r[k]); fixed += f != r[k]; r[k] = f for k in ("hard_negatives", "paraphrases"): new = [fix(x) for x in r.get(k, []) if isinstance(x, str) and x.strip()] fixed += sum(a != b for a, b in zip(new, r.get(k, []))) r[k] = new random.Random(0).shuffle(rows) n = len(rows); n_val = n_test = max(1, n // 20) split_of = lambda i: "validation" if i < n_val else "test" if i < n_val + n_test else "train" passages, queries, triplets, paras, pid_of = [], [], [], [], {} def pid(text, split): if text not in pid_of: pid_of[text] = f"p{len(pid_of):06d}" passages.append({"passage_id": pid_of[text], "text": text, "split": split}) return pid_of[text] for i, r in enumerate(rows): s, qid = split_of(i), f"q{i:05d}" pos = pid(r["positive"], s) negs = [pid(x, s) for x in r["hard_negatives"] if x != r["positive"]] queries.append({"query_id": qid, "text": r["query"], "positive_passage_id": pos, "split": s, "candidate_negative_passage_id": negs[0] if negs else None}) for ng in negs: triplets.append({"query_id": qid, "anchor": r["query"], "positive_passage_id": pos, "negative_passage_id": ng, "split": s}) for j, ptxt in enumerate(r["paraphrases"]): paras.append({"pair_id": f"{qid}_{j}", "query_id": qid, "text_a": r["query"], "text_b": ptxt, "split": s}) for name, data in [("queries", queries), ("passages", passages), ("triplets", triplets), ("paraphrases", paras)]: with open(os.path.join(out, f"{name}_{n}.jsonl"), "w", encoding="utf8") as f: for x in data: f.write(json.dumps(x, ensure_ascii=False) + "\n") cnt = lambda xs, s: sum(x["split"] == s for x in xs) print(f"{n} rows -> {len(queries)} queries (train {cnt(queries,'train')} / val {cnt(queries,'validation')} / test {cnt(queries,'test')}), " f"{len(passages)} passages, {len(triplets)} triplets, {len(paras)} paraphrase pairs; repaired {fixed} mis-encoded strings -> {out}")