File size: 2,921 Bytes
92f6dce
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
"""Convert a {query, positive, hard_negatives[], paraphrases[]} jsonl into the 4-file format
distill_stages.py reads (queries_/passages_/triplets_/paraphrases_*.jsonl with train/validation/test splits).

  python convert_generated.py <input.jsonl> <out_dir>
"""
import json, os, random, sys

src, out = sys.argv[1], sys.argv[2]
os.makedirs(out, exist_ok=True)

def fix(s):
    """Repair UTF-8 text that was decoded as cp1252/latin-1 (e.g. '“' -> '“')."""
    if "â" not in s and "Ã" not in s:
        return s
    for enc in ("cp1252", "latin-1"):
        try:
            t = s.encode(enc).decode("utf8")
            if "â" not in t and "Ã" not in t:
                return t
        except (UnicodeEncodeError, UnicodeDecodeError):
            pass
    return s

rows = [json.loads(l) for l in open(src, encoding="utf8") if l.strip()]
fixed = 0
for r in rows:
    for k in ("query", "positive"):
        f = fix(r[k]); fixed += f != r[k]; r[k] = f
    for k in ("hard_negatives", "paraphrases"):
        new = [fix(x) for x in r.get(k, []) if isinstance(x, str) and x.strip()]
        fixed += sum(a != b for a, b in zip(new, r.get(k, [])))
        r[k] = new

random.Random(0).shuffle(rows)
n = len(rows); n_val = n_test = max(1, n // 20)
split_of = lambda i: "validation" if i < n_val else "test" if i < n_val + n_test else "train"

passages, queries, triplets, paras, pid_of = [], [], [], [], {}
def pid(text, split):
    if text not in pid_of:
        pid_of[text] = f"p{len(pid_of):06d}"
        passages.append({"passage_id": pid_of[text], "text": text, "split": split})
    return pid_of[text]

for i, r in enumerate(rows):
    s, qid = split_of(i), f"q{i:05d}"
    pos = pid(r["positive"], s)
    negs = [pid(x, s) for x in r["hard_negatives"] if x != r["positive"]]
    queries.append({"query_id": qid, "text": r["query"], "positive_passage_id": pos, "split": s,
                    "candidate_negative_passage_id": negs[0] if negs else None})
    for ng in negs:
        triplets.append({"query_id": qid, "anchor": r["query"], "positive_passage_id": pos,
                         "negative_passage_id": ng, "split": s})
    for j, ptxt in enumerate(r["paraphrases"]):
        paras.append({"pair_id": f"{qid}_{j}", "query_id": qid, "text_a": r["query"], "text_b": ptxt, "split": s})

for name, data in [("queries", queries), ("passages", passages), ("triplets", triplets), ("paraphrases", paras)]:
    with open(os.path.join(out, f"{name}_{n}.jsonl"), "w", encoding="utf8") as f:
        for x in data:
            f.write(json.dumps(x, ensure_ascii=False) + "\n")
cnt = lambda xs, s: sum(x["split"] == s for x in xs)
print(f"{n} rows -> {len(queries)} queries (train {cnt(queries,'train')} / val {cnt(queries,'validation')} / test {cnt(queries,'test')}), "
      f"{len(passages)} passages, {len(triplets)} triplets, {len(paras)} paraphrase pairs; repaired {fixed} mis-encoded strings -> {out}")