File size: 1,383 Bytes
1c0d385 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | """Teacher-distillation dataset builder.
Merges hand-written teacher knowledge files (data/distill_*.jsonl) with the
existing forensic SFT set into data/sft_distill.jsonl, deduplicated by user
text. Each entry: {"persona": "analyst"|"skeptic", "user": ..., "assistant": ...}
Usage:
.venv/bin/python data/distill.py
"""
import hashlib
import json
import random
from pathlib import Path
HERE = Path(__file__).parent
OUT = HERE / "sft_distill.jsonl"
def main():
random.seed(13)
seen, examples = set(), []
sources = sorted(HERE.glob("distill_*.jsonl")) + [HERE / "seed_forensic.jsonl"]
for src in sources:
if not src.exists():
continue
for line in src.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
ex = json.loads(line)
h = hashlib.sha256(ex["user"].encode()).hexdigest()
if h in seen:
continue
seen.add(h)
examples.append(ex)
random.shuffle(examples)
with open(OUT, "w", encoding="utf-8") as f:
for ex in examples:
f.write(json.dumps(ex) + "\n")
n_p = {}
for ex in examples:
n_p[ex["persona"]] = n_p.get(ex["persona"], 0) + 1
print(f"wrote {len(examples)} examples -> {OUT} personas={n_p}")
if __name__ == "__main__":
main()
|