data/distill.py

"""Teacher-distillation dataset builder.

Merges hand-written teacher knowledge files (data/distill_*.jsonl) with the
existing forensic SFT set into data/sft_distill.jsonl, deduplicated by user
text. Each entry: {"persona": "analyst"|"skeptic", "user": ..., "assistant": ...}

Usage:
  .venv/bin/python data/distill.py
"""

import hashlib
import json
import random
from pathlib import Path

HERE = Path(__file__).parent
OUT = HERE / "sft_distill.jsonl"


def main():
    random.seed(13)
    seen, examples = set(), []
    sources = sorted(HERE.glob("distill_*.jsonl")) + [HERE / "seed_forensic.jsonl"]
    for src in sources:
        if not src.exists():
            continue
        for line in src.read_text(encoding="utf-8").splitlines():
            line = line.strip()
            if not line:
                continue
            ex = json.loads(line)
            h = hashlib.sha256(ex["user"].encode()).hexdigest()
            if h in seen:
                continue
            seen.add(h)
            examples.append(ex)
    random.shuffle(examples)
    with open(OUT, "w", encoding="utf-8") as f:
        for ex in examples:
            f.write(json.dumps(ex) + "\n")
    n_p = {}
    for ex in examples:
        n_p[ex["persona"]] = n_p.get(ex["persona"], 0) + 1
    print(f"wrote {len(examples)} examples -> {OUT}  personas={n_p}")


if __name__ == "__main__":
    main()