data/expand_v5.py

"""Expand v4 mix into v5: rephrase forensic/SOP users (same verified answers)
to multiply exposure to domain vocabulary, keep raw replay for fluency.

Output: data/sft_mix_v5.jsonl
"""
import json, random
from collections import Counter
from pathlib import Path

rng = random.Random(20260802)
OUT = Path("data/sft_mix_v5.jsonl")

def load(p): return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()]

PREFIXES = [
    "Verify:", "Check whether:", "Assess the claim:", "Is this true?",
    "Analyze:", "Examine:", "Look into:", "Evaluate the evidence for:",
    "What is wrong with this claim?", "Check the facts behind:",
    "Determine whether this holds:", "Scrutinize:",
]
SUFFIXES = [
    " Consider the available evidence.",
    " Check the sources and dates.",
    " Assess the strength of the evidence.",
    " Note any gaps.",
    " Compare the claims involved.",
]

def rephrase(user, assistant):
    variants = []
    for _ in range(2):
        u2 = user.strip()
        prefix = rng.choice(PREFIXES)
        suffix = rng.choice(SUFFIXES)
        u2 = f"{prefix} {u2}{suffix}"
        variants.append({"persona": "analyst", "user": u2, "assistant": assistant})
    return variants

def main():
    v4 = load("data/sft_mix_v4.jsonl")
    out = []
    n_reph = 0
    for ex in v4:
        out.append(ex)
        u = ex.get("user", "")
        if "raw" not in ex and len(u) > 80 and ex.get("persona") in ("analyst", "skeptic"):
            for v in rephrase(u, ex["assistant"]):
                out.append(v)
                n_reph += 1
    # dedupe by (persona, user)
    seen, final = set(), []
    for ex in out:
        k = ("raw", ex.get("raw", "")[:180]) if "raw" in ex else (ex.get("persona", "?"), ex.get("user", "")[:180])
        if k in seen:
            continue
        seen.add(k); final.append(ex)
    rng.shuffle(final)
    with open(OUT, "w", encoding="utf-8") as f:
        for ex in final:
            f.write(json.dumps(ex, ensure_ascii=False) + "\n")
    print("total", len(final), "rephrased_added", n_reph, dict(Counter(e.get("persona", "?") for e in final)))

if __name__ == "__main__":
    main()