| """Build evidence-in-context judgment dataset: claim + evidence -> verdict. |
| Dedupe across all SFT corpora; keeps rows where the user text contains both a |
| claim and a separate evidence marker, with a parseable Verdict label.""" |
| import json, re |
| from collections import Counter |
| from pathlib import Path |
|
|
| EVID = re.compile(r"records? show|permit|account [ab]|accounts? [ab]:|according to|filing|inspection|report[sed]? (by|that|the)|data show|documents|registered|confirmed|assessor|timeline|witness|source|wire|blog|notes? that|second investigation|reassessment|raised? [0-9]|rose [0-9]|increas[ed]? [0-9]", re.I) |
| VALS = ["true statement", "false statement", "supports", "refutes", "not_enough_info"] |
|
|
| def parse_v(text): |
| m = re.search(r"Verdict\s*:\s*([^.]+)\.", text, re.I) |
| if not m: return None |
| lab = m.group(1).strip().lower() |
| for cand in VALS: |
| if lab.startswith(cand): return cand |
| if lab.startswith("true"): return "true statement" |
| if lab.startswith("false"): return "false statement" |
| if lab.startswith("not"): return "not_enough_info" |
| if lab.startswith("support"): return "supports" |
| if lab.startswith("refut"): return "refutes" |
| return None |
|
|
| def load(p): |
| return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()] |
|
|
| files = ["sft_forensic.jsonl","sft_sop_mix.jsonl","sft_sop.jsonl","sft_distill_mix.jsonl", |
| "distill_method.jsonl","distill_analyst_a.jsonl","distill_analyst_b.jsonl", |
| "sft_mix_v2.jsonl","sft_mix_v3.jsonl"] |
| seen, out = set(), [] |
| for f in files: |
| for r in load(f"data/{f}"): |
| u, a = r.get("user", ""), r.get("assistant", "") |
| v = parse_v(a) |
| if not v or not EVID.search(u): |
| continue |
| key = u[:200] |
| if key in seen: |
| continue |
| seen.add(key) |
| out.append({"persona": r.get("persona", "analyst"), "user": u, "verdict": v}) |
| print("total", len(out), dict(Counter(o["verdict"] for o in out)), flush=True) |
| with open("data/evidence_judge.jsonl", "w", encoding="utf-8") as f: |
| for o in out: |
| f.write(json.dumps(o, ensure_ascii=False) + "\n") |
|
|
