File size: 2,113 Bytes
1c0d385 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Build evidence-in-context judgment dataset: claim + evidence -> verdict.
Dedupe across all SFT corpora; keeps rows where the user text contains both a
claim and a separate evidence marker, with a parseable Verdict label."""
import json, re
from collections import Counter
from pathlib import Path
EVID = re.compile(r"records? show|permit|account [ab]|accounts? [ab]:|according to|filing|inspection|report[sed]? (by|that|the)|data show|documents|registered|confirmed|assessor|timeline|witness|source|wire|blog|notes? that|second investigation|reassessment|raised? [0-9]|rose [0-9]|increas[ed]? [0-9]", re.I)
VALS = ["true statement", "false statement", "supports", "refutes", "not_enough_info"]
def parse_v(text):
m = re.search(r"Verdict\s*:\s*([^.]+)\.", text, re.I)
if not m: return None
lab = m.group(1).strip().lower()
for cand in VALS:
if lab.startswith(cand): return cand
if lab.startswith("true"): return "true statement"
if lab.startswith("false"): return "false statement"
if lab.startswith("not"): return "not_enough_info"
if lab.startswith("support"): return "supports"
if lab.startswith("refut"): return "refutes"
return None
def load(p):
return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()]
files = ["sft_forensic.jsonl","sft_sop_mix.jsonl","sft_sop.jsonl","sft_distill_mix.jsonl",
"distill_method.jsonl","distill_analyst_a.jsonl","distill_analyst_b.jsonl",
"sft_mix_v2.jsonl","sft_mix_v3.jsonl"]
seen, out = set(), []
for f in files:
for r in load(f"data/{f}"):
u, a = r.get("user", ""), r.get("assistant", "")
v = parse_v(a)
if not v or not EVID.search(u):
continue
key = u[:200]
if key in seen:
continue
seen.add(key)
out.append({"persona": r.get("persona", "analyst"), "user": u, "verdict": v})
print("total", len(out), dict(Counter(o["verdict"] for o in out)), flush=True)
with open("data/evidence_judge.jsonl", "w", encoding="utf-8") as f:
for o in out:
f.write(json.dumps(o, ensure_ascii=False) + "\n")
|