File size: 6,695 Bytes
507d891 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """Synthetic evidence-comparison SFT data with DETERMINISTIC verdict labels.
Pattern: "Claim: X {valueA}. Evidence: {source} {valueB}." -> verdict computed
by rules (equal/same -> supports; differ -> refutes; evidence silent -> not
enough information). Reasoning text cites the ACTUAL values from the claim and
the evidence, so the model MUST condition on content (the TinyStories lever
applied to forensic judgment).
Each case returns (claim, evidence, verdict, reason) so the reasoning text is
always generated from the same values the verdict was computed from.
"""
import json, random
from pathlib import Path
rng = random.Random(20260803)
OUT = Path("data/synth_evidence_v1.jsonl")
YEARS = list(range(1998, 2026))
NAMES = ["bridge", "hospital", "school", "plant", "mall",
"courthouse", "stadium", "airport", "tunnel", "tower"]
SOURCES = ["assessor record", "building permit", "filing", "inspection report",
"city registry", "maintenance log", "audit", "police report",
"warranty record", "insurance claim"]
VERBS = ["was built in", "was renovated in", "was inspected in", "was opened in",
"was closed in", "was registered in", "was last serviced in", "was painted in"]
PCTS = list(range(1, 96, 3))
GROUPS = ["crime", "spending", "enrollment", "revenue", "attendance", "emissions",
"incidents", "complaints", "cost", "output"]
INTROS = ["Compare claim against evidence.",
"Compare the claim to the record.",
"Weigh the claim against the evidence.",
"Check the claim against the record."]
def verb_word():
return rng.choice(["rose", "fell", "jumped", "dropped", "increased", "declined"])
def year_case():
name = rng.choice(NAMES)
verb = rng.choice(VERBS)
src = rng.choice(SOURCES)
a = rng.choice(YEARS)
b = a if rng.random() < 0.5 else rng.choice([y for y in YEARS if y != a])
claim = f"Verify: 'The {name} {verb} {a}.'"
ev = f"{src.capitalize()}: {verb} {b}."
if a == b:
verdict, reason = "supports", (
f"the claim says the {name} {verb} {a} and the {src} records the same year {b}, "
f"so the dates agree and the claim is directly supported by the evidence")
else:
verdict, reason = "refutes", (
f"the claim says the {name} {verb} {a} but the {src} records {b}, "
f"so the dates conflict and the claim is refuted by the evidence")
return claim, ev, verdict, reason
def pct_case():
group = rng.choice(GROUPS)
src = rng.choice(SOURCES)
verb = verb_word()
a = rng.choice(PCTS)
b = a if rng.random() < 0.5 else rng.choice([p for p in PCTS if p != a])
claim = f"Evaluate: '{group.capitalize()} {verb} {a}% last year.'"
ev = f"The {src} shows a {b}% change."
if a == b:
verdict, reason = "supports", (
f"the claim reports {group} {verb} by {a}% and the {src} confirms a {b}% change, "
f"so the figures agree and the claim is supported")
else:
verdict, reason = "refutes", (
f"the claim reports {group} {verb} by {a}% but the {src} shows {b}%, "
f"so the figures conflict and the claim is refuted")
return claim, ev, verdict, reason
def count_case():
item = rng.choice(["awards", "violations", "visitors", "complaints", "projects",
"tests", "citations", "failures", "upgrades", "repairs"])
who = rng.choice(["the department", "the company", "the city", "the agency", "the team"])
src = rng.choice(SOURCES)
a = rng.randint(3, 900) * 5
b = a if rng.random() < 0.5 else max(0, a + rng.randint(-4, 4) * 5)
claim = f"Check: '{who.capitalize()} reported {a} {item} last year.'"
ev = f"Per the {src}, the count was {b}."
if a == b:
verdict, reason = "supports", (
f"the claim reports {a} {item} and the {src} count is exactly {b}, "
f"so the figures match and the claim is supported")
else:
verdict, reason = "refutes", (
f"the claim reports {a} {item} but the {src} count is {b}, "
f"so the figures differ and the claim is refuted")
return claim, ev, verdict, reason
def time_case():
event = rng.choice(["the meeting", "the inspection", "the hearing", "the delivery", "the arrival"])
h1, m1 = rng.randint(8, 17), rng.choice([0, 15, 30, 45])
h2, m2 = rng.randint(8, 17), rng.choice([0, 15, 30, 45])
t1, t2 = at(h1, m1), at(h2, m2)
hh1, hh2 = f"{h1:02d}:{m1:02d}", f"{h2:02d}:{m2:02d}"
claim = f"Account A: '{event.capitalize()} {t1}.' Account B: '{event.capitalize()} {t2}.'"
ev = "Two accounts describe the same event; only the stated time differs."
if hh1 == hh2:
verdict, reason = "supports", (
f"Account A and Account B both say the event started at {hh1}, "
f"so the accounts agree and there is no discrepancy")
else:
verdict, reason = "refutes", (
f"Account A says the event started at {hh1} but Account B says {hh2}, "
f"so the accounts conflict on the time")
return claim, ev, verdict, reason
def missing_case():
name = rng.choice(NAMES)
verb = rng.choice(VERBS)
src = rng.choice(SOURCES)
a = rng.choice(YEARS)
claim = f"Verify: 'The {name} {verb} {a}.'"
ev = f"The {src} mentions the {name} but does not record a date for it."
verdict, reason = "not enough information", (
f"the {src} does not record a date for the {name}, "
f"so the claim can be neither confirmed nor refuted from this evidence")
return claim, ev, verdict, reason
def at(h, m):
return f"started at {h}:{m:02d}"
def gen(n=4000):
out = []
cases = {"year": year_case, "pct": pct_case, "count": count_case,
"time": time_case, "missing": missing_case}
for i in range(n):
kind = rng.choice(list(cases))
claim, ev, verdict, reason = cases[kind]()
conf = rng.choice(["HIGH", "MEDIUM"]) if verdict == "supports" else (
rng.choice(["HIGH", "MEDIUM", "LOW"]) if verdict == "refutes" else "LOW")
intro = rng.choice(INTROS)
asst = (f"<|scratchpad|>{intro} {reason}. "
f"<|final|>Verdict: {verdict}. Confidence: {conf}. "
f"Reasoning: {reason}.")
out.append({"persona": "analyst", "user": f"{claim} {ev}",
"assistant": asst, "synth": True, "kind": kind})
rng.shuffle(out)
with open(OUT, "w", encoding="utf-8") as f:
for o in out:
f.write(json.dumps(o, ensure_ascii=False) + "\n")
print(f"wrote {len(out)} examples -> {OUT}", flush=True)
if __name__ == "__main__":
gen()
|