data/synth/gen_evidence.py

"""Synthetic evidence-comparison SFT data with DETERMINISTIC verdict labels.

Pattern: "Claim: X {valueA}. Evidence: {source} {valueB}." -> verdict computed
by rules (equal/same -> supports; differ -> refutes; evidence silent -> not
enough information). Reasoning text cites the ACTUAL values from the claim and
the evidence, so the model MUST condition on content (the TinyStories lever
applied to forensic judgment).

Each case returns (claim, evidence, verdict, reason) so the reasoning text is
always generated from the same values the verdict was computed from.
"""
import json, random
from pathlib import Path

rng = random.Random(20260803)
OUT = Path("data/synth_evidence_v1.jsonl")

YEARS = list(range(1998, 2026))
NAMES = ["bridge", "hospital", "school", "plant", "mall",
         "courthouse", "stadium", "airport", "tunnel", "tower"]
SOURCES = ["assessor record", "building permit", "filing", "inspection report",
           "city registry", "maintenance log", "audit", "police report",
           "warranty record", "insurance claim"]
VERBS = ["was built in", "was renovated in", "was inspected in", "was opened in",
         "was closed in", "was registered in", "was last serviced in", "was painted in"]
PCTS = list(range(1, 96, 3))
GROUPS = ["crime", "spending", "enrollment", "revenue", "attendance", "emissions",
          "incidents", "complaints", "cost", "output"]
INTROS = ["Compare claim against evidence.",
          "Compare the claim to the record.",
          "Weigh the claim against the evidence.",
          "Check the claim against the record."]


def verb_word():
    return rng.choice(["rose", "fell", "jumped", "dropped", "increased", "declined"])


def year_case():
    name = rng.choice(NAMES)
    verb = rng.choice(VERBS)
    src = rng.choice(SOURCES)
    a = rng.choice(YEARS)
    b = a if rng.random() < 0.5 else rng.choice([y for y in YEARS if y != a])
    claim = f"Verify: 'The {name} {verb} {a}.'"
    ev = f"{src.capitalize()}: {verb} {b}."
    if a == b:
        verdict, reason = "supports", (
            f"the claim says the {name} {verb} {a} and the {src} records the same year {b}, "
            f"so the dates agree and the claim is directly supported by the evidence")
    else:
        verdict, reason = "refutes", (
            f"the claim says the {name} {verb} {a} but the {src} records {b}, "
            f"so the dates conflict and the claim is refuted by the evidence")
    return claim, ev, verdict, reason


def pct_case():
    group = rng.choice(GROUPS)
    src = rng.choice(SOURCES)
    verb = verb_word()
    a = rng.choice(PCTS)
    b = a if rng.random() < 0.5 else rng.choice([p for p in PCTS if p != a])
    claim = f"Evaluate: '{group.capitalize()} {verb} {a}% last year.'"
    ev = f"The {src} shows a {b}% change."
    if a == b:
        verdict, reason = "supports", (
            f"the claim reports {group} {verb} by {a}% and the {src} confirms a {b}% change, "
            f"so the figures agree and the claim is supported")
    else:
        verdict, reason = "refutes", (
            f"the claim reports {group} {verb} by {a}% but the {src} shows {b}%, "
            f"so the figures conflict and the claim is refuted")
    return claim, ev, verdict, reason


def count_case():
    item = rng.choice(["awards", "violations", "visitors", "complaints", "projects",
                       "tests", "citations", "failures", "upgrades", "repairs"])
    who = rng.choice(["the department", "the company", "the city", "the agency", "the team"])
    src = rng.choice(SOURCES)
    a = rng.randint(3, 900) * 5
    b = a if rng.random() < 0.5 else max(0, a + rng.randint(-4, 4) * 5)
    claim = f"Check: '{who.capitalize()} reported {a} {item} last year.'"
    ev = f"Per the {src}, the count was {b}."
    if a == b:
        verdict, reason = "supports", (
            f"the claim reports {a} {item} and the {src} count is exactly {b}, "
            f"so the figures match and the claim is supported")
    else:
        verdict, reason = "refutes", (
            f"the claim reports {a} {item} but the {src} count is {b}, "
            f"so the figures differ and the claim is refuted")
    return claim, ev, verdict, reason


def time_case():
    event = rng.choice(["the meeting", "the inspection", "the hearing", "the delivery", "the arrival"])
    h1, m1 = rng.randint(8, 17), rng.choice([0, 15, 30, 45])
    h2, m2 = rng.randint(8, 17), rng.choice([0, 15, 30, 45])
    t1, t2 = at(h1, m1), at(h2, m2)
    hh1, hh2 = f"{h1:02d}:{m1:02d}", f"{h2:02d}:{m2:02d}"
    claim = f"Account A: '{event.capitalize()} {t1}.' Account B: '{event.capitalize()} {t2}.'"
    ev = "Two accounts describe the same event; only the stated time differs."
    if hh1 == hh2:
        verdict, reason = "supports", (
            f"Account A and Account B both say the event started at {hh1}, "
            f"so the accounts agree and there is no discrepancy")
    else:
        verdict, reason = "refutes", (
            f"Account A says the event started at {hh1} but Account B says {hh2}, "
            f"so the accounts conflict on the time")
    return claim, ev, verdict, reason


def missing_case():
    name = rng.choice(NAMES)
    verb = rng.choice(VERBS)
    src = rng.choice(SOURCES)
    a = rng.choice(YEARS)
    claim = f"Verify: 'The {name} {verb} {a}.'"
    ev = f"The {src} mentions the {name} but does not record a date for it."
    verdict, reason = "not enough information", (
        f"the {src} does not record a date for the {name}, "
        f"so the claim can be neither confirmed nor refuted from this evidence")
    return claim, ev, verdict, reason


def at(h, m):
    return f"started at {h}:{m:02d}"


def gen(n=4000):
    out = []
    cases = {"year": year_case, "pct": pct_case, "count": count_case,
             "time": time_case, "missing": missing_case}
    for i in range(n):
        kind = rng.choice(list(cases))
        claim, ev, verdict, reason = cases[kind]()
        conf = rng.choice(["HIGH", "MEDIUM"]) if verdict == "supports" else (
            rng.choice(["HIGH", "MEDIUM", "LOW"]) if verdict == "refutes" else "LOW")
        intro = rng.choice(INTROS)
        asst = (f"<|scratchpad|>{intro} {reason}. "
                f"<|final|>Verdict: {verdict}. Confidence: {conf}. "
                f"Reasoning: {reason}.")
        out.append({"persona": "analyst", "user": f"{claim} {ev}",
                    "assistant": asst, "synth": True, "kind": kind})
    rng.shuffle(out)
    with open(OUT, "w", encoding="utf-8") as f:
        for o in out:
            f.write(json.dumps(o, ensure_ascii=False) + "\n")
    print(f"wrote {len(out)} examples -> {OUT}", flush=True)


if __name__ == "__main__":
    gen()