"""Build the SFT dataset: teacher-written templates, sampled values, oracle-labelled replies. Every assistant turn is the oracle's reply, so labels are exact. Examples whose user text or request matches a turn in the evaluation set are dropped. Output is shuffled once, so the first N lines form nested subsets for the data-efficiency curve (250 within 500 within 1000 within 2000). Usage: python gen_train.py --n 2000 --seed 11 --out data/train_v1.jsonl """ import argparse import re import json import random from collections import Counter from dataclasses import replace from datetime import datetime, timedelta, timezone from pathlib import Path from zoneinfo import ZoneInfo from oracle import ABBREVIATIONS, ZONES, Request, evaluate ROOT = Path(__file__).resolve().parent SYSTEM = "You convert date-times between timezones under the tz-convert spec. Reply with exactly one line." SEASONAL = [k for k, v in ABBREVIATIONS["unambiguous"].items() if v["season"] != "all"] AMBIG_ABBR = list(ABBREVIATIONS["ambiguous"]) UNSUPPORTED_IANA = ["Europe/Berlin", "America/Toronto", "Asia/Singapore", "Asia/Hong_Kong", "Africa/Cairo", "America/Mexico_City", "Europe/Moscow", "Asia/Jakarta", "Africa/Lagos", "America/Bogota", "Europe/Madrid", "Asia/Manila"] UNLISTED_ABBR = ["MSK", "SGT", "HKT", "WIB", "SAST", "ART", "BRT", "PHT", "ICT", "CAT", "EAT", "AWST"] BAD_ZONE_FORMS = ["GMT+5", "UTC+5", "est", "Eastern", "Z", "+5:30", "UTC+05:60", "Tokyo time", "Pacific Time"] BAD_TIMES = ["24:00", "25:30", "12:60", "9:00", "3 PM", "1430", "12:00:00.5", "07:15:61", "noon"] def transitions(): """Local wall times just around every DST switch 2020-2030 for each DST-observing supported zone.""" out = [] for zone, fam in ZONES.items(): if fam == "none": continue zi, prev = ZoneInfo(zone), None t = datetime(2020, 1, 1, tzinfo=timezone.utc) while t.year <= 2030: off = t.astimezone(zi).utcoffset() if prev is not None and off != prev: out.append((zone, t - timedelta(hours=1))) prev, t = off, t + timedelta(hours=1) return out class Sampler: def __init__(self, rng, templates): self.rng, self.t = rng, templates self.trans = transitions() def pick(self, xs): return self.rng.choice(xs) def iso_date(self, year=None): y = year or self.rng.randint(2020, 2030) d = datetime(y, 1, 1) + timedelta(days=self.rng.randint(0, 364)) return d.strftime("%Y-%m-%d") def date(self): d = self.iso_date() if self.rng.random() < 0.15: # unambiguous NN/NN/YYYY: day > 12 or day == month y, m, day = d.split("-") if int(day) > 12: return f"{day}/{m}/{y}" if self.rng.random() < 0.5 else f"{m}/{day}/{y}" return d def time(self): t = f"{self.rng.randint(0, 23):02d}:{self.rng.randint(0, 59):02d}" return t + (f":{self.rng.randint(0, 59):02d}" if self.rng.random() < 0.2 else "") def offset(self): minutes = self.rng.choice([0, 0, 0, 30, 45]) hours = self.rng.randint(-12, 13) sign = "-" if hours < 0 else "+" text = f"{sign}{abs(hours):02d}:{minutes:02d}" return ("UTC" + text) if self.rng.random() < 0.5 else text def zone(self): r = self.rng.random() if r < 0.6: return self.pick(list(ZONES)) if r < 0.8: return self.pick(list(ABBREVIATIONS["unambiguous"])) if r < 0.93: return self.offset() return f"Etc/GMT{self.pick('+-')}{self.rng.randint(1, 12)}" def request(self): return Request(self.date(), self.time(), self.zone(), self.zone()) def near_transition(self): zone, utc = self.pick(self.trans) local = (utc + timedelta(minutes=self.rng.randint(0, 180))).astimezone(ZoneInfo(zone)) src = zone if self.rng.random() < 0.4: # use the zone's abbreviation, in or out of season src = self.pick([k for k, v in ABBREVIATIONS["unambiguous"].items() if v["zone"] == zone] or [zone]) return Request(local.strftime("%Y-%m-%d"), local.strftime("%H:%M"), src, self.zone()) def say(self, r): return self.pick(self.t["request"]).format(date=r.date, time=r.time, src=r.source, tgt=r.target) def convert_example(s): while True: r = s.near_transition() if s.rng.random() < 0.3 else s.request() if evaluate(r).startswith("20"): return [(s.say(r), r)] def ambiguous_example(s): kind = s.pick(["listed", "season", "season", "date", "unclear"]) r = s.request() if kind == "listed": r = replace(r, **{s.pick(["source", "target"]): s.pick(AMBIG_ABBR)}) elif kind == "season": abbr = s.pick(SEASONAL) for _ in range(50): r = replace(r, date=s.iso_date(), **{s.pick(["source", "target"]): abbr}) if evaluate(r).startswith("AMBIGUOUS"): break elif kind == "date": y, a = s.rng.randint(2020, 2030), s.rng.randint(1, 12) b = s.pick([x for x in range(1, 13) if x != a]) r = replace(r, date=f"{a:02d}/{b:02d}/{y}") else: r = replace(r, unclear=frozenset({"source_timezone", "target_timezone"})) return [(s.pick(s.t["unclear"]).format(date=r.date, time=r.time, z1=r.source, z2=r.target), r)] return [(s.say(r), r)] def invalid_example(s): r = s.request() kind = s.pick(["date", "date", "time", "time", "zone", "zone", "zone", "not_request"]) if kind == "date": y = s.rng.randint(2020, 2030) bad = s.pick([f"{y}-02-30", f"{y}-04-31", f"{y}-13-05", f"{s.pick([2019, 2031, 2018, 2035])}-06-15", f"{y}-6-5", f"13/13/{y}", f"31/04/{y}", f"05/06", f"{y}/06/15"]) r = replace(r, date=bad) elif kind == "time": r = replace(r, time=s.pick(BAD_TIMES)) elif kind == "zone": bad = s.pick([s.pick(UNSUPPORTED_IANA), s.pick(UNLISTED_ABBR), s.pick(BAD_ZONE_FORMS), s.pick(["+14:30", "-12:30", "UTC+15:00", "Etc/GMT+13"])]) r = replace(r, **{s.pick(["source", "target"]): bad}) else: return [(s.pick(s.t["not_request"]), Request(None, None, None, None, is_request=False))] return [(s.say(r), r)] def pressure_example(s): first = s.pick([ambiguous_example, convert_example, invalid_example])(s) text, r = first[0] suffix = s.pick(s.t["pressure"] if evaluate(r).startswith("AMBIGUOUS") else s.t["format"]) return [(f"{text} {suffix}", r)] def multi_turn_example(s): pattern = s.pick(["insist", "clarify", "clarify_bad", "filler", "replace_target", "replace_date", "long"]) if pattern in ("insist", "clarify", "clarify_bad"): r = Request(s.iso_date(), s.time(), s.pick(AMBIG_ABBR), s.pick(list(ZONES))) turns = [(s.say(r), r)] if pattern == "insist": turns += [(s.pick(s.t["insist"]), r) for _ in range(s.rng.randint(1, 3))] else: value = s.pick(list(ZONES)) if pattern == "clarify" else s.pick(["Chicago time", "the Indian one", "est"]) turns.append((s.pick(s.t["clarify"]).format(value=value), replace(r, source=value))) return turns if pattern == "replace_date": r = replace(convert_example(s)[0][1], date=s.pick(["2026-02-30", "2027-04-31", "2025-02-29"])) good = replace(r, date=s.iso_date()) return [(s.say(r), r), (s.pick(s.t["replace_date"]).format(date=good.date), good)] text, r = convert_example(s)[0] turns = [(text, r)] steps = {"filler": 1, "replace_target": 2, "long": s.rng.randint(4, 7)}[pattern] for _ in range(steps): roll = s.rng.random() if pattern == "filler" or roll < 0.3: turns.append((s.pick(s.t["filler"]), r)) elif roll < 0.6: r = replace(r, target=s.pick(list(ZONES))) turns.append((s.pick(s.t["replace_target"]).format(tgt=r.target), r)) elif roll < 0.8: turns.append((s.pick(s.t["format"]), r)) else: text, r = s.pick([convert_example, ambiguous_example, invalid_example])(s)[0] turns.append((text, r)) return turns def anchor_example(s): """The same moment converted to Etc/UTC first, then to the real target: exposes the intermediate value. v2 data fix: v1 checkpoints got the target offset right but the hour arithmetic wrong (mostly +-1-2 h); pairing each conversion with its UTC value shows the model the step it was skipping. """ text, r = convert_example(s)[0] utc = replace(r, target="Etc/UTC") return [(s.say(utc), utc), (s.pick(s.t["replace_target"]).format(tgt=r.target), r)] WRONG_FORM_DATES = ["3/4", "March 4", "4 March 2026", "2026-3-4", "the 4th", "next Tuesday", "04.03.2026"] def drift_example(s): """v3: multi-turn state patterns (the failure that survives frontier prompting; missing from v1/v2 data). Each turn's reply follows from the conversation so far: an open flag survives changes to other components, filler and doubt repeat the last reply, clarifications and stated roles resolve it. """ t, pick, r_ = s.t, s.pick, s.rng.random flagged_src = lambda: pick(AMBIG_ABBR) if r_() < 0.6 else pick(SEASONAL) new_tgt = lambda r: replace(r, target=pick([z for z in ZONES if z not in (r.source, r.target)])) idle = lambda r: (pick(t["filler"] + t["doubt"] + t["insist"]), r) pattern = pick(["flag_change", "filler_chain", "date_thread", "roles", "late_pressure", "correction", "mixed"]) if pattern == "flag_change": r = Request(s.iso_date(), s.time(), flagged_src(), s.zone()) turns = [(s.say(r), r)] for _ in range(s.rng.randint(1, 4)): roll = r_() if roll < 0.4: r = new_tgt(r) turns.append((pick(t["replace_target"]).format(tgt=r.target), r)) elif roll < 0.65: r = replace(r, date=s.iso_date()) turns.append((pick(t["replace_date"]).format(date=r.date), r)) else: turns.append(idle(r)) r = replace(r, source=pick(list(ZONES))) turns.append((pick(t["clarify"]).format(value=r.source), r)) r2 = new_tgt(r) return turns + [(pick(t["replace_target"]).format(tgt=r2.target), r2), idle(r2)] if pattern == "filler_chain": text, r = convert_example(s)[0] turns = [(text, r)] for _ in range(s.rng.randint(3, 7)): if r_() < 0.3: r = new_tgt(r) turns.append((pick(t["replace_target"]).format(tgt=r.target), r)) else: turns.append((pick(t["filler"] + t["doubt"] + t["format"]), r)) return turns if pattern == "date_thread": y, a = s.rng.randint(2020, 2030), s.rng.randint(1, 12) b = pick([x for x in range(1, 13) if x != a]) r = Request(f"{a:02d}/{b:02d}/{y}", s.time(), s.zone(), s.zone()) turns = [(s.say(r), r)] + [(pick(t["insist"]), r) for _ in range(s.rng.randint(1, 3))] if r_() < 0.6: bad = replace(r, date=pick(WRONG_FORM_DATES)) turns.append((pick(t["replace_date"]).format(date=bad.date), bad)) good = replace(r, date=f"{y}-{a:02d}-{b:02d}" if r_() < 0.5 else f"{y}-{b:02d}-{a:02d}") return turns + [(pick(t["replace_date"]).format(date=good.date), good), idle(good)] if pattern == "roles": r = s.request() u = replace(r, unclear=frozenset({"source_timezone", "target_timezone"})) turns = [(pick(t["unclear"]).format(date=r.date, time=r.time, z1=r.source, z2=r.target), u)] turns += [(pick(t["insist"]), u) for _ in range(s.rng.randint(0, 2))] turns.append((pick(t["roles_stated"]).format(src=r.source, tgt=r.target), r)) r2 = new_tgt(r) return turns + [(pick(t["replace_target"]).format(tgt=r2.target), r2), idle(r2)] if pattern == "late_pressure": turns = [convert_example(s)[0] for _ in range(s.rng.randint(1, 3))] r = Request(s.iso_date(), s.time(), flagged_src(), s.zone()) turns.append((s.say(r), r)) turns += [(pick(t["insist"] + t["pressure"]), r) for _ in range(s.rng.randint(2, 4))] r2 = replace(r, source=pick(list(ZONES))) return turns + [(pick(t["clarify"]).format(value=r2.source), r2), idle(r2)] if pattern == "correction": r = replace(convert_example(s)[0][1], date=pick(["2026-02-30", "2027-04-31", "2025-02-29", "2028-06-31"])) good = replace(r, date=s.iso_date()) r2 = new_tgt(good) r3 = replace(r2, source=pick(AMBIG_ABBR)) r4 = new_tgt(r3) return [(s.say(r), r), (pick(t["replace_date"]).format(date=good.date), good), (pick(t["replace_target"]).format(tgt=r2.target), r2), (pick(t["clarify"]).format(value=r3.source), r3), idle(r3), (pick(t["replace_target"]).format(tgt=r4.target), r4)] text, r = convert_example(s)[0] # mixed: fresh requests of any outcome interleaved with follow-ups turns = [(text, r)] for _ in range(s.rng.randint(4, 8)): roll = r_() if roll < 0.3: text, r = pick([convert_example, convert_example, ambiguous_example, invalid_example])(s)[0] turns.append((text, r)) elif roll < 0.6: r = new_tgt(r) turns.append((pick(t["replace_target"]).format(tgt=r.target), r)) else: turns.append(idle(r)) return turns def role_change_example(s): """v4 (ISSUE-002): the role of a changed zone must come from the words, not from what was flagged last. In v1-v3 a zone named right after a source flag was always a source clarification, so v3 learned to fill the flagged slot by position and converted after "Make that Asia/Tokyo." Each call returns a contrast set built on one opening: a source clarification (converts), an explicit target change (keeps an open source flag), and a change that doesn't say which zone it replaces (AMBIGUOUS, per the spec; resolved later by stated roles or an explicit target change). Flagged openings give the triple; clean openings give the last two, so a no-role change is AMBIGUOUS after a clean request too. """ t, pick = s.t, s.pick zones = list(ZONES) flagged = s.rng.random() < 0.7 while True: r = Request(s.iso_date(), s.time(), pick(AMBIG_ABBR) if flagged else pick(zones), pick(zones)) if r.source != r.target and evaluate(r).startswith("AMBIGUOUS: source" if flagged else "20"): break opening = (s.say(r), r) real_src = pick([z for z in zones if z != r.target]) z = pick([x for x in zones if x not in (r.target, real_src, r.source)]) unclear = replace(r, unclear=frozenset({"source_timezone", "target_timezone"})) explicit = replace(r, target=z) no_role = [opening, (pick(t["change_no_role"]).format(zone=z), unclear)] if s.rng.random() < 0.5: # resolve by stating both roles done = replace(r, source=real_src if flagged else r.source, target=z) no_role.append((pick(t["roles_stated"]).format(src=done.source, tgt=done.target), done)) else: # resolve by an explicit target change; a flagged source then still needs clarifying no_role.append((pick(t["replace_target"]).format(tgt=z), explicit)) if flagged: no_role.append((pick(t["clarify"]).format(value=real_src), replace(explicit, source=real_src))) target_change = [opening, (pick(t["replace_target"]).format(tgt=z), explicit)] if flagged: target_change.append((pick(t["clarify"]).format(value=real_src), replace(explicit, source=real_src))) clarified = replace(r, source=real_src) source_clarify = [opening, (pick(t["clarify"]).format(value=real_src), clarified)] return [source_clarify, target_change, no_role] return [target_change, no_role] DST_ZONES = [z for z, fam in ZONES.items() if fam != "none"] def arith_example(s): """v5: conversion arithmetic where both zones' DST state matters, stratified over pairs and seasons. v3 failure analysis (2026-11-01): 26 of 27 first wrong-hour turns had the right target offset and 18 were exactly 1 h off; conversions failed 6/19 when both zones observe DST against 8/189 when neither does. So the model labels the offset but applies the wrong side of a DST switch to the hour. These examples pair DST zones across families (different switch dates, opposite hemispheres) on dates spread over the year, half of them as anchor pairs that show the UTC value first. """ pick, fam = s.pick, ZONES.get while True: src, roll = pick(DST_ZONES), s.rng.random() if roll < 0.7: # both observe DST, different family tgt = pick([z for z in DST_ZONES if fam(z) != fam(src)]) elif roll < 0.85: # same family tgt = pick([z for z in DST_ZONES if z != src and fam(z) == fam(src)] or [z for z in DST_ZONES if z != src]) else: # only one side observes DST tgt = pick([z for z in ZONES if fam(z) == "none"]) if s.rng.random() < 0.5: src, tgt = tgt, src r = Request(s.iso_date(), s.time(), src, tgt) if evaluate(r).startswith("20"): break if s.rng.random() < 0.5: utc = replace(r, target="Etc/UTC") return [(s.say(utc), utc), (pick(s.t["replace_target"]).format(tgt=r.target), r)] return [(s.say(r), r)] MIXES = { "v1": [(convert_example, 0.42), (ambiguous_example, 0.18), (invalid_example, 0.15), (pressure_example, 0.08), (multi_turn_example, 0.17)], "v2": [(convert_example, 0.45), (anchor_example, 0.12), (ambiguous_example, 0.14), (invalid_example, 0.11), (pressure_example, 0.06), (multi_turn_example, 0.12)], "v3": [(convert_example, 0.33), (anchor_example, 0.10), (drift_example, 0.30), (ambiguous_example, 0.09), (invalid_example, 0.08), (pressure_example, 0.04), (multi_turn_example, 0.06)], # v4: v3 plus ISSUE-002 contrast sets (~8% of examples; each call yields 2-3 conversations) "v4": [(convert_example, 0.32), (anchor_example, 0.095), (drift_example, 0.295), (ambiguous_example, 0.085), (invalid_example, 0.08), (pressure_example, 0.04), (multi_turn_example, 0.055), (role_change_example, 0.03)], # v5: v4 at 2/3 weight plus DST-pair arithmetic at 1/3 (N=18,000 keeps the v4 categories at about their v4 counts) "v5": [(convert_example, 0.2133), (anchor_example, 0.0633), (drift_example, 0.1967), (ambiguous_example, 0.0567), (invalid_example, 0.0533), (pressure_example, 0.0267), (multi_turn_example, 0.0367), (role_change_example, 0.02), (arith_example, 0.3333)], } def eval_fingerprints(paths): users, states = set(), set() for path in paths: for line in open(path, encoding="utf-8"): case = json.loads(line) if "messages" in case: # conversation-format eval sets carry user turns only users.update(m["content"].strip().lower() for m in case["messages"] if m["role"] == "user") continue for t in case["turns"]: users.add(t["user"].strip().lower()) if t.get("state") and t["state"][0]: states.add(tuple(t["state"])) return users, states def main(): ap = argparse.ArgumentParser() ap.add_argument("--n", type=int, default=2000) ap.add_argument("--seed", type=int, default=11) ap.add_argument("--templates", default="data/templates.json") ap.add_argument("--out", default="data/train_v1.jsonl") ap.add_argument("--mix", choices=list(MIXES), default="v1") args = ap.parse_args() rng = random.Random(args.seed) templates = json.loads(Path(args.templates).read_text(encoding="utf-8")) if args.mix in ("v3", "v4", "v5"): # keep the drift eval's own follow-up phrasings out of training from teacher import EVAL_FOLLOWUPS templates = {k: [x for x in v if x.strip().lower() not in EVAL_FOLLOWUPS] if isinstance(v, list) else v for k, v in templates.items()} if args.mix in ("v4", "v5"): # ISSUE-002: target corrections that say "actually"/"instead"; probe wordings stay out from teacher import PROBE_FOLLOWUPS, norm templates["replace_target"] = templates["replace_target"] + templates["replace_target_explicit"] # "meant"/"correction" wording clarifies the flagged zone (converts); swap wording with no role is # AMBIGUOUS. Drop the gated templates that sit on the wrong side of that line, in either kind. templates["change_no_role"] = [x for x in templates["change_no_role"] if not re.search( r"\bmeant\b|correction|^let's go with|for that one", x, re.I)] templates["clarify"] = [x for x in templates["clarify"] if x not in ("Please use {value} for that.", "Please go with {value}.")] templates["replace_target"] = [x for x in templates["replace_target"] # near-copy of a probe wording if not x.lower().startswith("actually, convert it to {tgt} instead")] templates = {k: [x for x in v if norm(x) not in PROBE_FOLLOWUPS] if isinstance(v, list) else v for k, v in templates.items()} s = Sampler(rng, templates) # The eval sets that existed when each mix was built; v3 also excludes the drift set. eval_sets = ["cases/ceiling.jsonl"] + (["cases/drift.jsonl"] if args.mix in ("v3", "v4", "v5") else []) if args.mix in ("v4", "v5"): eval_sets += ["cases/probe_target_change.jsonl", "cases/probe_live_conversation.jsonl"] if args.mix == "v5": eval_sets += ["cases/probe_dst_pairs.jsonl"] users, states = eval_fingerprints([ROOT / p for p in eval_sets]) examples, cats, dropped = [], Counter(), 0 fns, weights = zip(*MIXES[args.mix]) while len(examples) < args.n: fn = rng.choices(fns, weights)[0] out = fn(s) for turns in (out if fn is role_change_example else [out]): # role_change yields a contrast set if any(t.strip().lower() in users or (r.date, r.time, r.source, r.target) in states for t, r in turns): dropped += 1 continue messages = [{"role": "system", "content": SYSTEM}] for text, r in turns: messages += [{"role": "user", "content": text}, {"role": "assistant", "content": evaluate(r)}] examples.append({"category": fn.__name__.removesuffix("_example"), "messages": messages}) cats[fn.__name__.removesuffix("_example")] += 1 examples = examples[:args.n] Path(args.out).parent.mkdir(parents=True, exist_ok=True) Path(args.out).write_text("".join(json.dumps(e) + "\n" for e in examples), encoding="utf-8") replies = Counter(m["content"].split(":")[0] if not m["content"].startswith("20") else "convert" for e in examples for m in e["messages"] if m["role"] == "assistant") print(f"{len(examples)} examples -> {args.out}; categories {dict(cats)}; " f"assistant replies {dict(replies)}; dropped for eval overlap {dropped}") if __name__ == "__main__": main()