Download gen_train.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 23.8 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/gen_train.py
- Command line
-
hf download hf://debudash/slm-training/gen_train.py
-
curl -L -o gen_train.py https://huggingface.co/debudash/slm-training/resolve/main/gen_train.py
23.8 kB
| """Build the SFT dataset: teacher-written templates, sampled values, oracle-labelled replies. | |
| Every assistant turn is the oracle's reply, so labels are exact. Examples whose user text or request matches a | |
| turn in the evaluation set are dropped. Output is shuffled once, so the first N lines form nested subsets for | |
| the data-efficiency curve (250 within 500 within 1000 within 2000). | |
| Usage: python gen_train.py --n 2000 --seed 11 --out data/train_v1.jsonl | |
| """ | |
| import argparse | |
| import re | |
| import json | |
| import random | |
| from collections import Counter | |
| from dataclasses import replace | |
| from datetime import datetime, timedelta, timezone | |
| from pathlib import Path | |
| from zoneinfo import ZoneInfo | |
| from oracle import ABBREVIATIONS, ZONES, Request, evaluate | |
| ROOT = Path(__file__).resolve().parent | |
| SYSTEM = "You convert date-times between timezones under the tz-convert spec. Reply with exactly one line." | |
| SEASONAL = [k for k, v in ABBREVIATIONS["unambiguous"].items() if v["season"] != "all"] | |
| AMBIG_ABBR = list(ABBREVIATIONS["ambiguous"]) | |
| UNSUPPORTED_IANA = ["Europe/Berlin", "America/Toronto", "Asia/Singapore", "Asia/Hong_Kong", "Africa/Cairo", | |
| "America/Mexico_City", "Europe/Moscow", "Asia/Jakarta", "Africa/Lagos", "America/Bogota", | |
| "Europe/Madrid", "Asia/Manila"] | |
| UNLISTED_ABBR = ["MSK", "SGT", "HKT", "WIB", "SAST", "ART", "BRT", "PHT", "ICT", "CAT", "EAT", "AWST"] | |
| BAD_ZONE_FORMS = ["GMT+5", "UTC+5", "est", "Eastern", "Z", "+5:30", "UTC+05:60", "Tokyo time", "Pacific Time"] | |
| BAD_TIMES = ["24:00", "25:30", "12:60", "9:00", "3 PM", "1430", "12:00:00.5", "07:15:61", "noon"] | |
| def transitions(): | |
| """Local wall times just around every DST switch 2020-2030 for each DST-observing supported zone.""" | |
| out = [] | |
| for zone, fam in ZONES.items(): | |
| if fam == "none": | |
| continue | |
| zi, prev = ZoneInfo(zone), None | |
| t = datetime(2020, 1, 1, tzinfo=timezone.utc) | |
| while t.year <= 2030: | |
| off = t.astimezone(zi).utcoffset() | |
| if prev is not None and off != prev: | |
| out.append((zone, t - timedelta(hours=1))) | |
| prev, t = off, t + timedelta(hours=1) | |
| return out | |
| class Sampler: | |
| def __init__(self, rng, templates): | |
| self.rng, self.t = rng, templates | |
| self.trans = transitions() | |
| def pick(self, xs): | |
| return self.rng.choice(xs) | |
| def iso_date(self, year=None): | |
| y = year or self.rng.randint(2020, 2030) | |
| d = datetime(y, 1, 1) + timedelta(days=self.rng.randint(0, 364)) | |
| return d.strftime("%Y-%m-%d") | |
| def date(self): | |
| d = self.iso_date() | |
| if self.rng.random() < 0.15: # unambiguous NN/NN/YYYY: day > 12 or day == month | |
| y, m, day = d.split("-") | |
| if int(day) > 12: | |
| return f"{day}/{m}/{y}" if self.rng.random() < 0.5 else f"{m}/{day}/{y}" | |
| return d | |
| def time(self): | |
| t = f"{self.rng.randint(0, 23):02d}:{self.rng.randint(0, 59):02d}" | |
| return t + (f":{self.rng.randint(0, 59):02d}" if self.rng.random() < 0.2 else "") | |
| def offset(self): | |
| minutes = self.rng.choice([0, 0, 0, 30, 45]) | |
| hours = self.rng.randint(-12, 13) | |
| sign = "-" if hours < 0 else "+" | |
| text = f"{sign}{abs(hours):02d}:{minutes:02d}" | |
| return ("UTC" + text) if self.rng.random() < 0.5 else text | |
| def zone(self): | |
| r = self.rng.random() | |
| if r < 0.6: | |
| return self.pick(list(ZONES)) | |
| if r < 0.8: | |
| return self.pick(list(ABBREVIATIONS["unambiguous"])) | |
| if r < 0.93: | |
| return self.offset() | |
| return f"Etc/GMT{self.pick('+-')}{self.rng.randint(1, 12)}" | |
| def request(self): | |
| return Request(self.date(), self.time(), self.zone(), self.zone()) | |
| def near_transition(self): | |
| zone, utc = self.pick(self.trans) | |
| local = (utc + timedelta(minutes=self.rng.randint(0, 180))).astimezone(ZoneInfo(zone)) | |
| src = zone | |
| if self.rng.random() < 0.4: # use the zone's abbreviation, in or out of season | |
| src = self.pick([k for k, v in ABBREVIATIONS["unambiguous"].items() if v["zone"] == zone] or [zone]) | |
| return Request(local.strftime("%Y-%m-%d"), local.strftime("%H:%M"), src, self.zone()) | |
| def say(self, r): | |
| return self.pick(self.t["request"]).format(date=r.date, time=r.time, src=r.source, tgt=r.target) | |
| def convert_example(s): | |
| while True: | |
| r = s.near_transition() if s.rng.random() < 0.3 else s.request() | |
| if evaluate(r).startswith("20"): | |
| return [(s.say(r), r)] | |
| def ambiguous_example(s): | |
| kind = s.pick(["listed", "season", "season", "date", "unclear"]) | |
| r = s.request() | |
| if kind == "listed": | |
| r = replace(r, **{s.pick(["source", "target"]): s.pick(AMBIG_ABBR)}) | |
| elif kind == "season": | |
| abbr = s.pick(SEASONAL) | |
| for _ in range(50): | |
| r = replace(r, date=s.iso_date(), **{s.pick(["source", "target"]): abbr}) | |
| if evaluate(r).startswith("AMBIGUOUS"): | |
| break | |
| elif kind == "date": | |
| y, a = s.rng.randint(2020, 2030), s.rng.randint(1, 12) | |
| b = s.pick([x for x in range(1, 13) if x != a]) | |
| r = replace(r, date=f"{a:02d}/{b:02d}/{y}") | |
| else: | |
| r = replace(r, unclear=frozenset({"source_timezone", "target_timezone"})) | |
| return [(s.pick(s.t["unclear"]).format(date=r.date, time=r.time, z1=r.source, z2=r.target), r)] | |
| return [(s.say(r), r)] | |
| def invalid_example(s): | |
| r = s.request() | |
| kind = s.pick(["date", "date", "time", "time", "zone", "zone", "zone", "not_request"]) | |
| if kind == "date": | |
| y = s.rng.randint(2020, 2030) | |
| bad = s.pick([f"{y}-02-30", f"{y}-04-31", f"{y}-13-05", f"{s.pick([2019, 2031, 2018, 2035])}-06-15", | |
| f"{y}-6-5", f"13/13/{y}", f"31/04/{y}", f"05/06", f"{y}/06/15"]) | |
| r = replace(r, date=bad) | |
| elif kind == "time": | |
| r = replace(r, time=s.pick(BAD_TIMES)) | |
| elif kind == "zone": | |
| bad = s.pick([s.pick(UNSUPPORTED_IANA), s.pick(UNLISTED_ABBR), s.pick(BAD_ZONE_FORMS), | |
| s.pick(["+14:30", "-12:30", "UTC+15:00", "Etc/GMT+13"])]) | |
| r = replace(r, **{s.pick(["source", "target"]): bad}) | |
| else: | |
| return [(s.pick(s.t["not_request"]), Request(None, None, None, None, is_request=False))] | |
| return [(s.say(r), r)] | |
| def pressure_example(s): | |
| first = s.pick([ambiguous_example, convert_example, invalid_example])(s) | |
| text, r = first[0] | |
| suffix = s.pick(s.t["pressure"] if evaluate(r).startswith("AMBIGUOUS") else s.t["format"]) | |
| return [(f"{text} {suffix}", r)] | |
| def multi_turn_example(s): | |
| pattern = s.pick(["insist", "clarify", "clarify_bad", "filler", "replace_target", "replace_date", "long"]) | |
| if pattern in ("insist", "clarify", "clarify_bad"): | |
| r = Request(s.iso_date(), s.time(), s.pick(AMBIG_ABBR), s.pick(list(ZONES))) | |
| turns = [(s.say(r), r)] | |
| if pattern == "insist": | |
| turns += [(s.pick(s.t["insist"]), r) for _ in range(s.rng.randint(1, 3))] | |
| else: | |
| value = s.pick(list(ZONES)) if pattern == "clarify" else s.pick(["Chicago time", "the Indian one", "est"]) | |
| turns.append((s.pick(s.t["clarify"]).format(value=value), replace(r, source=value))) | |
| return turns | |
| if pattern == "replace_date": | |
| r = replace(convert_example(s)[0][1], date=s.pick(["2026-02-30", "2027-04-31", "2025-02-29"])) | |
| good = replace(r, date=s.iso_date()) | |
| return [(s.say(r), r), (s.pick(s.t["replace_date"]).format(date=good.date), good)] | |
| text, r = convert_example(s)[0] | |
| turns = [(text, r)] | |
| steps = {"filler": 1, "replace_target": 2, "long": s.rng.randint(4, 7)}[pattern] | |
| for _ in range(steps): | |
| roll = s.rng.random() | |
| if pattern == "filler" or roll < 0.3: | |
| turns.append((s.pick(s.t["filler"]), r)) | |
| elif roll < 0.6: | |
| r = replace(r, target=s.pick(list(ZONES))) | |
| turns.append((s.pick(s.t["replace_target"]).format(tgt=r.target), r)) | |
| elif roll < 0.8: | |
| turns.append((s.pick(s.t["format"]), r)) | |
| else: | |
| text, r = s.pick([convert_example, ambiguous_example, invalid_example])(s)[0] | |
| turns.append((text, r)) | |
| return turns | |
| def anchor_example(s): | |
| """The same moment converted to Etc/UTC first, then to the real target: exposes the intermediate value. | |
| v2 data fix: v1 checkpoints got the target offset right but the hour arithmetic wrong (mostly +-1-2 h); | |
| pairing each conversion with its UTC value shows the model the step it was skipping. | |
| """ | |
| text, r = convert_example(s)[0] | |
| utc = replace(r, target="Etc/UTC") | |
| return [(s.say(utc), utc), (s.pick(s.t["replace_target"]).format(tgt=r.target), r)] | |
| WRONG_FORM_DATES = ["3/4", "March 4", "4 March 2026", "2026-3-4", "the 4th", "next Tuesday", "04.03.2026"] | |
| def drift_example(s): | |
| """v3: multi-turn state patterns (the failure that survives frontier prompting; missing from v1/v2 data). | |
| Each turn's reply follows from the conversation so far: an open flag survives changes to other components, | |
| filler and doubt repeat the last reply, clarifications and stated roles resolve it. | |
| """ | |
| t, pick, r_ = s.t, s.pick, s.rng.random | |
| flagged_src = lambda: pick(AMBIG_ABBR) if r_() < 0.6 else pick(SEASONAL) | |
| new_tgt = lambda r: replace(r, target=pick([z for z in ZONES if z not in (r.source, r.target)])) | |
| idle = lambda r: (pick(t["filler"] + t["doubt"] + t["insist"]), r) | |
| pattern = pick(["flag_change", "filler_chain", "date_thread", "roles", "late_pressure", "correction", "mixed"]) | |
| if pattern == "flag_change": | |
| r = Request(s.iso_date(), s.time(), flagged_src(), s.zone()) | |
| turns = [(s.say(r), r)] | |
| for _ in range(s.rng.randint(1, 4)): | |
| roll = r_() | |
| if roll < 0.4: | |
| r = new_tgt(r) | |
| turns.append((pick(t["replace_target"]).format(tgt=r.target), r)) | |
| elif roll < 0.65: | |
| r = replace(r, date=s.iso_date()) | |
| turns.append((pick(t["replace_date"]).format(date=r.date), r)) | |
| else: | |
| turns.append(idle(r)) | |
| r = replace(r, source=pick(list(ZONES))) | |
| turns.append((pick(t["clarify"]).format(value=r.source), r)) | |
| r2 = new_tgt(r) | |
| return turns + [(pick(t["replace_target"]).format(tgt=r2.target), r2), idle(r2)] | |
| if pattern == "filler_chain": | |
| text, r = convert_example(s)[0] | |
| turns = [(text, r)] | |
| for _ in range(s.rng.randint(3, 7)): | |
| if r_() < 0.3: | |
| r = new_tgt(r) | |
| turns.append((pick(t["replace_target"]).format(tgt=r.target), r)) | |
| else: | |
| turns.append((pick(t["filler"] + t["doubt"] + t["format"]), r)) | |
| return turns | |
| if pattern == "date_thread": | |
| y, a = s.rng.randint(2020, 2030), s.rng.randint(1, 12) | |
| b = pick([x for x in range(1, 13) if x != a]) | |
| r = Request(f"{a:02d}/{b:02d}/{y}", s.time(), s.zone(), s.zone()) | |
| turns = [(s.say(r), r)] + [(pick(t["insist"]), r) for _ in range(s.rng.randint(1, 3))] | |
| if r_() < 0.6: | |
| bad = replace(r, date=pick(WRONG_FORM_DATES)) | |
| turns.append((pick(t["replace_date"]).format(date=bad.date), bad)) | |
| good = replace(r, date=f"{y}-{a:02d}-{b:02d}" if r_() < 0.5 else f"{y}-{b:02d}-{a:02d}") | |
| return turns + [(pick(t["replace_date"]).format(date=good.date), good), idle(good)] | |
| if pattern == "roles": | |
| r = s.request() | |
| u = replace(r, unclear=frozenset({"source_timezone", "target_timezone"})) | |
| turns = [(pick(t["unclear"]).format(date=r.date, time=r.time, z1=r.source, z2=r.target), u)] | |
| turns += [(pick(t["insist"]), u) for _ in range(s.rng.randint(0, 2))] | |
| turns.append((pick(t["roles_stated"]).format(src=r.source, tgt=r.target), r)) | |
| r2 = new_tgt(r) | |
| return turns + [(pick(t["replace_target"]).format(tgt=r2.target), r2), idle(r2)] | |
| if pattern == "late_pressure": | |
| turns = [convert_example(s)[0] for _ in range(s.rng.randint(1, 3))] | |
| r = Request(s.iso_date(), s.time(), flagged_src(), s.zone()) | |
| turns.append((s.say(r), r)) | |
| turns += [(pick(t["insist"] + t["pressure"]), r) for _ in range(s.rng.randint(2, 4))] | |
| r2 = replace(r, source=pick(list(ZONES))) | |
| return turns + [(pick(t["clarify"]).format(value=r2.source), r2), idle(r2)] | |
| if pattern == "correction": | |
| r = replace(convert_example(s)[0][1], date=pick(["2026-02-30", "2027-04-31", "2025-02-29", "2028-06-31"])) | |
| good = replace(r, date=s.iso_date()) | |
| r2 = new_tgt(good) | |
| r3 = replace(r2, source=pick(AMBIG_ABBR)) | |
| r4 = new_tgt(r3) | |
| return [(s.say(r), r), (pick(t["replace_date"]).format(date=good.date), good), | |
| (pick(t["replace_target"]).format(tgt=r2.target), r2), (pick(t["clarify"]).format(value=r3.source), r3), | |
| idle(r3), (pick(t["replace_target"]).format(tgt=r4.target), r4)] | |
| text, r = convert_example(s)[0] # mixed: fresh requests of any outcome interleaved with follow-ups | |
| turns = [(text, r)] | |
| for _ in range(s.rng.randint(4, 8)): | |
| roll = r_() | |
| if roll < 0.3: | |
| text, r = pick([convert_example, convert_example, ambiguous_example, invalid_example])(s)[0] | |
| turns.append((text, r)) | |
| elif roll < 0.6: | |
| r = new_tgt(r) | |
| turns.append((pick(t["replace_target"]).format(tgt=r.target), r)) | |
| else: | |
| turns.append(idle(r)) | |
| return turns | |
| def role_change_example(s): | |
| """v4 (ISSUE-002): the role of a changed zone must come from the words, not from what was flagged last. | |
| In v1-v3 a zone named right after a source flag was always a source clarification, so v3 learned to fill the | |
| flagged slot by position and converted after "Make that Asia/Tokyo." Each call returns a contrast set built on | |
| one opening: a source clarification (converts), an explicit target change (keeps an open source flag), and a | |
| change that doesn't say which zone it replaces (AMBIGUOUS, per the spec; resolved later by stated roles or an | |
| explicit target change). Flagged openings give the triple; clean openings give the last two, so a no-role | |
| change is AMBIGUOUS after a clean request too. | |
| """ | |
| t, pick = s.t, s.pick | |
| zones = list(ZONES) | |
| flagged = s.rng.random() < 0.7 | |
| while True: | |
| r = Request(s.iso_date(), s.time(), pick(AMBIG_ABBR) if flagged else pick(zones), pick(zones)) | |
| if r.source != r.target and evaluate(r).startswith("AMBIGUOUS: source" if flagged else "20"): | |
| break | |
| opening = (s.say(r), r) | |
| real_src = pick([z for z in zones if z != r.target]) | |
| z = pick([x for x in zones if x not in (r.target, real_src, r.source)]) | |
| unclear = replace(r, unclear=frozenset({"source_timezone", "target_timezone"})) | |
| explicit = replace(r, target=z) | |
| no_role = [opening, (pick(t["change_no_role"]).format(zone=z), unclear)] | |
| if s.rng.random() < 0.5: # resolve by stating both roles | |
| done = replace(r, source=real_src if flagged else r.source, target=z) | |
| no_role.append((pick(t["roles_stated"]).format(src=done.source, tgt=done.target), done)) | |
| else: # resolve by an explicit target change; a flagged source then still needs clarifying | |
| no_role.append((pick(t["replace_target"]).format(tgt=z), explicit)) | |
| if flagged: | |
| no_role.append((pick(t["clarify"]).format(value=real_src), replace(explicit, source=real_src))) | |
| target_change = [opening, (pick(t["replace_target"]).format(tgt=z), explicit)] | |
| if flagged: | |
| target_change.append((pick(t["clarify"]).format(value=real_src), replace(explicit, source=real_src))) | |
| clarified = replace(r, source=real_src) | |
| source_clarify = [opening, (pick(t["clarify"]).format(value=real_src), clarified)] | |
| return [source_clarify, target_change, no_role] | |
| return [target_change, no_role] | |
| DST_ZONES = [z for z, fam in ZONES.items() if fam != "none"] | |
| def arith_example(s): | |
| """v5: conversion arithmetic where both zones' DST state matters, stratified over pairs and seasons. | |
| v3 failure analysis (2026-11-01): 26 of 27 first wrong-hour turns had the right target offset and 18 were | |
| exactly 1 h off; conversions failed 6/19 when both zones observe DST against 8/189 when neither does. So the | |
| model labels the offset but applies the wrong side of a DST switch to the hour. These examples pair DST zones | |
| across families (different switch dates, opposite hemispheres) on dates spread over the year, half of them as | |
| anchor pairs that show the UTC value first. | |
| """ | |
| pick, fam = s.pick, ZONES.get | |
| while True: | |
| src, roll = pick(DST_ZONES), s.rng.random() | |
| if roll < 0.7: # both observe DST, different family | |
| tgt = pick([z for z in DST_ZONES if fam(z) != fam(src)]) | |
| elif roll < 0.85: # same family | |
| tgt = pick([z for z in DST_ZONES if z != src and fam(z) == fam(src)] or [z for z in DST_ZONES if z != src]) | |
| else: # only one side observes DST | |
| tgt = pick([z for z in ZONES if fam(z) == "none"]) | |
| if s.rng.random() < 0.5: | |
| src, tgt = tgt, src | |
| r = Request(s.iso_date(), s.time(), src, tgt) | |
| if evaluate(r).startswith("20"): | |
| break | |
| if s.rng.random() < 0.5: | |
| utc = replace(r, target="Etc/UTC") | |
| return [(s.say(utc), utc), (pick(s.t["replace_target"]).format(tgt=r.target), r)] | |
| return [(s.say(r), r)] | |
| MIXES = { | |
| "v1": [(convert_example, 0.42), (ambiguous_example, 0.18), (invalid_example, 0.15), | |
| (pressure_example, 0.08), (multi_turn_example, 0.17)], | |
| "v2": [(convert_example, 0.45), (anchor_example, 0.12), (ambiguous_example, 0.14), (invalid_example, 0.11), | |
| (pressure_example, 0.06), (multi_turn_example, 0.12)], | |
| "v3": [(convert_example, 0.33), (anchor_example, 0.10), (drift_example, 0.30), (ambiguous_example, 0.09), | |
| (invalid_example, 0.08), (pressure_example, 0.04), (multi_turn_example, 0.06)], | |
| # v4: v3 plus ISSUE-002 contrast sets (~8% of examples; each call yields 2-3 conversations) | |
| "v4": [(convert_example, 0.32), (anchor_example, 0.095), (drift_example, 0.295), (ambiguous_example, 0.085), | |
| (invalid_example, 0.08), (pressure_example, 0.04), (multi_turn_example, 0.055), (role_change_example, 0.03)], | |
| # v5: v4 at 2/3 weight plus DST-pair arithmetic at 1/3 (N=18,000 keeps the v4 categories at about their v4 counts) | |
| "v5": [(convert_example, 0.2133), (anchor_example, 0.0633), (drift_example, 0.1967), (ambiguous_example, 0.0567), | |
| (invalid_example, 0.0533), (pressure_example, 0.0267), (multi_turn_example, 0.0367), | |
| (role_change_example, 0.02), (arith_example, 0.3333)], | |
| } | |
| def eval_fingerprints(paths): | |
| users, states = set(), set() | |
| for path in paths: | |
| for line in open(path, encoding="utf-8"): | |
| case = json.loads(line) | |
| if "messages" in case: # conversation-format eval sets carry user turns only | |
| users.update(m["content"].strip().lower() for m in case["messages"] if m["role"] == "user") | |
| continue | |
| for t in case["turns"]: | |
| users.add(t["user"].strip().lower()) | |
| if t.get("state") and t["state"][0]: | |
| states.add(tuple(t["state"])) | |
| return users, states | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n", type=int, default=2000) | |
| ap.add_argument("--seed", type=int, default=11) | |
| ap.add_argument("--templates", default="data/templates.json") | |
| ap.add_argument("--out", default="data/train_v1.jsonl") | |
| ap.add_argument("--mix", choices=list(MIXES), default="v1") | |
| args = ap.parse_args() | |
| rng = random.Random(args.seed) | |
| templates = json.loads(Path(args.templates).read_text(encoding="utf-8")) | |
| if args.mix in ("v3", "v4", "v5"): # keep the drift eval's own follow-up phrasings out of training | |
| from teacher import EVAL_FOLLOWUPS | |
| templates = {k: [x for x in v if x.strip().lower() not in EVAL_FOLLOWUPS] if isinstance(v, list) else v | |
| for k, v in templates.items()} | |
| if args.mix in ("v4", "v5"): # ISSUE-002: target corrections that say "actually"/"instead"; probe wordings stay out | |
| from teacher import PROBE_FOLLOWUPS, norm | |
| templates["replace_target"] = templates["replace_target"] + templates["replace_target_explicit"] | |
| # "meant"/"correction" wording clarifies the flagged zone (converts); swap wording with no role is | |
| # AMBIGUOUS. Drop the gated templates that sit on the wrong side of that line, in either kind. | |
| templates["change_no_role"] = [x for x in templates["change_no_role"] if not re.search( | |
| r"\bmeant\b|correction|^let's go with|for that one", x, re.I)] | |
| templates["clarify"] = [x for x in templates["clarify"] | |
| if x not in ("Please use {value} for that.", "Please go with {value}.")] | |
| templates["replace_target"] = [x for x in templates["replace_target"] # near-copy of a probe wording | |
| if not x.lower().startswith("actually, convert it to {tgt} instead")] | |
| templates = {k: [x for x in v if norm(x) not in PROBE_FOLLOWUPS] if isinstance(v, list) else v | |
| for k, v in templates.items()} | |
| s = Sampler(rng, templates) | |
| # The eval sets that existed when each mix was built; v3 also excludes the drift set. | |
| eval_sets = ["cases/ceiling.jsonl"] + (["cases/drift.jsonl"] if args.mix in ("v3", "v4", "v5") else []) | |
| if args.mix in ("v4", "v5"): | |
| eval_sets += ["cases/probe_target_change.jsonl", "cases/probe_live_conversation.jsonl"] | |
| if args.mix == "v5": | |
| eval_sets += ["cases/probe_dst_pairs.jsonl"] | |
| users, states = eval_fingerprints([ROOT / p for p in eval_sets]) | |
| examples, cats, dropped = [], Counter(), 0 | |
| fns, weights = zip(*MIXES[args.mix]) | |
| while len(examples) < args.n: | |
| fn = rng.choices(fns, weights)[0] | |
| out = fn(s) | |
| for turns in (out if fn is role_change_example else [out]): # role_change yields a contrast set | |
| if any(t.strip().lower() in users or (r.date, r.time, r.source, r.target) in states for t, r in turns): | |
| dropped += 1 | |
| continue | |
| messages = [{"role": "system", "content": SYSTEM}] | |
| for text, r in turns: | |
| messages += [{"role": "user", "content": text}, {"role": "assistant", "content": evaluate(r)}] | |
| examples.append({"category": fn.__name__.removesuffix("_example"), "messages": messages}) | |
| cats[fn.__name__.removesuffix("_example")] += 1 | |
| examples = examples[:args.n] | |
| Path(args.out).parent.mkdir(parents=True, exist_ok=True) | |
| Path(args.out).write_text("".join(json.dumps(e) + "\n" for e in examples), encoding="utf-8") | |
| replies = Counter(m["content"].split(":")[0] if not m["content"].startswith("20") else "convert" | |
| for e in examples for m in e["messages"] if m["role"] == "assistant") | |
| print(f"{len(examples)} examples -> {args.out}; categories {dict(cats)}; " | |
| f"assistant replies {dict(replies)}; dropped for eval overlap {dropped}") | |
| if __name__ == "__main__": | |
| main() | |