slm-training / scripts /gen_probe_target_change.py
debudash's picture
Mirror slm-training @ 507b432 (course PDF excluded)
dc23a98 verified
Raw History Blame Contribute Delete
3.14 kB
"""ISSUE-002 probe: is a zone named right after a source flag read as the source, whatever the wording?
Found by a fresh grader-style conversation (cases/probe_live_conversation.jsonl): after
"AMBIGUOUS: source_timezone", tuned v3 read "Actually make that Asia/Tokyo instead." as a source replacement and
converted. Each scenario opens with a flagged source, then one second-turn wording; target-change wordings get a
third turn that resolves the source, which shows whether the new target was applied. The oracle writes every label.
A wording that doesn't say which component it fills ("Make that X.") leaves the flag open under the spec, as does
a target change.
Usage: python scripts/gen_probe_target_change.py > cases/probe_target_change.jsonl
python eval.py --model <repo> --eval-set cases/probe_target_change.jsonl
python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned
"""
import json
import sys
from dataclasses import replace
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from oracle import Request, evaluate # noqa: E402
VALUES = [ # (date, time, flagged source, first target, new target, real source)
("2026-06-12", "09:30", "CST", "Europe/London", "Asia/Tokyo", "America/Chicago"),
("2025-11-03", "14:15", "IST", "Europe/Paris", "America/New_York", "Asia/Kolkata"),
("2027-07-20", "08:00", "EST", "Asia/Dubai", "Australia/Sydney", "America/New_York"),
("2029-01-15", "18:45", "CST", "Asia/Seoul", "Europe/Athens", "America/Chicago"),
]
SECOND = [ # (slice, wording, what the second turn changes)
("tgt_training_phrase", "And in {tgt}?", "target"),
("tgt_explicit", "Change the target to {tgt}.", "target"),
("tgt_instead", "Actually, convert it to {tgt} instead.", "target"),
("tgt_referent", "Actually make that {tgt} instead.", "unclear"),
("tgt_referent_bare", "Make that {tgt}.", "unclear"),
("src_clarify_control", "I meant {src}.", "source"),
]
def main():
for i, (date, time, flagged, tgt1, tgt2, src) in enumerate(VALUES):
r1 = Request(date, time, flagged, tgt1)
first = f"Can you convert {date} {time} {flagged} to {tgt1}?"
for slice_, wording, kind in SECOND:
turns = [{"user": first, "expected": evaluate(r1)}]
if kind == "source":
r2 = replace(r1, source=src)
turns.append({"user": wording.format(src=src), "expected": evaluate(r2)})
else:
# The target reading and the unclear reading both leave the flagged source open.
r2 = replace(r1, target=tgt2)
turns.append({"user": wording.format(tgt=tgt2), "expected": evaluate(r2)})
if kind == "target":
r3 = replace(r2, source=src)
turns.append({"user": f"It's {src}.", "expected": evaluate(r3)})
print(json.dumps({"id": f"{slice_}-{i + 1}", "slice": slice_, "metric": "robustness", "turns": turns}))
if __name__ == "__main__":
main()