"""ISSUE-002 probe: is a zone named right after a source flag read as the source, whatever the wording? Found by a fresh grader-style conversation (cases/probe_live_conversation.jsonl): after "AMBIGUOUS: source_timezone", tuned v3 read "Actually make that Asia/Tokyo instead." as a source replacement and converted. Each scenario opens with a flagged source, then one second-turn wording; target-change wordings get a third turn that resolves the source, which shows whether the new target was applied. The oracle writes every label. A wording that doesn't say which component it fills ("Make that X.") leaves the flag open under the spec, as does a target change. Usage: python scripts/gen_probe_target_change.py > cases/probe_target_change.jsonl python eval.py --model --eval-set cases/probe_target_change.jsonl python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned """ import json import sys from dataclasses import replace from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) from oracle import Request, evaluate # noqa: E402 VALUES = [ # (date, time, flagged source, first target, new target, real source) ("2026-06-12", "09:30", "CST", "Europe/London", "Asia/Tokyo", "America/Chicago"), ("2025-11-03", "14:15", "IST", "Europe/Paris", "America/New_York", "Asia/Kolkata"), ("2027-07-20", "08:00", "EST", "Asia/Dubai", "Australia/Sydney", "America/New_York"), ("2029-01-15", "18:45", "CST", "Asia/Seoul", "Europe/Athens", "America/Chicago"), ] SECOND = [ # (slice, wording, what the second turn changes) ("tgt_training_phrase", "And in {tgt}?", "target"), ("tgt_explicit", "Change the target to {tgt}.", "target"), ("tgt_instead", "Actually, convert it to {tgt} instead.", "target"), ("tgt_referent", "Actually make that {tgt} instead.", "unclear"), ("tgt_referent_bare", "Make that {tgt}.", "unclear"), ("src_clarify_control", "I meant {src}.", "source"), ] def main(): for i, (date, time, flagged, tgt1, tgt2, src) in enumerate(VALUES): r1 = Request(date, time, flagged, tgt1) first = f"Can you convert {date} {time} {flagged} to {tgt1}?" for slice_, wording, kind in SECOND: turns = [{"user": first, "expected": evaluate(r1)}] if kind == "source": r2 = replace(r1, source=src) turns.append({"user": wording.format(src=src), "expected": evaluate(r2)}) else: # The target reading and the unclear reading both leave the flagged source open. r2 = replace(r1, target=tgt2) turns.append({"user": wording.format(tgt=tgt2), "expected": evaluate(r2)}) if kind == "target": r3 = replace(r2, source=src) turns.append({"user": f"It's {src}.", "expected": evaluate(r3)}) print(json.dumps({"id": f"{slice_}-{i + 1}", "slice": slice_, "metric": "robustness", "turns": turns})) if __name__ == "__main__": main()