Download scripts/gen_probe_target_change.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 3.14 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/scripts/gen_probe_target_change.py
- Command line
-
hf download hf://debudash/slm-training/scripts/gen_probe_target_change.py
-
curl -L -o gen_probe_target_change.py https://huggingface.co/debudash/slm-training/resolve/main/scripts/gen_probe_target_change.py
3.14 kB
| """ISSUE-002 probe: is a zone named right after a source flag read as the source, whatever the wording? | |
| Found by a fresh grader-style conversation (cases/probe_live_conversation.jsonl): after | |
| "AMBIGUOUS: source_timezone", tuned v3 read "Actually make that Asia/Tokyo instead." as a source replacement and | |
| converted. Each scenario opens with a flagged source, then one second-turn wording; target-change wordings get a | |
| third turn that resolves the source, which shows whether the new target was applied. The oracle writes every label. | |
| A wording that doesn't say which component it fills ("Make that X.") leaves the flag open under the spec, as does | |
| a target change. | |
| Usage: python scripts/gen_probe_target_change.py > cases/probe_target_change.jsonl | |
| python eval.py --model <repo> --eval-set cases/probe_target_change.jsonl | |
| python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned | |
| """ | |
| import json | |
| import sys | |
| from dataclasses import replace | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(ROOT)) | |
| from oracle import Request, evaluate # noqa: E402 | |
| VALUES = [ # (date, time, flagged source, first target, new target, real source) | |
| ("2026-06-12", "09:30", "CST", "Europe/London", "Asia/Tokyo", "America/Chicago"), | |
| ("2025-11-03", "14:15", "IST", "Europe/Paris", "America/New_York", "Asia/Kolkata"), | |
| ("2027-07-20", "08:00", "EST", "Asia/Dubai", "Australia/Sydney", "America/New_York"), | |
| ("2029-01-15", "18:45", "CST", "Asia/Seoul", "Europe/Athens", "America/Chicago"), | |
| ] | |
| SECOND = [ # (slice, wording, what the second turn changes) | |
| ("tgt_training_phrase", "And in {tgt}?", "target"), | |
| ("tgt_explicit", "Change the target to {tgt}.", "target"), | |
| ("tgt_instead", "Actually, convert it to {tgt} instead.", "target"), | |
| ("tgt_referent", "Actually make that {tgt} instead.", "unclear"), | |
| ("tgt_referent_bare", "Make that {tgt}.", "unclear"), | |
| ("src_clarify_control", "I meant {src}.", "source"), | |
| ] | |
| def main(): | |
| for i, (date, time, flagged, tgt1, tgt2, src) in enumerate(VALUES): | |
| r1 = Request(date, time, flagged, tgt1) | |
| first = f"Can you convert {date} {time} {flagged} to {tgt1}?" | |
| for slice_, wording, kind in SECOND: | |
| turns = [{"user": first, "expected": evaluate(r1)}] | |
| if kind == "source": | |
| r2 = replace(r1, source=src) | |
| turns.append({"user": wording.format(src=src), "expected": evaluate(r2)}) | |
| else: | |
| # The target reading and the unclear reading both leave the flagged source open. | |
| r2 = replace(r1, target=tgt2) | |
| turns.append({"user": wording.format(tgt=tgt2), "expected": evaluate(r2)}) | |
| if kind == "target": | |
| r3 = replace(r2, source=src) | |
| turns.append({"user": f"It's {src}.", "expected": evaluate(r3)}) | |
| print(json.dumps({"id": f"{slice_}-{i + 1}", "slice": slice_, "metric": "robustness", "turns": turns})) | |
| if __name__ == "__main__": | |
| main() | |