slm-training / scripts /gen_demo_script.py
debudash's picture
Mirror slm-training @ 507b432 (course PDF excluded)
dc23a98 verified
Raw History Blame Contribute Delete
3.07 kB
"""Write the demo's scenarios: featured ones chosen by a stated rule, and a set drawn by fixed seed from the eval sets.
Seeded rule: 3 clean conversions (adherence slices starting with "convert"), 1 pressure case (slice
pressure_single), and 1 drift conversation (any of the 70), each drawn uniformly with random.Random(SEED).
Featured (chosen, and labelled so in the app): one scenario per skill or fix, picked by a stated rule before any
final-model result, and kept whatever the outcome: the first insist_multi scenario (refusing to guess under
pressure); the seeded drift conversation (holding state); the ISSUE-002 probe's first role-free scenario; and
convert_iana-03, the conversion that exposed ISSUE-003. Every turn keeps the oracle's expected reply from the case
file. Committed before any recording.
Usage: python scripts/gen_demo_script.py (writes demo/script.jsonl)
"""
import json
import random
from pathlib import Path
SEED = 20261101
ROOT = Path(__file__).resolve().parent.parent
def load(name):
return [json.loads(l) for l in open(ROOT / "cases" / name, encoding="utf-8")]
def main():
rng = random.Random(SEED)
ceiling, drift, probe = load("ceiling.jsonl"), load("drift.jsonl"), load("probe_target_change.jsonl")
converts = [c for c in ceiling if c["metric"] == "adherence" and c["slice"].startswith("convert")]
pressure = [c for c in ceiling if c["slice"] == "pressure_single"]
seeded = ([("ceiling", c) for c in rng.sample(converts, 3)] + [("ceiling", rng.choice(pressure))]
+ [("drift", rng.choice(drift))])
by_id = {c["id"]: c for c in ceiling}
featured = [
("Refuses to guess under pressure", "ceiling", next(c for c in ceiling if c["slice"] == "insist_multi")),
("Holds state across a conversation", "drift", seeded[-1][1]),
('ISSUE-002: "make that {zone}" after a flag', "probe_target_change",
next(c for c in probe if c["slice"] == "tgt_referent")),
("ISSUE-003: DST pair, New York to Auckland", "ceiling", by_id["convert_iana-03"]),
]
rows = ([{"group": "featured", "label": label, "source": source, "case": c} for label, source, c in featured]
+ [{"group": "seeded", "label": c["id"], "source": source, "case": c} for source, c in seeded])
out = ROOT / "demo" / "script.jsonl"
with open(out, "w", encoding="utf-8", newline="\n") as f:
for r in rows:
c = r["case"]
f.write(json.dumps({"key": f"{r['group']}:{c['id']}", "group": r["group"], "label": r["label"],
"id": c["id"], "source": f"cases/{r['source']}.jsonl", "slice": c["slice"],
"seed": SEED if r["group"] == "seeded" else None,
"turns": [{"user": t["user"], "expected": t["expected"]} for t in c["turns"]]}) + "\n")
for r in rows:
print(f"{r['group']:<9} {r['case']['id']:<28} {len(r['case']['turns'])} turns {r['label']}")
if __name__ == "__main__":
main()