File size: 3,071 Bytes
dc23a98
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
"""Write the demo's scenarios: featured ones chosen by a stated rule, and a set drawn by fixed seed from the eval sets.



Seeded rule: 3 clean conversions (adherence slices starting with "convert"), 1 pressure case (slice

pressure_single), and 1 drift conversation (any of the 70), each drawn uniformly with random.Random(SEED).



Featured (chosen, and labelled so in the app): one scenario per skill or fix, picked by a stated rule before any

final-model result, and kept whatever the outcome: the first insist_multi scenario (refusing to guess under

pressure); the seeded drift conversation (holding state); the ISSUE-002 probe's first role-free scenario; and

convert_iana-03, the conversion that exposed ISSUE-003. Every turn keeps the oracle's expected reply from the case

file. Committed before any recording.



Usage: python scripts/gen_demo_script.py   (writes demo/script.jsonl)

"""
import json
import random
from pathlib import Path

SEED = 20261101
ROOT = Path(__file__).resolve().parent.parent


def load(name):
    return [json.loads(l) for l in open(ROOT / "cases" / name, encoding="utf-8")]


def main():
    rng = random.Random(SEED)
    ceiling, drift, probe = load("ceiling.jsonl"), load("drift.jsonl"), load("probe_target_change.jsonl")
    converts = [c for c in ceiling if c["metric"] == "adherence" and c["slice"].startswith("convert")]
    pressure = [c for c in ceiling if c["slice"] == "pressure_single"]
    seeded = ([("ceiling", c) for c in rng.sample(converts, 3)] + [("ceiling", rng.choice(pressure))]
              + [("drift", rng.choice(drift))])
    by_id = {c["id"]: c for c in ceiling}
    featured = [
        ("Refuses to guess under pressure", "ceiling", next(c for c in ceiling if c["slice"] == "insist_multi")),
        ("Holds state across a conversation", "drift", seeded[-1][1]),
        ('ISSUE-002: "make that {zone}" after a flag', "probe_target_change",
         next(c for c in probe if c["slice"] == "tgt_referent")),
        ("ISSUE-003: DST pair, New York to Auckland", "ceiling", by_id["convert_iana-03"]),
    ]
    rows = ([{"group": "featured", "label": label, "source": source, "case": c} for label, source, c in featured]
            + [{"group": "seeded", "label": c["id"], "source": source, "case": c} for source, c in seeded])
    out = ROOT / "demo" / "script.jsonl"
    with open(out, "w", encoding="utf-8", newline="\n") as f:
        for r in rows:
            c = r["case"]
            f.write(json.dumps({"key": f"{r['group']}:{c['id']}", "group": r["group"], "label": r["label"],
                                "id": c["id"], "source": f"cases/{r['source']}.jsonl", "slice": c["slice"],
                                "seed": SEED if r["group"] == "seeded" else None,
                                "turns": [{"user": t["user"], "expected": t["expected"]} for t in c["turns"]]}) + "\n")
    for r in rows:
        print(f"{r['group']:<9} {r['case']['id']:<28} {len(r['case']['turns'])} turns  {r['label']}")


if __name__ == "__main__":
    main()