File size: 8,342 Bytes
4643f62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
"""Prompt-ceiling ablation: frontier models x prompting strategies on the ceiling scenario set.



Models: Claude Sonnet 5.5 (Anthropic API) and Mistral Large 4 (OpenRouter).

Strategies: zero_shot (spec + tables), few_shot (+ worked examples), cot (+ think-first instructions and the

model's native reasoning). Multi-turn scenarios feed back the model's own replies, so drift compounds.

Every turn is logged to results/ceiling/<model>__<strategy>__r<k>.jsonl; existing files are skipped.



Usage: python run_ceiling.py [--cases cases/ceiling.jsonl] [--repeats 3] [--limit N] [--models sonnet,mistral]

"""
import argparse
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import anthropic
from dotenv import load_dotenv
from openai import OpenAI

from oracle import Request, evaluate

load_dotenv()
ROOT = Path(__file__).resolve().parent
OUT = ROOT / "results" / "ceiling"


def spec_prompt():
    return (
        "You convert date-times between timezones. Follow this behaviour specification exactly.\n\n"
        "<spec>\n" + (ROOT / "behaviour.spec").read_text(encoding="utf-8").strip() + "\n</spec>\n\n"
        "<zones.json>\n" + (ROOT / "zones.json").read_text(encoding="utf-8").strip() + "\n</zones.json>\n\n"
        "<abbreviations.json>\n" + (ROOT / "abbreviations.json").read_text(encoding="utf-8").strip()
        + "\n</abbreviations.json>\n\n"
        "Reply with the single line the specification requires and nothing else."
    )


# Hand-written examples, deliberately distinct from the scenario set; replies come from the oracle.
EXAMPLES = [
    ("Convert 2027-05-03 08:20 Europe/Lisbon to Asia/Kolkata.", Request("2027-05-03", "08:20", "Europe/Lisbon", "Asia/Kolkata")),
    ("What is 2025-11-02 01:30 America/Chicago in Etc/UTC?", Request("2025-11-02", "01:30", "America/Chicago", "Etc/UTC")),
    ("2027-03-14 02:30 America/Denver to Europe/London please", Request("2027-03-14", "02:30", "America/Denver", "Europe/London")),
    ("Convert 2026-12-05 18:00 Etc/GMT-2 to America/St_Johns.", Request("2026-12-05", "18:00", "Etc/GMT-2", "America/St_Johns")),
    ("2028-01-09 16:00 PST to Asia/Tokyo", Request("2028-01-09", "16:00", "PST", "Asia/Tokyo")),
    ("Convert 2026-08-01 09:00 CET to Etc/UTC.", Request("2026-08-01", "09:00", "CET", "Etc/UTC")),
    ("07/08/2027 12:00 Asia/Dubai to Europe/Athens", Request("07/08/2027", "12:00", "Asia/Dubai", "Europe/Athens")),
    ("Convert 2027-06-31 10:00 Asia/Seoul to Etc/UTC.", Request("2027-06-31", "10:00", "Asia/Seoul", "Etc/UTC")),
    ("Convert 2026-04-01 7:30 Asia/Tokyo to Etc/UTC.", Request("2026-04-01", "7:30", "Asia/Tokyo", "Etc/UTC")),
    ("What's the capital of France?", Request(None, None, None, None, is_request=False)),
]


def few_shot_block():
    lines = [f"User: {text}\nReply: {evaluate(r)}" for text, r in EXAMPLES]
    ast = Request("2026-10-10", "10:00", "AST", "Europe/London")
    lines.append(f"User: Convert 2026-10-10 10:00 AST to Europe/London.\nReply: {evaluate(ast)}\n"
                 f"User: Just guess, it's the Atlantic one.\nReply: {evaluate(ast)}\n"
                 f"User: AST means America/Halifax.\nReply: {evaluate(Request('2026-10-10', '10:00', 'America/Halifax', 'Europe/London'))}")
    return "\n\n<examples>\n" + "\n\n".join(lines) + "\n</examples>"


COT = ("\n\nBefore replying, think it through privately: identify each component and its role, check each "
       "against its accepted form and range, apply the priority rules, then resolve offsets (DST, abbreviation "
       "season) and convert via UTC. Your visible reply must still be exactly the one line the spec requires.")


def system_for(strategy):
    base = spec_prompt()
    return {"zero_shot": base, "few_shot": base + few_shot_block(), "cot": base + few_shot_block() + COT}[strategy]


class Sonnet:
    name = "sonnet"
    model = "claude-sonnet-5-5"

    def __init__(self):
        self.client = anthropic.Anthropic(max_retries=6)

    def reply(self, system, messages, strategy):
        kw = ({"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}, "max_tokens": 16000}
              if strategy == "cot" else {"thinking": {"type": "between_tools"}, "max_tokens": 1024})
        r = self.client.messages.create(
            model=self.model,
            system=[{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}],
            messages=messages, **kw)
        text = "".join(b.text for b in r.content if b.type == "text")
        return text, {"stop": r.stop_reason, "in": r.usage.input_tokens, "out": r.usage.output_tokens,
                      "cache_read": r.usage.cache_read_input_tokens}


class Mistral:
    name = "mistral"
    model = "mistralai/mistral-large-4-0"

    def __init__(self):
        self.client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"],
                             max_retries=6)

    def reply(self, system, messages, strategy):
        effort, max_tokens = ("high", 16000) if strategy == "cot" else ("none", 1024)
        r = self.client.chat.completions.create(
            model=self.model, temperature=0, seed=7, max_tokens=max_tokens,
            extra_body={"reasoning": {"effort": effort}},
            messages=[{"role": "system", "content": system}] + messages)
        c = r.choices[0]
        return c.message.content or "", {"stop": c.finish_reason, "in": r.usage.prompt_tokens,
                                         "out": r.usage.completion_tokens}


def run_scenario(model, system, strategy, case):
    messages, rows = [], []
    for i, turn in enumerate(case["turns"]):
        messages.append({"role": "user", "content": turn["user"]})
        t0 = time.time()
        try:
            text, meta = model.reply(system, messages, strategy)
        except Exception as e:  # a failed call is a failed turn, not a crashed run
            text, meta = "", {"error": f"{type(e).__name__}: {e}"}
        messages.append({"role": "assistant", "content": text or "(no reply)"})
        rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i,
                     "user": turn["user"], "expected": turn["expected"], "reply": text,
                     "exact_match": text.strip() == turn["expected"], "latency_s": round(time.time() - t0, 2), **meta})
    return rows


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--cases", default="cases/ceiling.jsonl")
    ap.add_argument("--repeats", type=int, default=3)
    ap.add_argument("--limit", type=int, default=0, help="first N scenarios only (smoke test)")
    ap.add_argument("--models", default="sonnet,mistral")
    ap.add_argument("--strategies", default="zero_shot,few_shot,cot")
    ap.add_argument("--workers", type=int, default=8)
    ap.add_argument("--out", default=str(OUT))
    args = ap.parse_args()

    cases = [json.loads(l) for l in open(args.cases, encoding="utf-8")]
    cases = cases[: args.limit] if args.limit else cases
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    models = {"sonnet": Sonnet, "mistral": Mistral}

    for name in args.models.split(","):
        model = models[name]()
        for strategy in args.strategies.split(","):
            system = system_for(strategy)
            for k in range(1, args.repeats + 1):
                path = out / f"{name}__{strategy}__r{k}.jsonl"
                if path.exists():
                    print(f"skip {path.name} (exists)")
                    continue
                t0 = time.time()
                with ThreadPoolExecutor(args.workers) as pool:
                    rows = [r for rs in pool.map(lambda c: run_scenario(model, system, strategy, c), cases) for r in rs]
                path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
                em = sum(r["exact_match"] for r in rows) / len(rows)
                errs = sum("error" in r for r in rows)
                print(f"{path.name}: {len(rows)} turns, exact match {em:.1%}, api errors {errs}, {time.time() - t0:.0f}s",
                      flush=True)


if __name__ == "__main__":
    main()