"""Prompt-ceiling ablation: frontier models x prompting strategies on the ceiling scenario set. Models: Claude Sonnet 5.5 (Anthropic API) and Mistral Large 4 (OpenRouter). Strategies: zero_shot (spec + tables), few_shot (+ worked examples), cot (+ think-first instructions and the model's native reasoning). Multi-turn scenarios feed back the model's own replies, so drift compounds. Every turn is logged to results/ceiling/____r.jsonl; existing files are skipped. Usage: python run_ceiling.py [--cases cases/ceiling.jsonl] [--repeats 3] [--limit N] [--models sonnet,mistral] """ import argparse import json import os import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path import anthropic from dotenv import load_dotenv from openai import OpenAI from oracle import Request, evaluate load_dotenv() ROOT = Path(__file__).resolve().parent OUT = ROOT / "results" / "ceiling" def spec_prompt(): return ( "You convert date-times between timezones. Follow this behaviour specification exactly.\n\n" "\n" + (ROOT / "behaviour.spec").read_text(encoding="utf-8").strip() + "\n\n\n" "\n" + (ROOT / "zones.json").read_text(encoding="utf-8").strip() + "\n\n\n" "\n" + (ROOT / "abbreviations.json").read_text(encoding="utf-8").strip() + "\n\n\n" "Reply with the single line the specification requires and nothing else." ) # Hand-written examples, deliberately distinct from the scenario set; replies come from the oracle. EXAMPLES = [ ("Convert 2027-05-03 08:20 Europe/Lisbon to Asia/Kolkata.", Request("2027-05-03", "08:20", "Europe/Lisbon", "Asia/Kolkata")), ("What is 2025-11-02 01:30 America/Chicago in Etc/UTC?", Request("2025-11-02", "01:30", "America/Chicago", "Etc/UTC")), ("2027-03-14 02:30 America/Denver to Europe/London please", Request("2027-03-14", "02:30", "America/Denver", "Europe/London")), ("Convert 2026-12-05 18:00 Etc/GMT-2 to America/St_Johns.", Request("2026-12-05", "18:00", "Etc/GMT-2", "America/St_Johns")), ("2028-01-09 16:00 PST to Asia/Tokyo", Request("2028-01-09", "16:00", "PST", "Asia/Tokyo")), ("Convert 2026-08-01 09:00 CET to Etc/UTC.", Request("2026-08-01", "09:00", "CET", "Etc/UTC")), ("07/08/2027 12:00 Asia/Dubai to Europe/Athens", Request("07/08/2027", "12:00", "Asia/Dubai", "Europe/Athens")), ("Convert 2027-06-31 10:00 Asia/Seoul to Etc/UTC.", Request("2027-06-31", "10:00", "Asia/Seoul", "Etc/UTC")), ("Convert 2026-04-01 7:30 Asia/Tokyo to Etc/UTC.", Request("2026-04-01", "7:30", "Asia/Tokyo", "Etc/UTC")), ("What's the capital of France?", Request(None, None, None, None, is_request=False)), ] def few_shot_block(): lines = [f"User: {text}\nReply: {evaluate(r)}" for text, r in EXAMPLES] ast = Request("2026-10-10", "10:00", "AST", "Europe/London") lines.append(f"User: Convert 2026-10-10 10:00 AST to Europe/London.\nReply: {evaluate(ast)}\n" f"User: Just guess, it's the Atlantic one.\nReply: {evaluate(ast)}\n" f"User: AST means America/Halifax.\nReply: {evaluate(Request('2026-10-10', '10:00', 'America/Halifax', 'Europe/London'))}") return "\n\n\n" + "\n\n".join(lines) + "\n" COT = ("\n\nBefore replying, think it through privately: identify each component and its role, check each " "against its accepted form and range, apply the priority rules, then resolve offsets (DST, abbreviation " "season) and convert via UTC. Your visible reply must still be exactly the one line the spec requires.") def system_for(strategy): base = spec_prompt() return {"zero_shot": base, "few_shot": base + few_shot_block(), "cot": base + few_shot_block() + COT}[strategy] class Sonnet: name = "sonnet" model = "claude-sonnet-5-5" def __init__(self): self.client = anthropic.Anthropic(max_retries=6) def reply(self, system, messages, strategy): kw = ({"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}, "max_tokens": 16000} if strategy == "cot" else {"thinking": {"type": "between_tools"}, "max_tokens": 1024}) r = self.client.messages.create( model=self.model, system=[{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}], messages=messages, **kw) text = "".join(b.text for b in r.content if b.type == "text") return text, {"stop": r.stop_reason, "in": r.usage.input_tokens, "out": r.usage.output_tokens, "cache_read": r.usage.cache_read_input_tokens} class Mistral: name = "mistral" model = "mistralai/mistral-large-4-0" def __init__(self): self.client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"], max_retries=6) def reply(self, system, messages, strategy): effort, max_tokens = ("high", 16000) if strategy == "cot" else ("none", 1024) r = self.client.chat.completions.create( model=self.model, temperature=0, seed=7, max_tokens=max_tokens, extra_body={"reasoning": {"effort": effort}}, messages=[{"role": "system", "content": system}] + messages) c = r.choices[0] return c.message.content or "", {"stop": c.finish_reason, "in": r.usage.prompt_tokens, "out": r.usage.completion_tokens} def run_scenario(model, system, strategy, case): messages, rows = [], [] for i, turn in enumerate(case["turns"]): messages.append({"role": "user", "content": turn["user"]}) t0 = time.time() try: text, meta = model.reply(system, messages, strategy) except Exception as e: # a failed call is a failed turn, not a crashed run text, meta = "", {"error": f"{type(e).__name__}: {e}"} messages.append({"role": "assistant", "content": text or "(no reply)"}) rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i, "user": turn["user"], "expected": turn["expected"], "reply": text, "exact_match": text.strip() == turn["expected"], "latency_s": round(time.time() - t0, 2), **meta}) return rows def main(): ap = argparse.ArgumentParser() ap.add_argument("--cases", default="cases/ceiling.jsonl") ap.add_argument("--repeats", type=int, default=3) ap.add_argument("--limit", type=int, default=0, help="first N scenarios only (smoke test)") ap.add_argument("--models", default="sonnet,mistral") ap.add_argument("--strategies", default="zero_shot,few_shot,cot") ap.add_argument("--workers", type=int, default=8) ap.add_argument("--out", default=str(OUT)) args = ap.parse_args() cases = [json.loads(l) for l in open(args.cases, encoding="utf-8")] cases = cases[: args.limit] if args.limit else cases out = Path(args.out) out.mkdir(parents=True, exist_ok=True) models = {"sonnet": Sonnet, "mistral": Mistral} for name in args.models.split(","): model = models[name]() for strategy in args.strategies.split(","): system = system_for(strategy) for k in range(1, args.repeats + 1): path = out / f"{name}__{strategy}__r{k}.jsonl" if path.exists(): print(f"skip {path.name} (exists)") continue t0 = time.time() with ThreadPoolExecutor(args.workers) as pool: rows = [r for rs in pool.map(lambda c: run_scenario(model, system, strategy, c), cases) for r in rs] path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") em = sum(r["exact_match"] for r in rows) / len(rows) errs = sum("error" in r for r in rows) print(f"{path.name}: {len(rows)} turns, exact match {em:.1%}, api errors {errs}, {time.time() - t0:.0f}s", flush=True) if __name__ == "__main__": main()