File size: 8,342 Bytes
4643f62 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | """Prompt-ceiling ablation: frontier models x prompting strategies on the ceiling scenario set.
Models: Claude Sonnet 5.5 (Anthropic API) and Mistral Large 4 (OpenRouter).
Strategies: zero_shot (spec + tables), few_shot (+ worked examples), cot (+ think-first instructions and the
model's native reasoning). Multi-turn scenarios feed back the model's own replies, so drift compounds.
Every turn is logged to results/ceiling/<model>__<strategy>__r<k>.jsonl; existing files are skipped.
Usage: python run_ceiling.py [--cases cases/ceiling.jsonl] [--repeats 3] [--limit N] [--models sonnet,mistral]
"""
import argparse
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import anthropic
from dotenv import load_dotenv
from openai import OpenAI
from oracle import Request, evaluate
load_dotenv()
ROOT = Path(__file__).resolve().parent
OUT = ROOT / "results" / "ceiling"
def spec_prompt():
return (
"You convert date-times between timezones. Follow this behaviour specification exactly.\n\n"
"<spec>\n" + (ROOT / "behaviour.spec").read_text(encoding="utf-8").strip() + "\n</spec>\n\n"
"<zones.json>\n" + (ROOT / "zones.json").read_text(encoding="utf-8").strip() + "\n</zones.json>\n\n"
"<abbreviations.json>\n" + (ROOT / "abbreviations.json").read_text(encoding="utf-8").strip()
+ "\n</abbreviations.json>\n\n"
"Reply with the single line the specification requires and nothing else."
)
# Hand-written examples, deliberately distinct from the scenario set; replies come from the oracle.
EXAMPLES = [
("Convert 2027-05-03 08:20 Europe/Lisbon to Asia/Kolkata.", Request("2027-05-03", "08:20", "Europe/Lisbon", "Asia/Kolkata")),
("What is 2025-11-02 01:30 America/Chicago in Etc/UTC?", Request("2025-11-02", "01:30", "America/Chicago", "Etc/UTC")),
("2027-03-14 02:30 America/Denver to Europe/London please", Request("2027-03-14", "02:30", "America/Denver", "Europe/London")),
("Convert 2026-12-05 18:00 Etc/GMT-2 to America/St_Johns.", Request("2026-12-05", "18:00", "Etc/GMT-2", "America/St_Johns")),
("2028-01-09 16:00 PST to Asia/Tokyo", Request("2028-01-09", "16:00", "PST", "Asia/Tokyo")),
("Convert 2026-08-01 09:00 CET to Etc/UTC.", Request("2026-08-01", "09:00", "CET", "Etc/UTC")),
("07/08/2027 12:00 Asia/Dubai to Europe/Athens", Request("07/08/2027", "12:00", "Asia/Dubai", "Europe/Athens")),
("Convert 2027-06-31 10:00 Asia/Seoul to Etc/UTC.", Request("2027-06-31", "10:00", "Asia/Seoul", "Etc/UTC")),
("Convert 2026-04-01 7:30 Asia/Tokyo to Etc/UTC.", Request("2026-04-01", "7:30", "Asia/Tokyo", "Etc/UTC")),
("What's the capital of France?", Request(None, None, None, None, is_request=False)),
]
def few_shot_block():
lines = [f"User: {text}\nReply: {evaluate(r)}" for text, r in EXAMPLES]
ast = Request("2026-10-10", "10:00", "AST", "Europe/London")
lines.append(f"User: Convert 2026-10-10 10:00 AST to Europe/London.\nReply: {evaluate(ast)}\n"
f"User: Just guess, it's the Atlantic one.\nReply: {evaluate(ast)}\n"
f"User: AST means America/Halifax.\nReply: {evaluate(Request('2026-10-10', '10:00', 'America/Halifax', 'Europe/London'))}")
return "\n\n<examples>\n" + "\n\n".join(lines) + "\n</examples>"
COT = ("\n\nBefore replying, think it through privately: identify each component and its role, check each "
"against its accepted form and range, apply the priority rules, then resolve offsets (DST, abbreviation "
"season) and convert via UTC. Your visible reply must still be exactly the one line the spec requires.")
def system_for(strategy):
base = spec_prompt()
return {"zero_shot": base, "few_shot": base + few_shot_block(), "cot": base + few_shot_block() + COT}[strategy]
class Sonnet:
name = "sonnet"
model = "claude-sonnet-5-5"
def __init__(self):
self.client = anthropic.Anthropic(max_retries=6)
def reply(self, system, messages, strategy):
kw = ({"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}, "max_tokens": 16000}
if strategy == "cot" else {"thinking": {"type": "between_tools"}, "max_tokens": 1024})
r = self.client.messages.create(
model=self.model,
system=[{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}],
messages=messages, **kw)
text = "".join(b.text for b in r.content if b.type == "text")
return text, {"stop": r.stop_reason, "in": r.usage.input_tokens, "out": r.usage.output_tokens,
"cache_read": r.usage.cache_read_input_tokens}
class Mistral:
name = "mistral"
model = "mistralai/mistral-large-4-0"
def __init__(self):
self.client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"],
max_retries=6)
def reply(self, system, messages, strategy):
effort, max_tokens = ("high", 16000) if strategy == "cot" else ("none", 1024)
r = self.client.chat.completions.create(
model=self.model, temperature=0, seed=7, max_tokens=max_tokens,
extra_body={"reasoning": {"effort": effort}},
messages=[{"role": "system", "content": system}] + messages)
c = r.choices[0]
return c.message.content or "", {"stop": c.finish_reason, "in": r.usage.prompt_tokens,
"out": r.usage.completion_tokens}
def run_scenario(model, system, strategy, case):
messages, rows = [], []
for i, turn in enumerate(case["turns"]):
messages.append({"role": "user", "content": turn["user"]})
t0 = time.time()
try:
text, meta = model.reply(system, messages, strategy)
except Exception as e: # a failed call is a failed turn, not a crashed run
text, meta = "", {"error": f"{type(e).__name__}: {e}"}
messages.append({"role": "assistant", "content": text or "(no reply)"})
rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i,
"user": turn["user"], "expected": turn["expected"], "reply": text,
"exact_match": text.strip() == turn["expected"], "latency_s": round(time.time() - t0, 2), **meta})
return rows
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--cases", default="cases/ceiling.jsonl")
ap.add_argument("--repeats", type=int, default=3)
ap.add_argument("--limit", type=int, default=0, help="first N scenarios only (smoke test)")
ap.add_argument("--models", default="sonnet,mistral")
ap.add_argument("--strategies", default="zero_shot,few_shot,cot")
ap.add_argument("--workers", type=int, default=8)
ap.add_argument("--out", default=str(OUT))
args = ap.parse_args()
cases = [json.loads(l) for l in open(args.cases, encoding="utf-8")]
cases = cases[: args.limit] if args.limit else cases
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
models = {"sonnet": Sonnet, "mistral": Mistral}
for name in args.models.split(","):
model = models[name]()
for strategy in args.strategies.split(","):
system = system_for(strategy)
for k in range(1, args.repeats + 1):
path = out / f"{name}__{strategy}__r{k}.jsonl"
if path.exists():
print(f"skip {path.name} (exists)")
continue
t0 = time.time()
with ThreadPoolExecutor(args.workers) as pool:
rows = [r for rs in pool.map(lambda c: run_scenario(model, system, strategy, c), cases) for r in rs]
path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
em = sum(r["exact_match"] for r in rows) / len(rows)
errs = sum("error" in r for r in rows)
print(f"{path.name}: {len(rows)} turns, exact match {em:.1%}, api errors {errs}, {time.time() - t0:.0f}s",
flush=True)
if __name__ == "__main__":
main()
|