slm-training / run_ceiling.py
debudash's picture
Mirror slm-training at 4376283
4643f62 verified
Raw History Blame Contribute Delete
8.34 kB
"""Prompt-ceiling ablation: frontier models x prompting strategies on the ceiling scenario set.
Models: Claude Sonnet 5.5 (Anthropic API) and Mistral Large 4 (OpenRouter).
Strategies: zero_shot (spec + tables), few_shot (+ worked examples), cot (+ think-first instructions and the
model's native reasoning). Multi-turn scenarios feed back the model's own replies, so drift compounds.
Every turn is logged to results/ceiling/<model>__<strategy>__r<k>.jsonl; existing files are skipped.
Usage: python run_ceiling.py [--cases cases/ceiling.jsonl] [--repeats 3] [--limit N] [--models sonnet,mistral]
"""
import argparse
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import anthropic
from dotenv import load_dotenv
from openai import OpenAI
from oracle import Request, evaluate
load_dotenv()
ROOT = Path(__file__).resolve().parent
OUT = ROOT / "results" / "ceiling"
def spec_prompt():
return (
"You convert date-times between timezones. Follow this behaviour specification exactly.\n\n"
"<spec>\n" + (ROOT / "behaviour.spec").read_text(encoding="utf-8").strip() + "\n</spec>\n\n"
"<zones.json>\n" + (ROOT / "zones.json").read_text(encoding="utf-8").strip() + "\n</zones.json>\n\n"
"<abbreviations.json>\n" + (ROOT / "abbreviations.json").read_text(encoding="utf-8").strip()
+ "\n</abbreviations.json>\n\n"
"Reply with the single line the specification requires and nothing else."
)
# Hand-written examples, deliberately distinct from the scenario set; replies come from the oracle.
EXAMPLES = [
("Convert 2027-05-03 08:20 Europe/Lisbon to Asia/Kolkata.", Request("2027-05-03", "08:20", "Europe/Lisbon", "Asia/Kolkata")),
("What is 2025-11-02 01:30 America/Chicago in Etc/UTC?", Request("2025-11-02", "01:30", "America/Chicago", "Etc/UTC")),
("2027-03-14 02:30 America/Denver to Europe/London please", Request("2027-03-14", "02:30", "America/Denver", "Europe/London")),
("Convert 2026-12-05 18:00 Etc/GMT-2 to America/St_Johns.", Request("2026-12-05", "18:00", "Etc/GMT-2", "America/St_Johns")),
("2028-01-09 16:00 PST to Asia/Tokyo", Request("2028-01-09", "16:00", "PST", "Asia/Tokyo")),
("Convert 2026-08-01 09:00 CET to Etc/UTC.", Request("2026-08-01", "09:00", "CET", "Etc/UTC")),
("07/08/2027 12:00 Asia/Dubai to Europe/Athens", Request("07/08/2027", "12:00", "Asia/Dubai", "Europe/Athens")),
("Convert 2027-06-31 10:00 Asia/Seoul to Etc/UTC.", Request("2027-06-31", "10:00", "Asia/Seoul", "Etc/UTC")),
("Convert 2026-04-01 7:30 Asia/Tokyo to Etc/UTC.", Request("2026-04-01", "7:30", "Asia/Tokyo", "Etc/UTC")),
("What's the capital of France?", Request(None, None, None, None, is_request=False)),
]
def few_shot_block():
lines = [f"User: {text}\nReply: {evaluate(r)}" for text, r in EXAMPLES]
ast = Request("2026-10-10", "10:00", "AST", "Europe/London")
lines.append(f"User: Convert 2026-10-10 10:00 AST to Europe/London.\nReply: {evaluate(ast)}\n"
f"User: Just guess, it's the Atlantic one.\nReply: {evaluate(ast)}\n"
f"User: AST means America/Halifax.\nReply: {evaluate(Request('2026-10-10', '10:00', 'America/Halifax', 'Europe/London'))}")
return "\n\n<examples>\n" + "\n\n".join(lines) + "\n</examples>"
COT = ("\n\nBefore replying, think it through privately: identify each component and its role, check each "
"against its accepted form and range, apply the priority rules, then resolve offsets (DST, abbreviation "
"season) and convert via UTC. Your visible reply must still be exactly the one line the spec requires.")
def system_for(strategy):
base = spec_prompt()
return {"zero_shot": base, "few_shot": base + few_shot_block(), "cot": base + few_shot_block() + COT}[strategy]
class Sonnet:
name = "sonnet"
model = "claude-sonnet-5-5"
def __init__(self):
self.client = anthropic.Anthropic(max_retries=6)
def reply(self, system, messages, strategy):
kw = ({"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}, "max_tokens": 16000}
if strategy == "cot" else {"thinking": {"type": "between_tools"}, "max_tokens": 1024})
r = self.client.messages.create(
model=self.model,
system=[{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}],
messages=messages, **kw)
text = "".join(b.text for b in r.content if b.type == "text")
return text, {"stop": r.stop_reason, "in": r.usage.input_tokens, "out": r.usage.output_tokens,
"cache_read": r.usage.cache_read_input_tokens}
class Mistral:
name = "mistral"
model = "mistralai/mistral-large-4-0"
def __init__(self):
self.client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"],
max_retries=6)
def reply(self, system, messages, strategy):
effort, max_tokens = ("high", 16000) if strategy == "cot" else ("none", 1024)
r = self.client.chat.completions.create(
model=self.model, temperature=0, seed=7, max_tokens=max_tokens,
extra_body={"reasoning": {"effort": effort}},
messages=[{"role": "system", "content": system}] + messages)
c = r.choices[0]
return c.message.content or "", {"stop": c.finish_reason, "in": r.usage.prompt_tokens,
"out": r.usage.completion_tokens}
def run_scenario(model, system, strategy, case):
messages, rows = [], []
for i, turn in enumerate(case["turns"]):
messages.append({"role": "user", "content": turn["user"]})
t0 = time.time()
try:
text, meta = model.reply(system, messages, strategy)
except Exception as e: # a failed call is a failed turn, not a crashed run
text, meta = "", {"error": f"{type(e).__name__}: {e}"}
messages.append({"role": "assistant", "content": text or "(no reply)"})
rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i,
"user": turn["user"], "expected": turn["expected"], "reply": text,
"exact_match": text.strip() == turn["expected"], "latency_s": round(time.time() - t0, 2), **meta})
return rows
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--cases", default="cases/ceiling.jsonl")
ap.add_argument("--repeats", type=int, default=3)
ap.add_argument("--limit", type=int, default=0, help="first N scenarios only (smoke test)")
ap.add_argument("--models", default="sonnet,mistral")
ap.add_argument("--strategies", default="zero_shot,few_shot,cot")
ap.add_argument("--workers", type=int, default=8)
ap.add_argument("--out", default=str(OUT))
args = ap.parse_args()
cases = [json.loads(l) for l in open(args.cases, encoding="utf-8")]
cases = cases[: args.limit] if args.limit else cases
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
models = {"sonnet": Sonnet, "mistral": Mistral}
for name in args.models.split(","):
model = models[name]()
for strategy in args.strategies.split(","):
system = system_for(strategy)
for k in range(1, args.repeats + 1):
path = out / f"{name}__{strategy}__r{k}.jsonl"
if path.exists():
print(f"skip {path.name} (exists)")
continue
t0 = time.time()
with ThreadPoolExecutor(args.workers) as pool:
rows = [r for rs in pool.map(lambda c: run_scenario(model, system, strategy, c), cases) for r in rs]
path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
em = sum(r["exact_match"] for r in rows) / len(rows)
errs = sum("error" in r for r in rows)
print(f"{path.name}: {len(rows)} turns, exact match {em:.1%}, api errors {errs}, {time.time() - t0:.0f}s",
flush=True)
if __name__ == "__main__":
main()