Download run_ceiling.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 8.34 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/run_ceiling.py
- Command line
-
hf download hf://debudash/slm-training/run_ceiling.py
-
curl -L -o run_ceiling.py https://huggingface.co/debudash/slm-training/resolve/main/run_ceiling.py
8.34 kB
| """Prompt-ceiling ablation: frontier models x prompting strategies on the ceiling scenario set. | |
| Models: Claude Sonnet 5.5 (Anthropic API) and Mistral Large 4 (OpenRouter). | |
| Strategies: zero_shot (spec + tables), few_shot (+ worked examples), cot (+ think-first instructions and the | |
| model's native reasoning). Multi-turn scenarios feed back the model's own replies, so drift compounds. | |
| Every turn is logged to results/ceiling/<model>__<strategy>__r<k>.jsonl; existing files are skipped. | |
| Usage: python run_ceiling.py [--cases cases/ceiling.jsonl] [--repeats 3] [--limit N] [--models sonnet,mistral] | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import anthropic | |
| from dotenv import load_dotenv | |
| from openai import OpenAI | |
| from oracle import Request, evaluate | |
| load_dotenv() | |
| ROOT = Path(__file__).resolve().parent | |
| OUT = ROOT / "results" / "ceiling" | |
| def spec_prompt(): | |
| return ( | |
| "You convert date-times between timezones. Follow this behaviour specification exactly.\n\n" | |
| "<spec>\n" + (ROOT / "behaviour.spec").read_text(encoding="utf-8").strip() + "\n</spec>\n\n" | |
| "<zones.json>\n" + (ROOT / "zones.json").read_text(encoding="utf-8").strip() + "\n</zones.json>\n\n" | |
| "<abbreviations.json>\n" + (ROOT / "abbreviations.json").read_text(encoding="utf-8").strip() | |
| + "\n</abbreviations.json>\n\n" | |
| "Reply with the single line the specification requires and nothing else." | |
| ) | |
| # Hand-written examples, deliberately distinct from the scenario set; replies come from the oracle. | |
| EXAMPLES = [ | |
| ("Convert 2027-05-03 08:20 Europe/Lisbon to Asia/Kolkata.", Request("2027-05-03", "08:20", "Europe/Lisbon", "Asia/Kolkata")), | |
| ("What is 2025-11-02 01:30 America/Chicago in Etc/UTC?", Request("2025-11-02", "01:30", "America/Chicago", "Etc/UTC")), | |
| ("2027-03-14 02:30 America/Denver to Europe/London please", Request("2027-03-14", "02:30", "America/Denver", "Europe/London")), | |
| ("Convert 2026-12-05 18:00 Etc/GMT-2 to America/St_Johns.", Request("2026-12-05", "18:00", "Etc/GMT-2", "America/St_Johns")), | |
| ("2028-01-09 16:00 PST to Asia/Tokyo", Request("2028-01-09", "16:00", "PST", "Asia/Tokyo")), | |
| ("Convert 2026-08-01 09:00 CET to Etc/UTC.", Request("2026-08-01", "09:00", "CET", "Etc/UTC")), | |
| ("07/08/2027 12:00 Asia/Dubai to Europe/Athens", Request("07/08/2027", "12:00", "Asia/Dubai", "Europe/Athens")), | |
| ("Convert 2027-06-31 10:00 Asia/Seoul to Etc/UTC.", Request("2027-06-31", "10:00", "Asia/Seoul", "Etc/UTC")), | |
| ("Convert 2026-04-01 7:30 Asia/Tokyo to Etc/UTC.", Request("2026-04-01", "7:30", "Asia/Tokyo", "Etc/UTC")), | |
| ("What's the capital of France?", Request(None, None, None, None, is_request=False)), | |
| ] | |
| def few_shot_block(): | |
| lines = [f"User: {text}\nReply: {evaluate(r)}" for text, r in EXAMPLES] | |
| ast = Request("2026-10-10", "10:00", "AST", "Europe/London") | |
| lines.append(f"User: Convert 2026-10-10 10:00 AST to Europe/London.\nReply: {evaluate(ast)}\n" | |
| f"User: Just guess, it's the Atlantic one.\nReply: {evaluate(ast)}\n" | |
| f"User: AST means America/Halifax.\nReply: {evaluate(Request('2026-10-10', '10:00', 'America/Halifax', 'Europe/London'))}") | |
| return "\n\n<examples>\n" + "\n\n".join(lines) + "\n</examples>" | |
| COT = ("\n\nBefore replying, think it through privately: identify each component and its role, check each " | |
| "against its accepted form and range, apply the priority rules, then resolve offsets (DST, abbreviation " | |
| "season) and convert via UTC. Your visible reply must still be exactly the one line the spec requires.") | |
| def system_for(strategy): | |
| base = spec_prompt() | |
| return {"zero_shot": base, "few_shot": base + few_shot_block(), "cot": base + few_shot_block() + COT}[strategy] | |
| class Sonnet: | |
| name = "sonnet" | |
| model = "claude-sonnet-5-5" | |
| def __init__(self): | |
| self.client = anthropic.Anthropic(max_retries=6) | |
| def reply(self, system, messages, strategy): | |
| kw = ({"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}, "max_tokens": 16000} | |
| if strategy == "cot" else {"thinking": {"type": "between_tools"}, "max_tokens": 1024}) | |
| r = self.client.messages.create( | |
| model=self.model, | |
| system=[{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}], | |
| messages=messages, **kw) | |
| text = "".join(b.text for b in r.content if b.type == "text") | |
| return text, {"stop": r.stop_reason, "in": r.usage.input_tokens, "out": r.usage.output_tokens, | |
| "cache_read": r.usage.cache_read_input_tokens} | |
| class Mistral: | |
| name = "mistral" | |
| model = "mistralai/mistral-large-4-0" | |
| def __init__(self): | |
| self.client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"], | |
| max_retries=6) | |
| def reply(self, system, messages, strategy): | |
| effort, max_tokens = ("high", 16000) if strategy == "cot" else ("none", 1024) | |
| r = self.client.chat.completions.create( | |
| model=self.model, temperature=0, seed=7, max_tokens=max_tokens, | |
| extra_body={"reasoning": {"effort": effort}}, | |
| messages=[{"role": "system", "content": system}] + messages) | |
| c = r.choices[0] | |
| return c.message.content or "", {"stop": c.finish_reason, "in": r.usage.prompt_tokens, | |
| "out": r.usage.completion_tokens} | |
| def run_scenario(model, system, strategy, case): | |
| messages, rows = [], [] | |
| for i, turn in enumerate(case["turns"]): | |
| messages.append({"role": "user", "content": turn["user"]}) | |
| t0 = time.time() | |
| try: | |
| text, meta = model.reply(system, messages, strategy) | |
| except Exception as e: # a failed call is a failed turn, not a crashed run | |
| text, meta = "", {"error": f"{type(e).__name__}: {e}"} | |
| messages.append({"role": "assistant", "content": text or "(no reply)"}) | |
| rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i, | |
| "user": turn["user"], "expected": turn["expected"], "reply": text, | |
| "exact_match": text.strip() == turn["expected"], "latency_s": round(time.time() - t0, 2), **meta}) | |
| return rows | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--cases", default="cases/ceiling.jsonl") | |
| ap.add_argument("--repeats", type=int, default=3) | |
| ap.add_argument("--limit", type=int, default=0, help="first N scenarios only (smoke test)") | |
| ap.add_argument("--models", default="sonnet,mistral") | |
| ap.add_argument("--strategies", default="zero_shot,few_shot,cot") | |
| ap.add_argument("--workers", type=int, default=8) | |
| ap.add_argument("--out", default=str(OUT)) | |
| args = ap.parse_args() | |
| cases = [json.loads(l) for l in open(args.cases, encoding="utf-8")] | |
| cases = cases[: args.limit] if args.limit else cases | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| models = {"sonnet": Sonnet, "mistral": Mistral} | |
| for name in args.models.split(","): | |
| model = models[name]() | |
| for strategy in args.strategies.split(","): | |
| system = system_for(strategy) | |
| for k in range(1, args.repeats + 1): | |
| path = out / f"{name}__{strategy}__r{k}.jsonl" | |
| if path.exists(): | |
| print(f"skip {path.name} (exists)") | |
| continue | |
| t0 = time.time() | |
| with ThreadPoolExecutor(args.workers) as pool: | |
| rows = [r for rs in pool.map(lambda c: run_scenario(model, system, strategy, c), cases) for r in rs] | |
| path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") | |
| em = sum(r["exact_match"] for r in rows) / len(rows) | |
| errs = sum("error" in r for r in rows) | |
| print(f"{path.name}: {len(rows)} turns, exact match {em:.1%}, api errors {errs}, {time.time() - t0:.0f}s", | |
| flush=True) | |
| if __name__ == "__main__": | |
| main() | |