"""Label unlabelled scenarios: an LLM extracts the request's components, the oracle computes the expected reply. Used by eval.py when an eval set (e.g. the staff held-out set) has prompts but no expected replies. The extractor only copies component strings as written and judges the two things the oracle can't compute (is it a request, are the roles unclear); every rule of the spec is applied by the oracle. Validate on a labelled set: python extract.py cases/ceiling.jsonl """ import json import sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path import anthropic from dotenv import load_dotenv from oracle import Request, evaluate load_dotenv() EXTRACTOR = "claude-sonnet-5-5" client = anthropic.Anthropic(max_retries=6) PROMPT = """You read a conversation with a timezone-conversion assistant and report the conversion request that is in effect after the LAST user message. Later messages may supply or replace components; messages that add nothing (thanks, pressure, doubt) leave the request unchanged. Copy each component EXACTLY as the user wrote it (same characters, no normalising, no fixing typos, no converting formats); use null if the user never gave it: - date, time: the date and time to convert, e.g. "03/04/2026", "3 PM", "2026-02-30" - source: the timezone the time is given in; target: the timezone to convert to. For a clarification like "CST means America/Chicago", the source becomes "America/Chicago". - is_request: false if the conversation has never asked for a conversion (questions about timezones, small talk). - unclear: list "source_timezone" and "target_timezone" if two zones are given but the wording does not say which is the source and which the target; otherwise an empty list. Word order alone never says it: a bare "2026-05-12 15:00 Asia/Tokyo Europe/London" is unclear (still copy both zones, first as source). {conversation} """ SCHEMA = { "type": "object", "properties": { "date": {"type": ["string", "null"]}, "time": {"type": ["string", "null"]}, "source": {"type": ["string", "null"]}, "target": {"type": ["string", "null"]}, "is_request": {"type": "boolean"}, "unclear": {"type": "array", "items": {"type": "string", "enum": ["source_timezone", "target_timezone"]}}, }, "required": ["date", "time", "source", "target", "is_request", "unclear"], "additionalProperties": False, } def extract(user_turns): convo = "\n".join(f"USER: {u}" for u in user_turns) r = client.messages.create(model=EXTRACTOR, max_tokens=2000, thinking={"type": "between_tools"}, output_config={"format": {"type": "json_schema", "schema": SCHEMA}}, messages=[{"role": "user", "content": PROMPT.format(conversation=convo)}]) f = json.loads(next(b.text for b in r.content if b.type == "text")) return Request(f["date"], f["time"], f["source"], f["target"], is_request=f["is_request"], unclear=frozenset(f["unclear"])) def label_case(case): """Fill in missing expected replies, turn by turn, from the user turns so far.""" users = [] for turn in case["turns"]: users.append(turn["user"]) if "expected" not in turn: req = extract(users) turn["expected"], turn["label_source"] = evaluate(req), "extracted" turn["state"] = [req.date, req.time, req.source, req.target] return case def label_missing(cases, workers=12): if all("expected" in t for c in cases for t in c["turns"]): return cases with ThreadPoolExecutor(workers) as pool: return list(pool.map(label_case, cases)) def normalise(raw): """Accept our scenario format, or {"prompt"/"input"/"question": str}, or {"messages": [...]}, or plain text.""" if isinstance(raw, str): raw = {"prompt": raw} if "turns" in raw: case = raw elif "messages" in raw: case = {"turns": [{"user": m["content"]} for m in raw["messages"] if m["role"] == "user"]} else: text = next(raw[k] for k in ("prompt", "input", "question", "user", "text") if k in raw) case = {"turns": [{"user": text}]} if "expected" in raw: case["turns"][0]["expected"] = raw["expected"] case.setdefault("id", raw.get("id")) case.setdefault("slice", raw.get("slice", "external")) case.setdefault("metric", raw.get("metric", "robustness")) return case def load_eval_set(path): cases = [] for i, line in enumerate(open(path, encoding="utf-8")): if not line.strip(): continue try: raw = json.loads(line) except json.JSONDecodeError: raw = line.strip() case = normalise(raw) case["id"] = case["id"] or f"case-{i + 1:03d}" cases.append(case) return cases if __name__ == "__main__": # Validation: strip the labels from a labelled set, re-label by extraction, compare with the oracle's labels. cases = [json.loads(l) for l in open(sys.argv[1], encoding="utf-8")] truth = [[t["expected"] for t in c["turns"]] for c in cases] stripped = [{**c, "turns": [{"user": t["user"]} for t in c["turns"]]} for c in cases] labelled = label_missing(stripped) pairs = [(e, t["expected"], c["id"], t["user"]) for c, es in zip(labelled, truth) for e, t in zip(es, c["turns"])] agree = sum(a == b for a, b, *_ in pairs) print(f"{Path(sys.argv[1]).name}: extracted labels match oracle labels on {agree}/{len(pairs)} turns " f"({agree / len(pairs):.1%})") for a, b, cid, u in [p for p in pairs if p[0] != p[1]][:15]: print(f" {cid}: {u[:70]!r} truth {a} | extracted {b}")