slm-training / extract.py
debudash's picture
Mirror slm-training at 4376283
4643f62 verified
Raw History Blame Contribute Delete
5.87 kB
"""Label unlabelled scenarios: an LLM extracts the request's components, the oracle computes the expected reply.
Used by eval.py when an eval set (e.g. the staff held-out set) has prompts but no expected replies. The extractor
only copies component strings as written and judges the two things the oracle can't compute (is it a request,
are the roles unclear); every rule of the spec is applied by the oracle.
Validate on a labelled set: python extract.py cases/ceiling.jsonl
"""
import json
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import anthropic
from dotenv import load_dotenv
from oracle import Request, evaluate
load_dotenv()
EXTRACTOR = "claude-sonnet-5-5"
client = anthropic.Anthropic(max_retries=6)
PROMPT = """You read a conversation with a timezone-conversion assistant and report the conversion request that is
in effect after the LAST user message. Later messages may supply or replace components; messages that add
nothing (thanks, pressure, doubt) leave the request unchanged.
Copy each component EXACTLY as the user wrote it (same characters, no normalising, no fixing typos, no
converting formats); use null if the user never gave it:
- date, time: the date and time to convert, e.g. "03/04/2026", "3 PM", "2026-02-30"
- source: the timezone the time is given in; target: the timezone to convert to. For a clarification like
"CST means America/Chicago", the source becomes "America/Chicago".
- is_request: false if the conversation has never asked for a conversion (questions about timezones, small talk).
- unclear: list "source_timezone" and "target_timezone" if two zones are given but the wording does not say which
is the source and which the target; otherwise an empty list. Word order alone never says it: a bare
"2026-05-12 15:00 Asia/Tokyo Europe/London" is unclear (still copy both zones, first as source).
<conversation>
{conversation}
</conversation>"""
SCHEMA = {
"type": "object",
"properties": {
"date": {"type": ["string", "null"]}, "time": {"type": ["string", "null"]},
"source": {"type": ["string", "null"]}, "target": {"type": ["string", "null"]},
"is_request": {"type": "boolean"},
"unclear": {"type": "array", "items": {"type": "string", "enum": ["source_timezone", "target_timezone"]}},
},
"required": ["date", "time", "source", "target", "is_request", "unclear"],
"additionalProperties": False,
}
def extract(user_turns):
convo = "\n".join(f"USER: {u}" for u in user_turns)
r = client.messages.create(model=EXTRACTOR, max_tokens=2000, thinking={"type": "between_tools"},
output_config={"format": {"type": "json_schema", "schema": SCHEMA}},
messages=[{"role": "user", "content": PROMPT.format(conversation=convo)}])
f = json.loads(next(b.text for b in r.content if b.type == "text"))
return Request(f["date"], f["time"], f["source"], f["target"], is_request=f["is_request"],
unclear=frozenset(f["unclear"]))
def label_case(case):
"""Fill in missing expected replies, turn by turn, from the user turns so far."""
users = []
for turn in case["turns"]:
users.append(turn["user"])
if "expected" not in turn:
req = extract(users)
turn["expected"], turn["label_source"] = evaluate(req), "extracted"
turn["state"] = [req.date, req.time, req.source, req.target]
return case
def label_missing(cases, workers=12):
if all("expected" in t for c in cases for t in c["turns"]):
return cases
with ThreadPoolExecutor(workers) as pool:
return list(pool.map(label_case, cases))
def normalise(raw):
"""Accept our scenario format, or {"prompt"/"input"/"question": str}, or {"messages": [...]}, or plain text."""
if isinstance(raw, str):
raw = {"prompt": raw}
if "turns" in raw:
case = raw
elif "messages" in raw:
case = {"turns": [{"user": m["content"]} for m in raw["messages"] if m["role"] == "user"]}
else:
text = next(raw[k] for k in ("prompt", "input", "question", "user", "text") if k in raw)
case = {"turns": [{"user": text}]}
if "expected" in raw:
case["turns"][0]["expected"] = raw["expected"]
case.setdefault("id", raw.get("id"))
case.setdefault("slice", raw.get("slice", "external"))
case.setdefault("metric", raw.get("metric", "robustness"))
return case
def load_eval_set(path):
cases = []
for i, line in enumerate(open(path, encoding="utf-8")):
if not line.strip():
continue
try:
raw = json.loads(line)
except json.JSONDecodeError:
raw = line.strip()
case = normalise(raw)
case["id"] = case["id"] or f"case-{i + 1:03d}"
cases.append(case)
return cases
if __name__ == "__main__":
# Validation: strip the labels from a labelled set, re-label by extraction, compare with the oracle's labels.
cases = [json.loads(l) for l in open(sys.argv[1], encoding="utf-8")]
truth = [[t["expected"] for t in c["turns"]] for c in cases]
stripped = [{**c, "turns": [{"user": t["user"]} for t in c["turns"]]} for c in cases]
labelled = label_missing(stripped)
pairs = [(e, t["expected"], c["id"], t["user"]) for c, es in zip(labelled, truth) for e, t in zip(es, c["turns"])]
agree = sum(a == b for a, b, *_ in pairs)
print(f"{Path(sys.argv[1]).name}: extracted labels match oracle labels on {agree}/{len(pairs)} turns "
f"({agree / len(pairs):.1%})")
for a, b, cid, u in [p for p in pairs if p[0] != p[1]][:15]:
print(f" {cid}: {u[:70]!r} truth {a} | extracted {b}")