Download extract.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 5.87 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/extract.py
- Command line
-
hf download hf://debudash/slm-training/extract.py
-
curl -L -o extract.py https://huggingface.co/debudash/slm-training/resolve/main/extract.py
5.87 kB
| """Label unlabelled scenarios: an LLM extracts the request's components, the oracle computes the expected reply. | |
| Used by eval.py when an eval set (e.g. the staff held-out set) has prompts but no expected replies. The extractor | |
| only copies component strings as written and judges the two things the oracle can't compute (is it a request, | |
| are the roles unclear); every rule of the spec is applied by the oracle. | |
| Validate on a labelled set: python extract.py cases/ceiling.jsonl | |
| """ | |
| import json | |
| import sys | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import anthropic | |
| from dotenv import load_dotenv | |
| from oracle import Request, evaluate | |
| load_dotenv() | |
| EXTRACTOR = "claude-sonnet-5-5" | |
| client = anthropic.Anthropic(max_retries=6) | |
| PROMPT = """You read a conversation with a timezone-conversion assistant and report the conversion request that is | |
| in effect after the LAST user message. Later messages may supply or replace components; messages that add | |
| nothing (thanks, pressure, doubt) leave the request unchanged. | |
| Copy each component EXACTLY as the user wrote it (same characters, no normalising, no fixing typos, no | |
| converting formats); use null if the user never gave it: | |
| - date, time: the date and time to convert, e.g. "03/04/2026", "3 PM", "2026-02-30" | |
| - source: the timezone the time is given in; target: the timezone to convert to. For a clarification like | |
| "CST means America/Chicago", the source becomes "America/Chicago". | |
| - is_request: false if the conversation has never asked for a conversion (questions about timezones, small talk). | |
| - unclear: list "source_timezone" and "target_timezone" if two zones are given but the wording does not say which | |
| is the source and which the target; otherwise an empty list. Word order alone never says it: a bare | |
| "2026-05-12 15:00 Asia/Tokyo Europe/London" is unclear (still copy both zones, first as source). | |
| <conversation> | |
| {conversation} | |
| </conversation>""" | |
| SCHEMA = { | |
| "type": "object", | |
| "properties": { | |
| "date": {"type": ["string", "null"]}, "time": {"type": ["string", "null"]}, | |
| "source": {"type": ["string", "null"]}, "target": {"type": ["string", "null"]}, | |
| "is_request": {"type": "boolean"}, | |
| "unclear": {"type": "array", "items": {"type": "string", "enum": ["source_timezone", "target_timezone"]}}, | |
| }, | |
| "required": ["date", "time", "source", "target", "is_request", "unclear"], | |
| "additionalProperties": False, | |
| } | |
| def extract(user_turns): | |
| convo = "\n".join(f"USER: {u}" for u in user_turns) | |
| r = client.messages.create(model=EXTRACTOR, max_tokens=2000, thinking={"type": "between_tools"}, | |
| output_config={"format": {"type": "json_schema", "schema": SCHEMA}}, | |
| messages=[{"role": "user", "content": PROMPT.format(conversation=convo)}]) | |
| f = json.loads(next(b.text for b in r.content if b.type == "text")) | |
| return Request(f["date"], f["time"], f["source"], f["target"], is_request=f["is_request"], | |
| unclear=frozenset(f["unclear"])) | |
| def label_case(case): | |
| """Fill in missing expected replies, turn by turn, from the user turns so far.""" | |
| users = [] | |
| for turn in case["turns"]: | |
| users.append(turn["user"]) | |
| if "expected" not in turn: | |
| req = extract(users) | |
| turn["expected"], turn["label_source"] = evaluate(req), "extracted" | |
| turn["state"] = [req.date, req.time, req.source, req.target] | |
| return case | |
| def label_missing(cases, workers=12): | |
| if all("expected" in t for c in cases for t in c["turns"]): | |
| return cases | |
| with ThreadPoolExecutor(workers) as pool: | |
| return list(pool.map(label_case, cases)) | |
| def normalise(raw): | |
| """Accept our scenario format, or {"prompt"/"input"/"question": str}, or {"messages": [...]}, or plain text.""" | |
| if isinstance(raw, str): | |
| raw = {"prompt": raw} | |
| if "turns" in raw: | |
| case = raw | |
| elif "messages" in raw: | |
| case = {"turns": [{"user": m["content"]} for m in raw["messages"] if m["role"] == "user"]} | |
| else: | |
| text = next(raw[k] for k in ("prompt", "input", "question", "user", "text") if k in raw) | |
| case = {"turns": [{"user": text}]} | |
| if "expected" in raw: | |
| case["turns"][0]["expected"] = raw["expected"] | |
| case.setdefault("id", raw.get("id")) | |
| case.setdefault("slice", raw.get("slice", "external")) | |
| case.setdefault("metric", raw.get("metric", "robustness")) | |
| return case | |
| def load_eval_set(path): | |
| cases = [] | |
| for i, line in enumerate(open(path, encoding="utf-8")): | |
| if not line.strip(): | |
| continue | |
| try: | |
| raw = json.loads(line) | |
| except json.JSONDecodeError: | |
| raw = line.strip() | |
| case = normalise(raw) | |
| case["id"] = case["id"] or f"case-{i + 1:03d}" | |
| cases.append(case) | |
| return cases | |
| if __name__ == "__main__": | |
| # Validation: strip the labels from a labelled set, re-label by extraction, compare with the oracle's labels. | |
| cases = [json.loads(l) for l in open(sys.argv[1], encoding="utf-8")] | |
| truth = [[t["expected"] for t in c["turns"]] for c in cases] | |
| stripped = [{**c, "turns": [{"user": t["user"]} for t in c["turns"]]} for c in cases] | |
| labelled = label_missing(stripped) | |
| pairs = [(e, t["expected"], c["id"], t["user"]) for c, es in zip(labelled, truth) for e, t in zip(es, c["turns"])] | |
| agree = sum(a == b for a, b, *_ in pairs) | |
| print(f"{Path(sys.argv[1]).name}: extracted labels match oracle labels on {agree}/{len(pairs)} turns " | |
| f"({agree / len(pairs):.1%})") | |
| for a, b, cid, u in [p for p in pairs if p[0] != p[1]][:15]: | |
| print(f" {cid}: {u[:70]!r} truth {a} | extracted {b}") | |