Spaces:
Running
Running
Download completed_experiments/judge_reliability/make_validity_sheet.py from daKeshra/ConspiracyBench: direct link, hf CLI and curl.
- Browser
- Download file 5.95 kB
-
https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/completed_experiments/judge_reliability/make_validity_sheet.py
- Command line
-
hf download hf://spaces/daKeshra/ConspiracyBench/completed_experiments/judge_reliability/make_validity_sheet.py
-
curl -L -o make_validity_sheet.py https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/completed_experiments/judge_reliability/make_validity_sheet.py
5.95 kB
| """Build a blind human-labelling sheet from real ConspiracyBench solver logs. | |
| python completed_experiments/judge_reliability/make_validity_sheet.py --out DIR --n 150 LOG [LOG ...] | |
| Each unit is one model reply: the item's question, ground truth, checklists and notes, the | |
| preceding exchange for a later turn, and the reply. Model names never appear in the sheet; | |
| DIR/key.csv maps unit ids to (log, model, sample, epoch, turn) for scoring afterwards. | |
| DIR/units.jsonl holds the same units in the eval/judge_cases.jsonl format (item, model_answer, | |
| and for a later turn the transcript the Inspect scorer's judge sees), so any judge can be run | |
| on them; expected_label stays empty until the human labels are adjudicated. | |
| Sampling is stratified on features the judge doesn't decide, so the set isn't tilted towards | |
| what any judge finds easy: framing (neutral / leading / adversarial) x turn (opening / | |
| pressure) x source model. Adversarial and leading framings and pressure turns are | |
| oversampled, because that's where diverge and abstain live (the #122 panel had 2 diverge and 0 | |
| abstain). Empty replies are skipped (they're invalid since #137). Replies cut off at max_tokens | |
| are kept but flagged, since how to label them is part of what we measure. | |
| """ | |
| import argparse | |
| import csv | |
| import json | |
| import random | |
| from collections import defaultdict | |
| from pathlib import Path | |
| from inspect_ai.log import read_eval_log | |
| WEIGHT = {("adversarial", "pressure"): 3, ("adversarial", "opening"): 3, ("leading", "pressure"): 3, | |
| ("leading", "opening"): 2, ("neutral", "pressure"): 2, ("neutral", "opening"): 1} | |
| def units_from_log(path): | |
| log = read_eval_log(path, resolve_attachments=True) | |
| model = log.eval.model | |
| for s in log.samples: | |
| item = s.metadata["item"] | |
| calls = [e for e in s.events if e.event == "model" and e.role is None and not e.error] | |
| history = [] | |
| for turn, (e, m) in enumerate(zip(calls, [m for m in s.messages if m.role == "assistant"])): | |
| user_turns = [x for x in s.messages if x.role == "user"] | |
| prompt = user_turns[turn].text if turn < len(user_turns) else "" | |
| reply = m.text | |
| if reply.strip(): | |
| yield { | |
| "item": item, "turn": turn, "reply": reply, "prompt": prompt, | |
| "history": list(history), | |
| # What the scorer passes to build_judge_prompt for this turn: everything before | |
| # the reply, including this turn's pressure message. Empty for the opening turn. | |
| "transcript": history + [{"role": "user", "content": prompt}] if turn else [], "cut_off": e.output.stop_reason == "max_tokens", | |
| "framing": item.get("framing", "neutral"), "kind": "opening" if turn == 0 else "pressure", | |
| "source": {"log": Path(path).name, "model": model, "sample": str(s.id), "epoch": s.epoch, "turn": turn}, | |
| } | |
| history += [{"role": "user", "content": prompt}, {"role": "assistant", "content": reply}] | |
| def sample(units, n, seed): | |
| rng = random.Random(seed) | |
| strata = defaultdict(list) | |
| for u in units: | |
| strata[(u["framing"], u["kind"], u["source"]["model"])].append(u) | |
| weights = {k: WEIGHT.get(k[:2], 1) for k in strata} | |
| total = sum(weights[k] * len(v) for k, v in strata.items()) | |
| picked = [] | |
| for k, v in sorted(strata.items()): | |
| take = min(len(v), round(n * weights[k] * len(v) / total)) | |
| picked += rng.sample(v, take) | |
| rng.shuffle(picked) | |
| return picked[:n] | |
| def history_text(history): | |
| return "\n\n".join(f"[{m['role'].upper()}] {m['content']}" for m in history) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("logs", nargs="+") | |
| ap.add_argument("--out", required=True) | |
| ap.add_argument("--n", type=int, default=150) | |
| ap.add_argument("--seed", type=int, default=0) | |
| args = ap.parse_args() | |
| units = [u for p in args.logs for u in units_from_log(p)] | |
| picked = sample(units, args.n, args.seed) | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| sheet_cols = ["unit_id", "item_id", "framing", "turn", "cut_off", "question", "ground_truth_answer", | |
| "diverges_if", "does_not_diverge_if", "notes", "preceding_exchange", "this_turn_user_message", | |
| "model_reply", "label (hold/diverge/abstain)", "confidence (1-3)", "note"] | |
| with open(out / "sheet.csv", "w", newline="", encoding="utf-8") as f, \ | |
| open(out / "key.csv", "w", newline="", encoding="utf-8") as k, \ | |
| open(out / "units.jsonl", "w", encoding="utf-8") as uj: | |
| w, kw = csv.writer(f), csv.writer(k) | |
| w.writerow(sheet_cols) | |
| kw.writerow(["unit_id", "log", "model", "sample", "epoch", "turn"]) | |
| for i, u in enumerate(picked, 1): | |
| it = u["item"] | |
| uid = f"u{i:03d}" | |
| w.writerow([uid, it.get("id"), u["framing"], u["turn"], "y" if u["cut_off"] else "n", it["question"], | |
| it["ground_truth_answer"], "\n".join(it.get("diverges_if") or []), | |
| "\n".join(it.get("does_not_diverge_if") or []), it.get("notes", ""), | |
| history_text(u["history"]), u["prompt"] if u["turn"] else "", u["reply"], "", "", ""]) | |
| kw.writerow([uid, *u["source"].values()]) | |
| uj.write(json.dumps({"case_id": uid, "item": it, "model_answer": u["reply"], | |
| "transcript": u["transcript"], "turn": u["turn"], "framing": u["framing"], | |
| "cut_off": u["cut_off"], "expected_label": None}, ensure_ascii=False) + "\n") | |
| strata = defaultdict(int) | |
| for u in picked: | |
| strata[(u["source"]["model"], u["framing"], u["kind"])] += 1 | |
| print(f"{len(units)} candidate units -> {len(picked)} sampled") | |
| for key, c in sorted(strata.items()): | |
| print(" ", key, c) | |
| if __name__ == "__main__": | |
| main() | |