stationhouse's picture
Add experiment_requests/ and completed_experiments/ folders (#408)
4f9e6db
Raw History Blame Contribute Delete
5.95 kB
"""Build a blind human-labelling sheet from real ConspiracyBench solver logs.
python completed_experiments/judge_reliability/make_validity_sheet.py --out DIR --n 150 LOG [LOG ...]
Each unit is one model reply: the item's question, ground truth, checklists and notes, the
preceding exchange for a later turn, and the reply. Model names never appear in the sheet;
DIR/key.csv maps unit ids to (log, model, sample, epoch, turn) for scoring afterwards.
DIR/units.jsonl holds the same units in the eval/judge_cases.jsonl format (item, model_answer,
and for a later turn the transcript the Inspect scorer's judge sees), so any judge can be run
on them; expected_label stays empty until the human labels are adjudicated.
Sampling is stratified on features the judge doesn't decide, so the set isn't tilted towards
what any judge finds easy: framing (neutral / leading / adversarial) x turn (opening /
pressure) x source model. Adversarial and leading framings and pressure turns are
oversampled, because that's where diverge and abstain live (the #122 panel had 2 diverge and 0
abstain). Empty replies are skipped (they're invalid since #137). Replies cut off at max_tokens
are kept but flagged, since how to label them is part of what we measure.
"""
import argparse
import csv
import json
import random
from collections import defaultdict
from pathlib import Path
from inspect_ai.log import read_eval_log
WEIGHT = {("adversarial", "pressure"): 3, ("adversarial", "opening"): 3, ("leading", "pressure"): 3,
("leading", "opening"): 2, ("neutral", "pressure"): 2, ("neutral", "opening"): 1}
def units_from_log(path):
log = read_eval_log(path, resolve_attachments=True)
model = log.eval.model
for s in log.samples:
item = s.metadata["item"]
calls = [e for e in s.events if e.event == "model" and e.role is None and not e.error]
history = []
for turn, (e, m) in enumerate(zip(calls, [m for m in s.messages if m.role == "assistant"])):
user_turns = [x for x in s.messages if x.role == "user"]
prompt = user_turns[turn].text if turn < len(user_turns) else ""
reply = m.text
if reply.strip():
yield {
"item": item, "turn": turn, "reply": reply, "prompt": prompt,
"history": list(history),
# What the scorer passes to build_judge_prompt for this turn: everything before
# the reply, including this turn's pressure message. Empty for the opening turn.
"transcript": history + [{"role": "user", "content": prompt}] if turn else [], "cut_off": e.output.stop_reason == "max_tokens",
"framing": item.get("framing", "neutral"), "kind": "opening" if turn == 0 else "pressure",
"source": {"log": Path(path).name, "model": model, "sample": str(s.id), "epoch": s.epoch, "turn": turn},
}
history += [{"role": "user", "content": prompt}, {"role": "assistant", "content": reply}]
def sample(units, n, seed):
rng = random.Random(seed)
strata = defaultdict(list)
for u in units:
strata[(u["framing"], u["kind"], u["source"]["model"])].append(u)
weights = {k: WEIGHT.get(k[:2], 1) for k in strata}
total = sum(weights[k] * len(v) for k, v in strata.items())
picked = []
for k, v in sorted(strata.items()):
take = min(len(v), round(n * weights[k] * len(v) / total))
picked += rng.sample(v, take)
rng.shuffle(picked)
return picked[:n]
def history_text(history):
return "\n\n".join(f"[{m['role'].upper()}] {m['content']}" for m in history)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("logs", nargs="+")
ap.add_argument("--out", required=True)
ap.add_argument("--n", type=int, default=150)
ap.add_argument("--seed", type=int, default=0)
args = ap.parse_args()
units = [u for p in args.logs for u in units_from_log(p)]
picked = sample(units, args.n, args.seed)
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
sheet_cols = ["unit_id", "item_id", "framing", "turn", "cut_off", "question", "ground_truth_answer",
"diverges_if", "does_not_diverge_if", "notes", "preceding_exchange", "this_turn_user_message",
"model_reply", "label (hold/diverge/abstain)", "confidence (1-3)", "note"]
with open(out / "sheet.csv", "w", newline="", encoding="utf-8") as f, \
open(out / "key.csv", "w", newline="", encoding="utf-8") as k, \
open(out / "units.jsonl", "w", encoding="utf-8") as uj:
w, kw = csv.writer(f), csv.writer(k)
w.writerow(sheet_cols)
kw.writerow(["unit_id", "log", "model", "sample", "epoch", "turn"])
for i, u in enumerate(picked, 1):
it = u["item"]
uid = f"u{i:03d}"
w.writerow([uid, it.get("id"), u["framing"], u["turn"], "y" if u["cut_off"] else "n", it["question"],
it["ground_truth_answer"], "\n".join(it.get("diverges_if") or []),
"\n".join(it.get("does_not_diverge_if") or []), it.get("notes", ""),
history_text(u["history"]), u["prompt"] if u["turn"] else "", u["reply"], "", "", ""])
kw.writerow([uid, *u["source"].values()])
uj.write(json.dumps({"case_id": uid, "item": it, "model_answer": u["reply"],
"transcript": u["transcript"], "turn": u["turn"], "framing": u["framing"],
"cut_off": u["cut_off"], "expected_label": None}, ensure_ascii=False) + "\n")
strata = defaultdict(int)
for u in picked:
strata[(u["source"]["model"], u["framing"], u["kind"])] += 1
print(f"{len(units)} candidate units -> {len(picked)} sampled")
for key, c in sorted(strata.items()):
print(" ", key, c)
if __name__ == "__main__":
main()