ConspiracyBench / analysis /designed-difficulty /sample_judge_audit.py
stationhouse's picture jiwon9703's picture
Add exploratory analysis of designed difficulty and model divergence (#449)
a91390f
Raw History Blame Contribute Delete
5.59 kB
from pathlib import Path
from collections import defaultdict
import json
import numpy as np
import pandas as pd
from inspect_ai.log import read_eval_log
OUT = Path('analysis/designed-difficulty')
ROOT = Path('completed_experiments/evals/full-2026-10-23/outputs')
rng = np.random.default_rng(20261031)
df = pd.read_csv(OUT / 'baseline_joined.csv')
df = df[
(df.variant == 'multiturn')
& (df.turn == 0)
& (df.status == 'ok')
& (df.dropped == 0)
& df.choice.isin(['hold', 'diverge', 'abstain'])
].copy()
prior = {
('qwen3.5-4b', 'cb-0090', 1),
('k2-horizon-3.7b', 'cb-0242', 1),
('qwen3.5-4b', 'cb-0242', 1),
}
df = df[~df.apply(
lambda x: (x.model, x.question_id, int(x.epoch)) in prior,
axis=1
)]
assert not df.duplicated(['model', 'case_id']).any()
# Balanced selection across model, verdict, and scarcity.
chosen = []
for (model, label), group in df.groupby(['model', 'choice'], sort=True):
group = group.sort_values(['question_id', 'epoch'])
ids = []
for score in (0, 1, 2):
g = group[group.evidence_scarcity == score]
if len(g) and len(ids) < 3:
ids.append(int(rng.choice(g.index.to_numpy())))
g = group[~group.index.isin(ids)]
ids += list(rng.choice(
g.index.to_numpy(),
size=min(3-len(ids), len(g)),
replace=False
))
if len(ids) < 3:
print('SHORT STRATUM:', model, label, len(ids))
chosen += ids
picked = df.loc[chosen].copy().reset_index(drop=True)
picked['audit_id'] = [
'AUD-%03d' % i for i in range(1, len(picked)+1)
]
print('Selected:', len(picked))
print(pd.crosstab(picked.model, picked.choice))
review, key, unresolved = [], [], []
for model, group in picked.groupby('model'):
wanted = {
(str(r.question_id), int(r.epoch))
for r in group.itertuples()
}
found = defaultdict(list)
for path in sorted((ROOT / model / 'multiturn').glob('*.eval')):
print('Reading', path.name)
log = read_eval_log(str(path))
for s in log.samples or []:
k = (str(s.id), int(s.epoch))
if k not in wanted:
continue
assistants = [
m for m in s.messages or []
if m.role == 'assistant'
]
if not assistants:
continue
content = assistants[0].content
if isinstance(content, str):
answer = content.strip()
else:
answer = '\n'.join(
p.text for p in content
if getattr(p, 'type', None) == 'text'
and isinstance(getattr(p, 'text', None), str)
).strip()
found[k].append((
answer, (s.metadata or {}).get('item', {}), str(path)
))
for r in group.itertuples():
sources = found.get(
(str(r.question_id), int(r.epoch)), []
)
uniq = {ans for ans, _, _ in sources if ans}
if len(uniq) != 1:
unresolved.append({
'audit_id': r.audit_id,
'case_id': r.case_id,
'source_files': ' | '.join(
p for _, _, p in sources
),
'distinct_answers': len(uniq),
})
continue
ans, item, path = next(
(a, i, p) for a, i, p in sources if a
)
if (
item.get('id') != r.question_id
or item.get('difficulty', {}).get('evidence_scarcity')
!= r.evidence_scarcity
):
raise ValueError('Metadata mismatch: ' + r.case_id)
review.append({
'audit_id': r.audit_id,
'model': model,
'question_id': r.question_id,
'evidence_scarcity': r.evidence_scarcity,
'question_openness': r.question_openness,
'question': item.get('question', ''),
'ground_truth': item.get('ground_truth_answer', ''),
'diverges_if': json.dumps(
item.get('diverges_if', []), ensure_ascii=False
),
'does_not_diverge_if': json.dumps(
item.get('does_not_diverge_if', []),
ensure_ascii=False
),
'visible_answer': ans,
'manual_choice': '',
'manual_reason': '',
})
key.append({
'audit_id': r.audit_id,
'judge_choice': r.choice,
'case_id': r.case_id,
'source_eval': path,
})
OUT.mkdir(parents=True, exist_ok=True)
pd.DataFrame(review).to_csv(
OUT / 'audit_review_BLIND.csv', index=False
)
pd.DataFrame(key).to_csv(
OUT / 'audit_judge_key.csv', index=False
)
pd.DataFrame(
unresolved,
columns=['audit_id','case_id','source_files','distinct_answers']
).to_csv(OUT / 'audit_unresolved.csv', index=False)
with (OUT / 'audit_review_BLIND.md').open('w') as f:
f.write('# Blinded Judge Audit\n\nJudge label hidden.\n')
for r in review:
f.write(
f"\n## {r['audit_id']} — "
f"{r['model']}, {r['question_id']}\n"
)
for name in [
'question', 'ground_truth', 'diverges_if',
'does_not_diverge_if', 'visible_answer'
]:
f.write(f"\n**{name}:**\n\n{r[name]}\n")
f.write(
'\n**Your verdict '
'(hold/diverge/abstain/uncertain):** ______\n\n---\n'
)
print('Extracted:', len(review))
print('Unresolved:', len(unresolved))
print('Saved blind review and Judge key.')