from pathlib import Path from collections import defaultdict import json import numpy as np import pandas as pd from inspect_ai.log import read_eval_log OUT = Path('analysis/designed-difficulty') ROOT = Path('completed_experiments/evals/full-2026-10-23/outputs') rng = np.random.default_rng(20261031) df = pd.read_csv(OUT / 'baseline_joined.csv') df = df[ (df.variant == 'multiturn') & (df.turn == 0) & (df.status == 'ok') & (df.dropped == 0) & df.choice.isin(['hold', 'diverge', 'abstain']) ].copy() prior = { ('qwen3.5-4b', 'cb-0090', 1), ('k2-horizon-3.7b', 'cb-0242', 1), ('qwen3.5-4b', 'cb-0242', 1), } df = df[~df.apply( lambda x: (x.model, x.question_id, int(x.epoch)) in prior, axis=1 )] assert not df.duplicated(['model', 'case_id']).any() # Balanced selection across model, verdict, and scarcity. chosen = [] for (model, label), group in df.groupby(['model', 'choice'], sort=True): group = group.sort_values(['question_id', 'epoch']) ids = [] for score in (0, 1, 2): g = group[group.evidence_scarcity == score] if len(g) and len(ids) < 3: ids.append(int(rng.choice(g.index.to_numpy()))) g = group[~group.index.isin(ids)] ids += list(rng.choice( g.index.to_numpy(), size=min(3-len(ids), len(g)), replace=False )) if len(ids) < 3: print('SHORT STRATUM:', model, label, len(ids)) chosen += ids picked = df.loc[chosen].copy().reset_index(drop=True) picked['audit_id'] = [ 'AUD-%03d' % i for i in range(1, len(picked)+1) ] print('Selected:', len(picked)) print(pd.crosstab(picked.model, picked.choice)) review, key, unresolved = [], [], [] for model, group in picked.groupby('model'): wanted = { (str(r.question_id), int(r.epoch)) for r in group.itertuples() } found = defaultdict(list) for path in sorted((ROOT / model / 'multiturn').glob('*.eval')): print('Reading', path.name) log = read_eval_log(str(path)) for s in log.samples or []: k = (str(s.id), int(s.epoch)) if k not in wanted: continue assistants = [ m for m in s.messages or [] if m.role == 'assistant' ] if not assistants: continue content = assistants[0].content if isinstance(content, str): answer = content.strip() else: answer = '\n'.join( p.text for p in content if getattr(p, 'type', None) == 'text' and isinstance(getattr(p, 'text', None), str) ).strip() found[k].append(( answer, (s.metadata or {}).get('item', {}), str(path) )) for r in group.itertuples(): sources = found.get( (str(r.question_id), int(r.epoch)), [] ) uniq = {ans for ans, _, _ in sources if ans} if len(uniq) != 1: unresolved.append({ 'audit_id': r.audit_id, 'case_id': r.case_id, 'source_files': ' | '.join( p for _, _, p in sources ), 'distinct_answers': len(uniq), }) continue ans, item, path = next( (a, i, p) for a, i, p in sources if a ) if ( item.get('id') != r.question_id or item.get('difficulty', {}).get('evidence_scarcity') != r.evidence_scarcity ): raise ValueError('Metadata mismatch: ' + r.case_id) review.append({ 'audit_id': r.audit_id, 'model': model, 'question_id': r.question_id, 'evidence_scarcity': r.evidence_scarcity, 'question_openness': r.question_openness, 'question': item.get('question', ''), 'ground_truth': item.get('ground_truth_answer', ''), 'diverges_if': json.dumps( item.get('diverges_if', []), ensure_ascii=False ), 'does_not_diverge_if': json.dumps( item.get('does_not_diverge_if', []), ensure_ascii=False ), 'visible_answer': ans, 'manual_choice': '', 'manual_reason': '', }) key.append({ 'audit_id': r.audit_id, 'judge_choice': r.choice, 'case_id': r.case_id, 'source_eval': path, }) OUT.mkdir(parents=True, exist_ok=True) pd.DataFrame(review).to_csv( OUT / 'audit_review_BLIND.csv', index=False ) pd.DataFrame(key).to_csv( OUT / 'audit_judge_key.csv', index=False ) pd.DataFrame( unresolved, columns=['audit_id','case_id','source_files','distinct_answers'] ).to_csv(OUT / 'audit_unresolved.csv', index=False) with (OUT / 'audit_review_BLIND.md').open('w') as f: f.write('# Blinded Judge Audit\n\nJudge label hidden.\n') for r in review: f.write( f"\n## {r['audit_id']} — " f"{r['model']}, {r['question_id']}\n" ) for name in [ 'question', 'ground_truth', 'diverges_if', 'does_not_diverge_if', 'visible_answer' ]: f.write(f"\n**{name}:**\n\n{r[name]}\n") f.write( '\n**Your verdict ' '(hold/diverge/abstain/uncertain):** ______\n\n---\n' ) print('Extracted:', len(review)) print('Unresolved:', len(unresolved)) print('Saved blind review and Judge key.')