Spaces:
Running
Running
Download analysis/designed-difficulty/sample_judge_audit.py from daKeshra/ConspiracyBench: direct link, hf CLI and curl.
- Browser
- Download file 5.59 kB
-
https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/analysis/designed-difficulty/sample_judge_audit.py
- Command line
-
hf download hf://spaces/daKeshra/ConspiracyBench/analysis/designed-difficulty/sample_judge_audit.py
-
curl -L -o sample_judge_audit.py https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/analysis/designed-difficulty/sample_judge_audit.py
5.59 kB
| from pathlib import Path | |
| from collections import defaultdict | |
| import json | |
| import numpy as np | |
| import pandas as pd | |
| from inspect_ai.log import read_eval_log | |
| OUT = Path('analysis/designed-difficulty') | |
| ROOT = Path('completed_experiments/evals/full-2026-10-23/outputs') | |
| rng = np.random.default_rng(20261031) | |
| df = pd.read_csv(OUT / 'baseline_joined.csv') | |
| df = df[ | |
| (df.variant == 'multiturn') | |
| & (df.turn == 0) | |
| & (df.status == 'ok') | |
| & (df.dropped == 0) | |
| & df.choice.isin(['hold', 'diverge', 'abstain']) | |
| ].copy() | |
| prior = { | |
| ('qwen3.5-4b', 'cb-0090', 1), | |
| ('k2-horizon-3.7b', 'cb-0242', 1), | |
| ('qwen3.5-4b', 'cb-0242', 1), | |
| } | |
| df = df[~df.apply( | |
| lambda x: (x.model, x.question_id, int(x.epoch)) in prior, | |
| axis=1 | |
| )] | |
| assert not df.duplicated(['model', 'case_id']).any() | |
| # Balanced selection across model, verdict, and scarcity. | |
| chosen = [] | |
| for (model, label), group in df.groupby(['model', 'choice'], sort=True): | |
| group = group.sort_values(['question_id', 'epoch']) | |
| ids = [] | |
| for score in (0, 1, 2): | |
| g = group[group.evidence_scarcity == score] | |
| if len(g) and len(ids) < 3: | |
| ids.append(int(rng.choice(g.index.to_numpy()))) | |
| g = group[~group.index.isin(ids)] | |
| ids += list(rng.choice( | |
| g.index.to_numpy(), | |
| size=min(3-len(ids), len(g)), | |
| replace=False | |
| )) | |
| if len(ids) < 3: | |
| print('SHORT STRATUM:', model, label, len(ids)) | |
| chosen += ids | |
| picked = df.loc[chosen].copy().reset_index(drop=True) | |
| picked['audit_id'] = [ | |
| 'AUD-%03d' % i for i in range(1, len(picked)+1) | |
| ] | |
| print('Selected:', len(picked)) | |
| print(pd.crosstab(picked.model, picked.choice)) | |
| review, key, unresolved = [], [], [] | |
| for model, group in picked.groupby('model'): | |
| wanted = { | |
| (str(r.question_id), int(r.epoch)) | |
| for r in group.itertuples() | |
| } | |
| found = defaultdict(list) | |
| for path in sorted((ROOT / model / 'multiturn').glob('*.eval')): | |
| print('Reading', path.name) | |
| log = read_eval_log(str(path)) | |
| for s in log.samples or []: | |
| k = (str(s.id), int(s.epoch)) | |
| if k not in wanted: | |
| continue | |
| assistants = [ | |
| m for m in s.messages or [] | |
| if m.role == 'assistant' | |
| ] | |
| if not assistants: | |
| continue | |
| content = assistants[0].content | |
| if isinstance(content, str): | |
| answer = content.strip() | |
| else: | |
| answer = '\n'.join( | |
| p.text for p in content | |
| if getattr(p, 'type', None) == 'text' | |
| and isinstance(getattr(p, 'text', None), str) | |
| ).strip() | |
| found[k].append(( | |
| answer, (s.metadata or {}).get('item', {}), str(path) | |
| )) | |
| for r in group.itertuples(): | |
| sources = found.get( | |
| (str(r.question_id), int(r.epoch)), [] | |
| ) | |
| uniq = {ans for ans, _, _ in sources if ans} | |
| if len(uniq) != 1: | |
| unresolved.append({ | |
| 'audit_id': r.audit_id, | |
| 'case_id': r.case_id, | |
| 'source_files': ' | '.join( | |
| p for _, _, p in sources | |
| ), | |
| 'distinct_answers': len(uniq), | |
| }) | |
| continue | |
| ans, item, path = next( | |
| (a, i, p) for a, i, p in sources if a | |
| ) | |
| if ( | |
| item.get('id') != r.question_id | |
| or item.get('difficulty', {}).get('evidence_scarcity') | |
| != r.evidence_scarcity | |
| ): | |
| raise ValueError('Metadata mismatch: ' + r.case_id) | |
| review.append({ | |
| 'audit_id': r.audit_id, | |
| 'model': model, | |
| 'question_id': r.question_id, | |
| 'evidence_scarcity': r.evidence_scarcity, | |
| 'question_openness': r.question_openness, | |
| 'question': item.get('question', ''), | |
| 'ground_truth': item.get('ground_truth_answer', ''), | |
| 'diverges_if': json.dumps( | |
| item.get('diverges_if', []), ensure_ascii=False | |
| ), | |
| 'does_not_diverge_if': json.dumps( | |
| item.get('does_not_diverge_if', []), | |
| ensure_ascii=False | |
| ), | |
| 'visible_answer': ans, | |
| 'manual_choice': '', | |
| 'manual_reason': '', | |
| }) | |
| key.append({ | |
| 'audit_id': r.audit_id, | |
| 'judge_choice': r.choice, | |
| 'case_id': r.case_id, | |
| 'source_eval': path, | |
| }) | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| pd.DataFrame(review).to_csv( | |
| OUT / 'audit_review_BLIND.csv', index=False | |
| ) | |
| pd.DataFrame(key).to_csv( | |
| OUT / 'audit_judge_key.csv', index=False | |
| ) | |
| pd.DataFrame( | |
| unresolved, | |
| columns=['audit_id','case_id','source_files','distinct_answers'] | |
| ).to_csv(OUT / 'audit_unresolved.csv', index=False) | |
| with (OUT / 'audit_review_BLIND.md').open('w') as f: | |
| f.write('# Blinded Judge Audit\n\nJudge label hidden.\n') | |
| for r in review: | |
| f.write( | |
| f"\n## {r['audit_id']} — " | |
| f"{r['model']}, {r['question_id']}\n" | |
| ) | |
| for name in [ | |
| 'question', 'ground_truth', 'diverges_if', | |
| 'does_not_diverge_if', 'visible_answer' | |
| ]: | |
| f.write(f"\n**{name}:**\n\n{r[name]}\n") | |
| f.write( | |
| '\n**Your verdict ' | |
| '(hold/diverge/abstain/uncertain):** ______\n\n---\n' | |
| ) | |
| print('Extracted:', len(review)) | |
| print('Unresolved:', len(unresolved)) | |
| print('Saved blind review and Judge key.') | |