Spaces:
Running
Running
File size: 11,405 Bytes
a91390f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 | #!/usr/bin/env python3
"""Claim-grouped predictive validity evaluation for ConspiracyBench baseline.
Run from the ConspiracyBench repository root:
python analysis/designed-difficulty/cv_predict.py
Dependencies: pandas numpy scikit-learn
Input: analysis/designed-difficulty/baseline_joined.csv
Outputs: cv_metrics.csv, cv_fold_metrics.csv, cv_oof_predictions.csv,
cv_bootstrap_delta.csv, RESEARCH_LOG.md (appended/replaced run section).
No new LLM inference or API calls.
"""
from __future__ import annotations
import json
import platform
import subprocess
from datetime import datetime, timezone
from pathlib import Path
import numpy as np
import pandas as pd
import sklearn
from sklearn.compose import ColumnTransformer
from sklearn.dummy import DummyClassifier
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import brier_score_loss, log_loss, roc_auc_score
from sklearn.model_selection import GroupKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder
ROOT = Path('analysis/designed-difficulty')
CSV = ROOT / 'baseline_joined.csv'
SEED = 42
FOLDS = 5
BOOTSTRAPS = 2000
EPS = 1e-12
SPECS = {
'M_null': ([], []),
'M0_base': (['model', 'category', 'framing'], []),
'M1_scarcity': (['model', 'category', 'framing'], ['evidence_scarcity']),
'M2_openness': (['model', 'category', 'framing'], ['question_openness']),
'M3_two_axes': (['model', 'category', 'framing'], ['evidence_scarcity', 'question_openness']),
'M4_band': (['model', 'category', 'framing', 'band'], []),
}
def metrics(y, p):
y = np.asarray(y, dtype=int)
p = np.clip(np.asarray(p, dtype=float), EPS, 1 - EPS)
return {
'n': int(len(y)),
'divergence_prevalence': float(y.mean()),
'log_loss': float(log_loss(y, p, labels=[0, 1])),
'brier': float(brier_score_loss(y, p)),
'auroc': float(roc_auc_score(y, p)) if np.unique(y).size == 2 else np.nan,
}
def fit_predict(train, test, categorical, numeric):
if not categorical and not numeric:
model = DummyClassifier(strategy='prior')
model.fit(np.zeros((len(train), 1)), train['y'])
return model.predict_proba(np.zeros((len(test), 1)))[:, list(model.classes_).index(1)]
blocks = []
if categorical:
blocks.append(('categorical', OneHotEncoder(handle_unknown='ignore'), categorical))
if numeric:
blocks.append(('numeric', SimpleImputer(strategy='median'), numeric))
preprocessor = ColumnTransformer(transformers=blocks)
pipeline = Pipeline([
('features', preprocessor),
('classifier', LogisticRegression(C=1.0, max_iter=3000, solver='lbfgs')),
])
columns = categorical + numeric
pipeline.fit(train[columns], train['y'])
return pipeline.predict_proba(test[columns])[:, list(pipeline.named_steps['classifier'].classes_).index(1)]
def paired_claim_bootstrap(df, target_name, metric_type, rng):
# Positive delta = lower error for target vs M0.
y = df['y'].to_numpy(float)
p0 = np.clip(df['M0_base'].to_numpy(float), EPS, 1 - EPS)
p1 = np.clip(df[target_name].to_numpy(float), EPS, 1 - EPS)
if metric_type == 'log_loss':
row_loss0 = -y * np.log(p0) - (1-y) * np.log(1-p0)
row_loss1 = -y * np.log(p1) - (1-y) * np.log(1-p1)
elif metric_type == 'brier':
row_loss0 = (y-p0)**2
row_loss1 = (y-p1)**2
else:
raise ValueError(metric_type)
claim_agg = pd.DataFrame({
'claim_id': df['cv_group'].to_numpy(),
'delta': row_loss0 - row_loss1,
}).groupby('claim_id')['delta'].agg(['sum', 'count'])
total = claim_agg['sum'].to_numpy()
counts = claim_agg['count'].to_numpy()
k = len(total)
samples = rng.integers(0, k, size=(BOOTSTRAPS, k))
improvements = total[samples].sum(axis=1) / counts[samples].sum(axis=1)
observed = float(total.sum() / counts.sum())
low, high = np.quantile(improvements, [0.025, 0.975])
return {'comparison': f'{target_name} vs M0_base', 'metric': metric_type,
'improvement': observed, 'ci_low': float(low), 'ci_high': float(high),
'num_claims': k, 'bootstrap_replicates': BOOTSTRAPS}
def run():
if not CSV.exists():
raise SystemExit(f'Missing {CSV}; run prepare_data.py first.')
df = pd.read_csv(CSV, keep_default_na=True)
criteria = (
df['variant'].eq('multiturn') & df['turn'].eq(0)
& df['status'].eq('ok') & df['dropped'].eq(0)
& df['choice'].isin(['hold', 'diverge', 'abstain'])
)
df = df.loc[criteria].copy().reset_index(drop=True)
if not len(df):
raise SystemExit('No valid multi-turn opening judgments.')
for field in ('question_id', 'model', 'evidence_scarcity', 'question_openness', 'band'):
if df[field].isna().any():
raise SystemExit(f'Missing required field: {field}')
if df.duplicated(['model', 'case_id']).any():
raise SystemExit('Duplicate model/case_id in input.')
df['y'] = df['choice'].eq('diverge').astype(int)
df['category'] = df['category'].fillna('unspecified').astype(str)
df['framing'] = df['framing'].fillna('unspecified').astype(str)
df['band'] = df['band'].astype(str)
for key in ('evidence_scarcity', 'question_openness'):
df[key] = pd.to_numeric(df[key], errors='raise')
# Missing claim IDs are grouped by question ID, never each epoch.
df['cv_group'] = df['claim_id'].where(df['claim_id'].notna(), df['question_id']).astype(str)
assert df.groupby('question_id')['cv_group'].nunique().max() == 1, 'Question crosses claim groups.'
n_claims = df['cv_group'].nunique()
if n_claims < FOLDS:
raise SystemExit(f'Only {n_claims} distinct claim groups; need >= {FOLDS}.')
print('=== ANALYSIS POPULATION ===')
print(f'Records: {len(df):,}; questions: {df.question_id.nunique()}; claims: {n_claims}; models: {df.model.nunique()}')
print(f'Pooled divergence prevalence: {df.y.mean():.4%}')
cv = GroupKFold(n_splits=FOLDS)
oof = df[['model', 'question_id', 'claim_id', 'cv_group', 'case_id', 'epoch', 'turn',
'category', 'framing', 'evidence_scarcity', 'question_openness', 'band', 'y']].copy()
oof['fold'] = -1
for name in SPECS:
oof[name] = np.nan
fold_rows = []
for fold, (tr, te) in enumerate(cv.split(df, df['y'], df['cv_group']), start=1):
train, test = df.iloc[tr], df.iloc[te]
if set(train['cv_group']) & set(test['cv_group']):
raise AssertionError('Claim-group leakage.')
if train['y'].nunique() != 2:
raise RuntimeError(f'Fold {fold}: training set has just one class.')
oof.loc[te, 'fold'] = fold
print(f'Fold {fold}: train={len(tr)}; test={len(te)}; held-out claims={test.cv_group.nunique()}; test event rate={test.y.mean():.2%}')
for name, (categorical, numeric) in SPECS.items():
probs = fit_predict(train, test, categorical, numeric)
oof.loc[te, name] = probs
fold_rows.append({'fold': fold, 'model_spec': name, 'n_claims': int(test.cv_group.nunique()), **metrics(test['y'], probs)})
assert oof['fold'].ge(1).all()
assert oof[list(SPECS)].notna().all().all()
assert (oof.groupby('cv_group')['fold'].nunique() == 1).all()
global_rows = []
for name in SPECS:
global_rows.append({'model_spec': name, **metrics(oof['y'], oof[name])})
global_metrics = pd.DataFrame(global_rows)
fold_metrics = pd.DataFrame(fold_rows)
rng = np.random.default_rng(SEED)
bootstrap = pd.DataFrame([
paired_claim_bootstrap(oof, name, metric, rng)
for name in SPECS if name not in ('M_null', 'M0_base')
for metric in ('log_loss', 'brier')
])
ROOT.mkdir(parents=True, exist_ok=True)
global_metrics.to_csv(ROOT / 'cv_metrics.csv', index=False)
fold_metrics.to_csv(ROOT / 'cv_fold_metrics.csv', index=False)
oof.to_csv(ROOT / 'cv_oof_predictions.csv', index=False)
bootstrap.to_csv(ROOT / 'cv_bootstrap_delta.csv', index=False)
print('\n=== 5-FOLD CLAIM-GROUPED OUT-OF-FOLD PERFORMANCE ===')
print(global_metrics.round(5).to_string(index=False))
print('\n=== PAIRED CLAIM BOOTSTRAP: POSITIVE = IMPROVEMENT OVER M0 ===')
print(bootstrap.round(6).to_string(index=False))
# Record run, including software/repository version. Re-running replaces only this section.
try:
head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True, stderr=subprocess.DEVNULL).strip()
except (FileNotFoundError, subprocess.CalledProcessError):
head = 'unknown'
stamp = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
log_path = ROOT / 'RESEARCH_LOG.md'
start_marker = '<!-- CV_RUN_START -->'
end_marker = '<!-- CV_RUN_END -->'
report = f'''{start_marker}
## Predictive validity experiment — latest local execution
- Run time: {stamp}
- Git commit: `{head}`
- Input: `{CSV}`
- Target: `choice == diverge` among `status == ok`, `dropped == 0`, `variant == multiturn`, `turn == 0`
- Observations: {len(df):,}; questions: {df.question_id.nunique()}; claims: {n_claims}; models: {df.model.nunique()}
- Splitting: 5-fold `GroupKFold` by `claim_id` (fallback `question_id`), keeping repeats and models within each fold
- Base features: model, category, framing. Compared added scarcity, openness, both, or original difficulty band
- Estimator: one-hot categories (unknown ignored), L2 logistic regression (`C=1`); null model uses training prevalence
- Metrics: out-of-fold log loss (primary), Brier score (primary), AUROC (secondary)
- Uncertainty: paired bootstrap of *out-of-fold error difference* over claim groups; {BOOTSTRAPS} draws, seed {SEED}
- Software: Python {platform.python_version()}, pandas {pd.__version__}, numpy {np.__version__}, scikit-learn {sklearn.__version__}
- Caution: *exploratory*, not preregistered; statistical association is not causation. All-fold metrics are pooled across samples that share questions and claims. Bootstrap quantifies uncertainty of the error difference under claim resampling, not all uncertainty due to judge quality or sampling design.
### Out-of-fold metrics
{global_metrics.round(6).to_markdown(index=False)}
### Improvement over M0 (positive means lower error)
{bootstrap.round(6).to_markdown(index=False)}
### Artifact files
- `cv_metrics.csv`: overall pooled out-of-fold performance
- `cv_fold_metrics.csv`: fold-specific performance and counts
- `cv_oof_predictions.csv`: individual held-out prediction rows, including fold and claim IDs
- `cv_bootstrap_delta.csv`: paired claim bootstrap confidence intervals for improvement
{end_marker}'''
if log_path.exists():
prior = log_path.read_text(encoding='utf-8')
if start_marker in prior and end_marker in prior:
prior = prior.split(start_marker)[0].rstrip() + '\n\n' + prior.split(end_marker, 1)[1].lstrip()
output_text = prior.rstrip() + '\n\n' + report + '\n'
else:
output_text = '# ConspiracyBench — Designed vs. Empirical Difficulty\n\n' + report + '\n'
# pandas.to_markdown depends on tabulate; provide a helpful fallback path if absent.
log_path.write_text(output_text, encoding='utf-8')
print(f'\nSaved outputs and updated: {log_path}')
if __name__ == '__main__':
run()
|