ConspiracyBench / analysis /designed-difficulty /critical_sensitivity.py
stationhouse's picture jiwon9703's picture
Add exploratory analysis of designed difficulty and model divergence (#449)
a91390f
Raw History Blame Contribute Delete
5.17 kB
from pathlib import Path
import warnings
import numpy as np
import pandas as pd
import statsmodels.api as sm
import statsmodels.formula.api as smf
from statsmodels.stats.multitest import multipletests
OUT = Path("analysis/designed-difficulty")
FORMULA = (
"y ~ evidence_scarcity + question_openness"
" + C(model) + C(category) + C(framing)"
)
df = pd.read_csv(OUT / "baseline_joined.csv")
df = df.loc[
df.variant.eq("multiturn")
& df.turn.eq(0)
& df.status.eq("ok")
& df.dropped.eq(0)
& df.choice.isin(["hold", "diverge", "abstain"])
].copy()
df["framing"] = df.framing.fillna("unspecified")
df["claim_cluster"] = df.claim_id.fillna(
df.question_id
).astype(str)
assert len(df) == 6320, f"Unexpected N: {len(df)}"
assert df.choice.eq("diverge").sum() == 506
# Judge-label distribution by model and scarcity.
counts = pd.crosstab(
[df.model, df.evidence_scarcity], df.choice
).reindex(
columns=["hold", "diverge", "abstain"],
fill_value=0
).reset_index()
counts["n"] = counts[
["hold", "diverge", "abstain"]
].sum(axis=1)
counts["diverge_pct_all"] = (
100 * counts.diverge / counts.n
)
counts["abstain_pct_all"] = (
100 * counts.abstain / counts.n
)
counts["diverge_pct_non_abstain"] = (
100 * counts.diverge /
(counts.hold + counts.diverge).replace(0, np.nan)
)
counts.to_csv(
OUT / "critical_label_counts.csv", index=False
)
print("=== JUDGE LABEL DISTRIBUTION ===")
print(counts.to_string(index=False))
coverage = df.groupby("evidence_scarcity").agg(
judgments=("choice", "size"),
questions=("question_id", "nunique"),
claims=("claim_cluster", "nunique"),
).reset_index()
coverage.to_csv(
OUT / "critical_score_coverage.csv", index=False
)
print("\n=== QUESTION / CLAIM COVERAGE ===")
print(coverage.to_string(index=False))
def run_fit(name, data, outcome="diverge", weight=None):
d = data.copy()
d["y"] = d.choice.eq(outcome).astype(int)
with warnings.catch_warnings(record=True) as warns:
warnings.simplefilter("always")
if weight is None:
fit = smf.glm(
FORMULA, data=d,
family=sm.families.Binomial()
).fit(
cov_type="cluster",
cov_kwds={"groups": d.claim_cluster},
maxiter=200
)
else:
unit = (
"question_id"
if weight == "question_equal"
else "claim_cluster"
)
w = 1.0 / d.groupby(unit).y.transform("size")
w = w / w.mean()
fit = smf.gee(
FORMULA,
groups="claim_cluster",
data=d,
family=sm.families.Binomial(),
cov_struct=sm.cov_struct.Independence(),
weights=w
).fit(maxiter=200)
rows = []
for term in ["evidence_scarcity", "question_openness"]:
beta = float(fit.params[term])
se = float(fit.bse[term])
rows.append({
"spec": name,
"weighting": weight or "none",
"outcome": outcome,
"term": term,
"n": len(d),
"questions": d.question_id.nunique(),
"claims": d.claim_cluster.nunique(),
"events": int(d.y.sum()),
"or": np.exp(beta),
"ci_low": np.exp(beta - 1.96 * se),
"ci_high": np.exp(beta + 1.96 * se),
"p_value": float(fit.pvalues[term]),
"converged": getattr(fit, "converged", True),
"warnings": " | ".join(
str(x.message) for x in warns[:2]
)
})
return rows
nonabstain = df[df.choice.isin(
["hold", "diverge"]
)].copy()
s01 = df[df.evidence_scarcity.isin([0, 1])].copy()
nonabstain_s01 = nonabstain[
nonabstain.evidence_scarcity.isin([0, 1])
].copy()
tests = [
("original", df, "diverge", None),
("non_abstain", nonabstain, "diverge", None),
("scarcity_0_1", s01, "diverge", None),
("non_abstain_s01", nonabstain_s01, "diverge", None),
("abstain_outcome", df, "abstain", None),
("question_equal", df, "diverge", "question_equal"),
("claim_equal", df, "diverge", "claim_equal"),
("non_abstain_question_equal", nonabstain,
"diverge", "question_equal"),
("non_abstain_claim_equal", nonabstain,
"diverge", "claim_equal"),
]
results = []
for name, data, outcome, weight in tests:
print(f"\nRunning {name}...")
results.extend(run_fit(name, data, outcome, weight))
results = pd.DataFrame(results)
# Holm correction for the original two difficulty axes.
base_mask = results.spec.eq("original")
adjusted = multipletests(
results.loc[base_mask, "p_value"],
method="holm"
)[1]
results["holm_p_two_axes"] = np.nan
results.loc[base_mask, "holm_p_two_axes"] = adjusted
results.to_csv(
OUT / "critical_sensitivity_results.csv",
index=False
)
print("\n=== FINAL SENSITIVITY RESULTS ===")
print(results[[
"spec", "term", "n", "events",
"or", "ci_low", "ci_high", "p_value",
"holm_p_two_axes", "converged"
]].to_string(index=False))
print("\nSaved all results.")