Spaces:
Running
Running
File size: 2,381 Bytes
a91390f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | from pathlib import Path
import numpy as np
import pandas as pd
import statsmodels.api as sm
import statsmodels.formula.api as smf
ROOT = Path("analysis/designed-difficulty")
df = pd.read_csv(ROOT / "baseline_joined.csv")
df = df[
(df["variant"] == "multiturn")
& (df["turn"] == 0)
& (df["status"] == "ok")
& (df["dropped"] == 0)
& df["choice"].isin(["hold", "diverge", "abstain"])
].copy()
df["y"] = (df["choice"] == "diverge").astype(int)
df["framing"] = df["framing"].fillna("unspecified")
df["category"] = df["category"].fillna("unspecified")
# Repeated epochs, shared questions across models, and
# questions sharing a claim are clustered together.
df["cluster"] = df["claim_id"].fillna(
df["question_id"]
)
print("Observations:", len(df))
print("Questions:", df["question_id"].nunique())
print("Claims:", df["cluster"].nunique())
print("Overall divergence:", round(df["y"].mean(), 4))
formulas = {
"A": (
"y ~ evidence_scarcity + question_openness "
"+ C(model)"
),
"B": (
"y ~ evidence_scarcity + question_openness "
"+ C(model) + C(category) + C(framing)"
),
}
results = []
for name, formula in formulas.items():
print(f"\n=== MODEL {name} ===")
model = smf.glm(
formula,
data=df,
family=sm.families.Binomial()
)
fit = model.fit(
cov_type="cluster",
cov_kwds={"groups": df["cluster"]}
)
for variable in [
"evidence_scarcity",
"question_openness",
]:
coef = fit.params[variable]
se = fit.bse[variable]
pval = fit.pvalues[variable]
odds_ratio = np.exp(coef)
ci_low = np.exp(coef - 1.96 * se)
ci_high = np.exp(coef + 1.96 * se)
results.append({
"model": name,
"predictor": variable,
"coefficient": coef,
"odds_ratio": odds_ratio,
"ci_low": ci_low,
"ci_high": ci_high,
"p_value": pval,
})
print(
f"{variable:20s} "
f"OR={odds_ratio:.3f} "
f"95% CI=[{ci_low:.3f}, {ci_high:.3f}] "
f"p={pval:.4g}"
)
print("AIC:", round(fit.aic, 2))
print("Converged:", fit.converged)
output = ROOT / "regression_results.csv"
pd.DataFrame(results).to_csv(output, index=False)
print("\nSaved:", output)
|