from pathlib import Path import warnings import numpy as np import pandas as pd import statsmodels.api as sm import statsmodels.formula.api as smf from statsmodels.stats.multitest import multipletests OUT = Path("analysis/designed-difficulty") FORMULA = ( "y ~ evidence_scarcity + question_openness" " + C(model) + C(category) + C(framing)" ) df = pd.read_csv(OUT / "baseline_joined.csv") df = df.loc[ df.variant.eq("multiturn") & df.turn.eq(0) & df.status.eq("ok") & df.dropped.eq(0) & df.choice.isin(["hold", "diverge", "abstain"]) ].copy() df["framing"] = df.framing.fillna("unspecified") df["claim_cluster"] = df.claim_id.fillna( df.question_id ).astype(str) assert len(df) == 6320, f"Unexpected N: {len(df)}" assert df.choice.eq("diverge").sum() == 506 # Judge-label distribution by model and scarcity. counts = pd.crosstab( [df.model, df.evidence_scarcity], df.choice ).reindex( columns=["hold", "diverge", "abstain"], fill_value=0 ).reset_index() counts["n"] = counts[ ["hold", "diverge", "abstain"] ].sum(axis=1) counts["diverge_pct_all"] = ( 100 * counts.diverge / counts.n ) counts["abstain_pct_all"] = ( 100 * counts.abstain / counts.n ) counts["diverge_pct_non_abstain"] = ( 100 * counts.diverge / (counts.hold + counts.diverge).replace(0, np.nan) ) counts.to_csv( OUT / "critical_label_counts.csv", index=False ) print("=== JUDGE LABEL DISTRIBUTION ===") print(counts.to_string(index=False)) coverage = df.groupby("evidence_scarcity").agg( judgments=("choice", "size"), questions=("question_id", "nunique"), claims=("claim_cluster", "nunique"), ).reset_index() coverage.to_csv( OUT / "critical_score_coverage.csv", index=False ) print("\n=== QUESTION / CLAIM COVERAGE ===") print(coverage.to_string(index=False)) def run_fit(name, data, outcome="diverge", weight=None): d = data.copy() d["y"] = d.choice.eq(outcome).astype(int) with warnings.catch_warnings(record=True) as warns: warnings.simplefilter("always") if weight is None: fit = smf.glm( FORMULA, data=d, family=sm.families.Binomial() ).fit( cov_type="cluster", cov_kwds={"groups": d.claim_cluster}, maxiter=200 ) else: unit = ( "question_id" if weight == "question_equal" else "claim_cluster" ) w = 1.0 / d.groupby(unit).y.transform("size") w = w / w.mean() fit = smf.gee( FORMULA, groups="claim_cluster", data=d, family=sm.families.Binomial(), cov_struct=sm.cov_struct.Independence(), weights=w ).fit(maxiter=200) rows = [] for term in ["evidence_scarcity", "question_openness"]: beta = float(fit.params[term]) se = float(fit.bse[term]) rows.append({ "spec": name, "weighting": weight or "none", "outcome": outcome, "term": term, "n": len(d), "questions": d.question_id.nunique(), "claims": d.claim_cluster.nunique(), "events": int(d.y.sum()), "or": np.exp(beta), "ci_low": np.exp(beta - 1.96 * se), "ci_high": np.exp(beta + 1.96 * se), "p_value": float(fit.pvalues[term]), "converged": getattr(fit, "converged", True), "warnings": " | ".join( str(x.message) for x in warns[:2] ) }) return rows nonabstain = df[df.choice.isin( ["hold", "diverge"] )].copy() s01 = df[df.evidence_scarcity.isin([0, 1])].copy() nonabstain_s01 = nonabstain[ nonabstain.evidence_scarcity.isin([0, 1]) ].copy() tests = [ ("original", df, "diverge", None), ("non_abstain", nonabstain, "diverge", None), ("scarcity_0_1", s01, "diverge", None), ("non_abstain_s01", nonabstain_s01, "diverge", None), ("abstain_outcome", df, "abstain", None), ("question_equal", df, "diverge", "question_equal"), ("claim_equal", df, "diverge", "claim_equal"), ("non_abstain_question_equal", nonabstain, "diverge", "question_equal"), ("non_abstain_claim_equal", nonabstain, "diverge", "claim_equal"), ] results = [] for name, data, outcome, weight in tests: print(f"\nRunning {name}...") results.extend(run_fit(name, data, outcome, weight)) results = pd.DataFrame(results) # Holm correction for the original two difficulty axes. base_mask = results.spec.eq("original") adjusted = multipletests( results.loc[base_mask, "p_value"], method="holm" )[1] results["holm_p_two_axes"] = np.nan results.loc[base_mask, "holm_p_two_axes"] = adjusted results.to_csv( OUT / "critical_sensitivity_results.csv", index=False ) print("\n=== FINAL SENSITIVITY RESULTS ===") print(results[[ "spec", "term", "n", "events", "or", "ci_low", "ci_high", "p_value", "holm_p_two_axes", "converged" ]].to_string(index=False)) print("\nSaved all results.")