Spaces:
Running
Running
Download analysis/designed-difficulty/critical_sensitivity.py from daKeshra/ConspiracyBench: direct link, hf CLI and curl.
- Browser
- Download file 5.17 kB
-
https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/analysis/designed-difficulty/critical_sensitivity.py
- Command line
-
hf download hf://spaces/daKeshra/ConspiracyBench/analysis/designed-difficulty/critical_sensitivity.py
-
curl -L -o critical_sensitivity.py https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/analysis/designed-difficulty/critical_sensitivity.py
5.17 kB
| from pathlib import Path | |
| import warnings | |
| import numpy as np | |
| import pandas as pd | |
| import statsmodels.api as sm | |
| import statsmodels.formula.api as smf | |
| from statsmodels.stats.multitest import multipletests | |
| OUT = Path("analysis/designed-difficulty") | |
| FORMULA = ( | |
| "y ~ evidence_scarcity + question_openness" | |
| " + C(model) + C(category) + C(framing)" | |
| ) | |
| df = pd.read_csv(OUT / "baseline_joined.csv") | |
| df = df.loc[ | |
| df.variant.eq("multiturn") | |
| & df.turn.eq(0) | |
| & df.status.eq("ok") | |
| & df.dropped.eq(0) | |
| & df.choice.isin(["hold", "diverge", "abstain"]) | |
| ].copy() | |
| df["framing"] = df.framing.fillna("unspecified") | |
| df["claim_cluster"] = df.claim_id.fillna( | |
| df.question_id | |
| ).astype(str) | |
| assert len(df) == 6320, f"Unexpected N: {len(df)}" | |
| assert df.choice.eq("diverge").sum() == 506 | |
| # Judge-label distribution by model and scarcity. | |
| counts = pd.crosstab( | |
| [df.model, df.evidence_scarcity], df.choice | |
| ).reindex( | |
| columns=["hold", "diverge", "abstain"], | |
| fill_value=0 | |
| ).reset_index() | |
| counts["n"] = counts[ | |
| ["hold", "diverge", "abstain"] | |
| ].sum(axis=1) | |
| counts["diverge_pct_all"] = ( | |
| 100 * counts.diverge / counts.n | |
| ) | |
| counts["abstain_pct_all"] = ( | |
| 100 * counts.abstain / counts.n | |
| ) | |
| counts["diverge_pct_non_abstain"] = ( | |
| 100 * counts.diverge / | |
| (counts.hold + counts.diverge).replace(0, np.nan) | |
| ) | |
| counts.to_csv( | |
| OUT / "critical_label_counts.csv", index=False | |
| ) | |
| print("=== JUDGE LABEL DISTRIBUTION ===") | |
| print(counts.to_string(index=False)) | |
| coverage = df.groupby("evidence_scarcity").agg( | |
| judgments=("choice", "size"), | |
| questions=("question_id", "nunique"), | |
| claims=("claim_cluster", "nunique"), | |
| ).reset_index() | |
| coverage.to_csv( | |
| OUT / "critical_score_coverage.csv", index=False | |
| ) | |
| print("\n=== QUESTION / CLAIM COVERAGE ===") | |
| print(coverage.to_string(index=False)) | |
| def run_fit(name, data, outcome="diverge", weight=None): | |
| d = data.copy() | |
| d["y"] = d.choice.eq(outcome).astype(int) | |
| with warnings.catch_warnings(record=True) as warns: | |
| warnings.simplefilter("always") | |
| if weight is None: | |
| fit = smf.glm( | |
| FORMULA, data=d, | |
| family=sm.families.Binomial() | |
| ).fit( | |
| cov_type="cluster", | |
| cov_kwds={"groups": d.claim_cluster}, | |
| maxiter=200 | |
| ) | |
| else: | |
| unit = ( | |
| "question_id" | |
| if weight == "question_equal" | |
| else "claim_cluster" | |
| ) | |
| w = 1.0 / d.groupby(unit).y.transform("size") | |
| w = w / w.mean() | |
| fit = smf.gee( | |
| FORMULA, | |
| groups="claim_cluster", | |
| data=d, | |
| family=sm.families.Binomial(), | |
| cov_struct=sm.cov_struct.Independence(), | |
| weights=w | |
| ).fit(maxiter=200) | |
| rows = [] | |
| for term in ["evidence_scarcity", "question_openness"]: | |
| beta = float(fit.params[term]) | |
| se = float(fit.bse[term]) | |
| rows.append({ | |
| "spec": name, | |
| "weighting": weight or "none", | |
| "outcome": outcome, | |
| "term": term, | |
| "n": len(d), | |
| "questions": d.question_id.nunique(), | |
| "claims": d.claim_cluster.nunique(), | |
| "events": int(d.y.sum()), | |
| "or": np.exp(beta), | |
| "ci_low": np.exp(beta - 1.96 * se), | |
| "ci_high": np.exp(beta + 1.96 * se), | |
| "p_value": float(fit.pvalues[term]), | |
| "converged": getattr(fit, "converged", True), | |
| "warnings": " | ".join( | |
| str(x.message) for x in warns[:2] | |
| ) | |
| }) | |
| return rows | |
| nonabstain = df[df.choice.isin( | |
| ["hold", "diverge"] | |
| )].copy() | |
| s01 = df[df.evidence_scarcity.isin([0, 1])].copy() | |
| nonabstain_s01 = nonabstain[ | |
| nonabstain.evidence_scarcity.isin([0, 1]) | |
| ].copy() | |
| tests = [ | |
| ("original", df, "diverge", None), | |
| ("non_abstain", nonabstain, "diverge", None), | |
| ("scarcity_0_1", s01, "diverge", None), | |
| ("non_abstain_s01", nonabstain_s01, "diverge", None), | |
| ("abstain_outcome", df, "abstain", None), | |
| ("question_equal", df, "diverge", "question_equal"), | |
| ("claim_equal", df, "diverge", "claim_equal"), | |
| ("non_abstain_question_equal", nonabstain, | |
| "diverge", "question_equal"), | |
| ("non_abstain_claim_equal", nonabstain, | |
| "diverge", "claim_equal"), | |
| ] | |
| results = [] | |
| for name, data, outcome, weight in tests: | |
| print(f"\nRunning {name}...") | |
| results.extend(run_fit(name, data, outcome, weight)) | |
| results = pd.DataFrame(results) | |
| # Holm correction for the original two difficulty axes. | |
| base_mask = results.spec.eq("original") | |
| adjusted = multipletests( | |
| results.loc[base_mask, "p_value"], | |
| method="holm" | |
| )[1] | |
| results["holm_p_two_axes"] = np.nan | |
| results.loc[base_mask, "holm_p_two_axes"] = adjusted | |
| results.to_csv( | |
| OUT / "critical_sensitivity_results.csv", | |
| index=False | |
| ) | |
| print("\n=== FINAL SENSITIVITY RESULTS ===") | |
| print(results[[ | |
| "spec", "term", "n", "events", | |
| "or", "ci_low", "ci_high", "p_value", | |
| "holm_p_two_axes", "converged" | |
| ]].to_string(index=False)) | |
| print("\nSaved all results.") | |