parsa-hke's picture
Redesign: the story (Two hours, thirty minutes), a live-run page and the record page (part 2)
98b213b verified
Raw History Blame Contribute Delete
5.87 kB
"""Student's t-tests beside the existing results (DEC-36, `docs/PHASE5_TTESTS.md`): the statistics.
`paired` and `welch` return one report row each: t, df, two-sided p, the mean difference with its 95% t interval, an
effect size (d_z, or Hedges' g), skewness and Shapiro-Wilk, and the rank test beside it (Wilcoxon signed-rank, or
Mann-Whitney U). `holm` adjusts the family's p-values. The case is the unit throughout; the callers give one value per
case, and a pair with a missing side is dropped and counted.
"""
import math
import numpy as np
from scipy import stats
ALPHA = 0.05
def _clean_pairs(a, b):
a, b = np.asarray(a, dtype=float), np.asarray(b, dtype=float)
if a.shape != b.shape:
raise ValueError("paired samples must have the same length")
keep = np.isfinite(a) & np.isfinite(b)
return a[keep], b[keep], int((~keep).sum())
def _shape(x):
x = np.asarray(x, dtype=float)
sw = stats.shapiro(x).pvalue if len(x) >= 3 and np.ptp(x) > 0 else float("nan")
sk = float(stats.skew(x)) if len(x) >= 3 and np.ptp(x) > 0 else float("nan")
return {"skewness": sk, "shapiro_p": float(sw)}
def paired(a, b, label="", alpha=ALPHA):
"""Paired t-test of a - b, per case."""
a, b, dropped = _clean_pairs(a, b)
d = a - b
n = len(d)
if n < 2:
raise ValueError("a paired t-test needs at least 2 pairs")
mean, sd = float(d.mean()), float(d.std(ddof=1))
df = n - 1
if sd == 0:
t, p, half = float("nan"), float("nan"), 0.0
else:
res = stats.ttest_rel(a, b)
t, p = float(res.statistic), float(res.pvalue)
half = stats.t.ppf(1 - alpha / 2, df) * sd / math.sqrt(n)
nonzero = d[d != 0]
w = stats.wilcoxon(nonzero) if len(nonzero) >= 1 else None
return {"_values": [d], "label": label, "form": "paired", "n": n, "dropped": dropped, "mean_a": float(a.mean()),
"mean_b": float(b.mean()), "mean_difference": mean, "ci": [mean - half, mean + half], "t": t, "df": df,
"p": p, "effect": "d_z", "effect_size": mean / sd if sd else float("nan"),
"assumptions": _shape(d),
"rank_test": {"name": "Wilcoxon signed-rank", "statistic": float(w.statistic) if w else float("nan"),
"p": float(w.pvalue) if w else float("nan"), "zero_differences": int(n - len(nonzero))}}
def welch(a, b, label="", alpha=ALPHA):
"""Welch's t-test of mean(a) - mean(b), unpaired."""
a, b = (np.asarray(x, dtype=float) for x in (a, b))
dropped = int((~np.isfinite(a)).sum() + (~np.isfinite(b)).sum())
a, b = a[np.isfinite(a)], b[np.isfinite(b)]
na, nb = len(a), len(b)
if na < 2 or nb < 2:
raise ValueError("Welch's t-test needs at least 2 cases per group")
va, vb = a.var(ddof=1), b.var(ddof=1)
se = math.sqrt(va / na + vb / nb)
mean = float(a.mean() - b.mean())
if se == 0:
t, p, df, half = float("nan"), float("nan"), float("nan"), 0.0
else:
df = (va / na + vb / nb) ** 2 / ((va / na) ** 2 / (na - 1) + (vb / nb) ** 2 / (nb - 1))
res = stats.ttest_ind(a, b, equal_var=False)
t, p = float(res.statistic), float(res.pvalue)
half = stats.t.ppf(1 - alpha / 2, df) * se
pooled = math.sqrt(((na - 1) * va + (nb - 1) * vb) / (na + nb - 2))
j = 1 - 3 / (4 * (na + nb) - 9) # Hedges' small-sample correction
u = stats.mannwhitneyu(a, b, alternative="two-sided")
return {"_values": [a, b], "label": label, "form": "Welch", "n": [na, nb], "dropped": dropped, "mean_a": float(a.mean()),
"mean_b": float(b.mean()), "mean_difference": mean, "ci": [mean - half, mean + half], "t": t,
"df": float(df), "p": p, "effect": "Hedges' g",
"effect_size": j * mean / pooled if pooled else float("nan"),
"assumptions": {"a": _shape(a), "b": _shape(b)},
"rank_test": {"name": "Mann-Whitney U", "statistic": float(u.statistic), "p": float(u.pvalue)}}
def holm(pvalues):
"""Holm's step-down adjusted p-values, in the input order. NaN stays NaN and is not counted in the family."""
p = np.asarray(pvalues, dtype=float)
out = np.full_like(p, np.nan)
idx = np.where(np.isfinite(p))[0]
m = len(idx)
order = idx[np.argsort(p[idx], kind="stable")]
running = 0.0
for rank, i in enumerate(order):
running = max(running, min(1.0, (m - rank) * p[i]))
out[i] = running
return out
def adjust(rows):
"""Add `p_holm` to each row, over the rows as one family."""
for row, ph in zip(rows, holm([r["p"] for r in rows])):
row["p_holm"] = float(ph)
return rows
def agrees_with_bootstrap(row, bootstrap_ci):
"""True when the t interval and the existing bootstrap interval agree on whether 0 is excluded."""
t_excl = row["ci"][0] > 0 or row["ci"][1] < 0
b_excl = bootstrap_ci[0] > 0 or bootstrap_ci[1] < 0
return bool(t_excl == b_excl)
def case_rate(per_case_row, numerator="hallucinated", denominator="claims"):
"""One case's rate from a metrics per_case row; NaN when it has no claims (a format failure)."""
den = per_case_row.get(denominator) or 0
return per_case_row[numerator] / den if den else float("nan")
def case_rates(records, metric):
"""Per-case rates, in record order, from verifier case records (compare.case_records): numerator over denominator
of `metric` (metrics.RATES) for each case on its own; NaN where the denominator is 0 (a format failure, or no claim
whose criticality is determined)."""
from strokecopilot.nlp.verifier import metrics as vm
if not records:
return []
table = vm._table(records)
num, den = vm.RATES[metric]
with np.errstate(invalid="ignore", divide="ignore"):
rates = np.where(table[den] > 0, table[num] / np.where(table[den] > 0, table[den], 1), np.nan)
return [float(x) for x in rates]