Spaces:
Running
Running
File size: 5,874 Bytes
98b213b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | """Student's t-tests beside the existing results (DEC-36, `docs/PHASE5_TTESTS.md`): the statistics.
`paired` and `welch` return one report row each: t, df, two-sided p, the mean difference with its 95% t interval, an
effect size (d_z, or Hedges' g), skewness and Shapiro-Wilk, and the rank test beside it (Wilcoxon signed-rank, or
Mann-Whitney U). `holm` adjusts the family's p-values. The case is the unit throughout; the callers give one value per
case, and a pair with a missing side is dropped and counted.
"""
import math
import numpy as np
from scipy import stats
ALPHA = 0.05
def _clean_pairs(a, b):
a, b = np.asarray(a, dtype=float), np.asarray(b, dtype=float)
if a.shape != b.shape:
raise ValueError("paired samples must have the same length")
keep = np.isfinite(a) & np.isfinite(b)
return a[keep], b[keep], int((~keep).sum())
def _shape(x):
x = np.asarray(x, dtype=float)
sw = stats.shapiro(x).pvalue if len(x) >= 3 and np.ptp(x) > 0 else float("nan")
sk = float(stats.skew(x)) if len(x) >= 3 and np.ptp(x) > 0 else float("nan")
return {"skewness": sk, "shapiro_p": float(sw)}
def paired(a, b, label="", alpha=ALPHA):
"""Paired t-test of a - b, per case."""
a, b, dropped = _clean_pairs(a, b)
d = a - b
n = len(d)
if n < 2:
raise ValueError("a paired t-test needs at least 2 pairs")
mean, sd = float(d.mean()), float(d.std(ddof=1))
df = n - 1
if sd == 0:
t, p, half = float("nan"), float("nan"), 0.0
else:
res = stats.ttest_rel(a, b)
t, p = float(res.statistic), float(res.pvalue)
half = stats.t.ppf(1 - alpha / 2, df) * sd / math.sqrt(n)
nonzero = d[d != 0]
w = stats.wilcoxon(nonzero) if len(nonzero) >= 1 else None
return {"_values": [d], "label": label, "form": "paired", "n": n, "dropped": dropped, "mean_a": float(a.mean()),
"mean_b": float(b.mean()), "mean_difference": mean, "ci": [mean - half, mean + half], "t": t, "df": df,
"p": p, "effect": "d_z", "effect_size": mean / sd if sd else float("nan"),
"assumptions": _shape(d),
"rank_test": {"name": "Wilcoxon signed-rank", "statistic": float(w.statistic) if w else float("nan"),
"p": float(w.pvalue) if w else float("nan"), "zero_differences": int(n - len(nonzero))}}
def welch(a, b, label="", alpha=ALPHA):
"""Welch's t-test of mean(a) - mean(b), unpaired."""
a, b = (np.asarray(x, dtype=float) for x in (a, b))
dropped = int((~np.isfinite(a)).sum() + (~np.isfinite(b)).sum())
a, b = a[np.isfinite(a)], b[np.isfinite(b)]
na, nb = len(a), len(b)
if na < 2 or nb < 2:
raise ValueError("Welch's t-test needs at least 2 cases per group")
va, vb = a.var(ddof=1), b.var(ddof=1)
se = math.sqrt(va / na + vb / nb)
mean = float(a.mean() - b.mean())
if se == 0:
t, p, df, half = float("nan"), float("nan"), float("nan"), 0.0
else:
df = (va / na + vb / nb) ** 2 / ((va / na) ** 2 / (na - 1) + (vb / nb) ** 2 / (nb - 1))
res = stats.ttest_ind(a, b, equal_var=False)
t, p = float(res.statistic), float(res.pvalue)
half = stats.t.ppf(1 - alpha / 2, df) * se
pooled = math.sqrt(((na - 1) * va + (nb - 1) * vb) / (na + nb - 2))
j = 1 - 3 / (4 * (na + nb) - 9) # Hedges' small-sample correction
u = stats.mannwhitneyu(a, b, alternative="two-sided")
return {"_values": [a, b], "label": label, "form": "Welch", "n": [na, nb], "dropped": dropped, "mean_a": float(a.mean()),
"mean_b": float(b.mean()), "mean_difference": mean, "ci": [mean - half, mean + half], "t": t,
"df": float(df), "p": p, "effect": "Hedges' g",
"effect_size": j * mean / pooled if pooled else float("nan"),
"assumptions": {"a": _shape(a), "b": _shape(b)},
"rank_test": {"name": "Mann-Whitney U", "statistic": float(u.statistic), "p": float(u.pvalue)}}
def holm(pvalues):
"""Holm's step-down adjusted p-values, in the input order. NaN stays NaN and is not counted in the family."""
p = np.asarray(pvalues, dtype=float)
out = np.full_like(p, np.nan)
idx = np.where(np.isfinite(p))[0]
m = len(idx)
order = idx[np.argsort(p[idx], kind="stable")]
running = 0.0
for rank, i in enumerate(order):
running = max(running, min(1.0, (m - rank) * p[i]))
out[i] = running
return out
def adjust(rows):
"""Add `p_holm` to each row, over the rows as one family."""
for row, ph in zip(rows, holm([r["p"] for r in rows])):
row["p_holm"] = float(ph)
return rows
def agrees_with_bootstrap(row, bootstrap_ci):
"""True when the t interval and the existing bootstrap interval agree on whether 0 is excluded."""
t_excl = row["ci"][0] > 0 or row["ci"][1] < 0
b_excl = bootstrap_ci[0] > 0 or bootstrap_ci[1] < 0
return bool(t_excl == b_excl)
def case_rate(per_case_row, numerator="hallucinated", denominator="claims"):
"""One case's rate from a metrics per_case row; NaN when it has no claims (a format failure)."""
den = per_case_row.get(denominator) or 0
return per_case_row[numerator] / den if den else float("nan")
def case_rates(records, metric):
"""Per-case rates, in record order, from verifier case records (compare.case_records): numerator over denominator
of `metric` (metrics.RATES) for each case on its own; NaN where the denominator is 0 (a format failure, or no claim
whose criticality is determined)."""
from strokecopilot.nlp.verifier import metrics as vm
if not records:
return []
table = vm._table(records)
num, den = vm.RATES[metric]
with np.errstate(invalid="ignore", divide="ignore"):
rates = np.where(table[den] > 0, table[num] / np.where(table[den] > 0, table[den], 1), np.nan)
return [float(x) for x in rates]
|