File size: 2,272 Bytes
6e802a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | # paired_stats.py: Paired WER contrasts between scored cells: bootstrap interval, exact sign test, Holm.
import argparse
import json
import random
from fractions import Fraction
BOOT_N, SEED = 10000, 20260704
def boot_ci(v, n_boot=BOOT_N, seed=SEED):
rng = random.Random(seed)
n = len(v)
ms = sorted(sum(rng.choices(v, k=n)) / n for _ in range(n_boot))
return ms[int(0.025 * n_boot)], ms[int(0.975 * n_boot)]
def sign_p(b, w):
n = b + w
if n == 0:
return 1.0
k, c, tot = min(b, w), 1, 0
for i in range(0, k + 1):
if i > 0:
c = c * (n - i + 1) // i
tot += c
return float(min(Fraction(1), Fraction(2 * tot, 2 ** n)))
def holm(ps):
order = sorted(range(len(ps)), key=lambda i: ps[i])
out, run = [0.0] * len(ps), 0.0
for rank, i in enumerate(order):
run = max(run, min(1.0, (len(ps) - rank) * ps[i]))
out[i] = run
return out
def contrast(wa, wb):
if len(wa) != len(wb):
raise ValueError(f"row counts differ ({len(wa)} vs {len(wb)}): not the same prompt rows")
d = [x - y for x, y in zip(wa, wb)]
lo, hi = boot_ci(d)
b, w = sum(1 for x in d if x < 0), sum(1 for x in d if x > 0)
return {"n": len(d), "delta": sum(d) / len(d), "ci": [lo, hi], "better": b, "worse": w, "tie": len(d) - b - w,
"sign_p": sign_p(b, w)}
def per_row_wer(spec):
path, cell = spec.rsplit(":", 1)
return json.load(open(path))[cell]["per_row"]["wer"]
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--pair", nargs=2, action="append", required=True, metavar=("A", "B"))
ap.add_argument("--out", default=None)
args = ap.parse_args()
rows = [dict(a=a, b=b, **contrast(per_row_wer(a), per_row_wer(b))) for a, b in args.pair]
for r, h in zip(rows, holm([r["sign_p"] for r in rows])):
r["holm_p"] = h
print(f"{r['a']} - {r['b']}: {r['delta']:+.3f} [{r['ci'][0]:+.3f}, {r['ci'][1]:+.3f}] "
f"{r['better']}/{r['worse']}/{r['tie']} sign p {r['sign_p']:.2g} Holm p {h:.2g} (over {len(rows)})")
if args.out:
json.dump(rows, open(args.out, "w"), indent=1)
if __name__ == "__main__":
main()
|