Buckets:
| """Paired bootstrap over test items (stratified by suite, 10,000 resamples) for group-score differences. | |
| Group score = equal-weight mean of suite accuracies (plan section 5). A comparison takes two | |
| (predictions file, K) pairs on the same items and returns the difference with a 95% CI. | |
| """ | |
| import json | |
| import sys | |
| import numpy as np | |
| sys.path.insert(0, __file__.rsplit("/", 1)[0]) | |
| from common import CORE14_EXCLUDE, GROUPS # noqa: E402 | |
| def load(path): | |
| return {json.loads(l)["id"]: json.loads(l) for l in open(path)} | |
| def correct(preds, K, ids): | |
| return np.array([int(np.argmax(preds[i]["probs"][str(K)]) == preds[i]["gold_idx"]) for i in ids], float) | |
| def suites_of(preds, group, subset=None): | |
| if group == "core17": | |
| names = sorted({p["suite"] for p in preds.values()}) | |
| elif group == "core14": | |
| names = sorted({p["suite"] for p in preds.values()} - set(CORE14_EXCLUDE)) | |
| elif group == "reasoning_no_mnli": | |
| names = [s for s in GROUPS["reasoning"] if s != "nimble_multinli"] | |
| else: | |
| names = GROUPS[group] | |
| by = {s: sorted(i for i, p in preds.items() if p["suite"] == s and (subset is None or subset(p))) for s in names} | |
| return {s: v for s, v in by.items() if v} | |
| def compare(A, KA, B, KB, group, R=10000, seed=0, subset=None): | |
| """mean over suites of acc(A) - acc(B), paired; returns dict(diff, lo, hi, a, b).""" | |
| by = suites_of(A, group, subset) | |
| rng = np.random.default_rng(seed) | |
| diffs = np.zeros(R) | |
| point_a, point_b = [], [] | |
| for s, ids in by.items(): | |
| ca, cb = correct(A, KA, ids), correct(B, KB, ids) | |
| point_a.append(ca.mean()) | |
| point_b.append(cb.mean()) | |
| idx = rng.integers(0, len(ids), size=(R, len(ids))) | |
| diffs += (ca[idx].mean(1) - cb[idx].mean(1)) | |
| diffs /= len(by) | |
| lo, hi = np.percentile(diffs, [2.5, 97.5]) | |
| return dict(diff=100 * (np.mean(point_a) - np.mean(point_b)), lo=100 * lo, hi=100 * hi, | |
| a=100 * np.mean(point_a), b=100 * np.mean(point_b), n_suites=len(by)) | |
| def fmt(r): | |
| return "%+.2f [%+.2f, %+.2f] (%.2f vs %.2f)" % (r["diff"], r["lo"], r["hi"], r["a"], r["b"]) | |
| if __name__ == "__main__": | |
| # quick CLI: bootstrap.py predsA KA predsB KB [group] | |
| A, B = load(sys.argv[1]), load(sys.argv[3]) | |
| group = sys.argv[5] if len(sys.argv) > 5 else "reasoning" | |
| print(group, fmt(compare(A, int(sys.argv[2]), B, int(sys.argv[4]), group))) | |
Xet Storage Details
- Size:
- 2.41 kB
- Xet hash:
- 1f9c4f4667f014ffc5c7723132d07aa0e940914817f2a59973cb02ecb8f2faec
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.