Revision 5: negative-result model card, report rev. 5 (PDF/MD/figure), CITATION.cff, iterations 020-021 code, logs, sealed tests and checkpoints
7420c83 verified | import json, math, random, collections, sys | |
| P = collections.defaultdict(dict) | |
| for f in ('gpu/predictionsA.jsonl', 'gpu/predictionsB.jsonl'): | |
| import os | |
| if not os.path.exists(f): continue | |
| for l in open(f): | |
| d = json.loads(l); P[d['sys']].setdefault(d['set'], {})[d['i']] = float(d['correct']) | |
| L = lambda s, st: [P[s][st][i] for i in sorted(P[s][st])] | |
| def wil(a): | |
| n = len(a); p = sum(a)/n; z = 1.96; d = 1+z*z/n; c = (p+z*z/(2*n))/d; h = z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/d; return f"{100*p:.2f} [{100*(c-h):.1f}, {100*(c+h):.1f}]" | |
| def boot(a, b, seed=0, B=10000): | |
| r = random.Random(seed); n = len(a); d = [x-y for x, y in zip(a, b)]; bs = sorted(sum(d[r.randrange(n)] for _ in range(n))/n*100 for _ in range(B)) | |
| return f"{sum(d)/n*100:+.2f} [{bs[int(.025*B)]:+.2f}, {bs[int(.975*B)]:+.2f}]" | |
| mean = lambda names, st: [sum(x)/len(x) for x in zip(*[L(n, st) for n in names])] | |
| SETS = ['sealed_id', 'sealed_ood', 'sealed_1_new', 'sealed_2_new', 'sealed_4_new'] | |
| groups = {'k2_021': [f'k2_021_s{s}' for s in (17, 29, 43)], 'k2_018': [f'k2_018_s{s}' for s in (17, 29, 43)]} | |
| for fn in ('values', 'values_csv', 'kv'): groups[f'sp_{fn}'] = [f'sp_{fn}_s{s}' for s in (17, 29, 43)] | |
| for p in ('ridge_aug', 'mlp_aug', 'mlp_ctrl_shuffled'): groups[p] = [f'{p}_s{s}' for s in (17, 29, 43)] | |
| singles = ['text_template'] + [k for k in P if k.startswith('fewshot_')] | |
| for st in SETS: | |
| print('==', st); T = L('text_template', st) | |
| for s in singles: print(f' {s}: {wil(L(s, st))} vs template {boot(L(s, st), T)}') | |
| for g, ns in groups.items(): | |
| if not all(st in P[n] for n in ns): continue | |
| per = ' / '.join(f"{100*sum(L(n, st))/len(L(n, st)):.1f}" for n in ns); m = mean(ns, st) | |
| extra = f" vs k2_021 {boot(m, mean(groups['k2_021'], st), 2)}" if g != 'k2_021' else '' | |
| print(f' {g}: {per} | mean {100*sum(m)/len(m):.2f} vs template {boot(m, T, 1)}{extra}') | |