import json, math, random, collections, sys
P = collections.defaultdict(dict)
for f in ('gpu/predictionsA.jsonl', 'gpu/predictionsB.jsonl'):
    import os
    if not os.path.exists(f): continue
    for l in open(f):
        d = json.loads(l); P[d['sys']].setdefault(d['set'], {})[d['i']] = float(d['correct'])
L = lambda s, st: [P[s][st][i] for i in sorted(P[s][st])]
def wil(a):
    n = len(a); p = sum(a)/n; z = 1.96; d = 1+z*z/n; c = (p+z*z/(2*n))/d; h = z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/d; return f"{100*p:.2f} [{100*(c-h):.1f}, {100*(c+h):.1f}]"
def boot(a, b, seed=0, B=10000):
    r = random.Random(seed); n = len(a); d = [x-y for x, y in zip(a, b)]; bs = sorted(sum(d[r.randrange(n)] for _ in range(n))/n*100 for _ in range(B))
    return f"{sum(d)/n*100:+.2f} [{bs[int(.025*B)]:+.2f}, {bs[int(.975*B)]:+.2f}]"
mean = lambda names, st: [sum(x)/len(x) for x in zip(*[L(n, st) for n in names])]
SETS = ['sealed_id', 'sealed_ood', 'sealed_1_new', 'sealed_2_new', 'sealed_4_new']
groups = {'k2_021': [f'k2_021_s{s}' for s in (17, 29, 43)], 'k2_018': [f'k2_018_s{s}' for s in (17, 29, 43)]}
for fn in ('values', 'values_csv', 'kv'): groups[f'sp_{fn}'] = [f'sp_{fn}_s{s}' for s in (17, 29, 43)]
for p in ('ridge_aug', 'mlp_aug', 'mlp_ctrl_shuffled'): groups[p] = [f'{p}_s{s}' for s in (17, 29, 43)]
singles = ['text_template'] + [k for k in P if k.startswith('fewshot_')]
for st in SETS:
    print('==', st); T = L('text_template', st)
    for s in singles: print(f'  {s}: {wil(L(s, st))}  vs template {boot(L(s, st), T)}')
    for g, ns in groups.items():
        if not all(st in P[n] for n in ns): continue
        per = ' / '.join(f"{100*sum(L(n, st))/len(L(n, st)):.1f}" for n in ns); m = mean(ns, st)
        extra = f" vs k2_021 {boot(m, mean(groups['k2_021'], st), 2)}" if g != 'k2_021' else ''
        print(f'  {g}: {per} | mean {100*sum(m)/len(m):.2f} vs template {boot(m, T, 1)}{extra}')