File size: 2,786 Bytes
e54feb4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | """Ansambl üye ağırlıkları ve son sıcaklık YALNIZ kalibrasyon bölümünde seçilir (decision-v7 calibration, 300 soru).
Dev/test'e (Jev'le kıyaslanan küme) hiç bakılmaz. Ölçüt: kalibrasyon Brier + ECE15 (eşitlikte NLL) — yayındaki Duo
tarifini (w .5/.5, T .8) birebir verir.
python select_cal.py run31.json run12.json runphi.json [--step 0.1]
Çıktı: {"weights": [...], "T": ..., "cal": {nll, brier, acc}} — jev_compare spec'ine aynen yazılır.
"""
import argparse, itertools, json, math
import numpy as np
from jev_compare import load_run, lsm
def main():
ap = argparse.ArgumentParser()
ap.add_argument("runs", nargs="+"); ap.add_argument("--step", type=float, default=0.1)
ap.add_argument("--min_w", type=float, default=0.0)
a = ap.parse_args()
R = [load_run(f) for f in a.runs]
cal = [r["runs_cal_items"] for r in R]
Ts = [float(r.get("runs_temperature") or r.get("temperature") or 1.0) for r in R]
n = len(cal[0]); assert all(len(c) == n for c in cal), "kalibrasyon madde sayısı farklı"
for k in range(n):
assert len({c[k]["y"] for c in cal}) == 1, "kalibrasyon madde sırası uyuşmuyor"
Z = [[lsm(lsm(np.array(c[k]["lp"])) / T) for k in range(n)] for c, T in zip(cal, Ts)]
Y = [cal[0][k]["y"] for k in range(n)]
grid = np.round(np.arange(0, 1 + 1e-9, a.step), 4)
best = None
for w in itertools.product(grid, repeat=len(R)):
if abs(sum(w) - 1) > 1e-6 or min(w) < a.min_w:
continue
comb = [sum(wi * Z[m][k] for m, wi in enumerate(w)) for k in range(n)]
for Tf in np.round(np.arange(0.5, 1.51, 0.05), 3):
nll = br = acc = 0.0; conf = []
for z, y in zip(comb, Y):
p = np.exp(lsm(z / Tf)); oh = np.zeros(len(p)); oh[y] = 1
nll -= math.log(max(p[y], 1e-12)); br += float(((p - oh) ** 2).sum()); acc += int(p.argmax()) == y
conf.append((float(p.max()), int(p.argmax()) == y))
ece = sum(len(b) / n * abs(np.mean([c[0] for c in b]) - np.mean([c[1] for c in b]))
for j in range(15) for b in [[c for c in conf if j / 15 < c[0] <= (j + 1) / 15]] if b)
# ölçüt: Brier + ECE (raporlanan iki kalibrasyon ölçüsü); eşitlikte NLL
key = (round(br / n + ece, 5), round(nll / n, 5))
if best is None or key < best[0]:
best = (key, [float(x) for x in w], float(Tf), acc / n, br / n, ece)
out = {"members": a.runs, "member_T": Ts, "weights": best[1], "T": best[2], "criterion": "cal Brier + ECE15",
"cal": {"n": n, "brier": round(best[4], 5), "ece15": round(best[5], 5), "nll": best[0][1], "accuracy": round(best[3], 4)}}
print(json.dumps(out))
if __name__ == "__main__":
main()
|