code/select_cal.py

"""Ansambl üye ağırlıkları ve son sıcaklık YALNIZ kalibrasyon bölümünde seçilir (decision-v7 calibration, 300 soru).
Dev/test'e (Jev'le kıyaslanan küme) hiç bakılmaz. Ölçüt: kalibrasyon Brier + ECE15 (eşitlikte NLL) — yayındaki Duo
tarifini (w .5/.5, T .8) birebir verir.

    python select_cal.py run31.json run12.json runphi.json [--step 0.1]
Çıktı: {"weights": [...], "T": ..., "cal": {nll, brier, acc}} — jev_compare spec'ine aynen yazılır.
"""
import argparse, itertools, json, math
import numpy as np
from jev_compare import load_run, lsm


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("runs", nargs="+"); ap.add_argument("--step", type=float, default=0.1)
    ap.add_argument("--min_w", type=float, default=0.0)
    a = ap.parse_args()
    R = [load_run(f) for f in a.runs]
    cal = [r["runs_cal_items"] for r in R]
    Ts = [float(r.get("runs_temperature") or r.get("temperature") or 1.0) for r in R]
    n = len(cal[0]); assert all(len(c) == n for c in cal), "kalibrasyon madde sayısı farklı"
    for k in range(n):
        assert len({c[k]["y"] for c in cal}) == 1, "kalibrasyon madde sırası uyuşmuyor"
    Z = [[lsm(lsm(np.array(c[k]["lp"])) / T) for k in range(n)] for c, T in zip(cal, Ts)]
    Y = [cal[0][k]["y"] for k in range(n)]
    grid = np.round(np.arange(0, 1 + 1e-9, a.step), 4)
    best = None
    for w in itertools.product(grid, repeat=len(R)):
        if abs(sum(w) - 1) > 1e-6 or min(w) < a.min_w:
            continue
        comb = [sum(wi * Z[m][k] for m, wi in enumerate(w)) for k in range(n)]
        for Tf in np.round(np.arange(0.5, 1.51, 0.05), 3):
            nll = br = acc = 0.0; conf = []
            for z, y in zip(comb, Y):
                p = np.exp(lsm(z / Tf)); oh = np.zeros(len(p)); oh[y] = 1
                nll -= math.log(max(p[y], 1e-12)); br += float(((p - oh) ** 2).sum()); acc += int(p.argmax()) == y
                conf.append((float(p.max()), int(p.argmax()) == y))
            ece = sum(len(b) / n * abs(np.mean([c[0] for c in b]) - np.mean([c[1] for c in b]))
                      for j in range(15) for b in [[c for c in conf if j / 15 < c[0] <= (j + 1) / 15]] if b)
            # ölçüt: Brier + ECE (raporlanan iki kalibrasyon ölçüsü); eşitlikte NLL
            key = (round(br / n + ece, 5), round(nll / n, 5))
            if best is None or key < best[0]:
                best = (key, [float(x) for x in w], float(Tf), acc / n, br / n, ece)
    out = {"members": a.runs, "member_T": Ts, "weights": best[1], "T": best[2], "criterion": "cal Brier + ECE15",
           "cal": {"n": n, "brier": round(best[4], 5), "ece15": round(best[5], 5), "nll": best[0][1], "accuracy": round(best[3], 4)}}
    print(json.dumps(out))


if __name__ == "__main__":
    main()