File size: 4,273 Bytes
5daf54a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | """AUBIN ansambl üye seçimi — DÜRÜST protokol: seçim yalnız dev bölümlerinde (kev_dev + kev_transfer_dev), test yalnız raporlanır.
Açgözlü ileri seçim (tekrarlı üye = ağırlık artışı): her adımda dev log-kaybını (NLL, sıcaklık cal'da) en çok düşüren üye eklenir;
iyileşme yoksa durur. Sonra seçilen ağırlıklarla kev_test / kev_transfer_test (ve dev) raporlanır.
python select_members.py out.json e12a.json e12b.json e31.json ... [--max 8] [--metric nll|acc]
"""
import argparse, json
import numpy as np
from ensemble import metrics
DEV = ("kev_dev", "kev_transfer_dev")
TEST = ("kev_test", "kev_transfer_test")
def runs(r):
d = dict(r["runs"] if "runs" in r else r)
if r.get("runs_cal_items") and "cal300" not in d: # kalibrasyon bölümü de seçim kümesi olabilir (--dev cal300)
d["cal300"] = {"items": [{"src": "cal", "y": it["y"], "lp": it["lp"]} for it in r["runs_cal_items"]],
"all": {"accuracy": float(np.mean([int(np.argmax(it["lp"])) == it["y"] for it in r["runs_cal_items"]]))}}
return d
def _n(lp, r):
"""Üyeyi kendi kalibrasyon sıcaklığıyla normalize et (farklı tabanların ham logit ölçekleri farklı)."""
x = np.asarray(lp, dtype=np.float64) / float(r.get("runs_temperature", 1.0))
return x - np.logaddexp.reduce(x)
def combo(R, w, s):
its = [runs(r)[s]["items"] for r in R]
rows = []
for k in range(len(its[0])):
lp = sum(wi * _n(x[k]["lp"], r) for wi, x, r in zip(w, its, R) if wi)
rows.append((its[0][k]["src"], its[0][k]["y"], lp))
return rows
def temp(R, w):
cal = [r.get("runs_cal_items") for r in R]
if not all(cal[i] for i in range(len(R)) if w[i]):
return 1.0
sel = [(wi, ci, r) for wi, ci, r in zip(w, cal, R) if wi]
comb = [(0, sel[0][1][k]["y"], sum(wi * _n(ci[k]["lp"], r) for wi, ci, r in sel)) for k in range(len(sel[0][1]))]
grid = np.concatenate([np.arange(0.3, 4.0, 0.05), np.arange(4.0, 20.01, 0.25)])
return float(min(grid, key=lambda t: metrics(comb, t)["all"]["nll"]))
def score(R, cnt, metric):
w = np.array(cnt, dtype=float); w = w / w.sum()
T = temp(R, w)
rows = [x for s in DEV for x in combo(R, w, s)]
m = metrics(rows, T)["all"]
return (-m["nll"] if metric == "nll" else m["accuracy"]), w, T
def main():
ap = argparse.ArgumentParser()
ap.add_argument("out"); ap.add_argument("files", nargs="+")
ap.add_argument("--max", type=int, default=8); ap.add_argument("--metric", default="nll", choices=["nll", "acc"])
ap.add_argument("--dev", default="kev_dev,kev_transfer_dev")
a = ap.parse_args()
global DEV
DEV = tuple(a.dev.split(","))
R = [json.load(open(f)) for f in a.files]
for f, r in zip(a.files, R):
miss = [s for s in DEV + ("kev_test",) if s not in runs(r)]
assert not miss, f"{f}: eksik küme {miss}"
cnt = [0] * len(R); best = None; path = []
for _ in range(a.max):
cands = []
for i in range(len(R)):
c = list(cnt); c[i] += 1
cands.append((score(R, c, a.metric)[0], i))
sc, i = max(cands)
if best is not None and sc <= best + 1e-6:
break
best = sc; cnt[i] += 1; path.append((a.files[i], round(sc, 5)))
print("ekle", a.files[i], "dev", a.metric, round(sc, 5), flush=True)
_, w, T = score(R, cnt, a.metric)
out = {"protocol": f"üye seçimi + ağırlık yalnız dev ({'+'.join(DEV)}); sıcaklık cal; test yalnız rapor",
"members": [{"file": f, "weight": round(float(x), 4), "model": r.get("model"), "lora": r.get("lora")}
for f, r, x in zip(a.files, R, w) if x],
"temperature": round(T, 2), "selection_path": path}
for s in DEV + TEST:
if not all(s in runs(r) for r, x in zip(R, w) if x):
continue
out[s] = metrics(combo(R, w, s), T)
print(s, json.dumps(out[s]["all"]), flush=True)
for f, r in zip(a.files, R): # tek üyeler (karşılaştırma için)
out.setdefault("single", {})[f] = {s: runs(r)[s]["all"]["accuracy"] for s in DEV + TEST if s in runs(r)}
json.dump(out, open(a.out, "w"), indent=1)
if __name__ == "__main__":
main()
|