File size: 2,871 Bytes
845f05c 9f60832 845f05c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | """AUBIN hız: durum KV önbelleği (aynı duruma sorulan sorularda durum bir kez işlenir) — doğruluk eşliği + soru-başı gecikme.
Aynı model, aynı sorular (Kev dev durumları, çok sorulu), önbellek KAPALI ↔ AÇIK:
- fark: log-olasılıklarda en büyük mutlak fark, argmax uyuşması
- gecikme: soru başına ms (tek soru, grup boyutu 1 — Jev'in API gecikmesiyle kıyaslanabilir ölçü)
python bench_cache.py --adapter emrevrg/AUBIN-12B --n_cases 60 --out cache.json
"""
import argparse, json, os, sys, time
import torch
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--adapter", default="emrevrg/AUBIN-12B"); ap.add_argument("--no4bit", action="store_true")
ap.add_argument("--device_map", default=None); ap.add_argument("--n_cases", type=int, default=60)
ap.add_argument("--suite", default="kev_dev"); ap.add_argument("--work", default="/tmp/kev"); ap.add_argument("--out", default="cache.json")
a = ap.parse_args()
import kevdata
from aubin.core import Aubin
from kev_llm import resolve_adapter
a.adapter = resolve_adapter(a.adapter)
K = kevdata.load(a.work)
cases = [c for c in K[a.suite] if len(c["questions"]) >= 2][: a.n_cases]
A = Aubin(a.adapter, four_bit=not a.no4bit, device_map=a.device_map, think_margin=0.0)
R = {"adapter": a.adapter, "four_bit": not a.no4bit, "gpu": torch.cuda.get_device_name(0), "suite": a.suite,
"cases": len(cases), "questions": sum(len(c["questions"]) for c in cases)}
A.cache_state = False; A.logprobs(cases[0]["state"], dict(list(cases[0]["questions"].items())[:1])) # ısınma
res = {}
for mode in (False, True):
A.cache_state, A._pc = mode, None
torch.cuda.synchronize(); t0 = time.time(); lps = []
for c in cases:
lp = A.logprobs(c["state"], c["questions"])
lps.append({k: v[1] for k, v in lp.items()})
torch.cuda.synchronize(); dt = time.time() - t0
res[mode] = lps
R["cache_on" if mode else "cache_off"] = {"ms_per_question": round(1000 * dt / R["questions"], 1), "seconds": round(dt, 1)}
print(mode, R["cache_on" if mode else "cache_off"], flush=True)
diffs, agree, n = [], 0, 0
for x, y in zip(res[False], res[True]):
for k in x:
diffs.append(float((x[k] - y[k]).abs().max())); agree += int(x[k].argmax() == y[k].argmax()); n += 1
R["max_abs_logprob_diff"] = round(max(diffs), 4); R["mean_abs_logprob_diff"] = round(sum(diffs) / len(diffs), 5)
R["argmax_agreement"] = round(agree / n, 4)
R["speedup"] = round(R["cache_off"]["ms_per_question"] / R["cache_on"]["ms_per_question"], 2)
json.dump(R, open(a.out, "w"), indent=1)
print("SONUC", json.dumps(R), flush=True)
if __name__ == "__main__":
main()
|