File size: 1,763 Bytes
87603cb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Score the adapter on the public TEST splits (never trained on). Usage (from model/): python eval_guard.py . out.json"""
import json, sys, time
from gliner2 import AutoExtractor
ck, out = sys.argv[1], sys.argv[2]
from load_guard import load
m = load(ck)
BARS = (0.5, 0.9)
def p_inj(text):
got = m.classify_text(text[:3000], {"injection": {"labels": ["injection", "benign"]}}, max_len=400, include_confidence=True)["injection"]
c = float(got["confidence"])
return c if got["label"] == "injection" else 1 - c
summary = {"checkpoint": ck, "sets": {}}
for name in ("boundary", "deepset"):
rows = [json.loads(l) for l in open(f"../data/{name}_test.jsonl") if l.strip()]
scored, secs = [], []
for r in rows:
t = time.time(); p = p_inj(r["text"]); secs.append(time.time() - t)
scored.append((r["label"], p))
s = {"n": len(rows), "injections": sum(y for y, _ in scored), "latency_p50_s": round(sorted(secs)[len(secs) // 2], 3)}
for bar in BARS:
tp = sum(1 for y, p in scored if y and p >= bar); fp = sum(1 for y, p in scored if not y and p >= bar)
fn = sum(1 for y, p in scored if y and p < bar); neg = sum(1 for y, _ in scored if not y)
prec = tp / (tp + fp) if tp + fp else 0; rec = tp / (tp + fn) if tp + fn else 0
s[f"bar{bar}"] = {"precision": round(prec, 3), "recall": round(rec, 3),
"f1": round(2 * prec * rec / (prec + rec), 3) if prec + rec else 0,
"false_flags": round(fp / neg, 3) if neg else 0,
"accuracy": round(sum(1 for y, p in scored if (p >= bar) == bool(y)) / len(scored), 3)}
summary["sets"][name] = s
json.dump(summary, open(out, "w"), indent=1)
print(json.dumps(summary, indent=1))
|