File size: 1,878 Bytes
e79d917 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """Swap the shipped decision rule to the cross-fitted winner.
Cross-fitted estimate (choose on one half of val, score on the other) says
threshold beats the bundle's ef05(0.4) by +0.000808. The PARAMETER is then picked
on all of val - legitimate here because we are shipping, not reporting; the honest
performance estimate remains the cross-fitted 0.982065.
"""
import sys, json, shutil
import numpy as np, pandas as pd, joblib
sys.path.insert(0, "src")
import decide as DC
from evaluate import macro_f05
ART = "work/results_A"
CACHE = "work/val_scored_20000.parquet"
b = joblib.load(f"{ART}/model_bundle.joblib")
cand = pd.read_parquet(CACHE)
truths = {int(k): set(v) for k, v in json.load(open(CACHE + ".truth.json")).items()}
ids = sorted(truths)
mm = b["miss_mass"]
def score(method, param):
p = DC.predict_sets(cand, "q", "c", "p", method, param, mm, True)
m, _ = macro_f05(p, truths, ids)
return m["f05"], m["precision"], m["recall"]
grid = np.round(np.arange(0.50, 0.95, 0.01), 3)
res = [(score("threshold", v)[0], v) for v in grid]
best_f, best_v = max(res)
f0, p0, r0 = score(b["method"], b["param"])
f1, p1, r1 = score("threshold", best_v)
print(f"current {b['method']}({b['param']}) F0.5 {f0:.6f} P {p0:.4f} R {r0:.4f}")
print(f"new threshold({best_v}) F0.5 {f1:.6f} P {p1:.4f} R {r1:.4f}")
print(f"delta on full val: {f1-f0:+.6f} (cross-fitted estimate was +0.000808)")
print()
print("nearby params:", [(v, round(f, 6)) for f, v in sorted(res, reverse=True)[:5]])
if f1 <= f0:
sys.exit("new rule is not better on full val - NOT changing the bundle")
shutil.copy(f"{ART}/model_bundle.joblib", f"{ART}/model_bundle.joblib.bak_ef05")
b["method"], b["param"] = "threshold", float(best_v)
joblib.dump(b, f"{ART}/model_bundle.joblib")
print(f"\nbundle updated -> threshold({best_v}) (previous saved as model_bundle.joblib.bak_ef05)")
|