Spaces:
Runtime error
Runtime error
File size: 2,388 Bytes
6eff894 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
#!/usr/bin/env python3
from pathlib import Path
import pandas as pd, numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_fscore_support, brier_score_loss
LOG = Path("logs/predictions.csv")
OUT = Path("results"); OUT.mkdir(exist_ok=True)
def week_key(ts): # ISO year-week
iso = ts.isocalendar()
return f"{iso.year}-W{iso.week:02d}"
def calibration_plot(p, y, bins=10, out_png="results/calibration.png"):
df = pd.DataFrame({"p":p, "y":y}).dropna()
df["bin"] = pd.qcut(df["p"], q=bins, duplicates="drop")
g = df.groupby("bin").agg(avg_p=("p","mean"), frac_pos=("y","mean"), n=("y","size")).reset_index(drop=True)
plt.figure()
plt.plot([0,1],[0,1], linestyle="--")
plt.plot(g["avg_p"], g["frac_pos"], marker="o")
plt.xlabel("Predicted probability")
plt.ylabel("Observed frequency")
plt.title("Calibration")
for i, n in enumerate(g["n"]):
plt.annotate(str(int(n)), (g["avg_p"].iloc[i], g["frac_pos"].iloc[i]))
plt.tight_layout()
plt.savefig(out_png, dpi=300); plt.close()
def main():
if not LOG.exists():
print("No logs yet.")
return
df = pd.read_csv(LOG, parse_dates=["ts_pred","logged_at"])
df = df[df["y_true"].astype(str).isin(["0","1"])].copy()
if df.empty:
print("No rows with y_true yet.")
return
df["y_true"] = df["y_true"].astype(int)
df["week"] = df["ts_pred"].apply(week_key)
# Weekly metrics per mode
rows = []
for (wk, mode), grp in df.groupby(["week","mode"]):
y = grp["y_true"].values
# decision at time of logging
yhat = (grp["p"].values >= grp["threshold"].values).astype(int)
P,R,F1,_ = precision_recall_fscore_support(y, yhat, average="binary", zero_division=0)
alerts = float(yhat.mean())
brier = brier_score_loss(y, grp["p"].values)
rows.append({"week":wk,"mode":mode,"n":len(grp),"precision":P,"recall":R,"f1":F1,"alert_rate":alerts,"brier":brier})
rep = pd.DataFrame(rows).sort_values(["week","mode"])
rep.to_csv(OUT/"weekly_report.csv", index=False)
print(rep)
# Overall calibration (all modes combined)
calibration_plot(df["p"].values, df["y_true"].values, bins=12, out_png=str(OUT/"calibration.png"))
print("Saved:", OUT/"weekly_report.csv", "and", OUT/"calibration.png")
if __name__ == "__main__":
main()
|