File size: 7,009 Bytes
710c7f2 24954d0 710c7f2 24954d0 710c7f2 77f1d1a 710c7f2 24954d0 710c7f2 24954d0 710c7f2 24954d0 710c7f2 24954d0 710c7f2 24954d0 710c7f2 c44f03d 710c7f2 24954d0 710c7f2 c44f03d 710c7f2 24954d0 710c7f2 24954d0 710c7f2 c44f03d 710c7f2 24954d0 710c7f2 24954d0 710c7f2 24954d0 710c7f2 24954d0 710c7f2 c44f03d 710c7f2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 | import os
import argparse
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import roc_curve, auc, precision_recall_curve, confusion_matrix
import mlflow
import mlflow.lightgbm
from ml.predict import predict_dataframe, load_model_and_encoders
def style_axis(ax):
"""Apply premium neon styling to matplotlib axes."""
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_color('#334155')
ax.spines['bottom'].set_color('#334155')
ax.tick_params(colors='#94a3b8', labelsize=10)
ax.grid(color='#334155', linestyle='--', alpha=0.4)
ax.set_facecolor('#0f172a')
def add_neon_glow(ax, x, y, color, lw=2):
"""Add a glowing effect to a line."""
ax.plot(x, y, color=color, lw=lw, zorder=5)
for n in range(1, 4):
ax.plot(x, y, color=color, lw=lw + (n*3), alpha=0.1, zorder=4)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--data-path", default="data_output/fastf1_races.parquet")
parser.add_argument("--output-dir", default="assets/metrics")
args = parser.parse_args()
os.makedirs(args.output_dir, exist_ok=True)
print("Loading data...")
df = pd.read_parquet(args.data_path)
test_df = df[df["season"] >= 2023].copy()
if test_df.empty:
test_df = df.copy()
print(f"Test data size: {len(test_df)}")
if "finish_position" in test_df.columns:
y_true = (test_df["finish_position"] == 1).astype(int)
else:
print("No target column found. Cannot generate proofs.")
return
print("Loading model and encoders...")
run_id = os.getenv("KRONECTOR_MODEL_RUN_ID")
if not run_id:
print("Please set KRONECTOR_MODEL_RUN_ID")
return
try:
model, encoders = load_model_and_encoders(run_id)
except Exception as e:
print(f"Failed to load model: {e}")
return
print("Generating predictions...")
preds_df = predict_dataframe(test_df, model, encoders, explain=True)
y_pred_prob = preds_df["win_probability"]
# Premium Dark Theme
plt.style.use('dark_background')
fig_color = "#0f172a"
cyan = "#00f0ff"
red = "#ff2a2a"
text_color = "#f8fafc"
# 1. ROC Curve
print("Plotting ROC Curve...")
fpr, tpr, _ = roc_curve(y_true, y_pred_prob)
roc_auc = auc(fpr, tpr)
fig, ax = plt.subplots(figsize=(8, 6), facecolor=fig_color)
style_axis(ax)
add_neon_glow(ax, fpr, tpr, cyan)
ax.fill_between(fpr, tpr, alpha=0.1, color=cyan)
ax.plot([0, 1], [0, 1], color='#475569', lw=2, linestyle='--')
ax.set_xlim([-0.02, 1.0])
ax.set_ylim([0.0, 1.05])
ax.set_xlabel('False Positive Rate', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
ax.set_ylabel('True Positive Rate', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
ax.set_title('Receiver Operating Characteristic (ROC)', color=text_color, fontsize=16, fontweight='bold', pad=20)
ax.legend([f'AUC = {roc_auc:.3f}'], loc="lower right", facecolor="#1e293b", edgecolor="#334155", labelcolor="white", fontsize=12)
plt.tight_layout()
plt.savefig(os.path.join(args.output_dir, "roc_curve.svg"), format="svg", transparent=True, bbox_inches='tight')
plt.close()
# 2. Precision-Recall Curve
print("Plotting Precision-Recall Curve...")
precision, recall, _ = precision_recall_curve(y_true, y_pred_prob)
pr_auc = auc(recall, precision)
fig, ax = plt.subplots(figsize=(8, 6), facecolor=fig_color)
style_axis(ax)
add_neon_glow(ax, recall, precision, cyan)
ax.fill_between(recall, precision, alpha=0.1, color=cyan)
ax.set_xlabel('Recall', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
ax.set_ylabel('Precision', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
ax.set_title('Precision-Recall Curve', color=text_color, fontsize=16, fontweight='bold', pad=20)
ax.legend([f'AUC = {pr_auc:.3f}'], loc="lower left", facecolor="#1e293b", edgecolor="#334155", labelcolor="white", fontsize=12)
plt.tight_layout()
plt.savefig(os.path.join(args.output_dir, "pr_curve.svg"), format="svg", transparent=True, bbox_inches='tight')
plt.close()
# 3. Confusion Matrix
print("Plotting Confusion Matrix...")
y_pred_class = (y_pred_prob > 0.5).astype(int)
cm = confusion_matrix(y_true, y_pred_class)
fig, ax = plt.subplots(figsize=(7, 6), facecolor=fig_color)
sns.heatmap(cm, annot=True, fmt="d", cmap=sns.color_palette("dark:#00f0ff", as_cmap=True),
cbar=False, ax=ax, annot_kws={"size": 18, "weight": "bold", "color": "white"},
linewidths=2, linecolor='#0f172a', square=True)
ax.set_facecolor('#0f172a')
ax.set_xlabel('Predicted Label', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
ax.set_ylabel('True Label', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
ax.set_title('Confusion Matrix (Threshold=0.5)', color=text_color, fontsize=16, fontweight='bold', pad=20)
ax.set_xticklabels(['Not Win', 'Win'], color='#94a3b8', fontsize=12)
ax.set_yticklabels(['Not Win', 'Win'], color='#94a3b8', fontsize=12, rotation=0)
plt.tight_layout()
plt.savefig(os.path.join(args.output_dir, "confusion_matrix.svg"), format="svg", transparent=True, bbox_inches='tight')
plt.close()
# 4. Global Feature Importance
print("Plotting Feature Importance...")
shap_dicts = preds_df["shap_values"]
shap_df = pd.DataFrame(shap_dicts.tolist())
mean_abs_shap = shap_df.abs().mean().sort_values(ascending=True).tail(12)
fig, ax = plt.subplots(figsize=(10, 8), facecolor=fig_color)
style_axis(ax)
ax.grid(False, axis='y') # Remove horizontal grid lines for bars
# Draw bars with gradient-like glowing effect
y_pos = np.arange(len(mean_abs_shap))
ax.barh(y_pos, mean_abs_shap.values, color=cyan, alpha=0.8, height=0.6, edgecolor=cyan, linewidth=1.5)
ax.set_yticks(y_pos)
ax.set_yticklabels([str(x).replace('_', ' ').title() for x in mean_abs_shap.index], color='#cbd5e1', fontsize=11, fontweight='bold')
ax.set_xlabel('Mean |SHAP Value| (Impact on Model Output)', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
ax.set_title('Global Feature Importance', color=text_color, fontsize=16, fontweight='bold', pad=20)
# Add neon values to bars
for i, v in enumerate(mean_abs_shap.values):
ax.text(v + (max(mean_abs_shap.values) * 0.02), i, f'{v:.3f}',
color=cyan, fontweight='bold', va='center', fontsize=11)
plt.tight_layout()
plt.savefig(os.path.join(args.output_dir, "feature_importance.svg"), format="svg", transparent=True, bbox_inches='tight')
plt.close()
print(f"Proofs generated successfully in {args.output_dir}")
if __name__ == "__main__":
main()
|