scripts/generate_model_proofs.py

import os
import argparse
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import roc_curve, auc, precision_recall_curve, confusion_matrix
import mlflow
import mlflow.lightgbm
from ml.predict import predict_dataframe, load_model_and_encoders

def style_axis(ax):
    """Apply premium neon styling to matplotlib axes."""
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['left'].set_color('#334155')
    ax.spines['bottom'].set_color('#334155')
    ax.tick_params(colors='#94a3b8', labelsize=10)
    ax.grid(color='#334155', linestyle='--', alpha=0.4)
    ax.set_facecolor('#0f172a')

def add_neon_glow(ax, x, y, color, lw=2):
    """Add a glowing effect to a line."""
    ax.plot(x, y, color=color, lw=lw, zorder=5)
    for n in range(1, 4):
        ax.plot(x, y, color=color, lw=lw + (n*3), alpha=0.1, zorder=4)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--data-path", default="data_output/fastf1_races.parquet")
    parser.add_argument("--output-dir", default="assets/metrics")
    args = parser.parse_args()

    os.makedirs(args.output_dir, exist_ok=True)
    
    print("Loading data...")
    df = pd.read_parquet(args.data_path)
    
    test_df = df[df["season"] >= 2023].copy()
    if test_df.empty:
        test_df = df.copy()
        
    print(f"Test data size: {len(test_df)}")

    if "finish_position" in test_df.columns:
        y_true = (test_df["finish_position"] == 1).astype(int)
    else:
        print("No target column found. Cannot generate proofs.")
        return

    print("Loading model and encoders...")
    run_id = os.getenv("KRONECTOR_MODEL_RUN_ID")
    if not run_id:
        print("Please set KRONECTOR_MODEL_RUN_ID")
        return
        
    try:
        model, encoders = load_model_and_encoders(run_id)
    except Exception as e:
        print(f"Failed to load model: {e}")
        return

    print("Generating predictions...")
    preds_df = predict_dataframe(test_df, model, encoders, explain=True)
    y_pred_prob = preds_df["win_probability"]
    
    # Premium Dark Theme
    plt.style.use('dark_background')
    fig_color = "#0f172a"
    cyan = "#00f0ff"
    red = "#ff2a2a"
    text_color = "#f8fafc"
    
    # 1. ROC Curve
    print("Plotting ROC Curve...")
    fpr, tpr, _ = roc_curve(y_true, y_pred_prob)
    roc_auc = auc(fpr, tpr)
    
    fig, ax = plt.subplots(figsize=(8, 6), facecolor=fig_color)
    style_axis(ax)
    
    add_neon_glow(ax, fpr, tpr, cyan)
    ax.fill_between(fpr, tpr, alpha=0.1, color=cyan)
    ax.plot([0, 1], [0, 1], color='#475569', lw=2, linestyle='--')
    
    ax.set_xlim([-0.02, 1.0])
    ax.set_ylim([0.0, 1.05])
    ax.set_xlabel('False Positive Rate', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
    ax.set_ylabel('True Positive Rate', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
    ax.set_title('Receiver Operating Characteristic (ROC)', color=text_color, fontsize=16, fontweight='bold', pad=20)
    
    ax.legend([f'AUC = {roc_auc:.3f}'], loc="lower right", facecolor="#1e293b", edgecolor="#334155", labelcolor="white", fontsize=12)
    plt.tight_layout()
    plt.savefig(os.path.join(args.output_dir, "roc_curve.svg"), format="svg", transparent=True, bbox_inches='tight')
    plt.close()

    # 2. Precision-Recall Curve
    print("Plotting Precision-Recall Curve...")
    precision, recall, _ = precision_recall_curve(y_true, y_pred_prob)
    pr_auc = auc(recall, precision)
    
    fig, ax = plt.subplots(figsize=(8, 6), facecolor=fig_color)
    style_axis(ax)
    
    add_neon_glow(ax, recall, precision, cyan)
    ax.fill_between(recall, precision, alpha=0.1, color=cyan)
    
    ax.set_xlabel('Recall', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
    ax.set_ylabel('Precision', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
    ax.set_title('Precision-Recall Curve', color=text_color, fontsize=16, fontweight='bold', pad=20)
    
    ax.legend([f'AUC = {pr_auc:.3f}'], loc="lower left", facecolor="#1e293b", edgecolor="#334155", labelcolor="white", fontsize=12)
    plt.tight_layout()
    plt.savefig(os.path.join(args.output_dir, "pr_curve.svg"), format="svg", transparent=True, bbox_inches='tight')
    plt.close()

    # 3. Confusion Matrix
    print("Plotting Confusion Matrix...")
    y_pred_class = (y_pred_prob > 0.5).astype(int)
    cm = confusion_matrix(y_true, y_pred_class)
    
    fig, ax = plt.subplots(figsize=(7, 6), facecolor=fig_color)
    sns.heatmap(cm, annot=True, fmt="d", cmap=sns.color_palette("dark:#00f0ff", as_cmap=True), 
                cbar=False, ax=ax, annot_kws={"size": 18, "weight": "bold", "color": "white"},
                linewidths=2, linecolor='#0f172a', square=True)
    
    ax.set_facecolor('#0f172a')
    ax.set_xlabel('Predicted Label', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
    ax.set_ylabel('True Label', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
    ax.set_title('Confusion Matrix (Threshold=0.5)', color=text_color, fontsize=16, fontweight='bold', pad=20)
    ax.set_xticklabels(['Not Win', 'Win'], color='#94a3b8', fontsize=12)
    ax.set_yticklabels(['Not Win', 'Win'], color='#94a3b8', fontsize=12, rotation=0)
    
    plt.tight_layout()
    plt.savefig(os.path.join(args.output_dir, "confusion_matrix.svg"), format="svg", transparent=True, bbox_inches='tight')
    plt.close()

    # 4. Global Feature Importance
    print("Plotting Feature Importance...")
    shap_dicts = preds_df["shap_values"]
    shap_df = pd.DataFrame(shap_dicts.tolist())
    
    mean_abs_shap = shap_df.abs().mean().sort_values(ascending=True).tail(12)
    
    fig, ax = plt.subplots(figsize=(10, 8), facecolor=fig_color)
    style_axis(ax)
    ax.grid(False, axis='y') # Remove horizontal grid lines for bars
    
    # Draw bars with gradient-like glowing effect
    y_pos = np.arange(len(mean_abs_shap))
    ax.barh(y_pos, mean_abs_shap.values, color=cyan, alpha=0.8, height=0.6, edgecolor=cyan, linewidth=1.5)
    
    ax.set_yticks(y_pos)
    ax.set_yticklabels([str(x).replace('_', ' ').title() for x in mean_abs_shap.index], color='#cbd5e1', fontsize=11, fontweight='bold')
    
    ax.set_xlabel('Mean |SHAP Value| (Impact on Model Output)', color='#cbd5e1', fontsize=12, fontweight='bold', labelpad=10)
    ax.set_title('Global Feature Importance', color=text_color, fontsize=16, fontweight='bold', pad=20)
    
    # Add neon values to bars
    for i, v in enumerate(mean_abs_shap.values):
        ax.text(v + (max(mean_abs_shap.values) * 0.02), i, f'{v:.3f}', 
                color=cyan, fontweight='bold', va='center', fontsize=11)

    plt.tight_layout()
    plt.savefig(os.path.join(args.output_dir, "feature_importance.svg"), format="svg", transparent=True, bbox_inches='tight')
    plt.close()

    print(f"Proofs generated successfully in {args.output_dir}")

if __name__ == "__main__":
    main()