Spaces:
Running
Running
| import pandas as pd | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| import os | |
| import sys | |
| import joblib | |
| from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc, f1_score | |
| # Add the project root to sys.path to import path_utils | |
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| import path_utils | |
| def evaluate_models(): | |
| # Load data | |
| preprocessed_path = path_utils.get_processed_data_path('preprocessed_data.pkl') | |
| if not os.path.exists(preprocessed_path): | |
| print(f"Error: Preprocessed data not found at {preprocessed_path}") | |
| return | |
| data = joblib.load(preprocessed_path) | |
| X_test = data['X_test'] | |
| y_test = data['y_test'] | |
| feature_names = data['feature_names'] | |
| print("Data loaded for evaluation.") | |
| # Load models | |
| models_to_eval = { | |
| 'Logistic Regression': joblib.load(path_utils.get_model_path('logistic_regression.pkl')), | |
| 'SVM': joblib.load(path_utils.get_model_path('svm_model.pkl')), | |
| 'Random Forest': joblib.load(path_utils.get_model_path('random_forest.pkl')), | |
| 'Decision Tree': joblib.load(path_utils.get_model_path('decision_tree.pkl')), | |
| 'XGBoost': joblib.load(path_utils.get_model_path('xgboost_model.pkl')), | |
| 'Isolation Forest': joblib.load(path_utils.get_model_path('isolation_forest.pkl')) | |
| } | |
| plt.figure(figsize=(10, 8)) | |
| results = [] | |
| for name, model in models_to_eval.items(): | |
| print(f"\nEvaluating {name}...") | |
| if name == 'Isolation Forest': | |
| # Isolation Forest returns -1 for anomaly, 1 for normal | |
| # Convert to 0 for normal, 1 for anomaly | |
| preds_raw = model.predict(X_test) | |
| y_pred = np.where(preds_raw == -1, 1, 0) | |
| # Anomaly score distribution | |
| scores = -model.decision_function(X_test) # Higher scores = more anomalous | |
| plt.figure(figsize=(8, 6)) | |
| sns.histplot(scores, bins=50, kde=True, color='purple') | |
| plt.title('Anomaly Scores (Isolation Forest)') | |
| plt.savefig(path_utils.get_output_path('anomaly_scores.png')) | |
| plt.close() | |
| else: | |
| y_pred = model.predict(X_test) | |
| # ROC Curve components | |
| if hasattr(model, "predict_proba"): | |
| y_prob = model.predict_proba(X_test)[:, 1] | |
| fpr, tpr, _ = roc_curve(y_test, y_prob) | |
| roc_auc = auc(fpr, tpr) | |
| plt.plot(fpr, tpr, label=f'{name} (AUC = {roc_auc:.2f})') | |
| # Metrics | |
| print(classification_report(y_test, y_pred)) | |
| f1 = f1_score(y_test, y_pred) | |
| results.append({'Model': name, 'F1-Score': f1}) | |
| # Confusion Matrix for the best model (XGBoost) | |
| if name == 'XGBoost': | |
| cm = confusion_matrix(y_test, y_pred) | |
| plt.figure(figsize=(8, 6)) | |
| sns.heatmap(cm, annot=True, fmt='d', cmap='Blues') | |
| plt.title('Confusion Matrix - XGBoost') | |
| plt.ylabel('Actual') | |
| plt.xlabel('Predicted') | |
| plt.savefig(path_utils.get_output_path('confusion_matrix_xgboost.png')) | |
| plt.close() | |
| # Feature Importance | |
| importances = model.feature_importances_ | |
| feat_imp = pd.Series(importances, index=feature_names).sort_values(ascending=False).head(10) | |
| plt.figure(figsize=(10, 7)) | |
| feat_imp.plot(kind='barh', color='teal') | |
| plt.title('Top 10 Feature Importances (XGBoost)') | |
| plt.savefig(path_utils.get_output_path('feature_importance.png')) | |
| plt.close() | |
| # Final ROC Plot formatting | |
| plt.plot([0, 1], [0, 1], 'k--') | |
| plt.xlabel('False Positive Rate') | |
| plt.ylabel('True Positive Rate') | |
| plt.title('ROC Curve Comparison') | |
| plt.legend(loc='lower right') | |
| plt.savefig(path_utils.get_output_path('roc_curve_comparison.png')) | |
| plt.close() | |
| # Summary Table | |
| res_df = pd.DataFrame(results) | |
| print("\nModel Performance Summary (F1-Score):") | |
| print(res_df.to_string(index=False)) | |
| print("\nEvaluation completed. All plots saved to 'outputs/' directory.") | |
| if __name__ == "__main__": | |
| evaluate_models() | |