Spaces:
Build error
Build error
File size: 5,174 Bytes
0a489c6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
# Contains classes and functions for evaluating trained
# models using the specified metrics.
from sklearn.metrics import accuracy_score, mean_squared_error, r2_score, mean_squared_error,classification_report, confusion_matrix, ConfusionMatrixDisplay
import json
import matplotlib.pyplot as plt
import streamlit as st
import pandas as pd
import numpy as np
class Evaluator:
def __init__(self, json_content, problem_type, target_variable):
self.json_content = json_content
self.problem_type = problem_type
def evaluate_model(self, models, X_test, y_test):
"""Evaluates the model using specified metrics and returns results."""
metrics = {}
for model_name, model in models.items():
metrics[model_name] = {}
print(f"Evaluating model: {model_name}")
predictions = model.predict(X_test)
# Choose evaluation metrics based on problem type
st.subheader(f"Model: {model_name}")
if self.problem_type == 'Classification':
self.log_confusion_matrix(y_test, predictions, model_name, model)
accuracy = self.log_classification_report(y_test, predictions, model_name)
metrics[model_name]["accuracy"] = accuracy
else: # 'regression'
rmse_score = self.log_rmse(y_test, predictions, model_name)
r2_score = self.log_r2(y_test, predictions, model_name)
adj_r2_score = self.log_adj_r2(X_test,y_test, predictions, model_name)
metrics[model_name]["rmse"] = rmse_score
metrics[model_name]["r2"] = r2_score
metrics[model_name]["adj_r2"] = adj_r2_score
return metrics
# return metrics
def save_metrics(self, metrics, file_path: str):
"""Saves evaluation metrics to a file."""
with open(file_path, 'w') as file:
json.dump(metrics, file)
def log_confusion_matrix(self, y_test, predictions, model_name, model):
"""Logs the confusion matrix."""
cm = confusion_matrix(y_test, predictions, labels=model.classes_)
# st.set_option('deprecation.showPyplotGlobalUse', False)
st.markdown(f"#### Confusion matrix for : {model_name} ")
fig, ax = plt.subplots()
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=model.classes_)
disp.plot(ax=ax)
st.pyplot(fig)
def log_classification_report(self, y_test, predictions, model_name):
"""Logs the classification report."""
st.markdown(f"#### Classification report for: {model_name} ")
accuracy = accuracy_score(y_test, predictions)
cr = classification_report(y_test, predictions, output_dict=True)
report_df = pd.DataFrame(cr).T
report_df = report_df.rename(columns={'precision': 'Precision', 'recall': 'Recall', 'f1-score': 'F1-Score', 'support': 'Support'})
st.table(report_df)
return round(accuracy,2)
def log_rmse(self, y_test, predictions, model_name):
"""Logs the root mean squared error."""
rmse = mean_squared_error(y_test, predictions, squared="False")
st.markdown(f"RMSE for {model_name}: {round(rmse,2)} ")
return round(rmse,2)
def log_r2(self, y_test, predictions, model_name):
"""Logs the R-squared score."""
r2 = r2_score(y_test, predictions)
st.markdown(f"R-squared score for {model_name}: {round(r2,2)} ")
return round(r2,2)
def log_adj_r2(self,X_test, y_test, predictions, model_name):
"""Logs the adjusted R-squared score."""
sample_size, n_variables = X_test.shape
r2 = r2_score(y_test, predictions)
adj_r2 = 1 - ((1 - r2) * (sample_size - 1)) / (sample_size - n_variables - 1)
print(f" model: {model_name}")
st.markdown(f"Adjusted R-squared score for {model_name}: {round(r2,2)} ")
return round(adj_r2,2)
def display_metrics(self, metrics):
available_metrics = list(next(iter(metrics.values())).keys())
available_models = list(metrics.keys())
num_models = len(available_models)
hue_colors = plt.cm.tab10(np.linspace(0, 1, num_models))
for metric in available_metrics:
fig, ax = plt.subplots(figsize=(10, 5))
for i, model in enumerate(available_models):
metric_value = metrics[model][metric]
bar = ax.bar(model, metric_value, color=hue_colors[i], label=model)
for rect in bar:
height = rect.get_height()
ax.annotate('{}'.format(round(height, 2)),
xy=(rect.get_x() + rect.get_width() / 2, height),
xytext=(0, 3), textcoords="offset points",
ha='center', va='bottom')
ax.set_xlabel('Algorithm Models')
ax.set_ylabel(metric.upper())
ax.legend()
plt.xticks(rotation=45)
plt.title(f'{metric.upper()}')
st.pyplot(fig)
|