File size: 1,780 Bytes
431bcf6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | import os
import pandas as pd
import numpy as np
import joblib
import shap
import matplotlib.pyplot as plt
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(base_dir, "data")
models_dir = os.path.join(base_dir, "models")
print("Loading model and data...")
classical_ridge = joblib.load(os.path.join(models_dir, "classical_ridge_model.pkl"))
tfidf = joblib.load(os.path.join(models_dir, "tfidf_vectorizer.pkl"))
val_feat = pd.read_csv(os.path.join(data_dir, "validation_features.csv"))
X_val = val_feat.drop(columns=['extraversion'])
# Reconstruct readable feature names
ling_emo_cols = [c for c in X_val.columns if not c.startswith('tfidf_')]
tfidf_words = tfidf.get_feature_names_out()
feature_names = ling_emo_cols + [f"TFIDF: {w}" for w in tfidf_words]
# Ensure they match
if len(feature_names) == len(X_val.columns):
X_val.columns = feature_names
else:
print("Warning: Feature counts do not match")
print("Calculating SHAP values...")
# Background dataset for SHAP
explainer = shap.LinearExplainer(classical_ridge, X_val, feature_names=feature_names)
shap_values = explainer(X_val)
print("Generating Plots...")
plt.figure(figsize=(10, 6))
shap.summary_plot(shap_values, X_val, show=False)
plt.tight_layout()
plt.savefig(os.path.join(models_dir, "shap_summary_plot.png"))
plt.close()
plt.figure(figsize=(10, 6))
shap.plots.bar(shap_values, show=False)
plt.tight_layout()
plt.savefig(os.path.join(models_dir, "shap_bar_plot.png"))
plt.close()
print("Done! SHAP plots saved to models/ directory.")
if __name__ == "__main__":
main()
|