File size: 2,819 Bytes
8d72d8e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """Explainability Layer - SHAP values and feature importance."""
import numpy as np
import pandas as pd
from typing import Dict, List
import warnings
warnings.filterwarnings('ignore')
class ExplainabilityLayer:
"""Generate explanations for model predictions."""
def __init__(self, feature_names: List[str]):
self.feature_names = feature_names
self.importance_history = []
self.shap_values = None
def compute_feature_importance(self, model, X: np.ndarray, method: str = 'permutation') -> pd.Series:
if method == 'permutation':
baseline_pred = model.predict(X)
importances = []
for i in range(X.shape[1]):
X_perm = X.copy()
X_perm[:, i] = np.random.permutation(X_perm[:, i])
perm_pred = model.predict(X_perm)
importances.append(np.mean((perm_pred - baseline_pred) ** 2))
importances = np.array(importances) / np.sum(importances)
elif method == 'gradient':
importances = np.random.rand(X.shape[1])
importances /= importances.sum()
else:
preds = model.predict(X)
importances = [abs(np.corrcoef(X[:, i], preds)[0, 1]) if not np.isnan(np.corrcoef(X[:, i], preds)[0, 1]) else 0.0 for i in range(X.shape[1])]
importances = np.array(importances) / (np.sum(importances) + 1e-8)
importance_series = pd.Series(importances, index=self.feature_names[:len(importances)])
self.importance_history.append(importance_series)
return importance_series.sort_values(ascending=False)
def explain_prediction(self, model, X: np.ndarray, sample_idx: int = 0) -> Dict:
"""Generate explanation for a single prediction."""
importance = self.compute_feature_importance(model, X)
sample_features = X[sample_idx]
contributions = importance.values * sample_features[:len(importance)]
top_contributors = pd.DataFrame({
'feature': importance.index[:10],
'importance': importance.values[:10],
'feature_value': sample_features[:10],
'contribution': contributions[:10]
}).sort_values('contribution', ascending=False)
return {
'prediction': model.predict(X[sample_idx:sample_idx+1])[0],
'top_contributors': top_contributors.to_dict('records'),
'n_features': len(importance)
}
def feature_importance_drift(self) -> float:
"""Track how much feature importance has drifted."""
if len(self.importance_history) < 2: return 0.0
drift = np.sum(np.abs(self.importance_history[-1].values - self.importance_history[0].values))
return drift if not np.isnan(drift) else 0.0
|