Spaces:
Runtime error
Runtime error
| import numpy as np | |
| import pandas as pd | |
| import matplotlib | |
| matplotlib.use('Agg') | |
| import matplotlib.pyplot as plt | |
| import sys, os | |
| sys.path.insert(0, '.') | |
| os.makedirs('docs', exist_ok=True) | |
| from xgboost import XGBClassifier | |
| from sklearn.metrics import confusion_matrix, classification_report | |
| from data.data_loader import load_data, preprocess, split_and_scale | |
| from src.smote import smote | |
| from src.hyperparameter_tuning import grid_search | |
| from src.preprocessing_experiment import find_best_threshold | |
| def run_error_analysis(): | |
| df = load_data() | |
| X, y, feature_cols = preprocess(df, use_domain_cleaning=True) | |
| X_train, X_val, X_test, y_train, y_val, y_test, _ = split_and_scale(X, y) | |
| X_train_s, y_train_s = smote(X_train, y_train, random_state=42) | |
| best_alpha, best_lambda = grid_search() | |
| model = XGBClassifier( | |
| n_estimators=100, max_depth=4, learning_rate=0.05, | |
| subsample=0.7, colsample_bytree=0.7, min_child_weight=5, | |
| reg_alpha=best_alpha, reg_lambda=best_lambda, | |
| eval_metric='logloss', early_stopping_rounds=15, random_state=42, | |
| ) | |
| model.fit(X_train_s, y_train_s, eval_set=[(X_val, y_val)], verbose=False) | |
| val_proba = model.predict_proba(X_val)[:, 1] | |
| threshold, _ = find_best_threshold(y_val, val_proba) | |
| proba = model.predict_proba(X_test)[:, 1] | |
| preds = (proba >= threshold).astype(int) | |
| test_df = pd.DataFrame(X_test, columns=feature_cols) | |
| test_df['true'] = y_test.values | |
| test_df['pred'] = preds | |
| test_df['proba'] = proba | |
| fn = test_df[(test_df['true']==1) & (test_df['pred']==0)] # missed burnout | |
| fp = test_df[(test_df['true']==0) & (test_df['pred']==1)] # wrong alarm | |
| tp = test_df[(test_df['true']==1) & (test_df['pred']==1)] # correctly caught | |
| correct = test_df[test_df['true'] == test_df['pred']] | |
| cm = confusion_matrix(y_test, preds) | |
| # exclude composite features — compare only raw inputs | |
| base_feats = [c for c in feature_cols if c not in | |
| ['RECOVERY_SCORE','SOCIAL_SUPPORT_SCORE','LIFESTYLE_SCORE','HEALTH_HABITS']] | |
| diff = (fn[base_feats].mean() - tp[base_feats].mean()).abs().sort_values(ascending=False) | |
| hard_feats = diff.head(6).index.tolist() | |
| margin = 0.10 | |
| borderline = test_df[(test_df['proba'] >= threshold-margin) & | |
| (test_df['proba'] <= threshold+margin)] | |
| confident = test_df[~test_df.index.isin(borderline.index)] | |
| border_acc = (borderline['true'] == borderline['pred']).mean() | |
| confid_acc = (confident['true'] == confident['pred']).mean() | |
| print(f"Threshold: {threshold:.2f} | FN: {len(fn)} | FP: {len(fp)}") | |
| print(classification_report(y_test, preds, target_names=['Low Risk','High Risk'])) | |
| fig, axes = plt.subplots(2, 2, figsize=(14, 10)) | |
| fig.suptitle( | |
| f'Error Analysis — Burnout Risk Classifier\n' | |
| f'Threshold={threshold:.2f} | Test n={len(test_df)} | ' | |
| f'FN={len(fn)} (missed burnout) FP={len(fp)} (wrong alarm)', | |
| fontsize=12, fontweight='bold' | |
| ) | |
| # ① confusion matrix | |
| ax = axes[0, 0] | |
| im = ax.imshow(cm, cmap='Blues') | |
| ax.set_title('① Failure Counts\nConfusion matrix at tuned threshold', fontweight='bold') | |
| ax.set_xticks([0,1]); ax.set_yticks([0,1]) | |
| ax.set_xticklabels(['Predicted\nLow Risk', 'Predicted\nHigh Risk'], fontsize=9) | |
| ax.set_yticklabels(['Actual\nLow Risk', 'Actual\nHigh Risk'], fontsize=9) | |
| cell_labels = [[f'TN\n{cm[0,0]}', f'FP\n{cm[0,1]}\n(wrong alarm)'], | |
| [f'FN\n{cm[1,0]}\n(missed)', f'TP\n{cm[1,1]}']] | |
| for i in range(2): | |
| for j in range(2): | |
| color = 'white' if cm[i,j] > cm.max()/2 else 'black' | |
| ax.text(j, i, cell_labels[i][j], ha='center', va='center', | |
| color=color, fontsize=11, fontweight='bold') | |
| plt.colorbar(im, ax=ax, fraction=0.04) | |
| # ② probability distributions | |
| ax = axes[0, 1] | |
| ax.set_title('② Why the Model Fails\nBoth error types cluster near the decision boundary', | |
| fontweight='bold') | |
| ax.hist(correct['proba'], bins=30, alpha=0.4, color='#2ecc71', label=f'Correct (n={len(correct)})') | |
| ax.hist(fn['proba'], bins=20, alpha=0.8, color='#e74c3c', label=f'False Negative — missed burnout (n={len(fn)})') | |
| ax.hist(fp['proba'], bins=20, alpha=0.8, color='#e67e22', label=f'False Positive — wrong alarm (n={len(fp)})') | |
| ax.axvline(threshold, color='black', linewidth=2, linestyle='--', | |
| label=f'Decision threshold = {threshold:.2f}') | |
| ax.set_xlabel('Predicted Probability of High Burnout Risk') | |
| ax.set_ylabel('Count') | |
| ax.legend(fontsize=8) | |
| ax.annotate('Errors concentrate\nhere — model is\nuncertain, not\nsystematically wrong', | |
| xy=(threshold, ax.get_ylim()[1]*0.6), | |
| xytext=(threshold+0.12, ax.get_ylim()[1]*0.75), | |
| arrowprops=dict(arrowstyle='->', color='black'), | |
| fontsize=8, color='black', | |
| bbox=dict(boxstyle='round,pad=0.3', facecolor='lightyellow', alpha=0.8)) | |
| # ③ feature profiles: FN vs TP | |
| ax = axes[1, 0] | |
| ax.set_title('③ Most Challenging Input Types\n' | |
| 'Features where missed cases differ most from correctly caught cases', | |
| fontweight='bold') | |
| x = np.arange(len(hard_feats)) | |
| w = 0.35 | |
| ax.bar(x-w/2, fn[base_feats].mean()[hard_feats], w, | |
| label='False Negatives — missed burnout', color='#e74c3c', alpha=0.85) | |
| ax.bar(x+w/2, tp[base_feats].mean()[hard_feats], w, | |
| label='True Positives — correctly caught', color='#2ecc71', alpha=0.85) | |
| ax.set_xticks(x) | |
| ax.set_xticklabels([f.replace('_','\n') for f in hard_feats], fontsize=8) | |
| ax.set_ylabel('Mean value (standardised)') | |
| ax.axhline(0, color='grey', linewidth=0.8, linestyle='--') | |
| ax.legend(fontsize=8) | |
| largest_gap_idx = int(np.argmax( | |
| np.abs(fn[base_feats].mean()[hard_feats].values - | |
| tp[base_feats].mean()[hard_feats].values) | |
| )) | |
| ax.annotate('Largest\ndifference', | |
| xy=(largest_gap_idx, fn[base_feats].mean()[hard_feats].iloc[largest_gap_idx]), | |
| xytext=(largest_gap_idx+0.6, | |
| fn[base_feats].mean()[hard_feats].iloc[largest_gap_idx]+0.3), | |
| arrowprops=dict(arrowstyle='->', color='black'), | |
| fontsize=8, | |
| bbox=dict(boxstyle='round,pad=0.3', facecolor='lightyellow', alpha=0.8)) | |
| # ④ borderline vs confident accuracy | |
| ax = axes[1, 1] | |
| ax.set_title('④ Hardest Input Zone\nAccuracy collapses near the decision boundary', | |
| fontweight='bold') | |
| groups = [f'Borderline\n(within ±{margin} of threshold)\nn={len(borderline)}', | |
| f'Confident\n(outside ±{margin})\nn={len(confident)}'] | |
| accs = [border_acc, confid_acc] | |
| bars = ax.bar(groups, accs, color=['#e67e22','#3498db'], alpha=0.85, width=0.4) | |
| ax.set_ylim(0, 1.05) | |
| ax.set_ylabel('Accuracy') | |
| ax.axhline(0.5, color='grey', linestyle='--', linewidth=1, label='Random chance') | |
| for bar, acc in zip(bars, accs): | |
| ax.text(bar.get_x() + bar.get_width()/2, acc + 0.03, | |
| f'{acc:.1%}', ha='center', fontsize=13, fontweight='bold') | |
| gap = confid_acc - border_acc | |
| ax.annotate(f'{gap:.1%} accuracy gap\n→ errors are not random;\nthey concentrate where\nthe model is uncertain', | |
| xy=(0, border_acc), xytext=(0.55, border_acc - 0.18), | |
| arrowprops=dict(arrowstyle='->', color='black'), | |
| fontsize=8.5, | |
| bbox=dict(boxstyle='round,pad=0.3', facecolor='lightyellow', alpha=0.8)) | |
| ax.legend(fontsize=8) | |
| plt.tight_layout() | |
| plt.savefig('docs/error_analysis.png', dpi=150, bbox_inches='tight') | |
| plt.close() | |
| print(f"\nSaved docs/error_analysis.png") | |
| print(f"\nKey findings:") | |
| print(f" Borderline accuracy : {border_acc:.1%} vs Confident accuracy: {confid_acc:.1%}") | |
| print(f" Hardest feature : {hard_feats[0]} (gap = {diff.iloc[0]:.3f})") | |
| print(f" FN > FP : {len(fn) > len(fp)} — model leans toward false alarms over misses") | |
| if __name__ == '__main__': | |
| run_error_analysis() |