Update cluster_utils.py

added more utils for back-projection

cluster_utils.py

@@ -18,9 +18,29 @@ def choose_k(X_pca, k_range=(2, 12)):
     return best_k
 
 #========== During Cluster Analysis ========== 
-# function to save cluster stats
-def compute_cluster_stats(X_processed, labels, feature_names):
-    df = pd.DataFrame(X_processed, columns=feature_names)
+def compute_cluster_centroids_pca(df_pca, labels):
+    df = pd.DataFrame(df_pca)
+    df['cluster'] = labels
+
+    return df.groupby('cluster').mean()
+
+# maps PCA-space centroids back to original feature space
+def inverse_project_centroids(pca_centroids, pca_model, scaler_model, original_feature_names):
+
+    scaled_centroids = pca_model.inverse_transform(pca_centroids.values) # back-project from PCA space to scaled feature space
+    original_space_centroids = scaler_model.inverse_transform(scaled_centroids) # undo scaling
+
+    df_original = pd.DataFrame(
+        original_space_centroids,
+        columns=original_feature_names,
+        index=pca_centroids.index
+    )
+
+    return df_original
+
+# function to compute and save cluster stats
+def compute_cluster_stats(df_pca, labels, feature_names):
+    df = pd.DataFrame(df_pca, columns=feature_names)
     df['cluster'] = labels
 
     stats = {}
@@ -35,31 +55,26 @@ def compute_cluster_stats(X_processed, labels, feature_names):
             "std": cluster_data.std().to_dict(),
             "min": cluster_data.min().to_dict(),
             "max": cluster_data.max().to_dict(),
-            "range": (cluster_data.max() - cluster_data.min()).to_dict(),
+            "range": (cluster_data.max() - cluster_data.min()).to_dict()
         }
 
     return stats
 
-# function to identify core features of the clusters
-def identify_extreme_features(X_processed, labels, feature_names, threshold=1.0):
-    df = pd.DataFrame(X_processed, columns=feature_names)
-    df['cluster'] = labels
-
-    global_mean = df.drop(columns=['cluster']).mean()
-    global_std = df.drop(columns=['cluster']).std()
+# ranks top drivers based on `top_n`
+def identify_top_drivers(original_space_centroids, top_n):
+    global_mean = original_space_centroids.mean()
 
-    extremes = {}
+    drivers = {}
 
-    for cluster_id in sorted(df['cluster'].unique()):
-        cluster_mean = df[df['cluster'] == cluster_id].drop(columns=['cluster']).mean()
-        
-        z_scores = ((cluster_mean - global_mean) / global_std).abs()
+    for cluster_id, row in original_space_centroids.iterrows():
+        deviation = row - global_mean
+        ranked = deviation.abs().sort_values(ascending=False)
 
-        extreme_features = z_scores[z_scores > threshold].sort_values(ascending=False)
+        top_features = ranked.head(top_n).index.tolist()
 
-        extremes[cluster_id] = {
-            "features": extreme_features.index.tolist(),
-            "z_scores": extreme_features.to_dict()
+        drivers[cluster_id] = {
+            "top_features": top_features,
+            "deviations": deviation[top_features].to_dict()
         }
 
-    return extremes
+    return drivers