Try Optimizing Silohuette Score

app.py

@@ -618,15 +618,22 @@ def compute_global_regression(df_combined, embedding_cols, tsne_params, df_f1, r
         silhouette = np.max(silhouette_vals)
     
     inertias = []
+    silohuettes_test = []
     K = range(1, 20)
 
+    # for k in K:
+    #     kmeans = KMeans(n_clusters=k, random_state=42)
+    #     kmeans.fit(reduced_real)
+    #     inertias.append(kmeans.inertia_)
+    # kl = KneeLocator(K, inertias, curve="convex", direction="decreasing")
+
     for k in K:
         kmeans = KMeans(n_clusters=k, random_state=42)
         kmeans.fit(reduced_real)
-        inertias.append(kmeans.inertia_)
-
+        labels_opt = kmeans_opt.fit_predict(X)
+        silohuettes_test.append(silhouette_score(X, labels_opt))
 
-    kl = KneeLocator(K, inertias, curve="convex", direction="decreasing")
+    kl = KneeLocator(K, silohuettes_test, curve="convex", direction="decreasing")
     elbow_k = kl.elbow
     kmeans_opt = KMeans(n_clusters=elbow_k, random_state=42, n_init=10)
     labels_opt = kmeans_opt.fit_predict(X)