Update app.py

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 import pandas as pd
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.cluster import KMeans
+from sklearn.metrics import silhouette_score, silhouette_samples
 import matplotlib.pyplot as plt
 from sklearn.decomposition import PCA
 import re
@@ -9,8 +10,6 @@ from io import BytesIO
 import tempfile
 import numpy as np
 from PIL import Image
-from wordcloud import WordCloud
-
 
 def preprocess_data(df):
     df.rename(columns={'Question Asked': 'texts'}, inplace=True)
@@ -126,12 +125,46 @@ def cluster_data(df, num_clusters):
 
     return df, X, kmeans
 
-def generate_wordcloud(df):
-    text = " ".join(df['texts'].tolist())
-    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text)
-    plt.figure(figsize=(10, 5))
-    plt.imshow(wordcloud, interpolation='bilinear')
-    plt.axis('off')
+def visualize_clusters(df):
+    plt.figure(figsize=(10, 6))
+    scatter = plt.scatter(df['PCA1'], df['PCA2'], c=df['Cluster'], cmap='viridis')
+    plt.legend(*scatter.legend_elements(), title="Clusters")
+    plt.title('Clusters of User Queries')
+    plt.xlabel('PCA Component 1')
+    plt.ylabel('PCA Component 2')
+    buf = BytesIO()
+    plt.savefig(buf, format='png')
+    buf.seek(0)
+    img = Image.open(buf)
+    return img
+
+def silhouette_analysis(X, labels, num_clusters):
+    fig, ax1 = plt.subplots(1, 1)
+    fig.set_size_inches(10, 6)
+    
+    ax1.set_xlim([-0.1, 1])
+    ax1.set_ylim([0, X.shape[0] + (num_clusters + 1) * 10])
+    
+    sample_silhouette_values = silhouette_samples(X, labels)
+    y_lower = 10
+    for i in range(num_clusters):
+        ith_cluster_silhouette_values = sample_silhouette_values[labels == i]
+        ith_cluster_silhouette_values.sort()
+        size_cluster_i = ith_cluster_silhouette_values.shape[0]
+        y_upper = y_lower + size_cluster_i
+        color = plt.cm.nipy_spectral(float(i) / num_clusters)
+        ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values,
+                          facecolor=color, edgecolor=color, alpha=0.7)
+        ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
+        y_lower = y_upper + 10
+    
+    ax1.set_title("The silhouette plot for the various clusters.")
+    ax1.set_xlabel("The silhouette coefficient values")
+    ax1.set_ylabel("Cluster label")
+    ax1.axvline(x=np.mean(sample_silhouette_values), color="red", linestyle="--")
+    ax1.set_yticks([])
+    ax1.set_xticks([i/10.0 for i in range(-1, 11)])
+    
     buf = BytesIO()
     plt.savefig(buf, format='png')
     buf.seek(0)
@@ -148,6 +181,8 @@ def main(file, num_clusters_to_display):
         df = preprocess_data(df)
         df, X, kmeans = cluster_data(df, num_clusters=15)
         
+        cluster_plot = visualize_clusters(df)
+
         cluster_sizes = df['Cluster'].value_counts()
         sorted_clusters = cluster_sizes.index.tolist()
         df['Cluster'] = pd.Categorical(df['Cluster'], categories=sorted_clusters, ordered=True)
@@ -161,16 +196,18 @@ def main(file, num_clusters_to_display):
         df['Cluster'] = pd.Categorical(df['Cluster'], categories=filtered_clusters, ordered=True)
         df = df.sort_values('Cluster')
 
-        wordcloud_img = generate_wordcloud(df)
+        silhouette_avg = silhouette_score(X, kmeans.labels_)
+        silhouette_plot = silhouette_analysis(X, kmeans.labels_, num_clusters=15)
+
+        # Convert silhouette score to percentage
+        silhouette_percentage = (silhouette_avg + 1) * 50
 
         with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
             df.to_csv(tmpfile.name, index=False)
-            csv_file_path = tmpfile.name
-
-        return csv_file_path, wordcloud_img
+            return tmpfile.name, silhouette_percentage, cluster_plot, silhouette_plot
     except Exception as e:
         print(f"Error: {e}")
-        return str(e), None
+        return str(e), None, None, None
 
 interface = gr.Interface(
     fn=main,
@@ -180,10 +217,13 @@ interface = gr.Interface(
     ],
     outputs=[
         gr.File(label="Clustered Data CSV"),
-        gr.Image(label="Word Cloud")
+        gr.Number(label="Clustering Quality (%)"),
+        gr.Image(label="Cluster Plot"),
+        gr.Image(label="Silhouette Plot")
     ],
     title="Unanswered User Queries Clustering",
     description="Unanswered User Query Categorization"
 )
 
 interface.launch()
+