Update app.py

app.py

@@ -2,14 +2,11 @@ import gradio as gr
 import pandas as pd
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.cluster import KMeans
-from sklearn.metrics import silhouette_score, silhouette_samples
-import matplotlib.pyplot as plt
-from sklearn.decomposition import PCA
+from sklearn.metrics import silhouette_score
+from sklearn.preprocessing import normalize
 import re
-from io import BytesIO
-import tempfile
-import numpy as np
-from PIL import Image
+from wordcloud import WordCloud
+import matplotlib.pyplot as plt
 
 def preprocess_data(df):
     df.rename(columns={'Question Asked': 'texts'}, inplace=True)
@@ -17,19 +14,6 @@ def preprocess_data(df):
     df['texts'] = df['texts'].str.lower()
     df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))
 
-    def remove_emoji(string):
-        emoji_pattern = re.compile("["
-                               u"\U0001F600-\U0001F64F"
-                               u"\U0001F300-\U0001F5FF"
-                               u"\U0001F680-\U0001F6FF"
-                               u"\U0001F1E0-\U0001F1FF"
-                               u"\U00002702-\U000027B0"
-                               u"\U000024C2-\U0001F251"
-                               "]+", flags=re.UNICODE)
-        return emoji_pattern.sub(r'', string) if isinstance(string, str) else string
-
-    df['texts'] = df['texts'].apply(remove_emoji)
-
     custom_synonyms = {
         'application': ['form'],
         'apply': ['fill', 'applied'],
@@ -113,113 +97,64 @@ def preprocess_data(df):
 def cluster_data(df, num_clusters):
     vectorizer = TfidfVectorizer(stop_words='english')
     X = vectorizer.fit_transform(df['texts'])
+    X = normalize(X)
 
     kmeans = KMeans(n_clusters=num_clusters, random_state=0)
     kmeans.fit(X)
     df['Cluster'] = kmeans.labels_
 
-    pca = PCA(n_components=2)
-    principal_components = pca.fit_transform(X.toarray())
-    df['PCA1'] = principal_components[:, 0]
-    df['PCA2'] = principal_components[:, 1]
-
     return df, X, kmeans
 
-def visualize_clusters(df):
-    plt.figure(figsize=(10, 6))
-    scatter = plt.scatter(df['PCA1'], df['PCA2'], c=df['Cluster'], cmap='viridis')
-    plt.legend(*scatter.legend_elements(), title="Clusters")
-    plt.title('Clusters of User Queries')
-    plt.xlabel('PCA Component 1')
-    plt.ylabel('PCA Component 2')
+def generate_wordcloud(texts):
+    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(" ".join(texts))
+    plt.figure(figsize=(10, 5))
+    plt.imshow(wordcloud, interpolation='bilinear')
+    plt.axis('off')
     buf = BytesIO()
     plt.savefig(buf, format='png')
     buf.seek(0)
-    img = Image.open(buf)
-    return img
+    return buf
 
-def silhouette_analysis(X, labels, num_clusters):
-    fig, ax1 = plt.subplots(1, 1)
-    fig.set_size_inches(10, 6)
+def main(file, num_clusters):
+    df = pd.read_csv(file)
     
-    ax1.set_xlim([-0.1, 1])
-    ax1.set_ylim([0, X.shape[0] + (num_clusters + 1) * 10])
+    # Filter by 'Fallback Message shown'
+    df = df[df['Answer'] == 'Fallback Message shown']
     
-    sample_silhouette_values = silhouette_samples(X, labels)
-    y_lower = 10
-    for i in range(num_clusters):
-        ith_cluster_silhouette_values = sample_silhouette_values[labels == i]
-        ith_cluster_silhouette_values.sort()
-        size_cluster_i = ith_cluster_silhouette_values.shape[0]
-        y_upper = y_lower + size_cluster_i
-        color = plt.cm.nipy_spectral(float(i) / num_clusters)
-        ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values,
-                          facecolor=color, edgecolor=color, alpha=0.7)
-        ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
-        y_lower = y_upper + 10
+    df = preprocess_data(df)
+    df, X, kmeans = cluster_data(df, num_clusters)
     
-    ax1.set_title("The silhouette plot for the various clusters.")
-    ax1.set_xlabel("The silhouette coefficient values")
-    ax1.set_ylabel("Cluster label")
-    ax1.axvline(x=np.mean(sample_silhouette_values), color="red", linestyle="--")
-    ax1.set_yticks([])
-    ax1.set_xticks([i/10.0 for i in range(-1, 11)])
+    clusters = df['Cluster'].unique()
+    wordclouds = []
+    for cluster in clusters:
+        texts = df[df['Cluster'] == cluster]['texts'].tolist()
+        wordcloud_image = generate_wordcloud(texts)
+        wordclouds.append((f"Cluster {cluster}", wordcloud_image))
+
+    cluster_sizes = df['Cluster'].value_counts()
+    top_clusters = cluster_sizes.head(num_clusters).index
+    top_queries = df[df['Cluster'].isin(top_clusters)][['Cluster', 'texts']]
     
-    buf = BytesIO()
-    plt.savefig(buf, format='png')
-    buf.seek(0)
-    img = Image.open(buf)
-    return img
-
-def main(file, num_clusters_to_display):
-    try:
-        df = pd.read_csv(file)
-        
-        # Filter by 'Fallback Message shown'
-        df = df[(df['Answer'] == 'Fallback Message shown')]
-        
-        df = preprocess_data(df)
-        df, X, kmeans = cluster_data(df, num_clusters=15)
-        
-        cluster_plot = visualize_clusters(df)
-
-        cluster_sizes = df['Cluster'].value_counts()
-        sorted_clusters = cluster_sizes.index.tolist()
-        df['Cluster'] = pd.Categorical(df['Cluster'], categories=sorted_clusters, ordered=True)
-        df = df.sort_values('Cluster')
-
-        # Filter out the largest cluster and get the next largest clusters
-        largest_cluster = sorted_clusters[0]
-        filtered_clusters = sorted_clusters[1:num_clusters_to_display+1]
-
-        df = df[df['Cluster'].isin(filtered_clusters)]
-        df['Cluster'] = pd.Categorical(df['Cluster'], categories=filtered_clusters, ordered=True)
-        df = df.sort_values('Cluster')
-
-        silhouette_avg = silhouette_score(X, kmeans.labels_)
-        silhouette_plot = silhouette_analysis(X, kmeans.labels_, num_clusters=15)
-
-        # Convert silhouette score to percentage
-        silhouette_percentage = (silhouette_avg + 1) * 50
-
-        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
-            df.to_csv(tmpfile.name, index=False)
-            return tmpfile.name, silhouette_percentage, cluster_plot, silhouette_plot
-    except Exception as e:
-        print(f"Error: {e}")
-        return str(e), None, None, None
+    return wordclouds, top_queries
+
+def display_results(wordclouds, top_queries):
+    for cluster, wordcloud in wordclouds:
+        print(cluster)
+        img = Image.open(wordcloud)
+        img.show()
+    
+    print("Top Queries by Cluster:")
+    print(top_queries.to_string(index=False))
 
 interface = gr.Interface(
     fn=main,
     inputs=[
         gr.File(label="Upload CSV File (.csv)"),
-        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=10, step=1, value=5)
+        gr.Slider(label="Number of Clusters", minimum=2, maximum=20, step=1, value=5)
     ],
     outputs=[
-        gr.File(label="Clustered Data CSV"),
-        gr.Number(label="Clustering Quality (%)"),
-        gr.Image(label="Cluster Plot"),
-        gr.Image(label="Silhouette Plot")
+        gr.Gallery(label="Word Clouds of Clusters"),
+        gr.Dataframe(label="Top Queries by Cluster")
     ],
     title="Unanswered User Queries Clustering",
     description="Unanswered User Query Categorization"