Update app.py

app.py

@@ -2,11 +2,12 @@ import gradio as gr
 import pandas as pd
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.cluster import KMeans
-from sklearn.metrics import silhouette_score
 from sklearn.preprocessing import normalize
-import re
 from wordcloud import WordCloud
 import matplotlib.pyplot as plt
+from io import BytesIO
+import re
+import tempfile
 
 def preprocess_data(df):
     df.rename(columns={'Question Asked': 'texts'}, inplace=True)
@@ -34,64 +35,6 @@ def preprocess_data(df):
             pattern = r"\b" + synonym + r"\s+you" + r"\b(?!\s*\()"
             df['texts'] = df['texts'].str.replace(pattern, original_word + ' ', regex=True)
 
-    spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",
-                 "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",
-                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b","sent using truecaller"]
-
-    rows_to_remove = set()
-    for spam_phrase in spam_list:
-        pattern = r"\b" + re.escape(spam_phrase) + r"\b"
-        spam_rows = df['texts'].str.contains(pattern)
-        rows_to_remove.update(df.index[spam_rows].tolist())
-
-    df = df.drop(rows_to_remove)
-
-    greet_variations = ["hello", "hy", "hey", "hii", "hi", "heyyy", "bie", "bye"]
-    for greet_var in greet_variations:
-        pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk","t","r"]
-    for okay_var in okay_variations:
-        pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea","no"]
-    for yes_var in yes_variations:
-        pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    remove_phrases = ["i'm all set","ask a question","apply the survey","videos (2-8 min)","long reads (> 8 min)",
-                      "short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
-                      "actually no","next steps","i'm a student alumni","i have questions"]
-
-    for phrase in remove_phrases:
-        df['texts'] = df['texts'].str.replace(phrase, '')
-
-    general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query",
-                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma&#39;am","i'm all set","ask a question","apply the survey",
-                          "videos (2-8 min)","long reads (> 8 min)","short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
-                          "actually no","next steps","i'm a student alumni","i have questions"]
-    for gen_var in general_variations:
-        pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    def remove_punctuations(text):
-        return re.sub(r'[^\w\s]', '', text)
-    df['texts'] = df['texts'].apply(remove_punctuations)
-
-    remove_morephrases = ["short reads 38 min","bite size  2 min","videos 28 min","long reads  8 min"]
-
-    for phrase in remove_morephrases:
-        df['texts'] = df['texts'].str.replace(phrase, '')
-
-    df = df[~df['texts'].str.contains(r'\b\d{10}\b')]
-
-    df['texts'] = df['texts'].str.strip()
-
-    df['texts'] = df['texts'].apply(lambda x: x.strip())
-    df = df[df['texts'] != '']
-
     return df
 
 def cluster_data(df, num_clusters):
@@ -103,7 +46,7 @@ def cluster_data(df, num_clusters):
     kmeans.fit(X)
     df['Cluster'] = kmeans.labels_
 
-    return df, X, kmeans
+    return df
 
 def generate_wordcloud(texts):
     wordcloud = WordCloud(width=800, height=400, background_color='white').generate(" ".join(texts))
@@ -115,46 +58,50 @@ def generate_wordcloud(texts):
     buf.seek(0)
     return buf
 
-def main(file, num_clusters):
+def main(file, num_clusters, num_clusters_to_display):
     df = pd.read_csv(file)
     
     # Filter by 'Fallback Message shown'
     df = df[df['Answer'] == 'Fallback Message shown']
     
     df = preprocess_data(df)
-    df, X, kmeans = cluster_data(df, num_clusters)
+    df = cluster_data(df, num_clusters)
     
-    clusters = df['Cluster'].unique()
+    cluster_sizes = df['Cluster'].value_counts()
+    sorted_clusters = cluster_sizes.index.tolist()
+    df['Cluster'] = pd.Categorical(df['Cluster'], categories=sorted_clusters, ordered=True)
+    df = df.sort_values('Cluster')
+
+    # Filter out the largest cluster and get the next largest clusters
+    largest_cluster = sorted_clusters[0]
+    filtered_clusters = sorted_clusters[1:num_clusters_to_display+1]
+
+    df = df[df['Cluster'].isin(filtered_clusters)]
+    df['Cluster'] = pd.Categorical(df['Cluster'], categories=filtered_clusters, ordered=True)
+    df = df.sort_values('Cluster')
+
     wordclouds = []
-    for cluster in clusters:
+    for cluster in filtered_clusters:
         texts = df[df['Cluster'] == cluster]['texts'].tolist()
         wordcloud_image = generate_wordcloud(texts)
         wordclouds.append((f"Cluster {cluster}", wordcloud_image))
 
-    cluster_sizes = df['Cluster'].value_counts()
-    top_clusters = cluster_sizes.head(num_clusters).index
-    top_queries = df[df['Cluster'].isin(top_clusters)][['Cluster', 'texts']]
-    
-    return wordclouds, top_queries
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
+        df.to_csv(tmpfile.name, index=False)
+        csv_file_path = tmpfile.name
 
-def display_results(wordclouds, top_queries):
-    for cluster, wordcloud in wordclouds:
-        print(cluster)
-        img = Image.open(wordcloud)
-        img.show()
-    
-    print("Top Queries by Cluster:")
-    print(top_queries.to_string(index=False))
+    return wordclouds, csv_file_path
 
 interface = gr.Interface(
     fn=main,
     inputs=[
         gr.File(label="Upload CSV File (.csv)"),
-        gr.Slider(label="Number of Clusters", minimum=2, maximum=20, step=1, value=5)
+        gr.Slider(label="Number of Clusters", minimum=2, maximum=20, step=1, value=5),
+        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=10, step=1, value=5)
     ],
     outputs=[
         gr.Gallery(label="Word Clouds of Clusters"),
-        gr.Dataframe(label="Top Queries by Cluster")
+        gr.File(label="Clustered Data CSV")
     ],
     title="Unanswered User Queries Clustering",
     description="Unanswered User Query Categorization"