Update app.py

app.py

@@ -5,127 +5,33 @@ from sklearn.cluster import KMeans
 import matplotlib.pyplot as plt
 from sklearn.decomposition import PCA
 import re
-
-def preprocess_excel_data(df):
-    df.rename(columns={'Queries': 'texts'}, inplace=True)
-    df['texts'] = df['texts'].astype(str)
-    df['texts'] = df['texts'].str.lower()
-    df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))
-
-    def remove_emoji(string):
-        emoji_pattern = re.compile("["
-                                   u"\U0001F600-\U0001F64F"
-                                   u"\U0001F300-\U0001F5FF"
-                                   u"\U0001F680-\U0001F6FF"
-                                   u"\U0001F1E0-\U0001F1FF"
-                                   u"\U00002702-\U000027B0"
-                                   u"\U000024C2-\U0001F251"
-                                   "]+", flags=re.UNICODE)
-        return emoji_pattern.sub(r'', string) if isinstance(string, str) else string
-
-    df['texts'] = df['texts'].apply(remove_emoji)
-
-    custom_synonyms = {
-        'application': ['form'],
-        'apply': ['fill', 'applied'],
-        'work': ['job'],
-        'salary': ['stipend', 'pay', 'payment', 'paid'],
-        'test': ['online test', 'amcat test', 'exam', 'assessment'],
-        'pass': ['clear', 'selected', 'pass or not'],
-        'result': ['outcome', 'mark', 'marks'],
-        'thanks': ["thanks a lot to you", "thankyou so much", "thank you so much", "tysm", "thank you",
-                   "okaythank", "thx", "ty", "thankyou", "thank", "thank u"],
-        'interview': ["pi"]
-    }
-
-    for original_word, synonym_list in custom_synonyms.items():
-        for synonym in synonym_list:
-            pattern = r"\b" + synonym + r"\b(?!\s*\()"
-            df['texts'] = df['texts'].str.replace(pattern, original_word, regex=True)
-            pattern = r"\b" + synonym + r"\s+you" + r"\b(?!\s*\()"
-            df['texts'] = df['texts'].str.replace(pattern, original_word + ' ', regex=True)
-
-    spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",
-                 "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",
-                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b", "sent using truecaller"]
-
-    rows_to_remove = set()
-    for spam_phrase in spam_list:
-        pattern = r"\b" + re.escape(spam_phrase) + r"\b"
-        spam_rows = df['texts'].str.contains(pattern)
-        rows_to_remove.update(df.index[spam_rows].tolist())
-
-    df = df.drop(rows_to_remove)
-
-    greet_variations = ["hello", "hy", "hey", "hii", "hi", "heyyy", "bie", "bye"]
-    for greet_var in greet_variations:
-        pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk", "t", "r"]
-    for okay_var in okay_variations:
-        pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea", "no"]
-    for yes_var in yes_variations:
-        pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    remove_phrases = ["i'm all set", "ask a question", "apply the survey", "videos (2-8 min)", "long reads (> 8 min)",
-                      "short reads (3-8 min)", "not a student alumni", "mock", "share feedback", "bite size (< 2 min)",
-                      "actually no", "next steps", "i'm a student alumni", "i have questions"]
-
-    for phrase in remove_phrases:
-        df['texts'] = df['texts'].str.replace(phrase, '')
-
-    general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query",
-                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma'am", "i'm all set", "ask a question", 
-                          "apply the survey", "videos (2-8 min)", "long reads (> 8 min)", "short reads (3-8 min)", "not a student alumni", 
-                          "mock", "share feedback", "bite size (< 2 min)", "actually no", "next steps", "i'm a student alumni", "i have questions"]
-
-    for gen_var in general_variations:
-        pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)"
-        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
-
-    def remove_punctuations(text):
-        return re.sub(r'[^\w\s]', '', text)
-    df['texts'] = df['texts'].apply(remove_punctuations)
-
-    remove_morephrases = ["short reads 38 min", "bite size  2 min", "videos 28 min", "long reads  8 min"]
-
-    for phrase in remove_morephrases:
-        df['texts'] = df['texts'].str.replace(phrase, '')
-
-    df = df[~df['texts'].str.contains(r'\b\d{10}\b')]
-
-    df['texts'] = df['texts'].str.strip()
-
-    df['texts'] = df['texts'].apply(lambda x: x.strip())
-    df = df[df['texts'] != '']
-
-    return df
-
-def preprocess_csv_data(df):
-    df = df[df['Answer'] == 'Fallback Message shown']
-    df.rename(columns={'User Query': 'texts'}, inplace=True)
+from io import BytesIO
+import tempfile
+from datetime import datetime
+
+def preprocess_data(df):
+    # Filter based on the 'Answer' column and the date
+    df = df[(df['Answer'] == 'Fallback Message shown') & (pd.to_datetime(df['Date'], dayfirst=True) > datetime(2024, 7, 1))]
+    
+    # Rename and preprocess the 'Question Asked' column
+    df.rename(columns={'Question Asked': 'texts'}, inplace=True)
     df['texts'] = df['texts'].astype(str)
     df['texts'] = df['texts'].str.lower()
     df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))
 
     def remove_emoji(string):
         emoji_pattern = re.compile("["
-                                   u"\U0001F600-\U0001F64F"
-                                   u"\U0001F300-\U0001F5FF"
-                                   u"\U0001F680-\U0001F6FF"
-                                   u"\U0001F1E0-\U0001F1FF"
-                                   u"\U00002702-\U000027B0"
-                                   u"\U000024C2-\U0001F251"
-                                   "]+", flags=re.UNICODE)
-        return emoji_pattern.sub(r'', string) if isinstance(string, str) else string
+                               u"\U0001F600-\U0001F64F"  # emoticons
+                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
+                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+                               u"\U00002702-\U000027B0"
+                               u"\U000024C2-\U0001F251"
+                               "]+", flags=re.UNICODE)
+        return emoji_pattern.sub(r'', string)
 
     df['texts'] = df['texts'].apply(remove_emoji)
-
+    
     custom_synonyms = {
         'application': ['form'],
         'apply': ['fill', 'applied'],
@@ -148,7 +54,7 @@ def preprocess_csv_data(df):
 
     spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",
                  "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",
-                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b", "sent using truecaller"]
+                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b","sent using truecaller"]
 
     rows_to_remove = set()
     for spam_phrase in spam_list:
@@ -163,28 +69,27 @@ def preprocess_csv_data(df):
         pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
 
-    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk", "t", "r"]
+    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk","t","r"]
     for okay_var in okay_variations:
         pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
 
-    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea", "no"]
+    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea","no"]
     for yes_var in yes_variations:
         pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
 
-    remove_phrases = ["i'm all set", "ask a question", "apply the survey", "videos (2-8 min)", "long reads (> 8 min)",
-                      "short reads (3-8 min)", "not a student alumni", "mock", "share feedback", "bite size (< 2 min)",
-                      "actually no", "next steps", "i'm a student alumni", "i have questions"]
+    remove_phrases = ["i'm all set","ask a question","apply the survey","videos (2-8 min)","long reads (> 8 min)",
+                      "short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
+                      "actually no","next steps","i'm a student alumni","i have questions"]
 
     for phrase in remove_phrases:
         df['texts'] = df['texts'].str.replace(phrase, '')
 
     general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query",
-                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma'am", "i'm all set", "ask a question", 
-                          "apply the survey", "videos (2-8 min)", "long reads (> 8 min)", "short reads (3-8 min)", "not a student alumni", 
-                          "mock", "share feedback", "bite size (< 2 min)", "actually no", "next steps", "i'm a student alumni", "i have questions"]
-
+                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma&#39;am","i'm all set","ask a question","apply the survey",
+                          "videos (2-8 min)","long reads (> 8 min)","short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
+                          "actually no","next steps","i'm a student alumni","i have questions"]
     for gen_var in general_variations:
         pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
@@ -193,7 +98,7 @@ def preprocess_csv_data(df):
         return re.sub(r'[^\w\s]', '', text)
     df['texts'] = df['texts'].apply(remove_punctuations)
 
-    remove_morephrases = ["short reads 38 min", "bite size  2 min", "videos 28 min", "long reads  8 min"]
+    remove_morephrases = ["short reads 38 min","bite size  2 min","videos 28 min","long reads  8 min"]
 
     for phrase in remove_morephrases:
         df['texts'] = df['texts'].str.replace(phrase, '')
@@ -207,47 +112,66 @@ def preprocess_csv_data(df):
 
     return df
 
-def kmeans_clustering(df, num_clusters):
+def cluster_data(df):
+    num_clusters = 10  # Set the number of clusters to 15
     vectorizer = TfidfVectorizer(stop_words='english')
     X = vectorizer.fit_transform(df['texts'])
 
-    kmeans = KMeans(n_clusters=num_clusters, random_state=42)
+    kmeans = KMeans(n_clusters=num_clusters, random_state=0)
     kmeans.fit(X)
     df['Cluster'] = kmeans.labels_
 
-    cluster_counts = df['Cluster'].value_counts().sort_values(ascending=False)
-    df_filtered = df[df['Cluster'].isin(cluster_counts.head(num_clusters).index)]
+    pca = PCA(n_components=2)
+    principal_components = pca.fit_transform(X.toarray())
+    df['PCA1'] = principal_components[:, 0]
+    df['PCA2'] = principal_components[:, 1]
 
-    return df_filtered
+    return df
+
+def visualize_clusters(df):
+    plt.figure(figsize=(10, 6))
+    scatter = plt.scatter(df['PCA1'], df['PCA2'], c=df['Cluster'], cmap='viridis')
+    plt.legend(*scatter.legend_elements(), title="Clusters")
+    plt.title('Clusters of User Queries')
+    plt.xlabel('PCA Component 1')
+    plt.ylabel('PCA Component 2')
+    plt.show()
 
-def main(file, num_clusters):
+def main(file, num_clusters_to_display):
     try:
-        if file.name.endswith('.xlsx'):
-            df = pd.read_excel(file.name)
-            df = preprocess_excel_data(df)
-        elif file.name.endswith('.csv'):
-            df = pd.read_csv(file.name)
-            df = preprocess_csv_data(df)
-        else:
-            return "Invalid file format. Please upload an Excel or CSV file."
-
-        df = kmeans_clustering(df, num_clusters)
+        df = pd.read_csv(file.name)
+        df = preprocess_data(df)
+        df = cluster_data(df)
+        visualize_clusters(df)
+
+        cluster_sizes = df['Cluster'].value_counts()
+        sorted_clusters = cluster_sizes.index.tolist()
+        df['Cluster'] = pd.Categorical(df['Cluster'], categories=sorted_clusters, ordered=True)
+        df = df.sort_values('Cluster')
+
+        # Filter out cluster 0 and get the largest clusters
+        filtered_clusters = [cluster for cluster in sorted_clusters if cluster != 0]
+        top_clusters = filtered_clusters[:num_clusters_to_display]
+
+        df = df[df['texts'].isin(top_clusters)]
+        df['Cluster'] = pd.Categorical(df['Cluster'], categories=top_clusters, ordered=True)
         df = df.sort_values('Cluster')
 
-        output_file = "filtered_clusters.csv"
-        df.to_csv(output_file, index=False)
-        return output_file
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
+            df.to_csv(tmpfile.name, index=False)
+            return tmpfile.name
     except Exception as e:
         return str(e)
 
-iface = gr.Interface(
+interface = gr.Interface(
     fn=main,
     inputs=[
-        gr.File(label="Upload Excel or CSV File (.xlsx or .csv)"),
-        gr.Slider(minimum=1, maximum=20, step=1, label="Number of Categories to Display")
+        gr.File(label="Upload CSV File (.csv)"),
+        gr.Slider(1, 10, step=1, label="Number of Categories to Display")
     ],
-    outputs=gr.File(label="Filtered CSV File")
+    outputs=gr.File(label="Clustered Data CSV"),
+    title="Unanswered User Queries Clustering",
+    description="Upload a CSV file (.csv) and select the number of largest clusters to display (excluding cluster 0)"
 )
 
-if __name__ == "__main__":
-    iface.launch()
+interface.launch()