Update app.py

app.py

@@ -16,13 +16,13 @@ def preprocess_data(df):
 
     def remove_emoji(string):
         emoji_pattern = re.compile("["
-                               u"\U0001F600-\U0001F64F"
-                               u"\U0001F300-\U0001F5FF"
-                               u"\U0001F680-\U0001F6FF"
-                               u"\U0001F1E0-\U0001F1FF"
-                               u"\U00002702-\U000027B0"
-                               u"\U000024C2-\U0001F251"
-                               "]+", flags=re.UNICODE)
+                                   u"\U0001F600-\U0001F64F"
+                                   u"\U0001F300-\U0001F5FF"
+                                   u"\U0001F680-\U0001F6FF"
+                                   u"\U0001F1E0-\U0001F1FF"
+                                   u"\U00002702-\U000027B0"
+                                   u"\U000024C2-\U0001F251"
+                                   "]+", flags=re.UNICODE)
         return emoji_pattern.sub(r'', string) if isinstance(string, str) else string
 
     df['texts'] = df['texts'].apply(remove_emoji)
@@ -49,7 +49,7 @@ def preprocess_data(df):
 
     spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",
                  "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",
-                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b","sent using truecaller"]
+                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b", "sent using truecaller"]
 
     rows_to_remove = set()
     for spam_phrase in spam_list:
@@ -64,27 +64,28 @@ def preprocess_data(df):
         pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
 
-    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk","t","r"]
+    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk", "t", "r"]
     for okay_var in okay_variations:
         pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
 
-    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea","no"]
+    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea", "no"]
     for yes_var in yes_variations:
         pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
 
-    remove_phrases = ["i'm all set","ask a question","apply the survey","videos (2-8 min)","long reads (> 8 min)",
-                      "short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
-                      "actually no","next steps","i'm a student alumni","i have questions"]
+    remove_phrases = ["i'm all set", "ask a question", "apply the survey", "videos (2-8 min)", "long reads (> 8 min)",
+                      "short reads (3-8 min)", "not a student alumni", "mock", "share feedback", "bite size (< 2 min)",
+                      "actually no", "next steps", "i'm a student alumni", "i have questions"]
 
     for phrase in remove_phrases:
         df['texts'] = df['texts'].str.replace(phrase, '')
 
     general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query",
-                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma&#39;am","i'm all set","ask a question","apply the survey",
-                          "videos (2-8 min)","long reads (> 8 min)","short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
-                          "actually no","next steps","i'm a student alumni","i have questions"]
+                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma'am", "i'm all set", "ask a question", 
+                          "apply the survey", "videos (2-8 min)", "long reads (> 8 min)", "short reads (3-8 min)", "not a student alumni", 
+                          "mock", "share feedback", "bite size (< 2 min)", "actually no", "next steps", "i'm a student alumni", "i have questions"]
+
     for gen_var in general_variations:
         pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)"
         df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
@@ -93,7 +94,7 @@ def preprocess_data(df):
         return re.sub(r'[^\w\s]', '', text)
     df['texts'] = df['texts'].apply(remove_punctuations)
 
-    remove_morephrases = ["short reads 38 min","bite size  2 min","videos 28 min","long reads  8 min"]
+    remove_morephrases = ["short reads 38 min", "bite size  2 min", "videos 28 min", "long reads  8 min"]
 
     for phrase in remove_morephrases:
         df['texts'] = df['texts'].str.replace(phrase, '')
@@ -108,7 +109,7 @@ def preprocess_data(df):
     return df
 
 def cluster_data(df):
-    num_clusters = 15  # Set the number of clusters to 15
+    num_clusters = 10  # Set the number of clusters to 15
     vectorizer = TfidfVectorizer(stop_words='english')
     X = vectorizer.fit_transform(df['texts'])
 
@@ -134,25 +135,15 @@ def visualize_clusters(df):
 
 def main(file, num_clusters_to_display):
     try:
-        # Determine the file type
-        file_extension = file.name.split('.')[-1].lower()
-        
-        # Load the file
-        if file_extension == 'xlsx':
-            df = pd.read_excel(file)
-        elif file_extension == 'csv':
-            df = pd.read_csv(file)
+        # Detect if the file is CSV or Excel
+        if file.name.endswith('.csv'):
+            df = pd.read_csv(file.name)
+            df = df[df['Answer'] == 'Fallback Message shown']  # Filter for 'Fallback Message shown' in 'Answer' column
+            df.rename(columns={'User Query': 'texts'}, inplace=True)  # Rename column to 'texts' for processing
         else:
-            return "Unsupported file type. Please upload an Excel or CSV file."
-        
-        # Process CSV file specifically
-        if file_extension == 'csv':
-            # Keep only rows where 'Answer' is 'Fallback Message shown'
-            df = df[df['Answer'] == 'Fallback Message shown']
-            # Focus on 'Query' column for text processing
-            df.rename(columns={'Query': 'texts'}, inplace=True)
-        
-        df = preprocess_data(df)
+            df = pd.read_excel(file.name)
+            df = preprocess_data(df)
+
         df = cluster_data(df)
         visualize_clusters(df)
 
@@ -165,35 +156,23 @@ def main(file, num_clusters_to_display):
         filtered_clusters = [cluster for cluster in sorted_clusters if cluster != 0]
         top_clusters = filtered_clusters[:num_clusters_to_display]
 
+        df = df[df['texts'] != '']
         df = df[df['Cluster'].isin(top_clusters)]
         df['Cluster'] = pd.Categorical(df['Cluster'], categories=top_clusters, ordered=True)
         df = df.sort_values('Cluster')
 
-        # Save the resulting DataFrame to a CSV file
-        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
-            df.to_csv(tmpfile.name, index=False)
-            tmpfile.flush()
-
-            return tmpfile.name
-
+        return df.to_csv(index=False)
     except Exception as e:
-        return f"An error occurred: {str(e)}"
-
-def upload_file(file, num_clusters_to_display):
-    result = main(file, num_clusters_to_display)
-    if result.endswith(".csv"):
-        return result
-    else:
-        return f"Error: {result}"
+        return str(e)
 
-interface = gr.Interface(
-    fn=upload_file,
+iface = gr.Interface(
+    fn=main,
     inputs=[
-        gr.inputs.File(label="Upload Excel or CSV File (.xlsx or .csv)", type="file"),
-        gr.inputs.Slider(minimum=1, maximum=20, step=1, default=5, label="Number of Categories to Display")
+        gr.File(label="Upload Excel or CSV File (.xlsx or .csv)"),
+        gr.Slider(minimum=1, maximum=20, step=1, label="Number of Categories to Display")
     ],
-    outputs=gr.outputs.File(label="Output CSV File"),
-    title="Unanswered User Queries Clustering"
+    outputs=gr.File(label="Filtered CSV File")
 )
 
-interface.launch(debug=True)
+if __name__ == "__main__":
+    iface.launch()