Create app.py

app.py

@@ -0,0 +1,181 @@
+import gradio as gr
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.cluster import KMeans
+import matplotlib.pyplot as plt
+from sklearn.decomposition import PCA
+import re
+
+def preprocess_data(df):
+    #Keep rows with "user" in the "sender" column
+    df = df[df['sender'].str.contains('user', case=False)]
+    df = df[df['Message Type'].isin(['TEXT']) | df['Message Type'].isin(['POSTBACK'])]
+
+    # Remove unnecessary columns
+    columns_to_drop = ["Timestamp", "sender", "User id", "User Name", "User Email", "User Phone No", "Platform", "Message Type"]
+    df = df.drop(columns=columns_to_drop, axis=1)
+
+    # Lowercase the "Message" column and rename it to "texts"
+    df['Message'] = df['Message'].str.lower()
+    df.rename(columns={'Message': 'texts'}, inplace=True)
+
+    # Convert the 'texts' column to string
+    df['texts'] = df['texts'].astype(str)
+
+    # Remove URL from text
+    df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))
+
+    # Remove emojis from text
+    def remove_emoji(string):
+        emoji_pattern = re.compile("["
+                               u"\U0001F600-\U0001F64F"
+                               u"\U0001F300-\U0001F5FF"
+                               u"\U0001F680-\U0001F6FF"
+                               u"\U0001F1E0-\U0001F1FF"
+                               u"\U00002702-\U000027B0"
+                               u"\U000024C2-\U0001F251"
+                               "]+", flags=re.UNICODE)
+        return emoji_pattern.sub(r'', string) if isinstance(string, str) else string
+
+    df['texts'] = df['texts'].apply(remove_emoji)
+
+    # Define synonyms
+    custom_synonyms = {
+        'application': ['form'],
+        'apply': ['fill', 'applied'],
+        'work': ['job'],
+        'salary': ['stipend', 'pay', 'payment', 'paid'],
+        'test': ['online test', 'amcat test', 'exam', 'assessment'],
+        'pass': ['clear', 'selected', 'pass or not'],
+        'result': ['outcome', 'mark', 'marks'],
+        'thanks': ["thanks a lot to you", "thankyou so much", "thank you so much", "tysm", "thank you",
+                   "okaythank", "thx", "ty", "thankyou", "thank", "thank u"],
+        'interview': ["pi"]
+    }
+
+    # Replace synonyms in the 'texts' column
+    for original_word, synonym_list in custom_synonyms.items():
+        for synonym in synonym_list:
+            pattern = r"\b" + synonym + r"\b(?!\s*\()"  # match whole word and exclude words in parentheses
+            df['texts'] = df['texts'].str.replace(pattern, original_word, regex=True)
+            pattern = r"\b" + synonym + r"\s+you" + r"\b(?!\s*\()"  # match whole word followed by optional whitespace and "you"
+            df['texts'] = df['texts'].str.replace(pattern, original_word + ' ', regex=True)
+
+    # Define list of spam words or phrases
+    spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",
+                 "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",
+                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b","sent using truecaller"]
+
+    # Remove any row that contains a spam phrase
+    rows_to_remove = set()
+    for spam_phrase in spam_list:
+        pattern = r"\b" + re.escape(spam_phrase) + r"\b"
+        spam_rows = df['texts'].str.contains(pattern)
+        rows_to_remove.update(df.index[spam_rows].tolist())
+
+    df = df.drop(rows_to_remove)
+
+    # Drop rows containing any greetings and its variations
+    greet_variations = ["hello", "hy", "hey", "hii", "hi", "heyyy", "bie", "bye"]
+    for greet_var in greet_variations:
+        pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b"
+        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
+
+    # Drop rows containing any okay response and its variations
+    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk","t","r"]
+    for okay_var in okay_variations:
+        pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b"
+        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
+
+    # Drop rows containing any yes response and its variations
+    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea","no"]
+    for yes_var in yes_variations:
+        pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b"
+        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
+
+    # Remove specific phrases from the "texts" column
+    remove_phrases = ["i'm all set","ask a question","apply the survey","videos (2-8 min)","long reads (> 8 min)",
+                      "short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
+                      "actually no","next steps","i'm a student alumni","i have questions"]
+
+    for phrase in remove_phrases:
+        df['texts'] = df['texts'].str.replace(phrase, '')
+
+
+    # Drop rows containing any general words from response and its variations
+    general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query",
+                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma&#39;am","i'm all set","ask a question","apply the survey",
+                          "videos (2-8 min)","long reads (> 8 min)","short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
+                          "actually no","next steps","i'm a student alumni","i have questions"]
+    for gen_var in general_variations:
+        pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)"
+        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)
+
+    def remove_punctuations(text):
+        return re.sub(r'[^\w\s]', '', text)
+    df['texts'] = df['texts'].apply(remove_punctuations)
+
+    # Remove specific phrases from the "texts" column
+    remove_morephrases = ["short reads 38 min","bite size  2 min","videos 28 min","long reads  8 min"]
+
+    for phrase in remove_morephrases:
+        df['texts'] = df['texts'].str.replace(phrase, '')
+
+    # Remove rows with phone numbers in the 'texts' column
+    df = df[~df['texts'].str.contains(r'\b\d{10}\b')]
+
+    # Remove any leading or trailing whitespaces
+    df['texts'] = df['texts'].str.strip()
+
+    # Remove blank rows
+    df['texts'] = df['texts'].apply(lambda x: x.strip())  # Remove leading and trailing whitespaces
+    df = df[df['texts'] != '']
+
+    return df
+
+def cluster_data(df, num_clusters=5):
+    # Vectorize the text data
+    vectorizer = TfidfVectorizer(stop_words='english')
+    X = vectorizer.fit_transform(df['texts'])
+
+    # Perform K-Means clustering
+    kmeans = KMeans(n_clusters=num_clusters, random_state=0)
+    kmeans.fit(X)
+    df['Cluster'] = kmeans.labels_
+
+    # Perform PCA to reduce dimensions for visualization
+    pca = PCA(n_components=2)
+    principal_components = pca.fit_transform(X.toarray())
+    df['PCA1'] = principal_components[:, 0]
+    df['PCA2'] = principal_components[:, 1]
+
+    return df
+
+def visualize_clusters(df):
+    plt.figure(figsize=(10, 6))
+    scatter = plt.scatter(df['PCA1'], df['PCA2'], c=df['Cluster'], cmap='viridis')
+    plt.legend(*scatter.legend_elements(), title="Clusters")
+    plt.title('Clusters of User Queries')
+    plt.xlabel('PCA Component 1')
+    plt.ylabel('PCA Component 2')
+    plt.show()
+
+def main(file, num_clusters):
+    df = pd.read_excel(file.name)
+    df = preprocess_data(df)
+    df = cluster_data(df, num_clusters)
+    visualize_clusters(df)
+    return df
+
+interface = gr.Interface(
+    fn=main,
+    inputs=[
+        gr.inputs.File(label="Upload Excel File (.xlsx)", type='file'),
+        gr.inputs.Number(default=5, label="Number of Clusters")
+    ],
+    outputs=gr.outputs.Dataframe(label="Clustered Data"),
+    title="Unanswered User Queries Clustering",
+    description="Upload an Excel file (.xlsx) containing user queries, and the algorithm will automatically cluster the queries into different categories."
+)
+
+interface.launch()