app.py

import gradio as gr
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.preprocessing import normalize
import re
from wordcloud import WordCloud
import matplotlib.pyplot as plt

def preprocess_data(df):
    df.rename(columns={'Question Asked': 'texts'}, inplace=True)
    df['texts'] = df['texts'].astype(str)
    df['texts'] = df['texts'].str.lower()
    df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))

    custom_synonyms = {
        'application': ['form'],
        'apply': ['fill', 'applied'],
        'work': ['job'],
        'salary': ['stipend', 'pay', 'payment', 'paid'],
        'test': ['online test', 'amcat test', 'exam', 'assessment'],
        'pass': ['clear', 'selected', 'pass or not'],
        'result': ['outcome', 'mark', 'marks'],
        'thanks': ["thanks a lot to you", "thankyou so much", "thank you so much", "tysm", "thank you",
                   "okaythank", "thx", "ty", "thankyou", "thank", "thank u"],
        'interview': ["pi"]
    }

    for original_word, synonym_list in custom_synonyms.items():
        for synonym in synonym_list:
            pattern = r"\b" + synonym + r"\b(?!\s*\()"
            df['texts'] = df['texts'].str.replace(pattern, original_word, regex=True)
            pattern = r"\b" + synonym + r"\s+you" + r"\b(?!\s*\()"
            df['texts'] = df['texts'].str.replace(pattern, original_word + ' ', regex=True)

    spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",
                 "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",
                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b","sent using truecaller"]

    rows_to_remove = set()
    for spam_phrase in spam_list:
        pattern = r"\b" + re.escape(spam_phrase) + r"\b"
        spam_rows = df['texts'].str.contains(pattern)
        rows_to_remove.update(df.index[spam_rows].tolist())

    df = df.drop(rows_to_remove)

    greet_variations = ["hello", "hy", "hey", "hii", "hi", "heyyy", "bie", "bye"]
    for greet_var in greet_variations:
        pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk","t","r"]
    for okay_var in okay_variations:
        pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea","no"]
    for yes_var in yes_variations:
        pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    remove_phrases = ["i'm all set","ask a question","apply the survey","videos (2-8 min)","long reads (> 8 min)",
                      "short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
                      "actually no","next steps","i'm a student alumni","i have questions"]

    for phrase in remove_phrases:
        df['texts'] = df['texts'].str.replace(phrase, '')

    general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query",
                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma&#39;am","i'm all set","ask a question","apply the survey",
                          "videos (2-8 min)","long reads (> 8 min)","short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
                          "actually no","next steps","i'm a student alumni","i have questions"]
    for gen_var in general_variations:
        pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    def remove_punctuations(text):
        return re.sub(r'[^\w\s]', '', text)
    df['texts'] = df['texts'].apply(remove_punctuations)

    remove_morephrases = ["short reads 38 min","bite size  2 min","videos 28 min","long reads  8 min"]

    for phrase in remove_morephrases:
        df['texts'] = df['texts'].str.replace(phrase, '')

    df = df[~df['texts'].str.contains(r'\b\d{10}\b')]

    df['texts'] = df['texts'].str.strip()

    df['texts'] = df['texts'].apply(lambda x: x.strip())
    df = df[df['texts'] != '']

    return df

def cluster_data(df, num_clusters):
    vectorizer = TfidfVectorizer(stop_words='english')
    X = vectorizer.fit_transform(df['texts'])
    X = normalize(X)

    kmeans = KMeans(n_clusters=num_clusters, random_state=0)
    kmeans.fit(X)
    df['Cluster'] = kmeans.labels_

    return df, X, kmeans

def generate_wordcloud(texts):
    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(" ".join(texts))
    plt.figure(figsize=(10, 5))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    buf = BytesIO()
    plt.savefig(buf, format='png')
    buf.seek(0)
    return buf

def main(file, num_clusters):
    df = pd.read_csv(file)
    
    # Filter by 'Fallback Message shown'
    df = df[df['Answer'] == 'Fallback Message shown']
    
    df = preprocess_data(df)
    df, X, kmeans = cluster_data(df, num_clusters)
    
    clusters = df['Cluster'].unique()
    wordclouds = []
    for cluster in clusters:
        texts = df[df['Cluster'] == cluster]['texts'].tolist()
        wordcloud_image = generate_wordcloud(texts)
        wordclouds.append((f"Cluster {cluster}", wordcloud_image))

    cluster_sizes = df['Cluster'].value_counts()
    top_clusters = cluster_sizes.head(num_clusters).index
    top_queries = df[df['Cluster'].isin(top_clusters)][['Cluster', 'texts']]
    
    return wordclouds, top_queries

def display_results(wordclouds, top_queries):
    for cluster, wordcloud in wordclouds:
        print(cluster)
        img = Image.open(wordcloud)
        img.show()
    
    print("Top Queries by Cluster:")
    print(top_queries.to_string(index=False))

interface = gr.Interface(
    fn=main,
    inputs=[
        gr.File(label="Upload CSV File (.csv)"),
        gr.Slider(label="Number of Clusters", minimum=2, maximum=20, step=1, value=5)
    ],
    outputs=[
        gr.Gallery(label="Word Clouds of Clusters"),
        gr.Dataframe(label="Top Queries by Cluster")
    ],
    title="Unanswered User Queries Clustering",
    description="Unanswered User Query Categorization"
)

interface.launch()