app.py

import gradio as gr
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score, silhouette_samples, davies_bouldin_score
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
import re
from io import BytesIO
import tempfile
import numpy as np
from PIL import Image
from nltk.stem import WordNetLemmatizer
from sklearn.preprocessing import normalize

def preprocess_data(df):
    df.rename(columns={'Question Asked': 'texts'}, inplace=True)
    df['texts'] = df['texts'].astype(str)
    df['texts'] = df['texts'].str.lower()
    df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))
    
    lemmatizer = WordNetLemmatizer()
    df['texts'] = df['texts'].apply(lambda text: ' '.join([lemmatizer.lemmatize(word) for word in text.split()]))

    def remove_emoji(string):
        emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"
                               u"\U0001F300-\U0001F5FF"
                               u"\U0001F680-\U0001F6FF"
                               u"\U0001F1E0-\U0001F1FF"
                               u"\U00002702-\U000027B0"
                               u"\U000024C2-\U0001F251"
                               "]+", flags=re.UNICODE)
        return emoji_pattern.sub(r'', string) if isinstance(string, str) else string

    df['texts'] = df['texts'].apply(remove_emoji)

    custom_synonyms = {
        'application': ['form'],
        'apply': ['fill', 'applied'],
        'work': ['job'],
        'salary': ['stipend', 'pay', 'payment', 'paid'],
        'test': ['online test', 'amcat test', 'exam', 'assessment'],
        'pass': ['clear', 'selected', 'pass or not'],
        'result': ['outcome', 'mark', 'marks'],
        'thanks': ["thanks a lot to you", "thankyou so much", "thank you so much", "tysm", "thank you",
                   "okaythank", "thx", "ty", "thankyou", "thank", "thank u"],
        'interview': ["pi"]
    }

    for original_word, synonym_list in custom_synonyms.items():
        for synonym in synonym_list:
            pattern = r"\b" + synonym + r"\b(?!\s*\()"
            df['texts'] = df['texts'].str.replace(pattern, original_word, regex=True)
            pattern = r"\b" + synonym + r"\s+you" + r"\b(?!\s*\()"
            df['texts'] = df['texts'].str.replace(pattern, original_word + ' ', regex=True)

    spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",
                 "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",
                 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b","sent using truecaller"]

    rows_to_remove = set()
    for spam_phrase in spam_list:
        pattern = r"\b" + re.escape(spam_phrase) + r"\b"
        spam_rows = df['texts'].str.contains(pattern)
        rows_to_remove.update(df.index[spam_rows].tolist())

    df = df.drop(rows_to_remove)

    greet_variations = ["hello", "hy", "hey", "hii", "hi", "heyyy", "bie", "bye"]
    for greet_var in greet_variations:
        pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk","t","r"]
    for okay_var in okay_variations:
        pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea","no"]
    for yes_var in yes_variations:
        pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    remove_phrases = ["i'm all set","ask a question","apply the survey","videos (2-8 min)","long reads (> 8 min)",
                      "short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
                      "actually no","next steps","i'm a student alumni","i have questions"]

    for phrase in remove_phrases:
        df['texts'] = df['texts'].str.replace(phrase, '')

    general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query",
                          "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma&#39;am","i'm all set","ask a question","apply the survey",
                          "videos (2-8 min)","long reads (> 8 min)","short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)",
                          "actually no","next steps","i'm a student alumni","i have questions"]
    for gen_var in general_variations:
        pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)"
        df['texts'] = df['texts'].str.replace(pattern, '', regex=True)

    def remove_punctuations(text):
        return re.sub(r'[^\w\s]', '', text)
    df['texts'] = df['texts'].apply(remove_punctuations)

    remove_morephrases = ["short reads 38 min","bite size  2 min","videos 28 min","long reads  8 min"]

    for phrase in remove_morephrases:
        df['texts'] = df['texts'].str.replace(phrase, '')

    df = df[~df['texts'].str.contains(r'\b\d{10}\b')]

    df['texts'] = df['texts'].str.strip()

    df['texts'] = df['texts'].apply(lambda x: x.strip())
    df = df[df['texts'] != '']

    return df

def cluster_data(df, num_clusters):
    vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1, 2), max_df=0.85, min_df=2)
    X = vectorizer.fit_transform(df['texts'])
    X = normalize(X)

    kmeans = KMeans(n_clusters=num_clusters, random_state=0)
    kmeans.fit(X)
    df['Cluster'] = kmeans.labels_

    pca = PCA(n_components=2)
    principal_components = pca.fit_transform(X.toarray())
    df['PCA1'] = principal_components[:, 0]
    df['PCA2'] = principal_components[:, 1]

    return df, X, kmeans

def visualize_clusters(df):
    plt.figure(figsize=(10, 6))
    scatter = plt.scatter(df['PCA1'], df['PCA2'], c=df['Cluster'], cmap='viridis')
    plt.legend(*scatter.legend_elements(), title="Clusters")
    plt.title('Clusters of User Queries')
    plt.xlabel('PCA Component 1')
    plt.ylabel('PCA Component 2')
    buf = BytesIO()
    plt.savefig(buf, format='png')
    buf.seek(0)
    img = Image.open(buf)
    return img

def silhouette_analysis(X, labels, num_clusters):
    fig, ax1 = plt.subplots(1, 1)
    fig.set_size_inches(10, 6)
    
    ax1.set_xlim([-0.1, 1])
    ax1.set_ylim([0, X.shape[0] + (num_clusters + 1) * 10])
    
    sample_silhouette_values = silhouette_samples(X, labels)
    y_lower = 10
    for i in range(num_clusters):
        ith_cluster_silhouette_values = sample_silhouette_values[labels == i]
        ith_cluster_silhouette_values.sort()
        size_cluster_i = ith_cluster_silhouette_values.shape[0]
        y_upper = y_lower + size_cluster_i
        color = plt.cm.nipy_spectral(float(i) / num_clusters)
        ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values,
                          facecolor=color, edgecolor=color, alpha=0.7)
        ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
        y_lower = y_upper + 10
    
    ax1.set_title("The silhouette plot for the various clusters.")
    ax1.set_xlabel("The silhouette coefficient values")
    ax1.set_ylabel("Cluster label")
    ax1.axvline(x=np.mean(sample_silhouette_values), color="red", linestyle="--")
    ax1.set_yticks([])
    ax1.set_xticks([i/10.0 for i in range(-1, 11)])
    
    buf = BytesIO()
    plt.savefig(buf, format='png')
    buf.seek(0)
    img = Image.open(buf)
    return img

def main(file, num_clusters_to_display):
    try:
        df = pd.read_csv(file)
        
        # Filter by 'Fallback Message shown'
        df = df[(df['Answer'] == 'Fallback Message shown')]
        
        df = preprocess_data(df)
        df, X, kmeans = cluster_data(df, num_clusters=15)
        
        cluster_plot = visualize_clusters(df)

        cluster_sizes = df['Cluster'].value_counts()
        sorted_clusters = cluster_sizes.index.tolist()
        df['Cluster'] = pd.Categorical(df['Cluster'], categories=sorted_clusters, ordered=True)
        df = df.sort_values('Cluster')

        # Filter out the largest cluster and get the next largest clusters
        largest_cluster = sorted_clusters[0]
        filtered_clusters = sorted_clusters[1:num_clusters_to_display+1]

        df = df[df['Cluster'].isin(filtered_clusters)]
        df['Cluster'] = pd.Categorical(df['Cluster'], categories=filtered_clusters, ordered=True)
        df = df.sort_values('Cluster')

        silhouette_avg = silhouette_score(X, kmeans.labels_)
        silhouette_plot = silhouette_analysis(X, kmeans.labels_, num_clusters=15)

        davies_bouldin = davies_bouldin_score(X, kmeans.labels_)

        # Convert silhouette score to percentage
        silhouette_percentage = (silhouette_avg + 1) * 50

        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
            df.to_csv(tmpfile.name, index=False)
            return tmpfile.name, silhouette_percentage, davies_bouldin, cluster_plot, silhouette_plot
    except Exception as e:
        print(f"Error: {e}")
        return str(e), None, None, None, None

interface = gr.Interface(
    fn=main,
    inputs=[
        gr.File(label="Upload CSV File (.csv)"),
        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=10, step=1, value=5)
    ],
    outputs=[
        gr.File(label="Clustered Data CSV"),
        gr.Number(label="Clustering Quality (%)"),
        gr.Number(label="Davies-Bouldin Index"),
        gr.Image(label="Cluster Plot"),
        gr.Image(label="Silhouette Plot")
    ],
    title="Unanswered User Queries Clustering",
    description="Unanswered User Query Categorization"
)

interface.launch()