Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pandas as pd | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.cluster import KMeans | |
| from sklearn.metrics import silhouette_score | |
| from sklearn.preprocessing import normalize | |
| import re | |
| from wordcloud import WordCloud | |
| import matplotlib.pyplot as plt | |
| def preprocess_data(df): | |
| df.rename(columns={'Question Asked': 'texts'}, inplace=True) | |
| df['texts'] = df['texts'].astype(str) | |
| df['texts'] = df['texts'].str.lower() | |
| df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text)) | |
| custom_synonyms = { | |
| 'application': ['form'], | |
| 'apply': ['fill', 'applied'], | |
| 'work': ['job'], | |
| 'salary': ['stipend', 'pay', 'payment', 'paid'], | |
| 'test': ['online test', 'amcat test', 'exam', 'assessment'], | |
| 'pass': ['clear', 'selected', 'pass or not'], | |
| 'result': ['outcome', 'mark', 'marks'], | |
| 'thanks': ["thanks a lot to you", "thankyou so much", "thank you so much", "tysm", "thank you", | |
| "okaythank", "thx", "ty", "thankyou", "thank", "thank u"], | |
| 'interview': ["pi"] | |
| } | |
| for original_word, synonym_list in custom_synonyms.items(): | |
| for synonym in synonym_list: | |
| pattern = r"\b" + synonym + r"\b(?!\s*\()" | |
| df['texts'] = df['texts'].str.replace(pattern, original_word, regex=True) | |
| pattern = r"\b" + synonym + r"\s+you" + r"\b(?!\s*\()" | |
| df['texts'] = df['texts'].str.replace(pattern, original_word + ' ', regex=True) | |
| spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein", | |
| "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar", | |
| "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b","sent using truecaller"] | |
| rows_to_remove = set() | |
| for spam_phrase in spam_list: | |
| pattern = r"\b" + re.escape(spam_phrase) + r"\b" | |
| spam_rows = df['texts'].str.contains(pattern) | |
| rows_to_remove.update(df.index[spam_rows].tolist()) | |
| df = df.drop(rows_to_remove) | |
| greet_variations = ["hello", "hy", "hey", "hii", "hi", "heyyy", "bie", "bye"] | |
| for greet_var in greet_variations: | |
| pattern = r"(?<!\S)" + greet_var + r"(?!\S)|\b" + greet_var + r"\b" | |
| df['texts'] = df['texts'].str.replace(pattern, '', regex=True) | |
| okay_variations = ["ok", "k", "kay", "okay", "okie", "kk", "ohhhk","t","r"] | |
| for okay_var in okay_variations: | |
| pattern = r"(?<!\S)" + okay_var + r"(?!\S)|\b" + okay_var + r"\b" | |
| df['texts'] = df['texts'].str.replace(pattern, '', regex=True) | |
| yes_variations = ["yes", "yeah", "yep", "yup", "yuh", "ya", "yes got it", "yeah it is", "yesss", "yea","no"] | |
| for yes_var in yes_variations: | |
| pattern = r"(?<!\S)" + yes_var + r"(?!\S)|\b" + yes_var + r"\b" | |
| df['texts'] = df['texts'].str.replace(pattern, '', regex=True) | |
| remove_phrases = ["i'm all set","ask a question","apply the survey","videos (2-8 min)","long reads (> 8 min)", | |
| "short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)", | |
| "actually no","next steps","i'm a student alumni","i have questions"] | |
| for phrase in remove_phrases: | |
| df['texts'] = df['texts'].str.replace(phrase, '') | |
| general_variations = ["good morning", "good evening", "good afternoon", "good night", "done", "sorry", "top", "query", | |
| "stop", "sir", "sure", "oh", "wow", "aaa", "maam", "mam", "ma'am","i'm all set","ask a question","apply the survey", | |
| "videos (2-8 min)","long reads (> 8 min)","short reads (3-8 min)","not a student alumni","mock","share feedback","bite size (< 2 min)", | |
| "actually no","next steps","i'm a student alumni","i have questions"] | |
| for gen_var in general_variations: | |
| pattern = r"(?<!\S)" + gen_var + r"(?!\S)|\b" + gen_var + r"\b(?=\W|$)" | |
| df['texts'] = df['texts'].str.replace(pattern, '', regex=True) | |
| def remove_punctuations(text): | |
| return re.sub(r'[^\w\s]', '', text) | |
| df['texts'] = df['texts'].apply(remove_punctuations) | |
| remove_morephrases = ["short reads 38 min","bite size 2 min","videos 28 min","long reads 8 min"] | |
| for phrase in remove_morephrases: | |
| df['texts'] = df['texts'].str.replace(phrase, '') | |
| df = df[~df['texts'].str.contains(r'\b\d{10}\b')] | |
| df['texts'] = df['texts'].str.strip() | |
| df['texts'] = df['texts'].apply(lambda x: x.strip()) | |
| df = df[df['texts'] != ''] | |
| return df | |
| def cluster_data(df, num_clusters): | |
| vectorizer = TfidfVectorizer(stop_words='english') | |
| X = vectorizer.fit_transform(df['texts']) | |
| X = normalize(X) | |
| kmeans = KMeans(n_clusters=num_clusters, random_state=0) | |
| kmeans.fit(X) | |
| df['Cluster'] = kmeans.labels_ | |
| return df, X, kmeans | |
| def generate_wordcloud(texts): | |
| wordcloud = WordCloud(width=800, height=400, background_color='white').generate(" ".join(texts)) | |
| plt.figure(figsize=(10, 5)) | |
| plt.imshow(wordcloud, interpolation='bilinear') | |
| plt.axis('off') | |
| buf = BytesIO() | |
| plt.savefig(buf, format='png') | |
| buf.seek(0) | |
| return buf | |
| def main(file, num_clusters): | |
| df = pd.read_csv(file) | |
| # Filter by 'Fallback Message shown' | |
| df = df[df['Answer'] == 'Fallback Message shown'] | |
| df = preprocess_data(df) | |
| df, X, kmeans = cluster_data(df, num_clusters) | |
| clusters = df['Cluster'].unique() | |
| wordclouds = [] | |
| for cluster in clusters: | |
| texts = df[df['Cluster'] == cluster]['texts'].tolist() | |
| wordcloud_image = generate_wordcloud(texts) | |
| wordclouds.append((f"Cluster {cluster}", wordcloud_image)) | |
| cluster_sizes = df['Cluster'].value_counts() | |
| top_clusters = cluster_sizes.head(num_clusters).index | |
| top_queries = df[df['Cluster'].isin(top_clusters)][['Cluster', 'texts']] | |
| return wordclouds, top_queries | |
| def display_results(wordclouds, top_queries): | |
| for cluster, wordcloud in wordclouds: | |
| print(cluster) | |
| img = Image.open(wordcloud) | |
| img.show() | |
| print("Top Queries by Cluster:") | |
| print(top_queries.to_string(index=False)) | |
| interface = gr.Interface( | |
| fn=main, | |
| inputs=[ | |
| gr.File(label="Upload CSV File (.csv)"), | |
| gr.Slider(label="Number of Clusters", minimum=2, maximum=20, step=1, value=5) | |
| ], | |
| outputs=[ | |
| gr.Gallery(label="Word Clouds of Clusters"), | |
| gr.Dataframe(label="Top Queries by Cluster") | |
| ], | |
| title="Unanswered User Queries Clustering", | |
| description="Unanswered User Query Categorization" | |
| ) | |
| interface.launch() | |