Update app.py

app.py

@@ -11,38 +11,46 @@ import plotly.express as px
 from PIL import Image
 
 categories_keywords = {
-    'Application Status': ['application', 'applied', 'update on my application', 'result of my application', 'selected', 'selection process', 'apply', 'fellow', 'lesson plan', 'status of my application', 'application update', 'application status', 'applied for'],
-    'Volunteering': ['volunteering', 'volunteer', 'volunteering certificate', 'resume my volunteering', 'volunteering journey', 'volunteering with TFI', 'volunteering opportunities', 'volunteer work', 'volunteer program'],
-    'Certificates': ['certificate', 'certificates', 'certificate of completion', 'volunteer certificate', 'issue certificate'],
-    'Job Opportunities': ['job', 'vacancy', 'Talent Acquisition Executive job', 'opportunity', 'job opening', 'job position', 'career opportunities'],
-    'Surveys and Forms': ['survey', 'form', 'fill out the survey', 'application form', 'survey link', 'survey form', 'form submission'],
-    'General Queries': ['query', 'queries', 'questions', 'feedback', 'loved', 'overwhelming', 'general question', 'inquiry', 'query about'],
-    'Spam': ['free recharge', 'offer', 'click the link', 'https'],
-    'Rescheduling and Postponing': ['reschedule', 'postpone', 'cancellation', 'date', 'time slot', 'change date', 'change time', 'reschedule appointment'],
-    'Contact and Communication Issues': ['call', 'phone', 'contact', 'not received', 'contact support', 'phone call', 'call back', 'internet'],
-    'Email and Credentials Issues': ['email', 'credentials', 'received', 'email issue', 'email problem', 'credential issue', 'login problem'],
-    'Timing and Scheduling': ['session', 'time', 'interview', 'baje', 'schedule time', 'meeting time', 'appointment time'],
-    'Salary and Benefits': ['salary', 'increment', 'accommodation', 'training period', 'reside', 'stipend', 'pay', 'wage', 'salary details', 'benefits information'],
-    'Technical Issues': ['network issues', 'zoom meeting', 'passcode', 'technical', 'issue','technical problem', 'system issue', 'technical support'],
-    'Complaint Handling': ['help', 'i need help', 'Help me', 'complaint', 'issue is unresolved', 'unsatisfied', 'bad experience'],
-    'End of Conversation': ['thanks', 'thankss', 'thank u', 'thank you', 'ok', 'okay', 'done', 'joining', 'sounds good', 'goodbye', 'end chat', 'end'],
-    'Miscellaneous': []
+    "Application Status": ["application status", "application", "status", "submitted", "processing", "pending", "approval", "rejected", "accepted"],
+    "Volunteering": ["volunteer", "volunteering", "help out", "assist", "volunteer work", "volunteer opportunities"],
+    "Certificates": ["certificate", "certificates", "completion", "certification", "accreditation", "proof", "document", "certified"],
+    "Job Opportunities": ["job", "opportunity", "career", "vacancy", "position", "employment", "hiring", "recruitment", "internship"],
+    "Surveys and Forms": ["survey", "form", "forms", "questionnaire", "feedback form", "response", "fill out", "submission"],
+    "General Queries": ["hello", "hi", "help", "general", "query", "question", "info", "information", "inquiry", "ask"],
+    "Spam": ["spam", "subscribe", "remove", "stop", "junk", "block", "opt-out", "watch my video", "click on this link", "win prize"],
+    "Rescheduling and Postponing": ["reschedule", "postpone", "delay", "change date", "new time", "rearrange", "shift", "adjust timing"],
+    "Contact and Communication Issues": ["contact", "communicate", "communication", "reach out", "phone", "email", "address", "details"],
+    "Email and Credentials Issues": ["email", "credentials", "login", "password", "access", "username", "account", "verification", "reset"],
+    "Timing and Scheduling": ["timing", "schedule", "scheduling", "time", "appointment", "availability", "calendar", "book", "slot"],
+    "Salary and Benefits": ["salary", "benefits", "pay", "compensation", "wages", "earnings", "package", "remuneration", "incentives"],
+    "Technical Issues": ["technical", "issue", "problem", "error", "bug", "glitch", "fix", "troubleshoot", "support"],
+    "End of Conversation": ["bye", "thank you", "thanks", "goodbye", "see you", "later", "end", "close", "sign off"],
+    "Start of Conversation": ["start", "begin", "hello", "hi", "initiate", "greet", "greeting", "open", "commence"],
+    "Feedback": ["feedback", "comments", "review", "opinion", "suggestion", "critique", "rating"],
+    "Online Meetings": ["meeting", "meeting code", "passcode", "join meeting", "zoom", "zoom link"],
+    "Event Inquiries": ["event", "webinar", "meeting", "conference", "session", "seminar", "workshop", "invitation"],
+    "Payment Issues": ["payment", "billing", "transaction", "charge", "fee", "invoice", "refund", "receipt"],
+    "Registration Issues": ["registration", "register", "sign up", "enroll", "join", "signup", "enrollment"],
+    "Service Requests": ["service", "support", "request", "assistance", "help", "aid", "maintenance"],
+    "Account Issues": ["account", "profile", "update", "activation", "deactivation", "credentials", "reset"],
+    "Product Information": ["product", "service", "details", "info", "information", "specifications", "features"],
+    "Order Status": ["order", "status", "tracking", "shipment", "delivery", "purchase", "dispatch"],
+    "Miscellaneous": []
 }
 
-
 def categorize_question(question):
+    words = question.split()
+    if len(words) == 1:
+        single_word = words[0].lower()
+        if any(single_word in keyword for keyword in categories_keywords["Start of Conversation"]):
+            return "Start of Conversation"
+        else:
+            return "End of Conversation"
+    
     for category, keywords in categories_keywords.items():
-        for keyword in keywords:
-            if keyword.lower() in question.lower():
-                # Check if the question is one word and belongs to 'End of Conversation'
-                if category == 'End of Conversation':
-                    return category
-                # If not 'End of Conversation', return the matched category
-                if category != 'End of Conversation':
-                    return category
-    return 'Miscellaneous'
-
-
+        if any(keyword.lower() in question.lower() for keyword in keywords):
+            return category
+    return "Miscellaneous"
 
 def preprocess_data(df):
     df.rename(columns={'Question Asked': 'texts'}, inplace=True)
@@ -95,7 +103,6 @@ def preprocess_data(df):
     df['texts'] = df['texts'].str.strip()
     df = df[df['texts'] != '']
 
-    # Categorize the texts
     df['Category'] = df['texts'].apply(categorize_question)
 
     return df
@@ -126,7 +133,7 @@ def generate_wordcloud(df):
         scale=2,
         relative_scaling=0.5,
         random_state=42
-    ).generate(text)
+    ).generate(text[:1000000])  # Limit the text length to 1,000,000 characters to avoid memory issues
     
     plt.figure(figsize=(15, 7))
     plt.imshow(wordcloud, interpolation='bilinear')
@@ -138,14 +145,21 @@ def generate_wordcloud(df):
     return img
 
 def generate_bar_chart(df, num_clusters_to_display):
-    # Exclude common words from the top words
     common_words = {'i', 'you', 'thanks', 'thank', 'ok', 'okay', 'sure', 'done'}
     
     top_categories = df['Category'].value_counts().index[:num_clusters_to_display]
     df_top_categories = df[df['Category'].isin(top_categories)]
     
-    category_top_words = df_top_categories.groupby('Category', observed=False)['texts'].apply(lambda x: ' '.join(x)).reset_index()
-    category_top_words['top_word'] = category_top_words['texts'].apply(lambda x: ' '.join([word for word in pd.Series(x.split()).value_counts().index if word not in common_words][:3]))
+    def extract_top_words(texts, n=1):
+        vec = TfidfVectorizer(stop_words='english').fit(texts)
+        sum_words = vec.transform(texts).sum(axis=0)
+        words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]
+        sorted_words = sorted(words_freq, key=lambda x: x[1], reverse=True)
+        return [word for word, freq in sorted_words if word not in common_words][:n]
+
+    category_top_words = df_top_categories.groupby('Category')['texts'].apply(lambda texts: extract_top_words(texts, 3)).reset_index()
+    category_top_words['top_word'] = category_top_words['texts'].apply(lambda words: ', '.join(words))
+    
     category_sizes = df_top_categories['Category'].value_counts().reset_index()
     category_sizes.columns = ['Category', 'Count']
     category_sizes = category_sizes.merge(category_top_words[['Category', 'top_word']], on='Category')
@@ -164,30 +178,15 @@ def main(file, num_clusters_to_display):
     try:
         df = pd.read_csv(file)
         
-        # Filter by 'Fallback Message shown'
         df = df[df['Answer'] == 'Fallback Message shown']
         
         df = preprocess_data(df)
         
-        # Get category sizes and sort by size in ascending order
-        category_sizes = df['Category'].value_counts().reset_index()
-        category_sizes.columns = ['Category', 'Count']
-        sorted_categories = category_sizes.sort_values(by='Count', ascending=True)['Category'].tolist()
-        
-        # Get the largest x categories as specified by num_clusters_to_display
-        largest_categories = sorted_categories[:num_clusters_to_display]
-        
-        # Filter the dataframe to include only the largest categories
-        filtered_df = df[df['Category'].isin(largest_categories)]
-        
-        # Sort the dataframe by Category
-        filtered_df = filtered_df.sort_values(by='Category')
-        
-        wordcloud_img = generate_wordcloud(filtered_df)
-        bar_chart_img = generate_bar_chart(filtered_df, num_clusters_to_display)
+        wordcloud_img = generate_wordcloud(df)
+        bar_chart_img = generate_bar_chart(df, num_clusters_to_display)
 
         with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
-            filtered_df.to_csv(tmpfile.name, index=False)
+            df.to_csv(tmpfile.name, index=False)
             csv_file_path = tmpfile.name
 
         return csv_file_path, wordcloud_img, bar_chart_img
@@ -199,7 +198,7 @@ interface = gr.Interface(
     fn=main,
     inputs=[
         gr.File(label="Upload CSV File (.csv)"),
-        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=10, step=1, value=5)
+        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=15, step=1, value=5)
     ],
     outputs=[
         gr.File(label="Categorized Data CSV"),