Update app.py

app.py

@@ -11,46 +11,38 @@ import plotly.express as px
 from PIL import Image
 
 categories_keywords = {
-    "Application Status": ["application status", "application", "status", "submitted", "processing", "pending", "approval", "rejected", "accepted"],
-    "Volunteering": ["volunteer", "volunteering", "help out", "assist", "volunteer work", "volunteer opportunities"],
-    "Certificates": ["certificate", "certificates", "completion", "certification", "accreditation", "proof", "document", "certified"],
-    "Job Opportunities": ["job", "opportunity", "career", "vacancy", "position", "employment", "hiring", "recruitment", "internship"],
-    "Surveys and Forms": ["survey", "form", "forms", "questionnaire", "feedback form", "response", "fill out", "submission"],
-    "General Queries": ["hello", "hi", "help", "general", "query", "question", "info", "information", "inquiry", "ask"],
-    "Spam": ["spam", "subscribe", "remove", "stop", "junk", "block", "opt-out", "watch my video", "click on this link", "win prize"],
-    "Rescheduling and Postponing": ["reschedule", "postpone", "delay", "change date", "new time", "rearrange", "shift", "adjust timing"],
-    "Contact and Communication Issues": ["contact", "communicate", "communication", "reach out", "phone", "email", "address", "details"],
-    "Email and Credentials Issues": ["email", "credentials", "login", "password", "access", "username", "account", "verification", "reset"],
-    "Timing and Scheduling": ["timing", "schedule", "scheduling", "time", "appointment", "availability", "calendar", "book", "slot"],
-    "Salary and Benefits": ["salary", "benefits", "pay", "compensation", "wages", "earnings", "package", "remuneration", "incentives"],
-    "Technical Issues": ["technical", "issue", "problem", "error", "bug", "glitch", "fix", "troubleshoot", "support"],
-    "End of Conversation": ["bye", "thank you", "thanks", "goodbye", "see you", "later", "end", "close", "sign off"],
-    "Start of Conversation": ["start", "begin", "hello", "hi", "initiate", "greet", "greeting", "open", "commence"],
-    "Feedback": ["feedback", "comments", "review", "opinion", "suggestion", "critique", "rating"],
-    "Online Meetings": ["meeting", "meeting code", "passcode", "join meeting", "zoom", "zoom link"],
-    "Event Inquiries": ["event", "webinar", "meeting", "conference", "session", "seminar", "workshop", "invitation"],
-    "Payment Issues": ["payment", "billing", "transaction", "charge", "fee", "invoice", "refund", "receipt"],
-    "Registration Issues": ["registration", "register", "sign up", "enroll", "join", "signup", "enrollment"],
-    "Service Requests": ["service", "support", "request", "assistance", "help", "aid", "maintenance"],
-    "Account Issues": ["account", "profile", "update", "activation", "deactivation", "credentials", "reset"],
-    "Product Information": ["product", "service", "details", "info", "information", "specifications", "features"],
-    "Order Status": ["order", "status", "tracking", "shipment", "delivery", "purchase", "dispatch"],
-    "Miscellaneous": []
+    'Application Status': ['application', 'applied', 'update on my application', 'result of my application', 'selected', 'selection process', 'apply', 'fellow', 'lesson plan', 'status of my application', 'application update', 'application status', 'applied for'],
+    'Volunteering': ['volunteering', 'volunteer', 'volunteering certificate', 'resume my volunteering', 'volunteering journey', 'volunteering with TFI', 'volunteering opportunities', 'volunteer work', 'volunteer program'],
+    'Certificates': ['certificate', 'certificates', 'certificate of completion', 'volunteer certificate', 'issue certificate'],
+    'Job Opportunities': ['job', 'vacancy', 'Talent Acquisition Executive job', 'opportunity', 'job opening', 'job position', 'career opportunities'],
+    'Surveys and Forms': ['survey', 'form', 'fill out the survey', 'application form', 'survey link', 'survey form', 'form submission'],
+    'General Queries': ['query', 'queries', 'questions', 'feedback', 'loved', 'overwhelming', 'general question', 'inquiry', 'query about'],
+    'Spam': ['free recharge', 'offer', 'click the link', 'https'],
+    'Rescheduling and Postponing': ['reschedule', 'postpone', 'cancellation', 'date', 'time slot', 'change date', 'change time', 'reschedule appointment'],
+    'Contact and Communication Issues': ['call', 'phone', 'contact', 'not received', 'contact support', 'phone call', 'call back', 'internet'],
+    'Email and Credentials Issues': ['email', 'credentials', 'received', 'email issue', 'email problem', 'credential issue', 'login problem'],
+    'Timing and Scheduling': ['session', 'time', 'interview', 'baje', 'schedule time', 'meeting time', 'appointment time'],
+    'Salary and Benefits': ['salary', 'increment', 'accommodation', 'training period', 'reside', 'stipend', 'pay', 'wage', 'salary details', 'benefits information'],
+    'Technical Issues': ['network issues', 'zoom meeting', 'passcode', 'technical', 'issue','technical problem', 'system issue', 'technical support'],
+    'Complaint Handling': ['help', 'i need help', 'Help me', 'complaint', 'issue is unresolved', 'unsatisfied', 'bad experience'],
+    'End of Conversation': ['thanks', 'thankss', 'thank u', 'thank you', 'ok', 'okay', 'done', 'joining', 'sounds good', 'goodbye', 'end chat', 'end'],
+    'Miscellaneous': []
 }
 
+
 def categorize_question(question):
-    words = question.split()
-    if len(words) == 1:
-        single_word = words[0].lower()
-        if any(single_word in keyword for keyword in categories_keywords["Start of Conversation"]):
-            return "Start of Conversation"
-        else:
-            return "End of Conversation"
-    
     for category, keywords in categories_keywords.items():
-        if any(keyword.lower() in question.lower() for keyword in keywords):
-            return category
-    return "Miscellaneous"
+        for keyword in keywords:
+            if keyword.lower() in question.lower():
+                # Check if the question is one word and belongs to 'End of Conversation'
+                if category == 'End of Conversation':
+                    return category
+                # If not 'End of Conversation', return the matched category
+                if category != 'End of Conversation':
+                    return category
+    return 'Miscellaneous'
+
+
 
 def preprocess_data(df):
     df.rename(columns={'Question Asked': 'texts'}, inplace=True)
@@ -103,6 +95,7 @@ def preprocess_data(df):
     df['texts'] = df['texts'].str.strip()
     df = df[df['texts'] != '']
 
+    # Categorize the texts
     df['Category'] = df['texts'].apply(categorize_question)
 
     return df
@@ -133,7 +126,7 @@ def generate_wordcloud(df):
         scale=2,
         relative_scaling=0.5,
         random_state=42
-    ).generate(text[:1000000])  # Limit the text length to 1,000,000 characters to avoid memory issues
+    ).generate(text)
     
     plt.figure(figsize=(15, 7))
     plt.imshow(wordcloud, interpolation='bilinear')
@@ -145,21 +138,14 @@ def generate_wordcloud(df):
     return img
 
 def generate_bar_chart(df, num_clusters_to_display):
+    # Exclude common words from the top words
     common_words = {'i', 'you', 'thanks', 'thank', 'ok', 'okay', 'sure', 'done'}
     
     top_categories = df['Category'].value_counts().index[:num_clusters_to_display]
     df_top_categories = df[df['Category'].isin(top_categories)]
     
-    def extract_top_words(texts, n=1):
-        vec = TfidfVectorizer(stop_words='english').fit(texts)
-        sum_words = vec.transform(texts).sum(axis=0)
-        words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]
-        sorted_words = sorted(words_freq, key=lambda x: x[1], reverse=True)
-        return [word for word, freq in sorted_words if word not in common_words][:n]
-
-    category_top_words = df_top_categories.groupby('Category')['texts'].apply(lambda texts: extract_top_words(texts, 3)).reset_index()
-    category_top_words['top_word'] = category_top_words['texts'].apply(lambda words: ', '.join(words))
-    
+    category_top_words = df_top_categories.groupby('Category', observed=False)['texts'].apply(lambda x: ' '.join(x)).reset_index()
+    category_top_words['top_word'] = category_top_words['texts'].apply(lambda x: ' '.join([word for word in pd.Series(x.split()).value_counts().index if word not in common_words][:3]))
     category_sizes = df_top_categories['Category'].value_counts().reset_index()
     category_sizes.columns = ['Category', 'Count']
     category_sizes = category_sizes.merge(category_top_words[['Category', 'top_word']], on='Category')
@@ -178,15 +164,30 @@ def main(file, num_clusters_to_display):
     try:
         df = pd.read_csv(file)
         
+        # Filter by 'Fallback Message shown'
         df = df[df['Answer'] == 'Fallback Message shown']
         
         df = preprocess_data(df)
         
-        wordcloud_img = generate_wordcloud(df)
-        bar_chart_img = generate_bar_chart(df, num_clusters_to_display)
+        # Get category sizes and sort by size in ascending order
+        category_sizes = df['Category'].value_counts().reset_index()
+        category_sizes.columns = ['Category', 'Count']
+        sorted_categories = category_sizes.sort_values(by='Count', ascending=True)['Category'].tolist()
+        
+        # Get the largest x categories as specified by num_clusters_to_display
+        largest_categories = sorted_categories[:num_clusters_to_display]
+        
+        # Filter the dataframe to include only the largest categories
+        filtered_df = df[df['Category'].isin(largest_categories)]
+        
+        # Sort the dataframe by Category
+        filtered_df = filtered_df.sort_values(by='Category')
+        
+        wordcloud_img = generate_wordcloud(filtered_df)
+        bar_chart_img = generate_bar_chart(filtered_df, num_clusters_to_display)
 
         with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
-            df.to_csv(tmpfile.name, index=False)
+            filtered_df.to_csv(tmpfile.name, index=False)
             csv_file_path = tmpfile.name
 
         return csv_file_path, wordcloud_img, bar_chart_img
@@ -198,7 +199,7 @@ interface = gr.Interface(
     fn=main,
     inputs=[
         gr.File(label="Upload CSV File (.csv)"),
-        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=15, step=1, value=5)
+        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=10, step=1, value=5)
     ],
     outputs=[
         gr.File(label="Categorized Data CSV"),
@@ -210,3 +211,72 @@ interface = gr.Interface(
 )
 
 interface.launch(share=True)
+
+
+
+
+
+—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------—--------
+
+
+
+
+
+def main(file, bot_name, num_clusters_to_display):
+    try:
+        global categories_keywords
+        if bot_name == "Teach For India":
+            categories_keywords = categories_keywords_tfi
+        else:
+            categories_keywords = categories_keywords_firki
+            
+        df = pd.read_csv(file.name)
+        
+        df = df[df['Answer'] == 'Fallback Message shown']
+        
+        df = preprocess_data(df, categories_keywords)
+        
+        category_sizes = df['Category'].value_counts().reset_index()
+        category_sizes.columns = ['Category', 'Count']
+        sorted_categories = category_sizes.sort_values(by='Count', ascending=True)['Category'].tolist()
+        
+        largest_categories = sorted_categories[:num_clusters_to_display]
+        
+        filtered_df = df[df['Category'].isin(largest_categories)]
+        
+        filtered_df = filtered_df.sort_values(by='Category')
+        
+        wordcloud_img = generate_wordcloud(filtered_df)
+        bar_chart_img = generate_bar_chart(df, num_clusters_to_display)
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
+            filtered_df.to_csv(tmpfile.name, index=False)
+            csv_file_path = tmpfile.name
+
+        return csv_file_path, wordcloud_img, bar_chart_img
+    except Exception as e:
+        print(f"Error: {e}")
+        return str(e), None, None
+
+def categorize_unanswered_queries(bot_name, file, num_clusters_to_display):
+    return main(file, bot_name, num_clusters_to_display)
+
+interface = gr.Interface(
+    fn=categorize_unanswered_queries,
+    inputs=[
+        gr.Radio(["Teach For India", "Firki"], label="Select ChatBot"),
+        gr.File(label="Upload CSV File (.csv)"),
+        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=10, step=1, value=5)
+    ],
+    outputs=[
+        gr.File(label="Categorized Data CSV"),
+        gr.Image(label="Word Cloud"),
+        gr.Image(label="Bar Chart")
+    ],
+    title="Unanswered User Queries Categorization",
+    description="Select the bot, upload the CSV file, and specify the number of categories to display to categorize unanswered user queries."
+)
+
+interface.launch()
+
+