Create app. py

app. py

@@ -0,0 +1,655 @@
+import gradio as gr
+import os
+import json
+import pandas as pd
+import numpy as np
+import datetime
+import plotly.express as px
+import plotly.graph_objects as go
+import msal
+import requests
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+import threading
+import time
+from transformers import pipeline
+import tempfile
+
+# Configuration
+MS_CLIENT_ID = os.getenv("MS_CLIENT_ID", "ff0d5b77-56a9-4fa0-bd59-5c7b4889186e")
+MS_TENANT_ID = os.getenv("MS_TENANT_ID", "677c00b7-cf19-4fef-9962-132a076ae325")
+MS_AUTHORITY = f"https://login.microsoftonline.com/{MS_TENANT_ID}"
+MS_REDIRECT_URI = os.getenv("MS_REDIRECT_URI", "https://huggingface.co/spaces/YOUR-USERNAME/email-thread-analyzer/")
+
+# Microsoft Graph API scopes
+SCOPES = [
+    "User.Read",
+    "Mail.Read",
+    "Mail.ReadBasic",
+]
+
+# Global variables
+auth_app = None
+current_user = None
+user_token = None
+emails = []
+email_threads = {}
+embeddings = {}
+qa_data = {}
+qa_model = None
+embedding_model = None
+search_results = []
+
+# Initialize MSAL app
+def init_auth_app():
+    global auth_app
+    auth_app = msal.PublicClientApplication(
+        client_id=MS_CLIENT_ID,
+        authority=MS_AUTHORITY
+    )
+
+# Initialize models
+def init_models():
+    global embedding_model, qa_model
+    try:
+        embedding_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
+        qa_model = pipeline("question-answering", model="deepset/roberta-base-squad2")
+        return "Models initialized successfully"
+    except Exception as e:
+        print(f"Error initializing models: {e}")
+        embedding_model = None
+        qa_model = None
+        return f"Error initializing models: {e}"
+
+# Get authorization URL
+def get_auth_url():
+    auth_url = auth_app.get_authorization_request_url(
+        scopes=SCOPES,
+        redirect_uri=MS_REDIRECT_URI,
+        state="state"
+    )
+    return auth_url
+
+# Process auth code
+def process_auth_code(auth_code):
+    global current_user, user_token
+    
+    try:
+        # Acquire token
+        token_response = auth_app.acquire_token_by_authorization_code(
+            code=auth_code,
+            scopes=SCOPES,
+            redirect_uri=MS_REDIRECT_URI
+        )
+        
+        if "error" in token_response:
+            return f"Error: {token_response['error_description']}"
+        
+        # Store token
+        user_token = token_response
+        
+        # Get user info
+        user_response = requests.get(
+            "https://graph.microsoft.com/v1.0/me",
+            headers={"Authorization": f"Bearer {user_token['access_token']}"}
+        )
+        
+        if user_response.status_code == 200:
+            current_user = user_response.json()
+            return f"Successfully authenticated as {current_user['displayName']}"
+        else:
+            return f"Error getting user info: {user_response.text}"
+    
+    except Exception as e:
+        return f"Error during authentication: {str(e)}"
+
+# Get mail folders
+def get_mail_folders():
+    if not user_token:
+        return [], "Not authenticated"
+    
+    try:
+        response = requests.get(
+            "https://graph.microsoft.com/v1.0/me/mailFolders",
+            headers={"Authorization": f"Bearer {user_token['access_token']}"}
+        )
+        
+        if response.status_code == 200:
+            folders = response.json()["value"]
+            return [(folder["displayName"], folder["id"]) for folder in folders], None
+        else:
+            return [], f"Error: {response.text}"
+    
+    except Exception as e:
+        return [], f"Error: {str(e)}"
+
+# Extract emails from folder
+def extract_emails(folder_id, max_emails=100, batch_size=25, start_date=None, end_date=None):
+    global emails, email_threads, embeddings
+    
+    if not user_token:
+        return "Not authenticated"
+    
+    try:
+        # Reset data
+        emails = []
+        email_threads = {}
+        embeddings = {}
+        
+        # Prepare filter
+        filter_query = ""
+        if start_date and end_date:
+            start_date_iso = datetime.datetime.strptime(start_date, "%Y-%m-%d").isoformat() + "Z"
+            end_date_iso = datetime.datetime.strptime(end_date, "%Y-%m-%d").isoformat() + "Z"
+            filter_query = f"receivedDateTime ge {start_date_iso} and receivedDateTime le {end_date_iso}"
+        
+        # Extract emails in batches
+        for i in range(0, max_emails, batch_size):
+            # Prepare request
+            url = f"https://graph.microsoft.com/v1.0/me/mailFolders/{folder_id}/messages"
+            headers = {"Authorization": f"Bearer {user_token['access_token']}"}
+            params = {
+                "$select": "id,subject,sender,from,toRecipients,ccRecipients,receivedDateTime,conversationId,bodyPreview,uniqueBody",
+                "$top": batch_size,
+                "$skip": i
+            }
+            
+            if filter_query:
+                params["$filter"] = filter_query
+            
+            # Make request
+            response = requests.get(url, headers=headers, params=params)
+            
+            if response.status_code != 200:
+                return f"Error: {response.text}"
+            
+            batch_emails = response.json()["value"]
+            
+            if not batch_emails:
+                break
+            
+            emails.extend(batch_emails)
+            
+            if len(emails) >= max_emails:
+                emails = emails[:max_emails]
+                break
+        
+        # Organize emails into threads
+        organize_email_threads()
+        
+        # Generate embeddings in background
+        threading.Thread(target=generate_embeddings).start()
+        
+        return f"Successfully extracted {len(emails)} emails organized into {len(email_threads)} threads"
+    
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+# Organize emails into threads
+def organize_email_threads():
+    global email_threads
+    
+    threads = {}
+    
+    for email in emails:
+        conversation_id = email["conversationId"]
+        
+        if conversation_id not in threads:
+            threads[conversation_id] = []
+        
+        threads[conversation_id].append(email)
+    
+    # Sort emails within each thread by date
+    for thread_id, thread_emails in threads.items():
+        thread_emails.sort(key=lambda x: x["receivedDateTime"])
+        
+        # Extract thread metadata
+        threads[thread_id] = {
+            "emails": thread_emails,
+            "subject": thread_emails[0]["subject"],
+            "start_date": thread_emails[0]["receivedDateTime"],
+            "end_date": thread_emails[-1]["receivedDateTime"],
+            "message_count": len(thread_emails),
+            "participants": get_unique_participants(thread_emails)
+        }
+    
+    email_threads = threads
+
+# Get unique participants
+def get_unique_participants(thread_emails):
+    participants = set()
+    
+    for email in thread_emails:
+        # Add sender
+        if "sender" in email and "emailAddress" in email["sender"]:
+            participants.add(email["sender"]["emailAddress"]["address"])
+        
+        # Add recipients
+        if "toRecipients" in email:
+            for recipient in email["toRecipients"]:
+                participants.add(recipient["emailAddress"]["address"])
+        
+        # Add CC recipients
+        if "ccRecipients" in email:
+            for recipient in email["ccRecipients"]:
+                participants.add(recipient["emailAddress"]["address"])
+    
+    return list(participants)
+
+# Generate embeddings for search
+def generate_embeddings():
+    global embeddings
+    
+    if not embedding_model or not email_threads:
+        return
+    
+    for thread_id, thread in email_threads.items():
+        # Create text representation of thread
+        text = thread["subject"] + " " + " ".join([email["bodyPreview"] for email in thread["emails"]])
+        
+        # Generate embedding
+        embedding = embedding_model.encode(text)
+        
+        # Store embedding
+        embeddings[thread_id] = embedding
+
+# Search threads
+def search_threads(query):
+    global search_results
+    
+    if not query or not embedding_model or not embeddings:
+        search_results = []
+        return "Please enter a search query and ensure emails have been extracted"
+    
+    try:
+        # Generate query embedding
+        query_embedding = embedding_model.encode(query)
+        
+        # Calculate similarity scores
+        scores = []
+        for thread_id, thread_embedding in embeddings.items():
+            similarity = cosine_similarity([query_embedding], [thread_embedding])[0][0]
+            scores.append((thread_id, similarity))
+        
+        # Sort by similarity and filter out low scores
+        scores.sort(key=lambda x: x[1], reverse=True)
+        relevant_threads = [thread_id for thread_id, score in scores if score > 0.2]
+        
+        # Get thread data
+        search_results = [email_threads[thread_id] for thread_id in relevant_threads]
+        
+        if not search_results:
+            return "No relevant threads found"
+        
+        return f"Found {len(search_results)} relevant threads"
+    
+    except Exception as e:
+        search_results = []
+        return f"Error: {str(e)}"
+
+# Generate Q&A for thread
+def generate_qa(thread_id):
+    if not qa_model or thread_id not in email_threads:
+        return "Unable to generate Q&A - model not loaded or thread not found"
+    
+    try:
+        thread = email_threads[thread_id]
+        
+        # Create thread context
+        context = f"Thread subject: {thread['subject']}\n\n"
+        for email in thread["emails"]:
+            sender = email["sender"]["emailAddress"]["address"]
+            context += f"From: {sender}\n"
+            context += f"Date: {email['receivedDateTime']}\n"
+            context += f"Content: {email['bodyPreview']}\n\n"
+        
+        # Generate sample questions
+        questions = [
+            f"What is the main topic of this email thread about '{thread['subject']}'?",
+            "Who are the key participants in this conversation?",
+            "What was the timeline of this discussion?",
+            "What were the main points discussed in this thread?"
+        ]
+        
+        # Generate answers
+        answers = []
+        for question in questions:
+            try:
+                result = qa_model(question=question, context=context)
+                answers.append(result["answer"])
+            except Exception as e:
+                answers.append(f"Error generating answer: {str(e)}")
+        
+        # Create summary
+        summary = f"This is an email thread with {thread['message_count']} messages about '{thread['subject']}'. "
+        summary += f"The conversation started on {thread['start_date']} and ended on {thread['end_date']}. "
+        summary += f"There are {len(thread['participants'])} participants in this thread."
+        
+        # Store Q&A data
+        qa_data[thread_id] = {
+            "questions": questions,
+            "answers": answers,
+            "summary": summary
+        }
+        
+        return f"Generated {len(questions)} Q&A pairs for thread"
+    
+    except Exception as e:
+        return f"Error generating Q&A: {str(e)}"
+
+# Get thread size distribution
+def get_thread_size_distribution():
+    if not email_threads:
+        return None
+    
+    # Count threads by size
+    sizes = {}
+    for thread in email_threads.values():
+        size = thread["message_count"]
+        if size in sizes:
+            sizes[size] += 1
+        else:
+            sizes[size] = 1
+    
+    # Convert to dataframe
+    df = pd.DataFrame([
+        {"Size": size, "Count": count}
+        for size, count in sizes.items()
+    ])
+    
+    # Sort by size
+    df = df.sort_values("Size")
+    
+    # Create chart
+    fig = px.bar(df, x="Size", y="Count", title="Thread Size Distribution")
+    return fig
+
+# Get activity over time
+def get_activity_over_time():
+    if not emails:
+        return None
+    
+    # Count emails by date
+    dates = {}
+    for email in emails:
+        date = email["receivedDateTime"].split("T")[0]
+        if date in dates:
+            dates[date] += 1
+        else:
+            dates[date] = 1
+    
+    # Convert to dataframe
+    df = pd.DataFrame([
+        {"Date": date, "Count": count}
+        for date, count in dates.items()
+    ])
+    
+    # Sort by date
+    df = df.sort_values("Date")
+    
+    # Create chart
+    fig = px.line(df, x="Date", y="Count", title="Activity Over Time")
+    return fig
+
+# Get participant activity
+def get_participant_activity():
+    if not emails:
+        return None
+    
+    # Count emails by sender
+    senders = {}
+    for email in emails:
+        if "sender" in email and "emailAddress" in email["sender"]:
+            sender = email["sender"]["emailAddress"]["address"]
+            if sender in senders:
+                senders[sender] += 1
+            else:
+                senders[sender] = 1
+    
+    # Convert to dataframe
+    df = pd.DataFrame([
+        {"Participant": sender, "Count": count}
+        for sender, count in senders.items()
+    ])
+    
+    # Sort by count
+    df = df.sort_values("Count", ascending=False).head(10)
+    
+    # Create chart
+    fig = px.bar(df, x="Count", y="Participant", title="Top 10 Participants", orientation='h')
+    return fig
+
+# Export thread data with Q&A
+def export_thread_data(thread_id):
+    if thread_id not in email_threads:
+        return None
+    
+    thread = email_threads[thread_id]
+    qa = qa_data.get(thread_id, {"questions": [], "answers": [], "summary": ""})
+    
+    export_data = {
+        "subject": thread["subject"],
+        "start_date": thread["start_date"],
+        "end_date": thread["end_date"],
+        "message_count": thread["message_count"],
+        "participants": thread["participants"],
+        "emails": [
+            {
+                "sender": email["sender"]["emailAddress"]["address"],
+                "received_date_time": email["receivedDateTime"],
+                "subject": email["subject"],
+                "body_preview": email["bodyPreview"]
+            }
+            for email in thread["emails"]
+        ],
+        "qa": qa
+    }
+    
+    # Save to temporary file
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.json', mode='w') as f:
+        json.dump(export_data, f, indent=2)
+        return f.name
+
+# Initialize
+init_auth_app()
+init_status = init_models()
+
+# Create the Gradio interface
+with gr.Blocks(title="Email Thread Analyzer with AI Q&A") as demo:
+    gr.Markdown("# Email Thread Analyzer with AI Q&A")
+    
+    # Authentication section
+    with gr.Tab("Authentication"):
+        with gr.Row():
+            with gr.Column(scale=2):
+                gr.Markdown("## Sign in with Microsoft")
+                gr.Markdown("1. Click 'Get Authentication URL' to start the sign-in process")
+                gr.Markdown("2. Copy the authorization code from the redirect URL")
+                gr.Markdown("3. Paste the code below and submit")
+            
+            with gr.Column(scale=3):
+                auth_url_button = gr.Button("Get Authentication URL")
+                auth_url_output = gr.Textbox(label="Authentication URL", interactive=False)
+                auth_code_input = gr.Textbox(label="Authorization Code")
+                auth_submit = gr.Button("Submit Authorization Code")
+                auth_status = gr.Textbox(label="Authentication Status", interactive=False, value=f"AI Models: {init_status}")
+    
+    # Email Extraction section
+    with gr.Tab("Email Extraction"):
+        with gr.Row():
+            with gr.Column():
+                folder_dropdown = gr.Dropdown(label="Select Mail Folder")
+                refresh_folders_button = gr.Button("Refresh Folders")
+                
+                with gr.Row():
+                    max_emails_input = gr.Number(label="Max Emails", value=100, minimum=1, maximum=1000)
+                    batch_size_input = gr.Number(label="Batch Size", value=25, minimum=1, maximum=100)
+                
+                with gr.Row():
+                    start_date_input = gr.Textbox(label="Start Date (YYYY-MM-DD)")
+                    end_date_input = gr.Textbox(label="End Date (YYYY-MM-DD)")
+                
+                extract_button = gr.Button("Extract Emails")
+                extraction_status = gr.Textbox(label="Extraction Status", interactive=False)
+    
+    # Thread Analysis section
+    with gr.Tab("Thread Analysis"):
+        with gr.Row():
+            with gr.Column():
+                analysis_status = gr.Textbox(label="Analysis Status")
+                
+                with gr.Tabs():
+                    with gr.Tab("Thread Size"):
+                        thread_size_plot = gr.Plot(label="Thread Size Distribution")
+                    
+                    with gr.Tab("Activity Over Time"):
+                        activity_plot = gr.Plot(label="Activity Over Time")
+                    
+                    with gr.Tab("Top Participants"):
+                        participants_plot = gr.Plot(label="Top Participants")
+                
+                generate_analytics_button = gr.Button("Generate Analytics")
+    
+    # Search section
+    with gr.Tab("Search"):
+        with gr.Row():
+            with gr.Column():
+                search_input = gr.Textbox(label="Search Query")
+                search_button = gr.Button("Search")
+                search_status = gr.Textbox(label="Search Status", interactive=False)
+            
+            with gr.Column():
+                search_results_dropdown = gr.Dropdown(label="Search Results")
+                view_thread_button = gr.Button("View Thread")
+    
+    # Q&A section
+    with gr.Tab("Q&A"):
+        with gr.Row():
+            with gr.Column():
+                thread_info = gr.Textbox(label="Thread Information", interactive=False)
+                qa_status = gr.Textbox(label="Q&A Status", interactive=False)
+                
+                with gr.Accordion("Thread Content", open=False):
+                    thread_content = gr.Textbox(label="Thread Content", interactive=False, lines=10)
+                
+                with gr.Row():
+                    question_dropdown = gr.Dropdown(label="Questions")
+                    gen_qa_button = gr.Button("Generate Q&A")
+                
+                answer_output = gr.Textbox(label="Answer", interactive=False, lines=5)
+                summary_output = gr.Textbox(label="Summary", interactive=False, lines=5)
+                
+                export_thread_button = gr.Button("Export Thread Data")
+                export_output = gr.File(label="Export Data")
+    
+    # Set up event handlers
+    # Authentication events
+    auth_url_button.click(
+        fn=get_auth_url,
+        outputs=auth_url_output
+    )
+    
+    auth_submit.click(
+        fn=process_auth_code,
+        inputs=auth_code_input,
+        outputs=auth_status
+    )
+    
+    # Folder refresh event
+    refresh_folders_button.click(
+        fn=lambda: get_mail_folders()[0],
+        outputs=folder_dropdown
+    )
+    
+    # Email extraction event
+    extract_button.click(
+        fn=extract_emails,
+        inputs=[folder_dropdown, max_emails_input, batch_size_input, start_date_input, end_date_input],
+        outputs=extraction_status
+    )
+    
+    # Analytics generation event
+    generate_analytics_button.click(
+        fn=lambda: (
+            "Analytics generated successfully",
+            get_thread_size_distribution(),
+            get_activity_over_time(),
+            get_participant_activity()
+        ),
+        outputs=[analysis_status, thread_size_plot, activity_plot, participants_plot]
+    )
+    
+    # Search events
+    search_button.click(
+        fn=lambda query: (
+            search_threads(query),
+            [f"{thread['subject']} ({thread['message_count']} messages)" for thread in search_results]
+        ),
+        inputs=search_input,
+        outputs=[search_status, search_results_dropdown]
+    )
+    
+    # Thread view event
+    def view_thread_details(thread_idx):
+        if not search_results or thread_idx < 0 or thread_idx >= len(search_results):
+            return "No thread selected", "", [], "", "", None
+        
+        thread = search_results[thread_idx]
+        thread_id = thread["emails"][0]["conversationId"]
+        
+        # Generate thread content
+        content = f"Subject: {thread['subject']}\n\n"
+        for email in thread["emails"]:
+            sender = email["sender"]["emailAddress"]["address"]
+            date = email["receivedDateTime"]
+            content += f"From: {sender} | Date: {date}\n"
+            content += f"Content: {email['bodyPreview']}\n\n"
+        
+        # Generate Q&A if not already generated
+        qa_result = "Q&A already generated"
+        if thread_id not in qa_data:
+            qa_result = generate_qa(thread_id)
+        
+        # Get questions, answer, summary
+        questions = qa_data.get(thread_id, {}).get("questions", [])
+        answer = qa_data.get(thread_id, {}).get("answers", [""])[0] if questions else ""
+        summary = qa_data.get(thread_id, {}).get("summary", "")
+        
+        # Export data
+        export_data = export_thread_data(thread_id)
+        
+        return f"Thread: {thread['subject']} ({thread['message_count']} messages)", content, questions, answer, summary, export_data
+    
+    view_thread_button.click(
+        fn=lambda: view_thread_details(0 if not search_results_dropdown.value else search_results_dropdown.index),
+        outputs=[thread_info, thread_content, question_dropdown, answer_output, summary_output, export_output]
+    )
+    
+    # Q&A events
+    question_dropdown.change(
+        fn=lambda q, thread_idx: qa_data.get(search_results[thread_idx]["emails"][0]["conversationId"], {}).get("answers", [""])[qa_data.get(search_results[thread_idx]["emails"][0]["conversationId"], {}).get("questions", []).index(q)] if q and thread_idx >= 0 and thread_idx < len(search_results) and search_results[thread_idx]["emails"][0]["conversationId"] in qa_data and q in qa_data.get(search_results[thread_idx]["emails"][0]["conversationId"], {}).get("questions", []) else "",
+        inputs=[question_dropdown, lambda: 0 if not search_results_dropdown.value else search_results_dropdown.index],
+        outputs=answer_output
+    )
+    
+    gen_qa_button.click(
+        fn=lambda thread_idx: (
+            generate_qa(search_results[thread_idx]["emails"][0]["conversationId"]) if thread_idx >= 0 and thread_idx < len(search_results) else "No thread selected",
+            qa_data.get(search_results[thread_idx]["emails"][0]["conversationId"], {}).get("questions", []) if thread_idx >= 0 and thread_idx < len(search_results) else [],
+            qa_data.get(search_results[thread_idx]["emails"][0]["conversationId"], {}).get("answers", [""])[0] if thread_idx >= 0 and thread_idx < len(search_results) and search_results[thread_idx]["emails"][0]["conversationId"] in qa_data and qa_data.get(search_results[thread_idx]["emails"][0]["conversationId"], {}).get("questions", []) else "",
+            qa_data.get(search_results[thread_idx]["emails"][0]["conversationId"], {}).get("summary", "") if thread_idx >= 0 and thread_idx < len(search_results) else ""
+        ),
+        inputs=lambda: 0 if not search_results_dropdown.value else search_results_dropdown.index,
+        outputs=[qa_status, question_dropdown, answer_output, summary_output]
+    )
+    
+    # Export event
+    export_thread_button.click(
+        fn=lambda thread_idx: export_thread_data(search_results[thread_idx]["emails"][0]["conversationId"]) if thread_idx >= 0 and thread_idx < len(search_results) else None,
+        inputs=lambda: 0 if not search_results_dropdown.value else search_results_dropdown.index,
+        outputs=export_output
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    demo.launch()