Update app.py

app.py

@@ -1,303 +1,355 @@
-from flask import Flask, request, jsonify
-from simple_salesforce import Salesforce
+import gradio as gr
 import pandas as pd
 from datetime import datetime
 import logging
-from sklearn.ensemble import IsolationForest
+import plotly.express as px
+from sklearn.ensemble import IsolationForest  # For anomaly detection
 from transformers import pipeline
-import torch
-import os
-import time
-import sys
-import requests
-from requests.exceptions import Timeout
+import torch  # For GPU availability check
+from simple_salesforce import Salesforce  # For Salesforce connection
 
-# Configure logging to console first
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout)
-    ]
-)
+# Configure logging for debugging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Add file handler for logging
+# Salesforce credentials (replace with your actual credentials or use environment variables)
+SALESFORCE_USERNAME = "your_username"
+SALESFORCE_PASSWORD = "your_password"
+SALESFORCE_SECURITY_TOKEN = "your_security_token"
+SALESFORCE_DOMAIN = "login"  # Use "test" for sandbox, "login" for production
+
+# Preload Hugging Face summarization model at startup
+logging.info("Preloading Hugging Face model...")
 try:
-    file_handler = logging.FileHandler('app.log')
-    file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
-    logging.getLogger().addHandler(file_handler)
-    logging.info("File logging enabled successfully")
+    device = 0 if torch.cuda.is_available() else -1
+    summarizer = pipeline("summarization", model="sshleifer/distilbart-cnn-12-6", device=device)  # Lighter model
+    logging.info(f"Hugging Face model preloaded successfully on device: {'GPU' if device == 0 else 'CPU'}")
 except Exception as e:
-    logging.warning(f"Failed to enable file logging: {str(e)}. Continuing with console logging only.")
-
-# Initialize Flask app
-app = Flask(__name__)
-logging.info("Flask app initialized")
-
-# Salesforce credentials (hardcoded for testing)
-SF_USERNAME = "multi-devicelabopsdashboard@sathkrutha.com"
-SF_PASSWORD = "Team@1234"
-SF_SECURITY_TOKEN = "BXgWWNXjvc3zJmVv2O7JfBqCc"
-SF_INSTANCE_URL = "https://multi-devicelabopsdashboard-dev-ed.develop.my.salesforce.com"
-
-# Global variable for Salesforce connection
-sf = None
-
-# Global variable for Hugging Face model (lazy initialization)
-summarizer = None
-logging.info("Hugging Face model set to lazy initialization")
+    logging.error(f"Failed to preload model: {str(e)}")
+    raise e
 
-# Health check endpoint
-@app.route('/health', methods=['GET'])
-def health_check():
-    return jsonify({"status": "App is running"}), 200
-
-# Connect to Salesforce with a timeout
+# Connect to Salesforce
 def connect_to_salesforce():
-    global sf
-    logging.info("Attempting to connect to Salesforce...")
-    start_time = time.time()
     try:
-        session = requests.Session()
-        adapter = requests.adapters.HTTPAdapter(max_retries=3)
-        session.mount('https://', adapter)
-        session.request('GET', SF_INSTANCE_URL, timeout=10)
         sf = Salesforce(
-            username=SF_USERNAME,
-            password=SF_PASSWORD,
-            security_token=SF_SECURITY_TOKEN,
-            instance_url=SF_INSTANCE_URL,
-            session=session
+            username=SALESFORCE_USERNAME,
+            password=SALESFORCE_PASSWORD,
+            security_token=SALESFORCE_SECURITY_TOKEN,
+            domain=SALESFORCE_DOMAIN
         )
-        logging.info(f"Connected to Salesforce successfully in {time.time() - start_time:.2f} seconds")
-    except Timeout:
-        logging.error("Salesforce connection timed out after 10 seconds")
-        sf = None
+        logging.info("Successfully connected to Salesforce")
+        return sf
     except Exception as e:
         logging.error(f"Failed to connect to Salesforce: {str(e)}")
-        sf = None
-
-# Lazy load the Hugging Face model
-def load_huggingface_model():
-    global summarizer
-    if summarizer is None:
-        logging.info("Loading Hugging Face model (t5-small)...")
-        start_time = time.time()
-        try:
-            device = 0 if torch.cuda.is_available() else -1
-            summarizer = pipeline("summarization", model="t5-small", device=device)
-            logging.info(f"Hugging Face model loaded successfully in {time.time() - start_time:.2f} seconds on device: {'GPU' if device == 0 else 'CPU'}")
-        except Exception as e:
-            logging.error(f"Failed to load Hugging Face model: {str(e)}")
-            summarizer = None
+        raise e
 
-# Fetch SmartLog records from Salesforce
-def fetch_smartlog_records(lab_site, start_date, end_date, equipment_type):
-    if sf is None:
-        raise Exception("Salesforce connection not established")
+# Fetch data from Salesforce
+def fetch_salesforce_data(sf, row_limit=10000, progress=gr.Progress()):
+    progress(0.05, "Fetching data from Salesforce...")
     try:
-        logging.info("Fetching SmartLog records from Salesforce...")
-        query = "SELECT Device_Id__c, Log_Type__c, Status__c, Timestamp__c, Usage_Hours__c, Downtime__c, AMC_Date__c FROM SmartLog__c WHERE "
-        conditions = []
-        params = {}
-        if lab_site:
-            conditions.append("Lab_Site__c = :lab_site")
-            params['lab_site'] = lab_site
-        if start_date:
-            conditions.append("Timestamp__c >= :start_date")
-            params['start_date'] = start_date
-        if end_date:
-            conditions.append("Timestamp__c <= :end_date")
-            params['end_date'] = end_date
-        if equipment_type:
-            conditions.append("Log_Type__c = :equipment_type")
-            params['equipment_type'] = equipment_type
-
-        if not conditions:
-            query = query.replace(" WHERE ", "")
-        else:
-            query += " AND ".join(conditions)
+        # Query Salesforce for LabEquipmentLog__c object
+        query = """
+            SELECT Device_ID__c, Log_Type__c, Status__c, Timestamp__c, 
+                   Usage_Hours__c, Downtime__c, AMC_Date__c 
+            FROM LabEquipmentLog__c
+            LIMIT {}
+        """.format(row_limit)
+        result = sf.query_all(query)
+        records = result["records"]
+        
+        # Convert to DataFrame
+        df = pd.DataFrame(records)
+        df = df.rename(columns={
+            "Device_ID__c": "device_id",
+            "Log_Type__c": "log_type",
+            "Status__c": "status",
+            "Timestamp__c": "timestamp",
+            "Usage_Hours__c": "usage_hours",
+            "Downtime__c": "downtime",
+            "AMC_Date__c": "amc_date"
+        })
+        
+        # Ensure proper data types
+        df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
+        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
+        df["usage_hours"] = df["usage_hours"].astype("float32", errors='ignore')
+        df["downtime"] = df["downtime"].astype("float32", errors='ignore')
+        df["device_id"] = df["device_id"].astype("string")
+        
+        logging.info(f"Fetched {len(df)} records from Salesforce")
+        return df
+    except Exception as e:
+        logging.error(f"Failed to fetch Salesforce data: {str(e)}")
+        raise e
 
-        result = sf.query_all(query, **params)
-        records = result['records']
+# Load data from CSV file
+def load_csv_data(file_obj, row_limit=10000, progress=gr.Progress()):
+    progress(0.05, "Loading CSV file...")
+    try:
+        file_name = file_obj.name if hasattr(file_obj, 'name') else file_obj
+        logging.info(f"Processing CSV file: {file_name}")
+        
+        if not file_name.endswith(".csv"):
+            logging.error("Unsupported file format")
+            raise ValueError("Unsupported file format. Please upload a CSV file.")
 
-        data = []
-        for record in records:
-            data.append({
-                'device_id': record['Device_Id__c'],
-                'log_type': record['Log_Type__c'],
-                'status': record['Status__c'],
-                'timestamp': record['Timestamp__c'],
-                'usage_hours': record['Usage_Hours__c'],
-                'downtime': record['Downtime__c'],
-                'amc_date': record['AMC_Date__c']
-            })
-        df = pd.DataFrame(data)
-        df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
-        df['amc_date'] = pd.to_datetime(df['amc_date'], errors='coerce')
-        logging.info(f"Fetched {len(df)} SmartLog records")
+        usecols = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
+        dtypes = {
+            "device_id": "string",
+            "log_type": "string",
+            "status": "string",
+            "usage_hours": "float32",
+            "downtime": "float32",
+            "amc_date": "string"
+        }
+        df = pd.read_csv(file_name, usecols=usecols, dtype=dtypes, nrows=row_limit)
+        
+        # Convert timestamps
+        df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
+        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
+        
+        logging.info(f"File loaded successfully with {len(df)} rows (limited to {row_limit} rows)")
         return df
     except Exception as e:
-        logging.error(f"Failed to fetch SmartLog records: {str(e)}")
+        logging.error(f"Failed to load CSV: {str(e)}")
         raise e
 
 # Format summary prompt and generate report
-def summarize_logs(df):
-    load_huggingface_model()
-    if summarizer is None:
-        return "Failed to load summarization model."
+def summarize_logs(df, progress=gr.Progress()):
+    progress(0.1, "Generating summary report...")
     try:
         total_devices = df["device_id"].nunique()
         most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
+
         prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
         summary = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
+        logging.info("Summary generated successfully")
         return summary
     except Exception as e:
         logging.error(f"Summary generation failed: {str(e)}")
         return "Failed to generate summary."
 
-# Anomaly Detection
-def detect_anomalies(df):
+# Anomaly Detection using Isolation Forest with sampling for large datasets
+def detect_anomalies(df, progress=gr.Progress()):
+    progress(0.4, "Detecting anomalies...")
     try:
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
-            return None, "Anomaly detection requires 'usage_hours' and 'downtime' columns."
+            logging.warning("Required columns for anomaly detection not found")
+            return "Anomaly detection requires 'usage_hours' and 'downtime' columns."
+
         if len(df) > 5000:
             df = df.sample(n=5000, random_state=42)
             logging.info("Sampled data for anomaly detection to 5,000 rows")
+
         features = df[["usage_hours", "downtime"]].fillna(0)
         iso_forest = IsolationForest(contamination=0.1, random_state=42, n_jobs=-1)
         df["anomaly"] = iso_forest.fit_predict(features)
+       
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
         if anomalies.empty:
-            return None, "No anomalies detected."
-        anomaly_lines = []
+            return "No anomalies detected."
+       
+        anomaly_lines = ["**Detected Anomalies:**"]
         for idx, row in anomalies.head(5).iterrows():
-            anomaly_lines.append({
-                "device_id": row['device_id'],
-                "usage_hours": float(row['usage_hours']),
-                "downtime": float(row['downtime']),
-                "timestamp": row['timestamp'].isoformat()
-            })
-        return anomaly_lines, None
+            anomaly_lines.append(f"- Device ID: {row['device_id']}, Usage Hours: {row['usage_hours']}, Downtime: {row['downtime']}, Timestamp: {row['timestamp']}")
+        anomaly_list = "\n".join(anomaly_lines)
+        logging.info("Anomalies detected successfully")
+        return anomaly_list
     except Exception as e:
         logging.error(f"Anomaly detection failed: {str(e)}")
-        return None, f"Anomaly detection failed: {str(e)}"
+        return f"Anomaly detection failed: {str(e)}"
 
-# AMC Reminders
-def check_amc_reminders(df, current_date):
+# AMC Reminders based on device and AMC date
+def check_amc_reminders(df, current_date, progress=gr.Progress()):
+    progress(0.6, "Checking AMC reminders...")
     try:
         if "device_id" not in df.columns or "amc_date" not in df.columns:
-            return None, "AMC reminders require 'device_id' and 'amc_date' columns."
+            logging.warning("Required columns for AMC reminders not found")
+            return "AMC reminders require 'device_id' and 'amc_date' columns."
+
+        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
         current_date = pd.to_datetime(current_date)
+       
         df["days_to_amc"] = (df["amc_date"] - current_date).dt.days
         reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "amc_date"]]
+       
         if reminders.empty:
-            return None, "No AMC reminders due within the next 30 days."
-        reminder_lines = []
+            return "No AMC reminders due within the next 30 days."
+       
+        reminder_lines = ["**Upcoming AMC Reminders:**"]
         for idx, row in reminders.head(5).iterrows():
-            reminder_lines.append({
-                "device_id": row['device_id'],
-                "amc_date": row['amc_date'].isoformat()
-            })
-        return reminder_lines, None
+            reminder_lines.append(f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}")
+        reminder_list = "\n".join(reminder_lines)
+        logging.info("AMC reminders generated successfully")
+        return reminder_list
     except Exception as e:
         logging.error(f"AMC reminder generation failed: {str(e)}")
-        return None, f"AMC reminder generation failed: {str(e)}"
+        return f"AMC reminder generation failed: {str(e)}"
 
-# Dashboard Insights
-def generate_dashboard_insights(df):
-    load_huggingface_model()
-    if summarizer is None:
-        return "Failed to load summarization model."
+# Dashboard Insights (AI-generated executive-level insights)
+def generate_dashboard_insights(df, progress=gr.Progress()):
+    progress(0.8, "Generating dashboard insights...")
     try:
         total_devices = df["device_id"].nunique()
         avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
         prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
         insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
+        logging.info("Dashboard insights generated successfully")
         return insights
     except Exception as e:
         logging.error(f"Dashboard insights generation failed: {str(e)}")
         return f"Dashboard insights generation failed: {str(e)}"
 
-# Create usage chart data for LaTeX table
-def create_usage_chart_data(df):
+# Create a bar chart for usage hours per device
+def create_usage_chart(df, progress=gr.Progress()):
+    progress(0.9, "Creating usage chart...")
     try:
         usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
         if len(usage_data) > 5:
             usage_data = usage_data.nlargest(5, "usage_hours")
             logging.info("Limited chart data to top 5 devices")
-        chart_lines = []
-        for idx, row in usage_data.iterrows():
-            chart_lines.append({
-                "device_id": row['device_id'],
-                "usage_hours": float(row['usage_hours'])
-            })
-        return chart_lines
+
+        custom_colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4']
+        fig = px.bar(
+            usage_data,
+            x="device_id",
+            y="usage_hours",
+            title="Usage Hours per Device",
+            labels={"device_id": "Device ID", "usage_hours": "Usage Hours"},
+            color="device_id",
+            color_discrete_sequence=custom_colors
+        )
+        fig.update_layout(
+            title_font_size=16,
+            margin=dict(l=20, r=20, t=40, b=20),
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            font=dict(size=12)
+        )
+        return fig
     except Exception as e:
-        logging.error(f"Failed to create usage chart data: {str(e)}")
-        return []
+        logging.error(f"Failed to create usage chart: {str(e)}")
+        return None
 
-# HTTP endpoint to process logs
-@app.route('/process_logs', methods=['POST'])
-def process_logs():
+# Main Gradio function
+async def process_logs(file_obj=None, progress=gr.Progress()):
     try:
-        if sf is None:
-            return jsonify({"error": "Salesforce connection not established. Check server logs for details."}), 500
-
-        data = request.get_json()
-        lab_site = data.get('lab_site')
-        start_date = data.get('start_date')
-        end_date = data.get('end_date')
-        equipment_type = data.get('equipment_type')
-        amc_threshold = data.get('amc_threshold', 30)
+        progress(0, "Starting data processing...")
+        
+        # Load data: prioritize CSV if uploaded, otherwise fetch from Salesforce
+        if file_obj:
+            df = load_csv_data(file_obj, row_limit=10000, progress=progress)
+        else:
+            progress(0.05, "No CSV uploaded, fetching from Salesforce...")
+            sf = connect_to_salesforce()
+            df = fetch_salesforce_data(sf, row_limit=10000, progress=progress)
 
-        df = fetch_smartlog_records(lab_site, start_date, end_date, equipment_type)
         if df.empty:
-            return jsonify({"error": "No data available in SmartLog__c."}), 400
+            logging.warning("No data available")
+            return "No data available.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated."
 
-        summary = summarize_logs(df)
+        # Step 1: Summary Report
+        progress(0.2, "Generating summary...")
+        summary = f"**Step 1: Summary Report**  \n{summarize_logs(df, progress)}"
 
-        preview_lines = []
-        for idx, row in df.head().iterrows():
-            preview_lines.append({
-                "row": idx + 1,
-                "device_id": row['device_id'],
-                "timestamp": row['timestamp'].isoformat() if pd.notnull(row['timestamp']) else None,
-                "usage_hours": float(row['usage_hours']) if pd.notnull(row['usage_hours']) else 0,
-                "downtime": float(row['downtime']) if pd.notnull(row['downtime']) else 0,
-                "amc_date": row['amc_date'].isoformat() if pd.notnull(row['amc_date']) else None
-            })
+        # Step 2: Log Preview
+        progress(0.3, "Previewing logs...")
+        if not df.empty:
+            preview_lines = ["**Step 2: Log Preview (First 5 Rows)**"]
+            for idx, row in df.head().iterrows():
+                preview_lines.append(f"**Row {idx + 1}:** Device ID: {row['device_id']}, Timestamp: {row['timestamp']}, Usage Hours: {row['usage_hours']}, Downtime: {row['downtime']}, AMC Date: {row['amc_date']}, Log Type: {row['log_type']}, Status: {row['status']}")
+            preview = "\n".join(preview_lines)
+        else:
+            preview = "**Step 2: Log Preview**  \nNo data available."
 
-        chart_data = create_usage_chart_data(df)
+        # Step 3: Usage Chart
+        chart = create_usage_chart(df, progress)
 
-        anomaly_lines, anomaly_error = detect_anomalies(df)
-        if anomaly_error:
-            anomaly_lines = [{"error": anomaly_error}]
+        # Step 4: Anomaly Detection
+        anomalies = f"**Step 3: Anomaly Detection**  \n{detect_anomalies(df, progress)}"
 
-        reminder_lines, reminder_error = check_amc_reminders(df, datetime.now())
-        if reminder_error:
-            reminder_lines = [{"error": reminder_error}]
+        # Step 5: AMC Reminders
+        amc_reminders = f"**Step 4: AMC Reminders**  \n{check_amc_reminders(df, datetime.now(), progress)}"
 
-        insights = generate_dashboard_insights(df)
+        # Step 6: Dashboard Insights
+        insights = f"**Step 5: Dashboard Insights (AI)**  \n{generate_dashboard_insights(df, progress)}"
 
-        response = {
-            "summary": summary,
-            "log_preview": preview_lines,
-            "usage_chart": chart_data,
-            "anomalies": anomaly_lines,
-            "amc_reminders": reminder_lines,
-            "insights": insights
-        }
+        progress(1.0, "Processing complete!")
+        return summary, preview, chart, anomalies, amc_reminders, insights
+    except Exception as e:
+        logging.error(f"Failed to process data: {str(e)}")
+        return f"Failed to process data: {str(e)}", None, None, None, None, None
 
-        return jsonify(response), 200
+# Gradio Interface with Step-by-Step Layout
+try:
+    logging.info("Initializing Gradio Blocks interface...")
+    with gr.Blocks(css="""
+        .dashboard-container {border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px; background-color: #f9f9f9;}
+        .dashboard-title {font-size: 24px; font-weight: bold; margin-bottom: 5px;}
+        .dashboard-section {margin-bottom: 5px;}
+        .dashboard-section h3 {font-size: 18px; margin-bottom: 2px;}
+        .dashboard-section p {margin: 1px 0; line-height: 1.2;}
+        .dashboard-section li {margin: 1px 0; line-height: 1.2;}
+        .dashboard-section ul {margin: 2px 0; padding-left: 20px;}
+    """) as iface:
+        gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Salesforce + Hugging Face AI)</h1>")
+        gr.Markdown("Upload a CSV file or fetch lab equipment logs from Salesforce to analyze usage.")
 
-    except Exception as e:
-        logging.error(f"Failed to process logs: {str(e)}")
-        return jsonify({"error": str(e)}), 500
+        with gr.Row():
+            with gr.Column(scale=1):
+                file_input = gr.File(label="Upload Logs (CSV)", file_types=[".csv"])
+                submit_button = gr.Button("Analyze Data", variant="primary")
+           
+            with gr.Column(scale=2):
+                with gr.Group(elem_classes="dashboard-container"):
+                    gr.Markdown("<div class='dashboard-title'>Analysis Results (Step-by-Step)</div>")
+                   
+                    # Step 1: Summary Report
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 1: Summary Report")
+                        summary_output = gr.Markdown()
+                   
+                    # Step 2: Log Preview
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 2: Log Preview")
+                        preview_output = gr.Markdown()
+                   
+                    # Step 3: Usage Chart
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 3: Usage Chart")
+                        chart_output = gr.Plot()
+                   
+                    # Step 4: Anomaly Detection
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 4: Anomaly Detection")
+                        anomaly_output = gr.Markdown()
+                   
+                    # Step 5: AMC Reminders
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 5: AMC Reminders")
+                        amc_output = gr.Markdown()
+                   
+                    # Step 6: Dashboard Insights
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 6: Dashboard Insights (AI)")
+                        insights_output = gr.Markdown()
+
+        submit_button.click(
+            fn=process_logs,
+            inputs=[file_input],
+            outputs=[summary_output, preview_output, chart_output, anomaly_output, amc_output, insights_output]
+        )
+
+    logging.info("Gradio interface initialized successfully")
+except Exception as e:
+    logging.error(f"Failed to initialize Gradio interface: {str(e)}")
+    raise e
 
 if __name__ == "__main__":
-    logging.info("Starting Flask application...")
-    start_time = time.time()
-    connect_to_salesforce()
-    logging.info(f"Flask application startup completed in {time.time() - start_time:.2f} seconds")
-    app.run(host="0.0.0.0", port=5000, debug=True)
+    try:
+        logging.info("Launching Gradio interface...")
+        iface.launch(server_name="0.0.0.0", server_port=7860, debug=True, share=False)
+        logging.info("Gradio interface launched successfully")
+    except Exception as e:
+        logging.error(f"Failed to launch Gradio interface: {str(e)}")
+        print(f"Error launching app: {str(e)}")
+        raise e