Update app.py

app.py

@@ -1,5 +1,5 @@
 """
-LabOps Log Analyzer Dashboard with CSV file upload and optional PDF generation
+LabOps Log Analyzer Dashboard with CSV file upload, PDF generation, and Salesforce integration
 """
 import gradio as gr
 import pandas as pd
@@ -9,6 +9,9 @@ import plotly.express as px
 from sklearn.ensemble import IsolationForest
 from transformers import pipeline
 import torch
+from concurrent.futures import ThreadPoolExecutor
+from simple_salesforce import Salesforce
+import os
 
 # Try to import reportlab for PDF generation
 try:
@@ -18,22 +21,65 @@ try:
     reportlab_available = True
     logging.info("reportlab module successfully imported")
 except ImportError:
-    logging.warning("reportlab module not found. PDF generation will be disabled. Install with: pip install reportlab")
+    logging.warning("reportlab module not found. PDF generation will be disabled.")
     reportlab_available = False
 
-# Configure logging for debugging
+# Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Preload Hugging Face summarization model at startup
+# Salesforce configuration (environment variables for security)
+try:
+    sf = Salesforce(
+        username=os.getenv('SF_USERNAME'),
+        password=os.getenv('SF_PASSWORD'),
+        security_token=os.getenv('SF_SECURITY_TOKEN'),
+        domain='login'  # Use 'test' for sandbox
+    )
+    logging.info("Salesforce connection established")
+except Exception as e:
+    logging.error(f"Failed to connect to Salesforce: {str(e)}")
+    sf = None
+
+# Preload Hugging Face summarization model
 logging.info("Preloading Hugging Face model...")
 try:
     device = 0 if torch.cuda.is_available() else -1
-    summarizer = pipeline("summarization", model="sshleifer/distilbart-cnn-12-6", device=device)
-    logging.info(f"Hugging Face model preloaded successfully on device: {'GPU' if device == 0 else 'CPU'}")
+    summarizer = pipeline(
+        "summarization",
+        model="facebook/bart-large-cnn",
+        device=device,
+        max_length=50,
+        min_length=10,
+        num_beams=4
+    )
+    logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
 except Exception as e:
     logging.error(f"Failed to preload model: {str(e)}")
     raise e
 
+# Save results to Salesforce SmartLog__c object
+def save_to_salesforce(df, summary, anomalies, amc_reminders, insights):
+    if sf is None:
+        logging.error("Salesforce connection not available")
+        return "Salesforce connection not available."
+    try:
+        for _, row in df.head(100).iterrows():  # Limit to 100 rows to avoid governor limits
+            record = {
+                'Device_Id__c': str(row['device_id'])[:50],
+                'Log_Type__c': str(row['log_type']),
+                'Status__c': str(row['status']),
+                'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
+                'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
+                'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
+                'AMC_Date__c': row['amc_date'].strftime('%Y-%m-%d') if pd.notna(row['amc_date']) else None
+            }
+            sf.SmartLog__c.create(record)
+        logging.info("Data saved to Salesforce successfully")
+        return "Data saved to Salesforce."
+    except Exception as e:
+        logging.error(f"Failed to save to Salesforce: {str(e)}")
+        return f"Failed to save to Salesforce: {str(e)}"
+
 # Format summary prompt and generate report
 def summarize_logs(df, progress=gr.Progress()):
     progress(0.1, "Generating summary report...")
@@ -53,11 +99,9 @@ def detect_anomalies(df, progress=gr.Progress()):
     progress(0.4, "Detecting anomalies...")
     try:
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
-            logging.warning("Required columns for anomaly detection not found")
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns."
-        if len(df) > 5000:
-            df = df.sample(n=5000, random_state=42)
-            logging.info("Sampled data for anomaly detection to 5,000 rows")
+        if len(df) > 1000:  # Reduced sample size for speed
+            df = df.sample(n=1000, random_state=42)
         features = df[["usage_hours", "downtime"]].fillna(0)
         iso_forest = IsolationForest(contamination=0.1, random_state=42, n_jobs=-1)
         df["anomaly"] = iso_forest.fit_predict(features)
@@ -65,21 +109,21 @@ def detect_anomalies(df, progress=gr.Progress()):
         if anomalies.empty:
             return "No anomalies detected."
         anomaly_lines = ["Detected Anomalies:"]
-        for idx, row in anomalies.head(5).iterrows():
-            anomaly_lines.append(f"- Device ID: {row['device_id']}, Usage Hours: {row['usage_hours']}, Downtime: {row['downtime']}, Timestamp: {row['timestamp']}")
-        anomaly_list = "\n".join(anomaly_lines)
-        logging.info("Anomalies detected successfully")
-        return anomaly_list
+        for _, row in anomalies.head(5).iterrows():
+            anomaly_lines.append(
+                f"- Device ID: {row['device_id']}, Usage Hours: {row['usage_hours']}, "
+                f"Downtime: {row['downtime']}, Timestamp: {row['timestamp']}"
+            )
+        return "\n".join(anomaly_lines)
     except Exception as e:
         logging.error(f"Anomaly detection failed: {str(e)}")
         return f"Anomaly detection failed: {str(e)}"
 
-# AMC Reminders based on device and AMC date
+# AMC Reminders
 def check_amc_reminders(df, current_date, progress=gr.Progress()):
     progress(0.6, "Checking AMC reminders...")
     try:
         if "device_id" not in df.columns or "amc_date" not in df.columns:
-            logging.warning("Required columns for AMC reminders not found")
             return "AMC reminders require 'device_id' and 'amc_date' columns."
         df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
         current_date = pd.to_datetime(current_date)
@@ -88,16 +132,14 @@ def check_amc_reminders(df, current_date, progress=gr.Progress()):
         if reminders.empty:
             return "No AMC reminders due within the next 30 days."
         reminder_lines = ["Upcoming AMC Reminders:"]
-        for idx, row in reminders.head(5).iterrows():
+        for _, row in reminders.head(5).iterrows():
             reminder_lines.append(f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}")
-        reminder_list = "\n".join(reminder_lines)
-        logging.info("AMC reminders generated successfully")
-        return reminder_list
+        return "\n".join(reminder_lines)
     except Exception as e:
         logging.error(f"AMC reminder generation failed: {str(e)}")
         return f"AMC reminder generation failed: {str(e)}"
 
-# Dashboard Insights (AI-generated)
+# Dashboard Insights
 def generate_dashboard_insights(df, progress=gr.Progress()):
     progress(0.8, "Generating dashboard insights...")
     try:
@@ -105,20 +147,18 @@ def generate_dashboard_insights(df, progress=gr.Progress()):
         avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
         prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
         insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
-        logging.info("Dashboard insights generated successfully")
         return insights
     except Exception as e:
         logging.error(f"Dashboard insights generation failed: {str(e)}")
         return f"Dashboard insights generation failed: {str(e)}"
 
-# Create a bar chart for usage hours per device
+# Create a bar chart
 def create_usage_chart(df, progress=gr.Progress()):
     progress(0.9, "Creating usage chart...")
     try:
         usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
         if len(usage_data) > 5:
             usage_data = usage_data.nlargest(5, "usage_hours")
-            logging.info("Limited chart data to top 5 devices")
         custom_colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4']
         fig = px.bar(
             usage_data,
@@ -141,54 +181,50 @@ def create_usage_chart(df, progress=gr.Progress()):
         logging.error(f"Failed to create usage chart: {str(e)}")
         return None
 
-# Generate PDF content using reportlab if available
+# Generate PDF content
 def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights):
     if not reportlab_available:
-        logging.warning("Skipping PDF generation: reportlab not installed")
         return None
     try:
-        pdf_path = "analysis_report.pdf"
+        pdf_path = f"analysis_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
         doc = SimpleDocTemplate(pdf_path, pagesize=letter)
         styles = getSampleStyleSheet()
         story = []
 
+        def safe_paragraph(text, style):
+            return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
+
         # Title
         story.append(Paragraph("LabOps Log Analysis Report", styles['Title']))
-        story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d')}", styles['Normal']))
+        story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
         story.append(Spacer(1, 12))
 
         # Summary Report
         story.append(Paragraph("Summary Report", styles['Heading2']))
-        for line in summary.split('\n'):
-            story.append(Paragraph(line, styles['Normal']))
+        story.append(safe_paragraph(summary or "No summary available.", styles['Normal']))
         story.append(Spacer(1, 12))
 
         # Log Preview
         story.append(Paragraph("Log Preview", styles['Heading2']))
-        for line in preview.split('\n'):
-            story.append(Paragraph(line, styles['Normal']))
+        story.append(safe_paragraph(preview or "No preview available.", styles['Normal']))
         story.append(Spacer(1, 12))
 
         # Anomaly Detection
         story.append(Paragraph("Anomaly Detection", styles['Heading2']))
-        for line in anomalies.split('\n'):
-            story.append(Paragraph(line, styles['Normal']))
+        story.append(safe_paragraph(anomalies or "No anomalies detected.", styles['Normal']))
         story.append(Spacer(1, 12))
 
         # AMC Reminders
         story.append(Paragraph("AMC Reminders", styles['Heading2']))
-        for line in amc_reminders.split('\n'):
-            story.append(Paragraph(line, styles['Normal']))
+        story.append(safe_paragraph(amc_reminders or "No AMC reminders.", styles['Normal']))
         story.append(Spacer(1, 12))
 
         # Dashboard Insights
         story.append(Paragraph("Dashboard Insights", styles['Heading2']))
-        for line in insights.split('\n'):
-            story.append(Paragraph(line, styles['Normal']))
+        story.append(safe_paragraph(insights or "No insights generated.", styles['Normal']))
 
-        # Build PDF
         doc.build(story)
-        logging.info(f"PDF generated successfully at {pdf_path}")
+        logging.info(f"PDF generated at {pdf_path}")
         return pdf_path
     except Exception as e:
         logging.error(f"Failed to generate PDF: {str(e)}")
@@ -199,95 +235,68 @@ async def process_logs(file_obj, progress=gr.Progress()):
     try:
         progress(0, "Starting file processing...")
         if not file_obj:
-            logging.warning("No file uploaded, returning empty results")
-            return "No file uploaded.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None
+            return "No file uploaded.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, "No Salesforce data saved."
 
         file_name = file_obj.name
         logging.info(f"Processing file: {file_name}")
         
         if not file_name.endswith(".csv"):
-            logging.error("Unsupported file format")
-            return "Unsupported file format. Please upload a CSV file.", "", None, "", "", "", None
-
-        try:
-            required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
-            dtypes = {
-                "device_id": "string",
-                "log_type": "string",
-                "status": "string",
-                "usage_hours": "float32",
-                "downtime": "float32",
-                "amc_date": "string"
-            }
-            df = pd.read_csv(file_obj, dtype=dtypes)
-            # Check for required columns
-            missing_columns = [col for col in required_columns if col not in df.columns]
-            if missing_columns:
-                logging.error(f"Missing columns: {missing_columns}")
-                return f"Missing required columns: {missing_columns}", None, None, None, None, None, None
-            logging.info(f"File loaded successfully with {len(df)} rows")
-        except Exception as e:
-            logging.error(f"Failed to load CSV: {str(e)}")
-            return f"Failed to load CSV: {str(e)}", None, None, None, None, None, None
-
-        try:
-            df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
-        except Exception as e:
-            logging.error(f"Date conversion failed: {str(e)}")
-            return f"Failed to convert timestamp to datetime: {str(e)}", None, None, None, None, None, None
-
+            return "Please upload a CSV file.", "", None, "", "", "", None, ""
+
+        # Load CSV
+        required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
+        dtypes = {
+            "device_id": "string",
+            "log_type": "string",
+            "status": "string",
+            "usage_hours": "float32",
+            "downtime": "float32",
+            "amc_date": "string"
+        }
+        df = pd.read_csv(file_obj, dtype=dtypes)
+        missing_columns = [col for col in required_columns if col not in df.columns]
+        if missing_columns:
+            return f"Missing columns: {missing_columns}", None, None, None, None, None, None, None
+        df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
         if df.empty:
-            logging.warning("No data provided in the file")
-            return "No data available in provided.", None, None, None, None, None, None
-
-        # Step 1: Summary
-        progress(0.2, "Summary...")
-        summary = f"Step 1: Summary Report\n{summarize_logs(df)}"
-
-        # Step 2: Log Preview
-        progress(0.3, "Previewing...")
-        if not df.empty:
-            preview_lines = ["Step 2: Log Preview (First 5 Rows)"]
-            for idx, row in df.head(5).iterrows():
-                preview_lines.append(
-                    f"Row {idx + 1}: Device ID: {row['device_id']}, "
-                    f"Log Type: {row['log_type']}, Status: {row['status']}, "
-                    f"Timestamp: {row['timestamp']}, Usage Hours: {row['usage_hours']}, "
-                    f"Downtime: {row['downtime']}, AMC Date: {row['amc_date']}"
-                )
-            preview = "\n".join(preview_lines)
-        else:
-            preview = "Step 2: Log Preview\nNo data available."
-
-        # Step 3: Usage Chart
-        progress(0.5, "Chart...")
-        chart = create_usage_chart(df)
-
-        # Step 4: Anomaly Detection
-        progress(0.7, "Anomalies...")
-        anomalies = f"Anomaly Detection\n{detect_anomalies(df)}"
-
-        # Step 5: AMC Reminders
-        progress(0.8, "AMC...")
-        amc_reminders = f"AMC Reminders\n{check_amc_reminders(df, datetime.now())}"
-
-        # Step 6: Insights
-        progress(0.9, "Insights...")
-        insights = f"Dashboard Insights (AI)\n{generate_dashboard_insights(df)}"
-
-        # Generate PDF if available
-        progress(0.95, "PDF...")
-        pdf_file = generate_pdf_content(summary, preview, anomalies, amc_reminders, insights) if reportlab_available else None
-        if pdf_file is None and reportlab_available:
-            logging.warning("PDF generation failed")
-        elif pdf_file is None:
-            logging.info("PDF skipped: no reportlab")
+            return "No data available.", None, None, None, None, None, None, None
+
+        # Parallel processing for speed
+        with ThreadPoolExecutor() as executor:
+            future_summary = executor.submit(summarize_logs, df)
+            future_anomalies = executor.submit(detect_anomalies, df)
+            future_amc = executor.submit(check_amc_reminders, df, datetime.now())
+            future_insights = executor.submit(generate_dashboard_insights, df)
+            future_chart = executor.submit(create_usage_chart, df)
+
+            summary = f"Step 1: Summary Report\n{future_summary.result()}"
+            anomalies = f"Anomaly Detection\n{future_anomalies.result()}"
+            amc_reminders = f"AMC Reminders\n{future_amc.result()}"
+            insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
+            chart = future_chart.result()
+
+        # Log Preview
+        preview_lines = ["Step 2: Log Preview (First 5 Rows)"]
+        for idx, row in df.head(5).iterrows():
+            preview_lines.append(
+                f"Row {idx + 1}: Device ID: {row['device_id']}, "
+                f"Log Type: {row['log_type']}, Status: {row['status']}, "
+                f"Timestamp: {row['timestamp']}, Usage Hours: {row['usage_hours']}, "
+                f"Downtime: {row['downtime']}, AMC Date: {row['amc_date']}"
+            )
+        preview = "\n".join(preview_lines)
+
+        # Save to Salesforce
+        salesforce_result = save_to_salesforce(df, summary, anomalies, amc_reminders, insights)
+
+        # Generate PDF
+        pdf_file = generate_pdf_content(summary, preview, anomalies, amc_reminders, insights)
 
         progress(1.0, "Done!")
-        return summary, preview, chart, anomalies, amc_reminders, insights, pdf_file
+        return summary, preview, chart, anomalies, amc_reminders, insights, pdf_file, salesforce_result
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error processing file: {str(e)}", None, None, None, None, None, None
+        return f"Error processing file: {str(e)}", None, None, None, None, None, None, None
 
 # Gradio Interface
 try:
@@ -312,37 +321,34 @@ try:
                 with gr.Group(elem_classes="dashboard-container"):
                     gr.Markdown("<div class='dashboard-title'>Analysis Results (Step-by-Step)</div>")
                     
-                    # Step 1: Summary Report
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Step 1: Summary Report")
                         summary_output = gr.Markdown()
                     
-                    # Step 2: Log Preview
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Step 2: Log Preview")
                         preview_output = gr.Markdown()
                         
-                    # Step 3: Usage Chart
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Step 3: Usage Chart")
                         chart_output = gr.Plot()
                         
-                    # Step 4: Anomaly Detection
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Step 4: Anomaly Detection")
                         anomaly_output = gr.Markdown()
                         
-                    # Step 5: AMC Reminders
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Step 5: AMC Reminders")
                         amc_output = gr.Markdown()
                         
-                    # Step 6: Dashboard Insights
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Step 6: Insights (AI)")
                         insights_output = gr.Markdown()
                         
-                    # PDF Download
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Salesforce Integration")
+                        salesforce_output = gr.Markdown()
+                        
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Download Report")
                         pdf_output = gr.File(label="Download Analysis Report as PDF")
@@ -350,7 +356,16 @@ try:
         submit_button.click(
             fn=process_logs,
             inputs=[file_input],
-            outputs=[summary_output, preview_output, chart_output, anomaly_output, amc_output, insights_output, pdf_output]
+            outputs=[
+                summary_output,
+                preview_output,
+                chart_output,
+                anomaly_output,
+                amc_output,
+                insights_output,
+                pdf_output,
+                salesforce_output
+            ]
         )
 
     logging.info("Gradio interface initialized successfully")
@@ -366,4 +381,4 @@ if __name__ == "__main__":
     except Exception as e:
         logging.error(f"Failed to launch Gradio interface: {str(e)}")
         print(f"Error launching app: {str(e)}")
-        raise e
+        raise e