Create app.py

app.py

@@ -0,0 +1,577 @@
+"""
+LabOps Log Analyzer Dashboard with CSV file upload, PDF generation, and Salesforce integration
+"""
+import gradio as gr
+import pandas as pd
+from datetime import datetime, timedelta
+import logging
+import plotly.express as px
+from sklearn.ensemble import IsolationForest
+from transformers import pipeline
+import torch
+from concurrent.futures import ThreadPoolExecutor
+from simple_salesforce import Salesforce
+import os
+import json
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Salesforce configuration
+try:
+    sf = Salesforce(
+        username=os.getenv('SF_USERNAME'),
+        password=os.getenv('SF_PASSWORD'),
+        security_token=os.getenv('SF_SECURITY_TOKEN'),
+        domain='login'
+    )
+    logging.info("Salesforce connection established")
+except Exception as e:
+    logging.error(f"Failed to connect to Salesforce: {str(e)}")
+    sf = None
+
+# Try to import reportlab
+try:
+    from reportlab.lib.pagesizes import letter
+    from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
+    from reportlab.lib.styles import getSampleStyleSheet
+    reportlab_available = True
+    logging.info("reportlab module successfully imported")
+except ImportError:
+    logging.warning("reportlab module not found. PDF generation disabled.")
+    reportlab_available = False
+
+# Preload Hugging Face model
+logging.info("Preloading Hugging Face model...")
+try:
+    device = 0 if torch.cuda.is_available() else -1
+    summarizer = pipeline(
+        "summarization",
+        model="facebook/bart-large-cnn",
+        device=device,
+        max_length=50,
+        min_length=10,
+        num_beams=4
+    )
+    logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
+except Exception as e:
+    logging.error(f"Failed to preload model: {str(e)}")
+    raise e
+
+# Fetch valid picklist values from Salesforce
+def get_picklist_values(field_name):
+    if sf is None:
+        return []
+    try:
+        obj_desc = sf.SmartLog__c.describe()
+        for field in obj_desc['fields']:
+            if field['name'] == field_name:
+                return [value['value'] for value in field['picklistValues'] if value['active']]
+        return []
+    except Exception as e:
+        logging.error(f"Failed to fetch picklist values for {field_name}: {str(e)}")
+        return []
+
+# Cache picklist values at startup
+status_values = get_picklist_values('Status__c') or ["Active", "Inactive", "Pending"]
+log_type_values = get_picklist_values('Log_Type__c') or ["Smart Log", "Cell Analysis", "UV Verification"]
+logging.info(f"Valid Status__c values: {status_values}")
+logging.info(f"Valid Log_Type__c values: {log_type_values}")
+
+# Map invalid picklist values to valid ones
+picklist_mapping = {
+    'Status__c': {
+        'normal': 'Active',
+        'error': 'Inactive',
+        'warning': 'Pending',
+        'ok': 'Active',
+        'failed': 'Inactive'
+    },
+    'Log_Type__c': {
+        'maint': 'Smart Log',
+        'error': 'Cell Analysis',
+        'ops': 'UV Verification',
+        'maintenance': 'Smart Log',
+        'cell': 'Cell Analysis',
+        'uv': 'UV Verification'
+    }
+}
+
+# Fetch folder ID for "LabOps Reports"
+def get_folder_id(folder_name):
+    if sf is None:
+        return None
+    try:
+        query = f"SELECT Id FROM Folder WHERE Name = '{folder_name}' AND Type = 'Report'"
+        result = sf.query(query)
+        if result['totalSize'] > 0:
+            folder_id = result['records'][0]['Id']
+            logging.info(f"Found folder ID for '{folder_name}': {folder_id}")
+            return folder_id
+        else:
+            logging.error(f"Folder '{folder_name}' not found in Salesforce.")
+            return None
+    except Exception as e:
+        logging.error(f"Failed to fetch folder ID for '{folder_name}': {str(e)}")
+        return None
+
+# Cache the folder ID at startup
+LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
+
+# Create Salesforce reports (Usage and AMC Reminders)
+def create_salesforce_reports(df):
+    if sf is None:
+        return "Salesforce connection not available."
+    if not LABOPS_REPORTS_FOLDER_ID:
+        return "Cannot create reports: 'LabOps Reports' folder not found in Salesforce."
+
+    try:
+        # Usage Report (Summary Report)
+        usage_report_metadata = {
+            "reportMetadata": {
+                "name": f"SmartLog_Usage_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
+                "developerName": f"SmartLog_Usage_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
+                "reportType": {
+                    "type": "CustomObject",
+                    "value": "SmartLog__c"
+                },
+                "reportFormat": "SUMMARY",
+                "reportBooleanFilter": None,
+                "reportFilters": [
+                    {
+                        "column": "SmartLog__c.Status__c",
+                        "operator": "equals",
+                        "value": "Active"
+                    },
+                    {
+                        "column": "SmartLog__c.Timestamp__c",
+                        "operator": "greaterOrEqual",
+                        "value": "THIS_MONTH"
+                    }
+                ],
+                "aggregates": ["s!SmartLog__c.Usage_Hours__c", "s!SmartLog__c.Downtime__c"],
+                "groupingsDown": [
+                    {
+                        "name": "Device_Id__c",
+                        "field": "SmartLog__c.Device_Id__c",
+                        "sortOrder": "Asc",
+                        "sortAggregate": None,
+                        "dateGranularity": "None"
+                    }
+                ],
+                "detailColumns": [
+                    "SmartLog__c.Device_Id__c",
+                    "SmartLog__c.Log_Type__c",
+                    "SmartLog__c.Status__c",
+                    "SmartLog__c.Timestamp__c",
+                    "SmartLog__c.Usage_Hours__c",
+                    "SmartLog__c.Downtime__c",
+                    "SmartLog__c.AMC_Date__c"
+                ],
+                "folderId": LABOPS_REPORTS_FOLDER_ID,
+                "currency": None
+            }
+        }
+        usage_result = sf.restful('analytics/reports', method='POST', json=usage_report_metadata)
+        usage_report_id = usage_result['id']
+        logging.info(f"Usage Report created: {usage_report_id}")
+
+        # AMC Reminders Report (Tabular Report)
+        amc_report_metadata = {
+            "reportMetadata": {
+                "name": f"SmartLog_AMC_Reminders_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
+                "developerName": f"SmartLog_AMC_Reminders_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
+                "reportType": {
+                    "type": "CustomObject",
+                    "value": "SmartLog__c"
+                },
+                "reportFormat": "TABULAR",
+                "reportBooleanFilter": None,
+                "reportFilters": [
+                    {
+                        "column": "SmartLog__c.Status__c",
+                        "operator": "equals",
+                        "value": "Active"
+                    },
+                    {
+                        "column": "SmartLog__c.AMC_Date__c",
+                        "operator": "greaterOrEqual",
+                        "value": "TODAY"
+                    },
+                    {
+                        "column": "SmartLog__c.AMC_Date__c",
+                        "operator": "lessOrEqual",
+                        "value": "NEXT_N_DAYS:30"
+                    }
+                ],
+                "detailColumns": [
+                    "SmartLog__c.Device_Id__c",
+                    "SmartLog__c.AMC_Date__c",
+                    "SmartLog__c.Status__c"
+                ],
+                "folderId": LABOPS_REPORTS_FOLDER_ID,
+                "currency": None
+            }
+        }
+        amc_result = sf.restful('analytics/reports', method='POST', json=amc_report_metadata)
+        amc_report_id = amc_result['id']
+        logging.info(f"AMC Reminders Report created: {amc_report_id}")
+
+        return f"Usage Report ID: {usage_report_id}, AMC Reminders Report ID: {amc_report_id}"
+    except Exception as e:
+        logging.error(f"Failed to create Salesforce reports: {str(e)}")
+        return f"Failed to create reports: {str(e)}"
+
+# Save results to Salesforce SmartLog__c
+def save_to_salesforce(df, summary, anomalies, amc_reminders, insights):
+    if sf is None:
+        return "Salesforce connection not available."
+    try:
+        records = []
+        current_date = datetime.now()
+        next_30_days = current_date + timedelta(days=30)
+        for _, row in df.head(100).iterrows():
+            # Validate and map picklist values
+            status = str(row['status'])
+            log_type = str(row['log_type'])
+            
+            # Map Status__c
+            if status not in status_values:
+                status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
+                if status is None:
+                    logging.warning(f"Skipping record with invalid Status__c: {row['status']}")
+                    continue
+            
+            # Map Log_Type__c
+            if log_type not in log_type_values:
+                log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
+                if log_type is None:
+                    logging.warning(f"Skipping record with invalid Log_Type__c: {row['log_type']}")
+                    continue
+
+            # Ensure AMC_Date__c is in correct format
+            amc_date_str = row['amc_date'].strftime('%Y-%m-%d') if pd.notna(row['amc_date']) else None
+            if amc_date_str:
+                amc_date = datetime.strptime(amc_date_str, '%Y-%m-%d')
+                # Log if this record qualifies for AMC Reminders
+                if status == "Active" and current_date.date() <= amc_date.date() <= next_30_days.date():
+                    logging.info(f"Record qualifies for AMC Reminders: Device ID {row['device_id']}, AMC Date {amc_date_str}")
+
+            record = {
+                'Device_Id__c': str(row['device_id'])[:50],
+                'Log_Type__c': log_type,
+                'Status__c': status,
+                'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
+                'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
+                'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
+                'AMC_Date__c': amc_date_str
+            }
+            records.append(record)
+        
+        # Bulk insert to reduce API calls
+        if records:
+            sf.bulk.SmartLog__c.insert(records)
+            logging.info(f"Saved {len(records)} records to Salesforce")
+        return f"Saved {len(records)} records to Salesforce."
+    except Exception as e:
+        logging.error(f"Failed to save to Salesforce: {str(e)}")
+        return f"Failed to save to Salesforce: {str(e)}"
+
+# Summarize logs
+def summarize_logs(df, progress=gr.Progress()):
+    progress(0.1, "Generating summary report...")
+    try:
+        total_devices = df["device_id"].nunique()
+        most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
+        prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
+        summary = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
+        logging.info("Summary generated successfully")
+        return summary
+    except Exception as e:
+        logging.error(f"Summary generation failed: {str(e)}")
+        return f"Failed to generate summary: {str(e)}"
+
+# Anomaly detection
+def detect_anomalies(df, progress=gr.Progress()):
+    progress(0.4, "Detecting anomalies...")
+    try:
+        if "usage_hours" not in df.columns or "downtime" not in df.columns:
+            return "Anomaly detection requires 'usage_hours' and 'downtime' columns."
+        if len(df) > 1000:
+            df = df.sample(n=1000, random_state=42)
+        features = df[["usage_hours", "downtime"]].fillna(0)
+        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_jobs=-1)
+        df["anomaly"] = iso_forest.fit_predict(features)
+        anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
+        if anomalies.empty:
+            return "No anomalies detected."
+        anomaly_lines = ["Detected Anomalies:"]
+        for _, row in anomalies.head(5).iterrows():
+            anomaly_lines.append(
+                f"- Device ID: {row['device_id']}, Usage Hours: {row['usage_hours']}, "
+                f"Downtime: {row['downtime']}, Timestamp: {row['timestamp']}"
+            )
+        return "\n".join(anomaly_lines)
+    except Exception as e:
+        logging.error(f"Anomaly detection failed: {str(e)}")
+        return f"Anomaly detection failed: {str(e)}"
+
+# AMC reminders (identify records for display)
+def check_amc_reminders(df, current_date, progress=gr.Progress()):
+    progress(0.6, "Checking AMC reminders...")
+    try:
+        if "device_id" not in df.columns or "amc_date" not in df.columns:
+            return "AMC reminders require 'device_id' and 'amc_date' columns."
+        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
+        current_date = pd.to_datetime(current_date)
+        df["days_to_amc"] = (df["amc_date"] - current_date).dt.days
+        reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "amc_date"]]
+        if reminders.empty:
+            return "No AMC reminders due within the next 30 days."
+        reminder_lines = ["Upcoming AMC Reminders:"]
+        for _, row in reminders.head(5).iterrows():
+            reminder_lines.append(f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}")
+        return "\n".join(reminder_lines)
+    except Exception as e:
+        logging.error(f"AMC reminder generation failed: {str(e)}")
+        return f"AMC reminder generation failed: {str(e)}"
+
+# Dashboard insights
+def generate_dashboard_insights(df, progress=gr.Progress()):
+    progress(0.8, "Generating dashboard insights...")
+    try:
+        total_devices = df["device_id"].nunique()
+        avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
+        prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
+        insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
+        return insights
+    except Exception as e:
+        logging.error(f"Dashboard insights generation failed: {str(e)}")
+        return f"Dashboard insights generation failed: {str(e)}"
+
+# Create usage chart
+def create_usage_chart(df, progress=gr.Progress()):
+    progress(0.9, "Creating usage chart...")
+    try:
+        usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
+        if len(usage_data) > 5:
+            usage_data = usage_data.nlargest(5, "usage_hours")
+        custom_colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4']
+        fig = px.bar(
+            usage_data,
+            x="device_id",
+            y="usage_hours",
+            title="Usage Hours per Device",
+            labels={"device_id": "Device ID", "usage_hours": "Usage Hours"},
+            color="device_id",
+            color_discrete_sequence=custom_colors
+        )
+        fig.update_layout(
+            title_font_size=16,
+            margin=dict(l=20, r=20, t=40, b=20),
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            font=dict(size=12)
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create usage chart: {str(e)}")
+        return None
+
+# Generate PDF content
+def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights):
+    if not reportlab_available:
+        return None
+    try:
+        pdf_path = f"analysis_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
+        doc = SimpleDocTemplate(pdf_path, pagesize=letter)
+        styles = getSampleStyleSheet()
+        story = []
+
+        def safe_paragraph(text, style):
+            return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
+
+        story.append(Paragraph("LabOps Log Analysis Report", styles['Title']))
+        story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
+        story.append(Spacer(1, 12))
+
+        story.append(Paragraph("Summary Report", styles['Heading2']))
+        story.append(safe_paragraph(summary or "No summary available.", styles['Normal']))
+        story.append(Spacer(1, 12))
+
+        story.append(Paragraph("Log Preview", styles['Heading2']))
+        story.append(safe_paragraph(preview or "No preview available.", styles['Normal']))
+        story.append(Spacer(1, 12))
+
+        story.append(Paragraph("Anomaly Detection", styles['Heading2']))
+        story.append(safe_paragraph(anomalies or "No anomalies detected.", styles['Normal']))
+        story.append(Spacer(1, 12))
+
+        story.append(Paragraph("AMC Reminders", styles['Heading2']))
+        story.append(safe_paragraph(amc_reminders or "No AMC reminders.", styles['Normal']))
+        story.append(Spacer(1, 12))
+
+        story.append(Paragraph("Dashboard Insights", styles['Heading2']))
+        story.append(safe_paragraph(insights or "No insights generated.", styles['Normal']))
+
+        doc.build(story)
+        logging.info(f"PDF generated at {pdf_path}")
+        return pdf_path
+    except Exception as e:
+        logging.error(f"Failed to generate PDF: {str(e)}")
+        return None
+
+# Main Gradio function
+async def process_logs(file_obj, progress=gr.Progress()):
+    try:
+        progress(0, "Starting file processing...")
+        if not file_obj:
+            return "No file uploaded.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, "No Salesforce data saved.", "No report created."
+
+        file_name = file_obj.name
+        logging.info(f"Processing file: {file_name}")
+        
+        if not file_name.endswith(".csv"):
+            return "Please upload a CSV file.", "", None, "", "", "", None, "", ""
+
+        required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
+        dtypes = {
+            "device_id": "string",
+            "log_type": "string",
+            "status": "string",
+            "usage_hours": "float32",
+            "downtime": "float32",
+            "amc_date": "string"
+        }
+        df = pd.read_csv(file_obj, dtype=dtypes)
+        missing_columns = [col for col in required_columns if col not in df.columns]
+        if missing_columns:
+            return f"Missing columns: {missing_columns}", None, None, None, None, None, None, None, None
+        df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
+        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
+        if df.empty:
+            return "No data available.", None, None, None, None, None, None, None, None
+
+        with ThreadPoolExecutor() as executor:
+            future_summary = executor.submit(summarize_logs, df)
+            future_anomalies = executor.submit(detect_anomalies, df)
+            future_amc = executor.submit(check_amc_reminders, df, datetime.now())
+            future_insights = executor.submit(generate_dashboard_insights, df)
+            future_chart = executor.submit(create_usage_chart, df)
+            future_reports = executor.submit(create_salesforce_reports, df)
+
+            summary = f"Step 1: Summary Report\n{future_summary.result()}"
+            anomalies = f"Anomaly Detection\n{future_anomalies.result()}"
+            amc_reminders = f"AMC Reminders\n{future_amc.result()}"
+            insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
+            chart = future_chart.result()
+            report_result = future_reports.result()
+
+        preview_lines = ["Step 2: Log Preview (First 5 Rows)"]
+        for idx, row in df.head(5).iterrows():
+            preview_lines.append(
+                f"Row {idx + 1}: Device ID: {row['device_id']}, "
+                f"Log Type: {row['log_type']}, Status: {row['status']}, "
+                f"Timestamp: {row['timestamp']}, Usage Hours: {row['usage_hours']}, "
+                f"Downtime: {row['downtime']}, AMC Date: {row['amc_date']}"
+            )
+        preview = "\n".join(preview_lines)
+
+        salesforce_result = save_to_salesforce(df, summary, anomalies, amc_reminders, insights)
+        pdf_file = generate_pdf_content(summary, preview, anomalies, amc_reminders, insights)
+
+        progress(1.0, "Done!")
+        return summary, preview, chart, anomalies, amc_reminders, insights, pdf_file, salesforce_result, report_result
+    except Exception as e:
+        logging.error(f"Failed to process file: {str(e)}")
+        return f"Error: {str(e)}", None, None, None, None, None, None, None, None
+
+# Gradio Interface
+try:
+    logging.info("Initializing Gradio interface...")
+    with gr.Blocks(css="""
+        .dashboard-container {border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px;}
+        .dashboard-title {font-size: 24px; font-weight: bold; margin-bottom: 5px;}
+        .dashboard-section {margin-bottom: 20px;}
+        .dashboard-section h3 {font-size: 18px; margin-bottom: 2px;}
+        .dashboard-section p {margin: 1px 0; line-height: 1.2;}
+        .dashboard-section ul {margin: 2px 0; padding-left: 20px;}
+    """) as iface:
+        gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
+        gr.Markdown("Upload a CSV file to analyze and generate Salesforce reports.")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                file_input = gr.File(label="Upload Logs (CSV)", file_types=[".csv"])
+                submit_button = gr.Button("Analyze", variant="primary")
+
+            with gr.Column(scale=2):
+                with gr.Group(elem_classes="dashboard-container"):
+                    gr.Markdown("<div class='dashboard-title'>Analysis Results</div>")
+                    
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 1: Summary Report")
+                        summary_output = gr.Markdown()
+                    
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 2: Log Preview")
+                        preview_output = gr.Markdown()
+                        
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 3: Usage Chart")
+                        chart_output = gr.Plot()
+                        
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 4: Anomaly Detection")
+                        anomaly_output = gr.Markdown()
+                        
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 5: AMC Reminders")
+                        amc_output = gr.Markdown()
+                        
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Step 6: Insights (AI)")
+                        insights_output = gr.Markdown()
+                        
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Salesforce Integration")
+                        salesforce_output = gr.Markdown()
+                        
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Salesforce Reports")
+                        report_output = gr.Markdown()
+                        
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Download Report")
+                        pdf_output = gr.File(label="Download Analysis Report as PDF")
+
+        submit_button.click(
+            fn=process_logs,
+            inputs=[file_input],
+            outputs=[
+                summary_output,
+                preview_output,
+                chart_output,
+                anomaly_output,
+                amc_output,
+                insights_output,
+                pdf_output,
+                salesforce_output,
+                report_output
+            ]
+        )
+
+    logging.info("Gradio interface initialized successfully")
+except Exception as e:
+    logging.error(f"Failed to initialize Gradio interface: {str(e)}")
+    raise e
+
+if __name__ == "__main__":
+    try:
+        logging.info("Launching Gradio interface...")
+        iface.launch(server_name="0.0.0.0", server_port=7860, debug=True, share=False)
+        logging.info("Gradio interface launched successfully")
+    except Exception as e:
+        logging.error(f"Failed to launch Gradio interface: {str(e)}")
+        print(f"Error launching app: {str(e)}")
+        raise e