Update app.py

app.py

@@ -10,10 +10,24 @@ import os
 import io
 import time
 import asyncio
+from simple_salesforce import Salesforce
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
+# Salesforce configuration
+try:
+    sf = Salesforce(
+        username='multi-devicelabopsdashboard@sathkrutha.com',
+        password='Team@1234',
+        security_token=os.getenv('SF_SECURITY_TOKEN', ''),
+        domain='login'
+    )
+    logging.info("Salesforce connection established")
+except Exception as e:
+    logging.error(f"Failed to connect to Salesforce: {str(e)}")
+    sf = None
+
 # Try to import reportlab
 try:
     from reportlab.lib.pagesizes import letter
@@ -26,6 +40,166 @@ except ImportError:
     logging.warning("reportlab module not found. PDF generation disabled.")
     reportlab_available = False
 
+# Cache picklist values at startup
+def get_picklist_values(field_name):
+    if sf is None:
+        return []
+    try:
+        obj_desc = sf.SmartLog__c.describe()
+        for field in obj_desc['fields']:
+            if field['name'] == field_name:
+                return [value['value'] for value in field['picklistValues'] if value['active']]
+        return []
+    except Exception as e:
+        logging.error(f"Failed to fetch picklist values for {field_name}: {str(e)}")
+        return []
+
+status_values = get_picklist_values('Status__c') or ["Active", "Inactive", "Pending"]
+log_type_values = get_picklist_values('Log_Type__c') or ["Smart Log", "Cell Analysis", "UV Verification"]
+logging.info(f"Valid Status__c values: {status_values}")
+logging.info(f"Valid Log_Type__c values: {log_type_values}")
+
+# Map invalid picklist values
+picklist_mapping = {
+    'Status__c': {
+        'normal': 'Active',
+        'error': 'Inactive',
+        'warning': 'Pending',
+        'ok': 'Active',
+        'failed': 'Inactive'
+    },
+    'Log_Type__c': {
+        'maint': 'Smart Log',
+        'error': 'Cell Analysis',
+        'ops': 'UV Verification',
+        'maintenance': 'Smart Log',
+        'cell': 'Cell Analysis',
+        'uv': 'UV Verification',
+        'weight log': 'Smart Log'
+    }
+}
+
+# Cache folder ID for Salesforce reports
+def get_folder_id(folder_name):
+    if sf is None:
+        return None
+    try:
+        query = f"SELECT Id FROM Folder WHERE Name = '{folder_name}' AND Type = 'Report'"
+        result = sf.query(query)
+        if result['totalSize'] > 0:
+            folder_id = result['records'][0]['Id']
+            logging.info(f"Found folder ID for '{folder_name}': {folder_id}")
+            return folder_id
+        else:
+            logging.error(f"Folder '{folder_name}' not found in Salesforce.")
+            return None
+    except Exception as e:
+        logging.error(f"Failed to fetch folder ID for '{folder_name}': {str(e)}")
+        return None
+
+LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
+
+# Salesforce report creation
+def create_salesforce_reports(df):
+    if sf is None or not LABOPS_REPORTS_FOLDER_ID:
+        logging.error("Cannot create Salesforce reports: No connection or folder ID")
+        return
+    try:
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        reports = [
+            {
+                "reportMetadata": {
+                    "name": f"SmartLog_Usage_Report_{timestamp}",
+                    "developerName": f"SmartLog_Usage_Report_{timestamp}",
+                    "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
+                    "reportFormat": "TABULAR",
+                    "reportBooleanFilter": None,
+                    "reportFilters": [],
+                    "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.Usage_Hours__c"],
+                    "folderId": LABOPS_REPORTS_FOLDER_ID
+                }
+            },
+            {
+                "reportMetadata": {
+                    "name": f"SmartLog_AMC_Reminders_{timestamp}",
+                    "developerName": f"SmartLog_AMC_Reminders_{timestamp}",
+                    "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
+                    "reportFormat": "TABULAR",
+                    "reportBooleanFilter": None,
+                    "reportFilters": [],
+                    "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.AMC_Date__c"],
+                    "folderId": LABOPS_REPORTS_FOLDER_ID
+                }
+            }
+        ]
+        for report in reports:
+            sf.restful('analytics/reports', method='POST', json=report)
+        logging.info("Salesforce reports created successfully")
+    except Exception as e:
+        logging.error(f"Failed to create Salesforce reports: {str(e)}")
+
+# Save to Salesforce
+def save_to_salesforce(df, reminders_df):
+    if sf is None:
+        logging.error("No Salesforce connection available")
+        return
+    try:
+        logging.info("Starting Salesforce save operation")
+        current_date = datetime.now()
+        next_30_days = current_date + timedelta(days=30)
+        records = []
+        reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
+        logging.info(f"Processing {len(df)} records for Salesforce")
+
+        for idx, row in df.iterrows():
+            status = str(row['status']).lower()
+            log_type = str(row['log_type']).lower()
+            status_mapped = picklist_mapping['Status__c'].get(status, status_values[0] if status_values else 'Active')
+            log_type_mapped = picklist_mapping['Log_Type__c'].get(log_type, log_type_values[0] if log_type_values else 'Smart Log')
+            
+            if not status_mapped or not log_type_mapped:
+                logging.warning(f"Skipping record {idx}: Invalid status ({status}) or log_type ({log_type})")
+                continue
+
+            amc_date_str = None
+            if pd.notna(row['amc_date']):
+                try:
+                    amc_date = pd.to_datetime(row['amc_date']).strftime('%Y-%m-%d')
+                    amc_date_str = amc_date
+                    amc_date_dt = datetime.strptime(amc_date, '%Y-%m-%d')
+                    if status_mapped == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
+                        logging.info(f"AMC Reminder for Device ID {row['device_id']}: {amc_date}")
+                except Exception as e:
+                    logging.warning(f"Invalid AMC date for Device ID {row['device_id']}: {str(e)}")
+
+            record = {
+                'Device_Id__c': str(row['device_id'])[:50],
+                'Log_Type__c': log_type_mapped,
+                'Status__c': status_mapped,
+                'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
+                'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
+                'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
+                'AMC_Date__c': amc_date_str
+            }
+            records.append(record)
+
+        if records:
+            batch_size = 100
+            for i in range(0, len(records), batch_size):
+                batch = records[i:i + batch_size]
+                try:
+                    result = sf.bulk.SmartLog__c.insert(batch)
+                    logging.info(f"Saved {len(batch)} records to Salesforce in batch {i//batch_size + 1}")
+                    for res in result:
+                        if not res['success']:
+                            logging.error(f"Failed to save record: {res['errors']}")
+                except Exception as e:
+                    logging.error(f"Failed to save batch {i//batch_size + 1}: {str(e)}")
+        else:
+            logging.warning("No records to save to Salesforce")
+    except Exception as e:
+        logging.error(f"Failed to save to Salesforce: {str(e)}")
+
 # Summarize logs
 def summarize_logs(df):
     try:
@@ -65,8 +239,7 @@ def check_amc_reminders(df, current_date):
         reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
         if reminders.empty:
             return "No AMC reminders due within the next 30 days.", reminders
-        # Modified to show all reminders instead of limiting to 5
-        return "\n".join([f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}" for _, row in reminders.iterrows()]), reminders
+        return "\n".join([f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}" for _, row in reminders.head(5).iterrows()]), reminders
     except Exception as e:
         logging.error(f"AMC reminder generation failed: {str(e)}")
         return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
@@ -232,12 +405,12 @@ def generate_device_cards(df):
             health_color = {'Healthy': 'green', 'Unhealthy': 'red', 'Warning': 'orange', 'Unknown': 'gray'}.get(row['health'], 'gray')
             timestamp_str = str(row['timestamp']) if pd.notna(row['timestamp']) else 'Unknown'
             cards_html += f"""
-            <div style="border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px; width: 200px;">
-                <h4>Device: {row['device_id']}</h4>
-                <p><b>Health:</b> <span style="color: {health_color}">{row['health']}</span></p>
-                <p><b>Usage Count:</b> {row['count']}</p>
-                <p><b>Last Log:</b> {timestamp_str}</p>
-            </div>
+                <div style="border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px; width: 200px;">
+                    <h4>Device: {row['device_id']}</h4>
+                    <p><b>Health:</b> <span style="color: {health_color}">{row['health']}</span></p>
+                    <p><b>Usage Count:</b> {row['count']}</p>
+                    <p><b>Last Log:</b> {timestamp_str}</p>
+                </div>
             """
         cards_html += '</div>'
         return cards_html
@@ -315,25 +488,21 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         logging.error(f"Failed to generate PDF: {str(e)}")
         return None
 
-# Main processing function (Updated)
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, cached_df_state, cached_filtered_df_state):
+# Main processing function
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, cached_df_state, last_modified_state):
     start_time = time.time()
     try:
         if not file_obj:
-            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state, None
-
-        # Check if date range is provided
-        if not date_range or len(date_range) != 2:
-            return "Please provide a valid date range.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state, None
+            return "No file uploaded.", "<p>No data available.</p>", None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, cached_df_state, last_modified_state
 
         file_path = file_obj.name
         current_modified_time = os.path.getmtime(file_path)
 
-        # Load or use cached original dataframe
+        # Read file only if it's new or modified
         if cached_df_state is None or current_modified_time != last_modified_state:
-            logging.info(f"Processing file: {file_path}")
+            logging.info(f"Processing new or modified file: {file_path}")
             if not file_path.endswith(".csv"):
-                return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state, cached_df_state, None
+                return "Please upload a CSV file.", "<p>Invalid file format.</p>", None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, cached_df_state, last_modified_state
 
             required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
             dtypes = {
@@ -347,19 +516,18 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             df = pd.read_csv(file_path, dtype=dtypes)
             missing_columns = [col for col in required_columns if col not in df.columns]
             if missing_columns:
-                return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, None
+                return f"Missing columns: {missing_columns}", "<p>Missing required columns.</p>", None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, cached_df_state, last_modified_state
 
             df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
             df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
             if df["timestamp"].dt.tz is None:
                 df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
             if df.empty:
-                return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, None
-            cached_df_state = df
+                return "No data available.", "<p>No data available.</p>", None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, df, current_modified_time
         else:
             df = cached_df_state
 
-        # Always apply current filters to the original dataframe
+        # Apply filters
         filtered_df = df.copy()
         if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
             filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
@@ -367,37 +535,34 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             filtered_df = filtered_df[filtered_df['equipment_type'] == equipment_type_filter]
         if date_range and len(date_range) == 2:
             days_start, days_end = date_range
-            today = pd.to_datetime(datetime.now().date()).tz_localize('Asia/Kolkata')
+            today = pd.to_datetime(datetime.now()).tz_localize('Asia/Kolkata')
             start_date = today + pd.Timedelta(days=days_start)
             end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
+            start_date = start_date.tz_convert('Asia/Kolkata') if start_date.tzinfo else start_date.tz_localize('Asia/Kolkata')
+            end_date = end_date.tz_convert('Asia/Kolkata') if end_date.tzinfo else end_date.tz_localize('Asia/Kolkata')
+            logging.info(f"Date range filter: start_date={start_date}, end_date={end_date}")
+            logging.info(f"Before date filter: {len(filtered_df)} rows")
             filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
+            logging.info(f"After date filter: {len(filtered_df)} rows")
 
         if filtered_df.empty:
-            return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, None
+            return "No data after applying filters.", "<p>No data after filters.</p>", None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, df, current_modified_time
 
         # Generate table for preview
         preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
         preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
         # Run critical tasks concurrently
-        try:
-            with ThreadPoolExecutor(max_workers=2) as executor:
-                future_anomalies = executor.submit(detect_anomalies, filtered_df)
-                future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
-
-                summary = f"Step 1: Summary Report\n{summarize_logs(filtered_df)}"
-                anomalies, anomalies_df = future_anomalies.result()
-                anomalies = f"Anomaly Detection\n{anomalies}"
-                amc_reminders, reminders_df = future_amc.result()
-                amc_reminders = f"AMC Reminders\n{amc_reminders}"
-                insights = f"Dashboard Insights\n{generate_dashboard_insights(filtered_df)}"
-        except Exception as e:
-            logging.error(f"Concurrent task execution failed: {str(e)}")
-            summary = "Failed to generate summary due to processing error."
-            anomalies = "Anomaly detection failed due to processing error."
-            amc_reminders = "AMC reminders failed due to processing error."
-            insights = "Insights generation failed due to processing error."
-            anomalies_df = pd.DataFrame()
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            future_anomalies = executor.submit(detect_anomalies, filtered_df)
+            future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
+
+            summary = f"Step 1: Summary Report\n{summarize_logs(filtered_df)}"
+            anomalies, anomalies_df = future_anomalies.result()
+            anomalies = f"Anomaly Detection\n{anomalies}"
+            amc_reminders, reminders_df = future_amc.result()
+            amc_reminders = f"AMC Reminders\n{amc_reminders}"
+            insights = f"Dashboard Insights\n{generate_dashboard_insights(filtered_df)}"
 
         # Generate charts sequentially
         usage_chart = create_usage_chart(filtered_df)
@@ -407,15 +572,19 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
         device_cards = generate_device_cards(filtered_df)
 
+        # Save to Salesforce after all other processing
+        save_to_salesforce(filtered_df, reminders_df)
+        create_salesforce_reports(filtered_df)
+
         elapsed_time = time.time() - start_time
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
         if elapsed_time > 3:
             logging.warning(f"Processing time exceeded 3 seconds: {elapsed_time:.2f} seconds")
 
-        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, None, current_modified_time, cached_df_state, filtered_df)
+        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, None, df, current_modified_time)
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, None
+        return f"Error: {str(e)}", "<p>Error processing data.</p>", None, '<p>Error processing data.</p>', None, None, None, None, "", "", "", None, cached_df_state, last_modified_state
 
 # Generate PDF separately
 async def generate_pdf(summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights):
@@ -466,7 +635,6 @@ try:
         last_modified_state = gr.State(value=None)
         current_file_state = gr.State(value=None)
         cached_df_state = gr.State(value=None)
-        cached_filtered_df_state = gr.State(value=None)
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -475,9 +643,9 @@ try:
                     gr.Markdown("### Filters")
                     lab_site_filter = gr.Dropdown(label="Lab Site", choices=['All'], value='All', interactive=True)
                     equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
-                    date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
-                submit_button = gr.Button("Analyze", variant="primary")
-                pdf_button = gr.Button("Export PDF", variant="secondary")
+                    date_range_filter = gr.Slider(label="Date Range (Days from Today, e.g., -7 to 0 means last 7 days)", minimum=-365, maximum=0, step=1, value=[-7, 0])
+                    submit_button = gr.Button("Analyze", variant="primary")
+                    pdf_button = gr.Button("Export PDF", variant="secondary")
 
             with gr.Column(scale=2):
                 with gr.Group(elem_classes="dashboard-container"):
@@ -514,7 +682,7 @@ try:
                         insights_output = gr.Markdown()
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Export Report")
-                        pdf_output = gr.File(label="Download Report as PDF")
+                        pdf_output = gr.File(label="Download Status Report as PDF")
 
         file_input.change(
             fn=update_filters,
@@ -525,8 +693,8 @@ try:
 
         submit_button.click(
             fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state]
+            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, cached_df_state, last_modified_state],
+            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, cached_df_state, last_modified_state]
         )
 
         pdf_button.click(
@@ -535,7 +703,7 @@ try:
             outputs=[pdf_output]
         )
 
-    logging.info("Gradio interface initialized successfully")
+        logging.info("Gradio interface initialized successfully")
 except Exception as e:
     logging.error(f"Failed to initialize Gradio interface: {str(e)}")
     raise e