Update app.py

app.py

@@ -11,6 +11,7 @@ from simple_salesforce import Salesforce
 import os
 import io
 import time
+import uuid
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -200,7 +201,7 @@ def save_to_salesforce(df, reminders_df):
             records.append(record)
 
         if records:
-            batch_size = 200  # Smaller batch size for faster processing
+            batch_size = 100  # Reduced batch size for faster processing
             for i in range(0, len(records), batch_size):
                 batch = records[i:i + batch_size]
                 try:
@@ -237,9 +238,9 @@ def detect_anomalies(df):
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
         features = df[["usage_hours", "downtime"]].fillna(0)
-        if len(features) > 500:
-            features = features.sample(n=500, random_state=42)
-        iso_forest = IsolationForest(contamination=0.1, random_state=42)
+        if len(features) > 200:  # Reduced sample size for faster processing
+            features = features.sample(n=200, random_state=42)
+        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=50)  # Reduced n_estimators
         df["anomaly"] = iso_forest.fit_predict(features)
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
         if anomalies.empty:
@@ -263,7 +264,7 @@ def check_amc_reminders(df, current_date):
         reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
         if reminders.empty:
             return "No AMC reminders due within the next 30 days.", reminders
-        result = "\n".join([f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}" for _, row in reminders.head(5).iterrows()])
+        result = "\n".join([f"- Device ID: {row['device_id']}, Log Type: {row['log_type']}, Status: {row['status']}, Usage: {row['usage_hours']}, Downtime: {row['downtime']}, AMC Date: {row['amc_date']}" for _, row in reminders.head(5).iterrows()])
         logging.info(f"AMC reminders generation took {time.time() - start_time:.2f} seconds")
         return result, reminders
     except Exception as e:
@@ -419,32 +420,12 @@ def generate_device_cards(df):
         logging.error(f"Failed to generate device cards: {str(e)}")
         return f'<p>Error generating device cards: {str(e)}</p>'
 
-# Generate monthly status
-def generate_monthly_status(df, selected_month):
-    try:
-        total_devices = df['device_id'].nunique()
-        total_usage_hours = df['usage_hours'].sum()
-        total_downtime = df['downtime'].sum()
-        avg_usage = total_usage_hours / total_devices if total_devices > 0 else 0
-        avg_downtime = total_downtime / total_devices if total_devices > 0 else 0
-        return f"""
-        Monthly Status for {selected_month}:
-        - Total Devices: {total_devices}
-        - Total Usage Hours: {total_usage_hours:.2f}
-        - Total Downtime Hours: {total_downtime:.2f}
-        - Average Usage per Device: {avg_usage:.2f} hours
-        - Average Downtime per Device: {avg_downtime:.2f} hours
-        """
-    except Exception as e:
-        logging.error(f"Failed to generate monthly status: {str(e)}")
-        return f"Failed to generate monthly status: {str(e)}"
-
 # Generate PDF content
-def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, df, selected_month):
+def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart):
     if not reportlab_available:
         return None
     try:
-        pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
+        pdf_path = f"status_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
         doc = SimpleDocTemplate(pdf_path, pagesize=letter)
         styles = getSampleStyleSheet()
         story = []
@@ -452,16 +433,10 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         def safe_paragraph(text, style):
             return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
 
-        story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
+        story.append(Paragraph("LabOps Status Report", styles['Title']))
         story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
         story.append(Spacer(1, 12))
 
-        if selected_month != "All":
-            monthly_status = generate_monthly_status(df, selected_month)
-            story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
-            story.append(safe_paragraph(monthly_status, styles['Normal']))
-            story.append(Spacer(1, 12))
-
         story.append(Paragraph("Summary Report", styles['Heading2']))
         story.append(safe_paragraph(summary, styles['Normal']))
         story.append(Spacer(1, 12))
@@ -516,7 +491,7 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         return None
 
 # Main processing function
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state):
     start_time = time.time()
     try:
         if not file_obj:
@@ -540,7 +515,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             "downtime": "float32",
             "amc_date": "string"
         }
-        df = pd.read_csv(file_path, dtype=dtypes)
+        df = pd.read_csv(file_path, dtype=dtypes, usecols=required_columns)
         missing_columns = [col for col in required_columns if col not in df.columns]
         if missing_columns:
             return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
@@ -564,12 +539,6 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             start_date = today + pd.Timedelta(days=days_start)
             end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
             filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
-        if month_filter and month_filter != "All":
-            selected_date = pd.to_datetime(month_filter, format="%B %Y")
-            filtered_df = filtered_df[
-                (filtered_df['timestamp'].dt.year == selected_date.year) &
-                (filtered_df['timestamp'].dt.month == selected_date.month)
-            ]
 
         if filtered_df.empty:
             return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
@@ -579,7 +548,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
         # Run tasks concurrently
-        with ThreadPoolExecutor(max_workers=6) as executor:
+        with ThreadPoolExecutor(max_workers=8) as executor:  # Increased workers for better parallelization
             future_summary = executor.submit(summarize_logs, filtered_df)
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
@@ -601,16 +570,16 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             downtime_chart = future_downtime_chart.result()
             daily_log_chart = future_daily_log_chart.result()
             weekly_uptime_chart = future_weekly_uptime_chart.result()
-            anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)  # Use anomalies_df
+            anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
             device_cards = future_device_cards.result()
 
         save_to_salesforce(filtered_df, reminders_df)
-        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, filtered_df, month_filter)
+        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart)
 
         elapsed_time = time.time() - start_time
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
-        if elapsed_time > 10:
-            logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
+        if elapsed_time > 30:
+            logging.warning(f"Processing time exceeded 30 seconds: {elapsed_time:.2f} seconds")
 
         return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
     except Exception as e:
@@ -620,7 +589,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
 # Update filters
 def update_filters(file_obj):
     if not file_obj:
-        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
+        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
     try:
         with open(file_obj.name, 'rb') as f:
             csv_content = f.read().decode('utf-8')
@@ -629,12 +598,11 @@ def update_filters(file_obj):
 
         lab_site_options = ['All'] + [site for site in df['lab_site'].dropna().astype(str).unique().tolist() if site.strip()] if 'lab_site' in df.columns else ['All']
         equipment_type_options = ['All'] + [equip for equip in df['equipment_type'].dropna().astype(str).unique().tolist() if equip.strip()] if 'equipment_type' in df.columns else ['All']
-        month_options = ['All'] + sorted(df['timestamp'].dt.strftime('%B %Y').dropna().unique().tolist()) if 'timestamp' in df.columns else ['All']
 
-        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
+        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All')
     except Exception as e:
         logging.error(f"Failed to update filters: {str(e)}")
-        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
+        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
 
 # Gradio Interface
 try:
@@ -664,7 +632,6 @@ try:
                     lab_site_filter = gr.Dropdown(label="Lab Site", choices=['All'], value='All', interactive=True)
                     equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
                     date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
-                    month_filter = gr.Dropdown(label="Select Month for Report", choices=['All'], value='All', interactive=True)
                 submit_button = gr.Button("Analyze", variant="primary")
 
             with gr.Column(scale=2):
@@ -702,19 +669,19 @@ try:
                         insights_output = gr.Markdown()
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Export Report")
-                        pdf_output = gr.File(label="Download Monthly Status Report as PDF")
+                        pdf_output = gr.File(label="Download Status Report as PDF")
 
         file_input.change(
             fn=update_filters,
             inputs=[file_input],
-            outputs=[lab_site_filter, equipment_type_filter, month_filter],
+            outputs=[lab_site_filter, equipment_type_filter],
             queue=False
         )
 
         submit_button.click(
             fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, month_filter, last_modified_state],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
+            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state],
+            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_chart, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
         )
 
     logging.info("Gradio interface initialized successfully")