Update app.py

app.py

@@ -5,7 +5,7 @@ import logging
 import plotly.express as px
 import plotly.graph_objects as go
 from sklearn.ensemble import IsolationForest
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor  # Added missing import
 import os
 import io
 import time
@@ -314,70 +314,22 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         logging.error(f"Failed to generate PDF: {str(e)}")
         return None
 
-# Update filters
-def update_filters(current_file_path, cached_df_state):
-    try:
-        if not current_file_path or cached_df_state is None:
-            logging.warning("No file or cached DataFrame available for filter update")
-            return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
-        
-        df = cached_df_state
-        if df.empty:
-            logging.warning("Cached DataFrame is empty")
-            return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
-
-        lab_site_options = ['All']
-        if 'lab_site' in df.columns:
-            sites = df['lab_site'].dropna().astype(str).unique().tolist()
-            lab_site_options.extend([site for site in sites if site.strip()])
-            logging.info(f"Lab site options populated: {lab_site_options}")
-
-        equipment_type_options = ['All']
-        if 'equipment_type' in df.columns:
-            types = df['equipment_type'].dropna().astype(str).unique().tolist()
-            equipment_type_options.extend([t for t in types if t.strip()])
-            logging.info(f"Equipment type options populated: {equipment_type_options}")
-
-        if len(lab_site_options) == 1:
-            logging.warning("No valid lab_site values found in DataFrame")
-        if len(equipment_type_options) == 1:
-            logging.warning("No valid equipment_type values found in DataFrame")
-
-        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All')
-    except Exception as e:
-        logging.error(f"Failed to update filters: {str(e)}")
-        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
-
 # Main processing function
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path):
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, cached_df_state, cached_filtered_df_state):
     start_time = time.time()
     try:
-        if not file_obj and not current_file_path:
-            logging.warning("No file provided or cached.")
-            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path
-
-        file_path = file_obj.name if file_obj else current_file_path
-        if not os.path.exists(file_path):
-            logging.error(f"File path does not exist: {file_path}")
-            return "File not found.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path
+        if not file_obj:
+            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state, cached_filtered_df_state
 
+        file_path = file_obj.name
         current_modified_time = os.path.getmtime(file_path)
-        df = None  # Initialize df to avoid undefined reference
-
-        # Check if we can use cached filtered data
-        if (last_modified_state == current_modified_time and 
-            cached_filtered_df_state is not None and 
-            file_path == current_file_path):
-            logging.info("Using cached filtered DataFrame")
+        if last_modified_state and current_modified_time == last_modified_state and cached_filtered_df_state is not None:
             filtered_df = cached_filtered_df_state
         else:
-            # Load or use cached raw DataFrame
-            if (cached_df_state is None or 
-                current_modified_time != last_modified_state or 
-                file_path != current_file_path):
-                logging.info(f"Reading new CSV file: {file_path}")
+            if cached_df_state is None or current_modified_time != last_modified_state:
+                logging.info(f"Processing file: {file_path}")
                 if not file_path.endswith(".csv"):
-                    return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state, cached_df_state, cached_filtered_df_state, file_path
+                    return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state, cached_df_state, cached_filtered_df_state
 
                 required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
                 dtypes = {
@@ -391,16 +343,15 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
                 df = pd.read_csv(file_path, dtype=dtypes)
                 missing_columns = [col for col in required_columns if col not in df.columns]
                 if missing_columns:
-                    return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, cached_filtered_df_state, file_path
+                    return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, cached_filtered_df_state
 
                 df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
                 df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
                 if df["timestamp"].dt.tz is None:
                     df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
                 if df.empty:
-                    return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, cached_filtered_df_state, file_path
+                    return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, cached_filtered_df_state
             else:
-                logging.info("Using cached raw DataFrame")
                 df = cached_df_state
 
             # Apply filters
@@ -417,7 +368,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
                 filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
 
             if filtered_df.empty:
-                return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, filtered_df, file_path
+                return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, filtered_df
 
         # Generate table for preview
         preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
@@ -456,10 +407,10 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         if elapsed_time > 3:
             logging.warning(f"Processing time exceeded 3 seconds: {elapsed_time:.2f} seconds")
 
-        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, None, current_modified_time, df, filtered_df, file_path)
+        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, None, current_modified_time, df, filtered_df)
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path
+        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, cached_filtered_df_state
 
 # Generate PDF separately
 async def generate_pdf(summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights):
@@ -471,6 +422,24 @@ async def generate_pdf(summary, preview_html, usage_chart, device_cards, daily_l
         logging.error(f"Failed to generate PDF: {str(e)}")
         return None
 
+# Update filters
+def update_filters(file_obj, current_file_state):
+    if not file_obj or file_obj.name == current_file_state:
+        return gr.update(), gr.update(), current_file_state
+    try:
+        with open(file_obj.name, 'rb') as f:
+            csv_content = f.read().decode('utf-8')
+        df = pd.read_csv(io.StringIO(csv_content))
+        df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
+
+        lab_site_options = ['All'] + [site for site in df['lab_site'].dropna().astype(str).unique().tolist() if site.strip()] if 'lab_site' in df.columns else ['All']
+        equipment_type_options = ['All'] + [equip for equip in df['equipment_type'].dropna().astype(str).unique().tolist() if equip.strip()] if 'equipment_type' in df.columns else ['All']
+
+        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), file_obj.name
+    except Exception as e:
+        logging.error(f"Failed to update filters: {str(e)}")
+        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), current_file_state
+
 # Gradio Interface
 try:
     logging.info("Initializing Gradio interface...")
@@ -487,12 +456,12 @@ try:
         .table tr:nth-child(even) {background-color: #f9f9f9;}
     """) as iface:
         gr.Markdown("<h1>LabOps Log Analyzer Dashboard</h1>")
-        gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard. Use 'Export PDF' for report download. Filters update without re-uploading the CSV.")
+        gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard. Use 'Export PDF' for report download.")
 
         last_modified_state = gr.State(value=None)
+        current_file_state = gr.State(value=None)
         cached_df_state = gr.State(value=None)
         cached_filtered_df_state = gr.State(value=None)
-        current_file_path = gr.State(value=None)
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -542,48 +511,17 @@ try:
                         gr.Markdown("### Export Report")
                         pdf_output = gr.File(label="Download Status Report as PDF")
 
-        # Update file path and filters when CSV is uploaded
         file_input.change(
-            fn=lambda file_obj, cached_df, file_path: (file_obj.name if file_obj else file_path, cached_df, file_obj.name if file_obj else file_path),
-            inputs=[file_input, cached_df_state, current_file_path],
-            outputs=[current_file_path, cached_df_state, current_file_path],
-            queue=False
-        ).then(
-            fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path],
-            queue=False
-        ).then(
             fn=update_filters,
-            inputs=[current_file_path, cached_df_state],
-            outputs=[lab_site_filter, equipment_type_filter],
+            inputs=[file_input, current_file_state],
+            outputs=[lab_site_filter, equipment_type_filter, current_file_state],
             queue=False
         )
 
-        # Process logs on submit or filter change
         submit_button.click(
             fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path]
-        )
-
-        # Update on filter change without requiring new file
-        lab_site_filter.change(
-            fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path]
-        )
-
-        equipment_type_filter.change(
-            fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path]
-        )
-
-        date_range_filter.change(
-            fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state, current_file_path]
+            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state],
+            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state]
         )
 
         pdf_button.click(