Update app.py

app.py

@@ -3,15 +3,16 @@ import pandas as pd
 from datetime import datetime, timedelta
 import logging
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.ensemble import IsolationForest
 from transformers import pipeline
 import torch
 from concurrent.futures import ThreadPoolExecutor
 from simple_salesforce import Salesforce
 import os
-import json
 import io
 import time
+import asyncio
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -49,7 +50,7 @@ try:
         "summarization",
         model="t5-small",
         device=device,
-        max_length=30,  # Reduced for faster processing
+        max_length=20,  # Further reduced for speed
         min_length=10,
         num_beams=2
     )
@@ -149,8 +150,8 @@ def summarize_logs(df):
     try:
         total_devices = df["device_id"].nunique()
         most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
-        prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
-        summary = summarizer(prompt, max_length=30, min_length=10, do_sample=False)[0]["summary_text"]
+        prompt = f"Logs: {total_devices} devices. Most used: {most_used}."
+        summary = summarizer(prompt, max_length=20, min_length=10, do_sample=False)[0]["summary_text"]
         return summary
     except Exception as e:
         logging.error(f"Summary generation failed: {str(e)}")
@@ -162,8 +163,8 @@ def detect_anomalies(df):
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
         features = df[["usage_hours", "downtime"]].fillna(0)
-        if len(features) > 200:  # Reduced sample size
-            features = features.sample(n=200, random_state=42)
+        if len(features) > 100:  # Further reduced sample size
+            features = features.sample(n=100, random_state=42)
         iso_forest = IsolationForest(contamination=0.1, random_state=42)
         df["anomaly"] = iso_forest.fit_predict(features)
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
@@ -196,17 +197,30 @@ def generate_dashboard_insights(df):
         total_devices = df["device_id"].nunique()
         avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
         prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
-        insights = summarizer(prompt, max_length=30, min_length=10, do_sample=False)[0]["summary_text"]
+        insights = summarizer(prompt, max_length=20, min_length=10, do_sample=False)[0]["summary_text"]
         return insights
     except Exception as e:
         logging.error(f"Dashboard insights generation failed: {str(e)}")
         return f"Dashboard insights generation failed: {str(e)}"
 
+# Placeholder chart for empty data
+def create_placeholder_chart(title):
+    fig = go.Figure()
+    fig.add_annotation(
+        text="No data available for this chart",
+        xref="paper", yref="paper",
+        x=0.5, y=0.5, showarrow=False,
+        font=dict(size=16)
+    )
+    fig.update_layout(title=title, margin=dict(l=20, r=20, t=40, b=20))
+    return fig
+
 # Create usage chart
 def create_usage_chart(df):
     try:
-        if df.empty:
-            return None
+        if df.empty or "usage_hours" not in df.columns or "device_id" not in df.columns:
+            logging.warning("Insufficient data for usage chart")
+            return create_placeholder_chart("Usage Hours per Device")
         usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
         if len(usage_data) > 5:
             usage_data = usage_data.nlargest(5, "usage_hours")
@@ -221,11 +235,14 @@ def create_usage_chart(df):
         return fig
     except Exception as e:
         logging.error(f"Failed to create usage chart: {str(e)}")
-        return None
+        return create_placeholder_chart("Usage Hours per Device")
 
 # Create downtime chart
 def create_downtime_chart(df):
     try:
+        if df.empty or "downtime" not in df.columns or "device_id" not in df.columns:
+            logging.warning("Insufficient data for downtime chart")
+            return create_placeholder_chart("Downtime per Device")
         downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
         if len(downtime_data) > 5:
             downtime_data = downtime_data.nlargest(5, "downtime")
@@ -240,13 +257,18 @@ def create_downtime_chart(df):
         return fig
     except Exception as e:
         logging.error(f"Failed to create downtime chart: {str(e)}")
-        return None
+        return create_placeholder_chart("Downtime per Device")
 
 # Create daily log trends chart
 def create_daily_log_trends_chart(df):
     try:
-        df['date'] = df['timestamp'].dt.date
+        if df.empty or "timestamp" not in df.columns:
+            logging.warning("Insufficient data for daily log trends chart")
+            return create_placeholder_chart("Daily Log Trends")
+        df['date'] = pd.to_datetime(df['timestamp']).dt.date
         daily_logs = df.groupby('date').size().reset_index(name='log_count')
+        if daily_logs.empty:
+            return create_placeholder_chart("Daily Log Trends")
         fig = px.line(
             daily_logs,
             x='date',
@@ -258,19 +280,24 @@ def create_daily_log_trends_chart(df):
         return fig
     except Exception as e:
         logging.error(f"Failed to create daily log trends chart: {str(e)}")
-        return None
+        return create_placeholder_chart("Daily Log Trends")
 
 # Create weekly uptime chart
 def create_weekly_uptime_chart(df):
     try:
-        df['week'] = df['timestamp'].dt.isocalendar().week
-        df['year'] = df['timestamp'].dt.year
+        if df.empty or "timestamp" not in df.columns or "usage_hours" not in df.columns or "downtime" not in df.columns:
+            logging.warning("Insufficient data for weekly uptime chart")
+            return create_placeholder_chart("Weekly Uptime Percentage")
+        df['week'] = pd.to_datetime(df['timestamp']).dt.isocalendar().week
+        df['year'] = pd.to_datetime(df['timestamp']).dt.year
         weekly_data = df.groupby(['year', 'week']).agg({
             'usage_hours': 'sum',
             'downtime': 'sum'
         }).reset_index()
         weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
         weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
+        if weekly_data.empty:
+            return create_placeholder_chart("Weekly Uptime Percentage")
         fig = px.bar(
             weekly_data,
             x='year_week',
@@ -282,15 +309,18 @@ def create_weekly_uptime_chart(df):
         return fig
     except Exception as e:
         logging.error(f"Failed to create weekly uptime chart: {str(e)}")
-        return None
+        return create_placeholder_chart("Weekly Uptime Percentage")
 
 # Create anomaly alerts chart
 def create_anomaly_alerts_chart(anomalies_df):
     try:
-        if anomalies_df.empty:
-            return None
-        anomalies_df['date'] = anomalies_df['timestamp'].dt.date
+        if anomalies_df is None or anomalies_df.empty or "timestamp" not in anomalies_df.columns:
+            logging.warning("Insufficient data for anomaly alerts chart")
+            return create_placeholder_chart("Anomaly Alerts Over Time")
+        anomalies_df['date'] = pd.to_datetime(anomalies_df['timestamp']).dt.date
         anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
+        if anomaly_counts.empty:
+            return create_placeholder_chart("Anomaly Alerts Over Time")
         fig = px.scatter(
             anomaly_counts,
             x='date',
@@ -302,7 +332,7 @@ def create_anomaly_alerts_chart(anomalies_df):
         return fig
     except Exception as e:
         logging.error(f"Failed to create anomaly alerts chart: {str(e)}")
-        return None
+        return create_placeholder_chart("Anomaly Alerts Over Time")
 
 # Generate device cards
 def generate_device_cards(df):
@@ -408,41 +438,41 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         return None
 
 # Main processing function
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state):
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, cached_df_state):
     start_time = time.time()
     try:
         if not file_obj:
-            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
+            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state
 
         file_path = file_obj.name
         current_modified_time = os.path.getmtime(file_path)
-        if last_modified_state and current_modified_time == last_modified_state:
-            return None, None, None, None, None, None, None, None, None, None, None, None, last_modified_state
-
-        logging.info(f"Processing file: {file_path}")
-        if not file_path.endswith(".csv"):
-            return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
-
-        required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
-        dtypes = {
-            "device_id": "string",
-            "log_type": "string",
-            "status": "string",
-            "usage_hours": "float32",
-            "downtime": "float32",
-            "amc_date": "string"
-        }
-        df = pd.read_csv(file_path, dtype=dtypes)
-        missing_columns = [col for col in required_columns if col not in df.columns]
-        if missing_columns:
-            return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
-
-        df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
-        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
-        if df["timestamp"].dt.tz is None:
-            df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
-        if df.empty:
-            return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
+        if last_modified_state and current_modified_time == last_modified_state and cached_df_state is not None:
+            df = cached_df_state
+        else:
+            logging.info(f"Processing file: {file_path}")
+            if not file_path.endswith(".csv"):
+                return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state, cached_df_state
+
+            required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
+            dtypes = {
+                "device_id": "string",
+                "log_type": "string",
+                "status": "string",
+                "usage_hours": "float32",
+                "downtime": "float32",
+                "amc_date": "string"
+            }
+            df = pd.read_csv(file_path, dtype=dtypes)
+            missing_columns = [col for col in required_columns if col not in df.columns]
+            if missing_columns:
+                return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state
+
+            df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
+            df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
+            if df["timestamp"].dt.tz is None:
+                df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
+            if df.empty:
+                return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state
 
         # Apply filters
         filtered_df = df.copy()
@@ -458,23 +488,22 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
 
         if filtered_df.empty:
-            return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
+            return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df
 
         # Generate table for preview
         preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
         preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
-        # Run tasks concurrently
-        with ThreadPoolExecutor(max_workers=6) as executor:
+        # Run critical tasks concurrently
+        with ThreadPoolExecutor(max_workers=4) as executor:  # Reduced workers
             future_summary = executor.submit(summarize_logs, filtered_df)
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
             future_insights = executor.submit(generate_dashboard_insights, filtered_df)
             future_usage_chart = executor.submit(create_usage_chart, filtered_df)
             future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
-            future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
+            future_daily_log_chart = executor.submit(create_daily_log_chart, filtered_df)
             future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
-            future_anomaly_alerts_chart = executor.submit(create_anomaly_alerts_chart, future_anomalies.result()[1])  # Pass anomalies_df
             future_device_cards = executor.submit(generate_device_cards, filtered_df)
 
             summary = f"Step 1: Summary Report\n{future_summary.result()}"
@@ -487,21 +516,32 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             downtime_chart = future_downtime_chart.result()
             daily_log_chart = future_daily_log_chart.result()
             weekly_uptime_chart = future_weekly_uptime_chart.result()
-            anomaly_alerts_chart = future_anomaly_alerts_chart.result()
             device_cards = future_device_cards.result()
 
+        # Generate anomaly alerts chart after anomalies are computed
+        anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
+
         save_to_salesforce(filtered_df, reminders_df)
-        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart)
 
         elapsed_time = time.time() - start_time
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
-        if elapsed_time > 10:
-            logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
+        if elapsed_time > 5:  # Stricter threshold
+            logging.warning(f"Processing time exceeded 5 seconds: {elapsed_time:.2f} seconds")
 
-        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
+        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, None, current_modified_time, df)
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state
+        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state
+
+# Generate PDF separately
+async def generate_pdf(summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights):
+    try:
+        preview_df = pd.read_html(preview_html)[0]
+        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart)
+        return pdf_file
+    except Exception as e:
+        logging.error(f"Failed to generate PDF: {str(e)}")
+        return None
 
 # Update filters
 def update_filters(file_obj, current_file_state):
@@ -537,10 +577,11 @@ try:
         .table tr:nth-child(even) {background-color: #f9f9f9;}
     """) as iface:
         gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
-        gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard with the latest data.")
+        gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard. Use 'Export PDF' for report download.")
 
         last_modified_state = gr.State(value=None)
         current_file_state = gr.State(value=None)
+        cached_df_state = gr.State(value=None)
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -551,6 +592,7 @@ try:
                     equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
                     date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
                 submit_button = gr.Button("Analyze", variant="primary")
+                pdf_button = gr.Button("Export PDF", variant="secondary")
 
             with gr.Column(scale=2):
                 with gr.Group(elem_classes="dashboard-container"):
@@ -598,8 +640,14 @@ try:
 
         submit_button.click(
             fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
+            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state],
+            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state]
+        )
+
+        pdf_button.click(
+            fn=generate_pdf,
+            inputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output],
+            outputs=[pdf_output]
         )
 
     logging.info("Gradio interface initialized successfully")