Update app.py

app.py

@@ -17,7 +17,8 @@ import functools
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Salesforce configuration
+# Salesforce configuration (Disabled for now)
+"""
 try:
     sf = Salesforce(
         username='multi-devicelabopsdashboard@sathkrutha.com',
@@ -29,6 +30,8 @@ try:
 except Exception as e:
     logging.error(f"Failed to connect to Salesforce: {str(e)}")
     sf = None
+"""
+sf = None  # Temporarily disable Salesforce
 
 # Try to import reportlab
 try:
@@ -50,9 +53,9 @@ try:
         "summarization",
         model="t5-small",
         device=device,
-        max_length=50,
+        max_length=30,  # Reduced for faster inference
         min_length=10,
-        num_beams=2
+        num_beams=1  # Reduced for faster inference
     )
     logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
 except Exception as e:
@@ -118,7 +121,11 @@ def get_folder_id(folder_name):
 
 LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
 
-# Salesforce report creation
+# Salesforce report creation (Disabled for now)
+def create_salesforce_reports(df):
+    logging.info("Salesforce report creation skipped for optimization")
+    return
+"""
 def create_salesforce_reports(df):
     if sf is None or not LABOPS_REPORTS_FOLDER_ID:
         return
@@ -155,8 +162,13 @@ def create_salesforce_reports(df):
         logging.info("Salesforce reports created")
     except Exception as e:
         logging.error(f"Failed to create Salesforce reports: {str(e)}")
+"""
 
-# Save to Salesforce
+# Save to Salesforce (Disabled for now)
+def save_to_salesforce(df, reminders_df):
+    logging.info("Salesforce save operation skipped for optimization")
+    return
+"""
 def save_to_salesforce(df, reminders_df):
     if sf is None:
         logging.error("No Salesforce connection available")
@@ -217,15 +229,31 @@ def save_to_salesforce(df, reminders_df):
             logging.warning("No records to save to Salesforce")
     except Exception as e:
         logging.error(f"Failed to save to Salesforce: {str(e)}")
+"""
+
+# Cache summarization results
+def cache_summary(func):
+    @functools.wraps(func)
+    def wrapper(df, *args, **kwargs):
+        cache_key = f"{id(df)}_{func.__name__}"
+        if not hasattr(wrapper, 'cache'):
+            wrapper.cache = {}
+        if cache_key in wrapper.cache:
+            return wrapper.cache[cache_key]
+        result = func(df, *args, **kwargs)
+        wrapper.cache[cache_key] = result
+        return result
+    return wrapper
 
 # Summarize logs
+@cache_summary
 def summarize_logs(df):
     start_time = time.time()
     try:
         total_devices = df["device_id"].nunique()
         most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
         prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
-        summary = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
+        summary = summarizer(prompt, max_length=30, min_length=10, do_sample=False)[0]["summary_text"]
         logging.info(f"Summary generation took {time.time() - start_time:.2f} seconds")
         return summary
     except Exception as e:
@@ -239,9 +267,9 @@ def detect_anomalies(df):
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
         features = df[["usage_hours", "downtime"]].fillna(0)
-        if len(features) > 200:
-            features = features.sample(n=200, random_state=42)
-        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=50)
+        if len(features) > 100:  # Further reduced sample size
+            features = features.sample(n=100, random_state=42)
+        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=30)  # Further reduced n_estimators
         df["anomaly"] = iso_forest.fit_predict(features)
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
         if anomalies.empty:
@@ -273,13 +301,14 @@ def check_amc_reminders(df, current_date):
         return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
 
 # Dashboard insights
+@cache_summary
 def generate_dashboard_insights(df):
     start_time = time.time()
     try:
         total_devices = df["device_id"].nunique()
         avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
         prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
-        insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
+        insights = summarizer(prompt, max_length=30, min_length=10, do_sample=False)[0]["summary_text"]
         logging.info(f"Insights generation took {time.time() - start_time:.2f} seconds")
         return insights
     except Exception as e:
@@ -300,7 +329,7 @@ def cache_dataframe(func):
         return result
     return wrapper
 
-# Create usage chart
+# Create usage chart (Only this chart will be generated to save time)
 @cache_dataframe
 def create_usage_chart(df):
     try:
@@ -322,90 +351,18 @@ def create_usage_chart(df):
         logging.error(f"Failed to create usage chart: {str(e)}")
         return None
 
-# Create downtime chart
-@cache_dataframe
+# Skipped other chart functions to save time
 def create_downtime_chart(df):
-    try:
-        downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
-        if len(downtime_data) > 5:
-            downtime_data = downtime_data.nlargest(5, "downtime")
-        fig = px.bar(
-            downtime_data,
-            x="device_id",
-            y="downtime",
-            title="Downtime per Device",
-            labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"}
-        )
-        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
-        return fig
-    except Exception as e:
-        logging.error(f"Failed to create downtime chart: {str(e)}")
-        return None
+    return None
 
-# Create daily log trends chart
-@cache_dataframe
 def create_daily_log_trends_chart(df):
-    try:
-        df['date'] = df['timestamp'].dt.date
-        daily_logs = df.groupby('date').size().reset_index(name='log_count')
-        fig = px.line(
-            daily_logs,
-            x='date',
-            y='log_count',
-            title="Daily Log Trends",
-            labels={"date": "Date", "log_count": "Number of Logs"}
-        )
-        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
-        return fig
-    except Exception as e:
-        logging.error(f"Failed to create daily log trends chart: {str(e)}")
-        return None
+    return None
 
-# Create weekly uptime chart
-@cache_dataframe
 def create_weekly_uptime_chart(df):
-    try:
-        df['week'] = df['timestamp'].dt.isocalendar().week
-        df['year'] = df['timestamp'].dt.year
-        weekly_data = df.groupby(['year', 'week']).agg({
-            'usage_hours': 'sum',
-            'downtime': 'sum'
-        }).reset_index()
-        weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
-        weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
-        fig = px.bar(
-            weekly_data,
-            x='year_week',
-            y='uptime_percent',
-            title="Weekly Uptime Percentage",
-            labels={"year_week": "Year-Week", "uptime_percent": "Uptime %"}
-        )
-        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
-        return fig
-    except Exception as e:
-        logging.error(f"Failed to create weekly uptime chart: {str(e)}")
-        return None
+    return None
 
-# Create anomaly alerts chart
-@cache_dataframe
 def create_anomaly_alerts_chart(anomalies_df):
-    try:
-        if anomalies_df.empty:
-            return None
-        anomalies_df['date'] = anomalies_df['timestamp'].dt.date
-        anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
-        fig = px.scatter(
-            anomaly_counts,
-            x='date',
-            y='anomaly_count',
-            title="Anomaly Alerts Over Time",
-            labels={"date": "Date", "anomaly_count": "Number of Anomalies"}
-        )
-        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
-        return fig
-    except Exception as e:
-        logging.error(f"Failed to create anomaly alerts chart: {str(e)}")
-        return None
+    return None
 
 # Generate device cards
 def generate_device_cards(df):
@@ -440,8 +397,8 @@ def generate_device_cards(df):
         logging.error(f"Failed to generate device cards: {str(e)}")
         return f'<p>Error generating device cards: {str(e)}</p>'
 
-# Generate PDF content
-def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart):
+# Generate PDF content (Simplified to reduce time)
+def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html):
     if not reportlab_available:
         return None
     try:
@@ -500,9 +457,6 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         story.append(safe_paragraph(insights, styles['Normal']))
         story.append(Spacer(1, 12))
 
-        story.append(Paragraph("Charts", styles['Heading2']))
-        story.append(Paragraph("[Chart placeholders - see dashboard for visuals]", styles['Normal']))
-
         doc.build(story)
         logging.info(f"PDF generated at {pdf_path}")
         return pdf_path
@@ -511,8 +465,9 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         return None
 
 # Main processing function
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state):
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, progress=gr.Progress()):
     start_time = time.time()
+    progress(0, desc="Starting processing...")
     try:
         if not file_obj:
             return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
@@ -526,6 +481,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         if not file_path.endswith(".csv"):
             return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
 
+        progress(0.1, desc="Loading CSV file...")
         required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
         dtypes = {
             "device_id": "string",
@@ -536,14 +492,15 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             "amc_date": "string"
         }
         df = pd.read_csv(file_path, dtype=dtypes, usecols=required_columns)
-        if len(df) > 10000:  # Early exit for large datasets
-            df = df.sample(n=10000, random_state=42)
-            logging.warning("Dataset too large, sampled to 10,000 rows")
+        if len(df) > 5000:  # More aggressive sampling
+            df = df.sample(n=5000, random_state=42)
+            logging.warning("Dataset too large, sampled to 5,000 rows")
 
         missing_columns = [col for col in required_columns if col not in df.columns]
         if missing_columns:
             return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
+        progress(0.2, desc="Processing timestamps...")
         df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
         df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
         if df["timestamp"].dt.tz is None:
@@ -552,7 +509,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
         # Apply filters
-        filtered_df = df.copy()
+        filtered_df = df
         if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
             filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
         if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
@@ -568,43 +525,51 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
         # Generate table for preview
+        progress(0.3, desc="Generating log preview...")
         preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
         preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
         # Run tasks concurrently
-        with ThreadPoolExecutor(max_workers=8) as executor:
+        progress(0.4, desc="Running analysis tasks...")
+        with ThreadPoolExecutor(max_workers=4) as executor:  # Reduced workers to avoid overhead
             future_summary = executor.submit(summarize_logs, filtered_df)
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
             future_insights = executor.submit(generate_dashboard_insights, filtered_df)
             future_usage_chart = executor.submit(create_usage_chart, filtered_df)
-            future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
-            future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
-            future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
             future_device_cards = executor.submit(generate_device_cards, filtered_df)
-            future_reports = executor.submit(create_salesforce_reports, filtered_df)
 
+            progress(0.5, desc="Collecting summary results...")
             summary = f"Step 1: Summary Report\n{future_summary.result()}"
+            progress(0.6, desc="Collecting anomaly detection results...")
             anomalies, anomalies_df = future_anomalies.result()
             anomalies = f"Anomaly Detection\n{anomalies}"
+            progress(0.7, desc="Collecting AMC reminders...")
             amc_reminders, reminders_df = future_amc.result()
             amc_reminders = f"AMC Reminders\n{amc_reminders}"
+            progress(0.8, desc="Collecting insights...")
             insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
+            progress(0.9, desc="Generating chart...")
             usage_chart = future_usage_chart.result()
-            downtime_chart = future_downtime_chart.result()
-            daily_log_chart = future_daily_log_chart.result()
-            weekly_uptime_chart = future_weekly_uptime_chart.result()
-            anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
+            downtime_chart = None
+            daily_log_chart = None
+            weekly_uptime_chart = None
+            anomaly_alerts_chart = None
             device_cards = future_device_cards.result()
 
-        save_to_salesforce(filtered_df, reminders_df)
-        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart)
+        # Skip Salesforce operations
+        # save_to_salesforce(filtered_df, reminders_df)
+        # create_salesforce_reports(filtered_df)
+
+        progress(0.95, desc="Generating PDF...")
+        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards)
 
         elapsed_time = time.time() - start_time
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
-        if elapsed_time > 30:
-            logging.warning(f"Processing time exceeded 30 seconds: {elapsed_time:.2f} seconds")
+        if elapsed_time > 10:
+            logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
 
+        progress(1.0, desc="Processing complete!")
         return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")