Update app.py

app.py

@@ -12,6 +12,7 @@ import os
 import io
 import time
 import uuid
+import functools
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -201,7 +202,7 @@ def save_to_salesforce(df, reminders_df):
             records.append(record)
 
         if records:
-            batch_size = 100  # Reduced batch size for faster processing
+            batch_size = 100
             for i in range(0, len(records), batch_size):
                 batch = records[i:i + batch_size]
                 try:
@@ -238,9 +239,9 @@ def detect_anomalies(df):
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
         features = df[["usage_hours", "downtime"]].fillna(0)
-        if len(features) > 200:  # Reduced sample size for faster processing
+        if len(features) > 200:
             features = features.sample(n=200, random_state=42)
-        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=50)  # Reduced n_estimators
+        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=50)
         df["anomaly"] = iso_forest.fit_predict(features)
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
         if anomalies.empty:
@@ -285,7 +286,22 @@ def generate_dashboard_insights(df):
         logging.error(f"Dashboard insights generation failed: {str(e)}")
         return f"Dashboard insights generation failed: {str(e)}"
 
+# Cache DataFrame processing
+def cache_dataframe(func):
+    @functools.wraps(func)
+    def wrapper(df, *args, **kwargs):
+        cache_key = f"{id(df)}_{func.__name__}"
+        if not hasattr(wrapper, 'cache'):
+            wrapper.cache = {}
+        if cache_key in wrapper.cache:
+            return wrapper.cache[cache_key]
+        result = func(df, *args, **kwargs)
+        wrapper.cache[cache_key] = result
+        return result
+    return wrapper
+
 # Create usage chart
+@cache_dataframe
 def create_usage_chart(df):
     try:
         if df.empty:
@@ -307,6 +323,7 @@ def create_usage_chart(df):
         return None
 
 # Create downtime chart
+@cache_dataframe
 def create_downtime_chart(df):
     try:
         downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
@@ -326,6 +343,7 @@ def create_downtime_chart(df):
         return None
 
 # Create daily log trends chart
+@cache_dataframe
 def create_daily_log_trends_chart(df):
     try:
         df['date'] = df['timestamp'].dt.date
@@ -344,6 +362,7 @@ def create_daily_log_trends_chart(df):
         return None
 
 # Create weekly uptime chart
+@cache_dataframe
 def create_weekly_uptime_chart(df):
     try:
         df['week'] = df['timestamp'].dt.isocalendar().week
@@ -368,6 +387,7 @@ def create_weekly_uptime_chart(df):
         return None
 
 # Create anomaly alerts chart
+@cache_dataframe
 def create_anomaly_alerts_chart(anomalies_df):
     try:
         if anomalies_df.empty:
@@ -516,6 +536,10 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             "amc_date": "string"
         }
         df = pd.read_csv(file_path, dtype=dtypes, usecols=required_columns)
+        if len(df) > 10000:  # Early exit for large datasets
+            df = df.sample(n=10000, random_state=42)
+            logging.warning("Dataset too large, sampled to 10,000 rows")
+
         missing_columns = [col for col in required_columns if col not in df.columns]
         if missing_columns:
             return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
@@ -548,7 +572,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
         # Run tasks concurrently
-        with ThreadPoolExecutor(max_workers=8) as executor:  # Increased workers for better parallelization
+        with ThreadPoolExecutor(max_workers=8) as executor:
             future_summary = executor.submit(summarize_logs, filtered_df)
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
@@ -681,7 +705,7 @@ try:
         submit_button.click(
             fn=process_logs,
             inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_chart, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
+            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
         )
 
     logging.info("Gradio interface initialized successfully")