Update app.py

app.py

@@ -18,19 +18,6 @@ import functools
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # Salesforce configuration (Disabled for now)
-"""
-try:
-    sf = Salesforce(
-        username='multi-devicelabopsdashboard@sathkrutha.com',
-        password='Team@1234',
-        security_token=os.getenv('SF_SECURITY_TOKEN', ''),
-        domain='login'
-    )
-    logging.info("Salesforce connection established")
-except Exception as e:
-    logging.error(f"Failed to connect to Salesforce: {str(e)}")
-    sf = None
-"""
 sf = None  # Temporarily disable Salesforce
 
 # Try to import reportlab
@@ -53,9 +40,9 @@ try:
         "summarization",
         model="t5-small",
         device=device,
-        max_length=30,  # Reduced for faster inference
+        max_length=30,
         min_length=10,
-        num_beams=1  # Reduced for faster inference
+        num_beams=1
     )
     logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
 except Exception as e:
@@ -125,111 +112,11 @@ LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
 def create_salesforce_reports(df):
     logging.info("Salesforce report creation skipped for optimization")
     return
-"""
-def create_salesforce_reports(df):
-    if sf is None or not LABOPS_REPORTS_FOLDER_ID:
-        return
-    try:
-        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-        reports = [
-            {
-                "reportMetadata": {
-                    "name": f"SmartLog_Usage_Report_{timestamp}",
-                    "developerName": f"SmartLog_Usage_Report_{timestamp}",
-                    "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
-                    "reportFormat": "TABULAR",
-                    "reportBooleanFilter": None,
-                    "reportFilters": [],
-                    "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.Usage_Hours__c"],
-                    "folderId": LABOPS_REPORTS_FOLDER_ID
-                }
-            },
-            {
-                "reportMetadata": {
-                    "name": f"SmartLog_AMC_Reminders_{timestamp}",
-                    "developerName": f"SmartLog_AMC_Reminders_{timestamp}",
-                    "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
-                    "reportFormat": "TABULAR",
-                    "reportBooleanFilter": None,
-                    "reportFilters": [],
-                    "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.AMC_Date__c"],
-                    "folderId": LABOPS_REPORTS_FOLDER_ID
-                }
-            }
-        ]
-        for report in reports:
-            sf.restful('analytics/reports', method='POST', json=report)
-        logging.info("Salesforce reports created")
-    except Exception as e:
-        logging.error(f"Failed to create Salesforce reports: {str(e)}")
-"""
 
 # Save to Salesforce (Disabled for now)
 def save_to_salesforce(df, reminders_df):
     logging.info("Salesforce save operation skipped for optimization")
     return
-"""
-def save_to_salesforce(df, reminders_df):
-    if sf is None:
-        logging.error("No Salesforce connection available")
-        return
-    try:
-        logging.info("Starting Salesforce save operation")
-        current_date = datetime.now()
-        next_30_days = current_date + timedelta(days=30)
-        records = []
-        reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
-        logging.info(f"Processing {len(df)} records for Salesforce")
-
-        for idx, row in df.iterrows():
-            status = str(row['status']).lower()
-            log_type = str(row['log_type']).lower()
-            status_mapped = picklist_mapping['Status__c'].get(status, status_values[0] if status_values else 'Active')
-            log_type_mapped = picklist_mapping['Log_Type__c'].get(log_type, log_type_values[0] if log_type_values else 'Smart Log')
-            
-            if not status_mapped or not log_type_mapped:
-                logging.warning(f"Skipping record {idx}: Invalid status ({status}) or log_type ({log_type})")
-                continue
-
-            amc_date_str = None
-            if pd.notna(row['amc_date']):
-                try:
-                    amc_date = pd.to_datetime(row['amc_date']).strftime('%Y-%m-%d')
-                    amc_date_str = amc_date
-                    amc_date_dt = datetime.strptime(amc_date, '%Y-%m-%d')
-                    if status_mapped == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
-                        logging.info(f"AMC Reminder for Device ID {row['device_id']}: {amc_date}")
-                except Exception as e:
-                    logging.warning(f"Invalid AMC date for Device ID {row['device_id']}: {str(e)}")
-
-            record = {
-                'Device_Id__c': str(row['device_id'])[:50],
-                'Log_Type__c': log_type_mapped,
-                'Status__c': status_mapped,
-                'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
-                'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
-                'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
-                'AMC_Date__c': amc_date_str
-            }
-            records.append(record)
-
-        if records:
-            batch_size = 100
-            for i in range(0, len(records), batch_size):
-                batch = records[i:i + batch_size]
-                try:
-                    result = sf.bulk.SmartLog__c.insert(batch)
-                    logging.info(f"Saved {len(batch)} records to Salesforce in batch {i//batch_size + 1}")
-                    for res in result:
-                        if not res['success']:
-                            logging.error(f"Failed to save record: {res['errors']}")
-                except Exception as e:
-                    logging.error(f"Failed to save batch {i//batch_size + 1}: {str(e)}")
-        else:
-            logging.warning("No records to save to Salesforce")
-    except Exception as e:
-        logging.error(f"Failed to save to Salesforce: {str(e)}")
-"""
 
 # Cache summarization results
 def cache_summary(func):
@@ -267,9 +154,9 @@ def detect_anomalies(df):
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
         features = df[["usage_hours", "downtime"]].fillna(0)
-        if len(features) > 100:  # Further reduced sample size
+        if len(features) > 100:
             features = features.sample(n=100, random_state=42)
-        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=30)  # Further reduced n_estimators
+        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=30)
         df["anomaly"] = iso_forest.fit_predict(features)
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
         if anomalies.empty:
@@ -329,7 +216,7 @@ def cache_dataframe(func):
         return result
     return wrapper
 
-# Create usage chart (Only this chart will be generated to save time)
+# Create usage chart
 @cache_dataframe
 def create_usage_chart(df):
     try:
@@ -351,18 +238,102 @@ def create_usage_chart(df):
         logging.error(f"Failed to create usage chart: {str(e)}")
         return None
 
-# Skipped other chart functions to save time
+# Create downtime chart (Re-enabled with optimization)
+@cache_dataframe
 def create_downtime_chart(df):
-    return None
+    try:
+        if df.empty:
+            return None
+        downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
+        if len(downtime_data) > 5:
+            downtime_data = downtime_data.nlargest(5, "downtime")
+        fig = px.bar(
+            downtime_data,
+            x="device_id",
+            y="downtime",
+            title="Downtime per Device",
+            labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"}
+        )
+        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create downtime chart: {str(e)}")
+        return None
 
+# Create daily log trends chart (Re-enabled with optimization)
+@cache_dataframe
 def create_daily_log_trends_chart(df):
-    return None
+    try:
+        if df.empty:
+            return None
+        df['date'] = df['timestamp'].dt.date
+        daily_logs = df.groupby('date').size().reset_index(name='log_count')
+        if len(daily_logs) > 30:  # Limit to 30 days for faster plotting
+            daily_logs = daily_logs.tail(30)
+        fig = px.line(
+            daily_logs,
+            x='date',
+            y='log_count',
+            title="Daily Log Trends",
+            labels={"date": "Date", "log_count": "Number of Logs"}
+        )
+        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create daily log trends chart: {str(e)}")
+        return None
 
+# Create weekly uptime chart (Re-enabled with optimization)
+@cache_dataframe
 def create_weekly_uptime_chart(df):
-    return None
+    try:
+        if df.empty:
+            return None
+        df['week'] = df['timestamp'].dt.isocalendar().week
+        df['year'] = df['timestamp'].dt.year
+        weekly_data = df.groupby(['year', 'week']).agg({
+            'usage_hours': 'sum',
+            'downtime': 'sum'
+        }).reset_index()
+        if len(weekly_data) > 12:  # Limit to 12 weeks for faster plotting
+            weekly_data = weekly_data.tail(12)
+        weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
+        weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
+        fig = px.bar(
+            weekly_data,
+            x='year_week',
+            y='uptime_percent',
+            title="Weekly Uptime Percentage",
+            labels={"year_week": "Year-Week", "uptime_percent": "Uptime %"}
+        )
+        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create weekly uptime chart: {str(e)}")
+        return None
 
+# Create anomaly alerts chart (Re-enabled with optimization)
+@cache_dataframe
 def create_anomaly_alerts_chart(anomalies_df):
-    return None
+    try:
+        if anomalies_df.empty:
+            return None
+        anomalies_df['date'] = anomalies_df['timestamp'].dt.date
+        anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
+        if len(anomaly_counts) > 30:  # Limit to 30 days for faster plotting
+            anomaly_counts = anomaly_counts.tail(30)
+        fig = px.scatter(
+            anomaly_counts,
+            x='date',
+            y='anomaly_count',
+            title="Anomaly Alerts Over Time",
+            labels={"date": "Date", "anomaly_count": "Number of Anomalies"}
+        )
+        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create anomaly alerts chart: {str(e)}")
+        return None
 
 # Generate device cards
 def generate_device_cards(df):
@@ -397,7 +368,7 @@ def generate_device_cards(df):
         logging.error(f"Failed to generate device cards: {str(e)}")
         return f'<p>Error generating device cards: {str(e)}</p>'
 
-# Generate PDF content (Simplified to reduce time)
+# Generate PDF content
 def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html):
     if not reportlab_available:
         return None
@@ -470,7 +441,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
     progress(0, desc="Starting processing...")
     try:
         if not file_obj:
-            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
+            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cardsPEM available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
 
         file_path = file_obj.name
         current_modified_time = os.path.getmtime(file_path)
@@ -492,7 +463,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             "amc_date": "string"
         }
         df = pd.read_csv(file_path, dtype=dtypes, usecols=required_columns)
-        if len(df) > 5000:  # More aggressive sampling
+        if len(df) > 5000:
             df = df.sample(n=5000, random_state=42)
             logging.warning("Dataset too large, sampled to 5,000 rows")
 
@@ -531,12 +502,15 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
 
         # Run tasks concurrently
         progress(0.4, desc="Running analysis tasks...")
-        with ThreadPoolExecutor(max_workers=4) as executor:  # Reduced workers to avoid overhead
+        with ThreadPoolExecutor(max_workers=4) as executor:
             future_summary = executor.submit(summarize_logs, filtered_df)
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
             future_insights = executor.submit(generate_dashboard_insights, filtered_df)
             future_usage_chart = executor.submit(create_usage_chart, filtered_df)
+            future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
+            future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
+            future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
             future_device_cards = executor.submit(generate_device_cards, filtered_df)
 
             progress(0.5, desc="Collecting summary results...")
@@ -549,25 +523,21 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             amc_reminders = f"AMC Reminders\n{amc_reminders}"
             progress(0.8, desc="Collecting insights...")
             insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
-            progress(0.9, desc="Generating chart...")
+            progress(0.9, desc="Generating charts...")
             usage_chart = future_usage_chart.result()
-            downtime_chart = None
-            daily_log_chart = None
-            weekly_uptime_chart = None
-            anomaly_alerts_chart = None
+            downtime_chart = future_downtime_chart.result()
+            daily_log_chart = future_daily_log_chart.result()
+            weekly_uptime_chart = future_weekly_uptime_chart.result()
+            anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
             device_cards = future_device_cards.result()
 
-        # Skip Salesforce operations
-        # save_to_salesforce(filtered_df, reminders_df)
-        # create_salesforce_reports(filtered_df)
-
         progress(0.95, desc="Generating PDF...")
         pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards)
 
         elapsed_time = time.time() - start_time
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
-        if elapsed_time > 10:
-            logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
+        if elapsed_time > 30:
+            logging.warning(f"Processing time exceeded 30 seconds: {elapsed_time:.2f} seconds")
 
         progress(1.0, desc="Processing complete!")
         return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)