Update app.py

app.py

@@ -48,9 +48,9 @@ try:
         "summarization",
         model="t5-small",
         device=device,
-        max_length=40,
+        max_length=50,
         min_length=10,
-        num_beams=1
+        num_beams=2
     )
     logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
 except Exception as e:
@@ -164,12 +164,10 @@ def save_to_salesforce(df, reminders_df):
         current_date = datetime.now()
         next_30_days = current_date + timedelta(days=30)
         records = []
-        # Cap the number of records to save
-        max_records_to_save = 1000
-        df_to_save = df.head(max_records_to_save)
-        logging.info(f"Processing {len(df_to_save)} records for Salesforce (capped at {max_records_to_save})")
+        reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
+        logging.info(f"Processing {len(df)} records for Salesforce")
 
-        for idx, row in df_to_save.iterrows():
+        for idx, row in df.iterrows():
             status = str(row['status']).lower()
             log_type = str(row['log_type']).lower()
             status_mapped = picklist_mapping['Status__c'].get(status, status_values[0] if status_values else 'Active')
@@ -202,7 +200,7 @@ def save_to_salesforce(df, reminders_df):
             records.append(record)
 
         if records:
-            batch_size = 100  # Reduced batch size for faster inserts
+            batch_size = 200  # Smaller batch size for faster processing
             for i in range(0, len(records), batch_size):
                 batch = records[i:i + batch_size]
                 try:
@@ -222,12 +220,10 @@ def save_to_salesforce(df, reminders_df):
 def summarize_logs(df):
     start_time = time.time()
     try:
-        # Sample data for summarization if large
-        sample_df = df.sample(n=min(500, len(df)), random_state=42) if len(df) > 500 else df
-        total_devices = sample_df["device_id"].nunique()
-        most_used = sample_df.groupby("device_id")["usage_hours"].sum().idxmax() if not sample_df.empty else "N/A"
+        total_devices = df["device_id"].nunique()
+        most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
         prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
-        summary = summarizer(prompt, max_length=40, min_length=10, do_sample=False)[0]["summary_text"]
+        summary = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
         logging.info(f"Summary generation took {time.time() - start_time:.2f} seconds")
         return summary
     except Exception as e:
@@ -278,12 +274,10 @@ def check_amc_reminders(df, current_date):
 def generate_dashboard_insights(df):
     start_time = time.time()
     try:
-        # Sample data for insights if large
-        sample_df = df.sample(n=min(500, len(df)), random_state=42) if len(df) > 500 else df
-        total_devices = sample_df["device_id"].nunique()
-        avg_usage = sample_df["usage_hours"].mean() if "usage_hours" in sample_df.columns else 0
+        total_devices = df["device_id"].nunique()
+        avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
         prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
-        insights = summarizer(prompt, max_length=40, min_length=10, do_sample=False)[0]["summary_text"]
+        insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
         logging.info(f"Insights generation took {time.time() - start_time:.2f} seconds")
         return insights
     except Exception as e:
@@ -296,13 +290,13 @@ def create_usage_chart(df):
         if df.empty:
             return None
         usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
-        if len(usage_data) > 3:  # Reduced to top 3
-            usage_data = usage_data.nlargest(3, "usage_hours")
+        if len(usage_data) > 5:
+            usage_data = usage_data.nlargest(5, "usage_hours")
         fig = px.bar(
             usage_data,
             x="device_id",
             y="usage_hours",
-            title="Usage Hours per Device (Top 3)",
+            title="Usage Hours per Device",
             labels={"device_id": "Device ID", "usage_hours": "Usage Hours"}
         )
         fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
@@ -315,13 +309,13 @@ def create_usage_chart(df):
 def create_downtime_chart(df):
     try:
         downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
-        if len(downtime_data) > 3:  # Reduced to top 3
-            downtime_data = downtime_data.nlargest(3, "downtime")
+        if len(downtime_data) > 5:
+            downtime_data = downtime_data.nlargest(5, "downtime")
         fig = px.bar(
             downtime_data,
             x="device_id",
             y="downtime",
-            title="Downtime per Device (Top 3)",
+            title="Downtime per Device",
             labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"}
         )
         fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
@@ -348,6 +342,30 @@ def create_daily_log_trends_chart(df):
         logging.error(f"Failed to create daily log trends chart: {str(e)}")
         return None
 
+# Create weekly uptime chart
+def create_weekly_uptime_chart(df):
+    try:
+        df['week'] = df['timestamp'].dt.isocalendar().week
+        df['year'] = df['timestamp'].dt.year
+        weekly_data = df.groupby(['year', 'week']).agg({
+            'usage_hours': 'sum',
+            'downtime': 'sum'
+        }).reset_index()
+        weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
+        weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
+        fig = px.bar(
+            weekly_data,
+            x='year_week',
+            y='uptime_percent',
+            title="Weekly Uptime Percentage",
+            labels={"year_week": "Year-Week", "uptime_percent": "Uptime %"}
+        )
+        fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create weekly uptime chart: {str(e)}")
+        return None
+
 # Create anomaly alerts chart
 def create_anomaly_alerts_chart(anomalies_df):
     try:
@@ -401,12 +419,32 @@ def generate_device_cards(df):
         logging.error(f"Failed to generate device cards: {str(e)}")
         return f'<p>Error generating device cards: {str(e)}</p>'
 
+# Generate monthly status
+def generate_monthly_status(df, selected_month):
+    try:
+        total_devices = df['device_id'].nunique()
+        total_usage_hours = df['usage_hours'].sum()
+        total_downtime = df['downtime'].sum()
+        avg_usage = total_usage_hours / total_devices if total_devices > 0 else 0
+        avg_downtime = total_downtime / total_devices if total_devices > 0 else 0
+        return f"""
+        Monthly Status for {selected_month}:
+        - Total Devices: {total_devices}
+        - Total Usage Hours: {total_usage_hours:.2f}
+        - Total Downtime Hours: {total_downtime:.2f}
+        - Average Usage per Device: {avg_usage:.2f} hours
+        - Average Downtime per Device: {avg_downtime:.2f} hours
+        """
+    except Exception as e:
+        logging.error(f"Failed to generate monthly status: {str(e)}")
+        return f"Failed to generate monthly status: {str(e)}"
+
 # Generate PDF content
-def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, anomaly_alerts_chart, downtime_chart, df):
+def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, df, selected_month):
     if not reportlab_available:
         return None
     try:
-        pdf_path = f"status_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
+        pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
         doc = SimpleDocTemplate(pdf_path, pagesize=letter)
         styles = getSampleStyleSheet()
         story = []
@@ -414,38 +452,38 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         def safe_paragraph(text, style):
             return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
 
-        story.append(Paragraph("LabOps Status Report", styles['Title']))
+        story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
         story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
         story.append(Spacer(1, 12))
 
+        if selected_month != "All":
+            monthly_status = generate_monthly_status(df, selected_month)
+            story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
+            story.append(safe_paragraph(monthly_status, styles['Normal']))
+            story.append(Spacer(1, 12))
+
         story.append(Paragraph("Summary Report", styles['Heading2']))
         story.append(safe_paragraph(summary, styles['Normal']))
         story.append(Spacer(1, 12))
 
         story.append(Paragraph("Log Preview", styles['Heading2']))
         if not preview_df.empty:
-            # Limit PDF table to 100 rows to avoid excessive rendering time
-            max_pdf_rows = 100
-            pdf_preview_df = preview_df.head(max_pdf_rows)
-            data = [pdf_preview_df.columns.tolist()] + pdf_preview_df.values.tolist()
+            data = [preview_df.columns.tolist()] + preview_df.head(5).values.tolist()
             table = Table(data)
             table.setStyle(TableStyle([
                 ('BACKGROUND', (0, 0), (-1, 0), colors.grey),
                 ('TEXTCOLOR', (0, 0), (-1, 0), colors.whitesmoke),
                 ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
                 ('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'),
-                ('FONTSIZE', (0, 0), (-1, 0), 8),
-                ('BOTTOMPADDING', (0, 0), (-1, 0), 10),
+                ('FONTSIZE', (0, 0), (-1, 0), 12),
+                ('BOTTOMPADDING', (0, 0), (-1, 0), 12),
                 ('BACKGROUND', (0, 1), (-1, -1), colors.beige),
                 ('TEXTCOLOR', (0, 1), (-1, -1), colors.black),
                 ('FONTNAME', (0, 1), (-1, -1), 'Helvetica'),
-                ('FONTSIZE', (0, 1), (-1, -1), 6),
+                ('FONTSIZE', (0, 1), (-1, -1), 10),
                 ('GRID', (0, 0), (-1, -1), 1, colors.black)
             ]))
             story.append(table)
-            if len(preview_df) > max_pdf_rows:
-                story.append(Spacer(1, 12))
-                story.append(safe_paragraph(f"Note: Log Preview in PDF limited to {max_pdf_rows} rows. See dashboard for full table.", styles['Normal']))
         else:
             story.append(safe_paragraph("No preview available.", styles['Normal']))
         story.append(Spacer(1, 12))
@@ -478,23 +516,21 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         return None
 
 # Main processing function
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state):
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
     start_time = time.time()
     try:
         if not file_obj:
-            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
+            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
 
         file_path = file_obj.name
         current_modified_time = os.path.getmtime(file_path)
         if last_modified_state and current_modified_time == last_modified_state:
-            return None, None, None, None, None, None, None, None, None, None, None, last_modified_state
+            return None, None, None, None, None, None, None, None, None, None, None, None, last_modified_state
 
         logging.info(f"Processing file: {file_path}")
         if not file_path.endswith(".csv"):
-            return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, "", "", "", None, last_modified_state
+            return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
 
-        # Cap the CSV size to avoid excessive processing
-        max_rows = 5000
         required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
         dtypes = {
             "device_id": "string",
@@ -504,19 +540,17 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             "downtime": "float32",
             "amc_date": "string"
         }
-        df = pd.read_csv(file_path, dtype=dtypes, low_memory=False, nrows=max_rows)
-        if len(df) == max_rows:
-            logging.warning(f"CSV exceeds {max_rows} rows. Only the first {max_rows} rows will be processed.")
+        df = pd.read_csv(file_path, dtype=dtypes)
         missing_columns = [col for col in required_columns if col not in df.columns]
         if missing_columns:
-            return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, last_modified_state
+            return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
         df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
         df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
         if df["timestamp"].dt.tz is None:
             df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
         if df.empty:
-            return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, last_modified_state
+            return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
         # Apply filters
         filtered_df = df.copy()
@@ -530,16 +564,22 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             start_date = today + pd.Timedelta(days=days_start)
             end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
             filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
+        if month_filter and month_filter != "All":
+            selected_date = pd.to_datetime(month_filter, format="%B %Y")
+            filtered_df = filtered_df[
+                (filtered_df['timestamp'].dt.year == selected_date.year) &
+                (filtered_df['timestamp'].dt.month == selected_date.month)
+            ]
 
         if filtered_df.empty:
-            return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, last_modified_state
+            return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
-        # Generate table for preview with all rows
-        preview_df = filtered_df  # Show all rows
+        # Generate table for preview
+        preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
         preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
         # Run tasks concurrently
-        with ThreadPoolExecutor(max_workers=8) as executor:  # Increased workers
+        with ThreadPoolExecutor(max_workers=6) as executor:
             future_summary = executor.submit(summarize_logs, filtered_df)
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
@@ -547,6 +587,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             future_usage_chart = executor.submit(create_usage_chart, filtered_df)
             future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
             future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
+            future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
             future_device_cards = executor.submit(generate_device_cards, filtered_df)
             future_reports = executor.submit(create_salesforce_reports, filtered_df)
 
@@ -559,39 +600,41 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             usage_chart = future_usage_chart.result()
             downtime_chart = future_downtime_chart.result()
             daily_log_chart = future_daily_log_chart.result()
-            anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
+            weekly_uptime_chart = future_weekly_uptime_chart.result()
+            anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)  # Use anomalies_df
             device_cards = future_device_cards.result()
 
         save_to_salesforce(filtered_df, reminders_df)
-        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, anomaly_alerts_chart, downtime_chart, filtered_df)
+        pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, filtered_df, month_filter)
 
         elapsed_time = time.time() - start_time
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
-        if elapsed_time > 20:
-            logging.warning(f"Processing time exceeded 20 seconds: {elapsed_time:.2f} seconds")
+        if elapsed_time > 10:
+            logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
 
-        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
+        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, last_modified_state
+        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
 # Update filters
 def update_filters(file_obj):
     if not file_obj:
-        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
+        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
     try:
         with open(file_obj.name, 'rb') as f:
             csv_content = f.read().decode('utf-8')
-        df = pd.read_csv(io.StringIO(csv_content), low_memory=False, nrows=5000)
+        df = pd.read_csv(io.StringIO(csv_content))
         df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
 
         lab_site_options = ['All'] + [site for site in df['lab_site'].dropna().astype(str).unique().tolist() if site.strip()] if 'lab_site' in df.columns else ['All']
         equipment_type_options = ['All'] + [equip for equip in df['equipment_type'].dropna().astype(str).unique().tolist() if equip.strip()] if 'equipment_type' in df.columns else ['All']
+        month_options = ['All'] + sorted(df['timestamp'].dt.strftime('%B %Y').dropna().unique().tolist()) if 'timestamp' in df.columns else ['All']
 
-        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All')
+        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
     except Exception as e:
         logging.error(f"Failed to update filters: {str(e)}")
-        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
+        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
 
 # Gradio Interface
 try:
@@ -621,6 +664,7 @@ try:
                     lab_site_filter = gr.Dropdown(label="Lab Site", choices=['All'], value='All', interactive=True)
                     equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
                     date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
+                    month_filter = gr.Dropdown(label="Select Month for Report", choices=['All'], value='All', interactive=True)
                 submit_button = gr.Button("Analyze", variant="primary")
 
             with gr.Column(scale=2):
@@ -643,6 +687,8 @@ try:
                             downtime_chart_output = gr.Plot()
                         with gr.Tab("Daily Log Trends"):
                             daily_log_trends_output = gr.Plot()
+                        with gr.Tab("Weekly Uptime Percentage"):
+                            weekly_uptime_output = gr.Plot()
                         with gr.Tab("Anomaly Alerts"):
                             anomaly_alerts_output = gr.Plot()
                     with gr.Group(elem_classes="dashboard-section"):
@@ -656,19 +702,19 @@ try:
                         insights_output = gr.Markdown()
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Export Report")
-                        pdf_output = gr.File(label="Download Status Report as PDF")
+                        pdf_output = gr.File(label="Download Monthly Status Report as PDF")
 
         file_input.change(
             fn=update_filters,
             inputs=[file_input],
-            outputs=[lab_site_filter, equipment_type_filter],
+            outputs=[lab_site_filter, equipment_type_filter, month_filter],
             queue=False
         )
 
         submit_button.click(
             fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
+            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, month_filter, last_modified_state],
+            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
         )
 
     logging.info("Gradio interface initialized successfully")