RathodHarish commited on
Commit
4340807
·
verified ·
1 Parent(s): 7a4c424

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +165 -114
app.py CHANGED
@@ -9,16 +9,25 @@ import torch
9
  from concurrent.futures import ThreadPoolExecutor
10
  from simple_salesforce import Salesforce
11
  import os
 
12
  import io
13
  import time
14
- import uuid
15
- import functools
16
 
17
  # Configure logging
18
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
19
 
20
- # Salesforce configuration (Disabled for now)
21
- sf = None # Temporarily disable Salesforce
 
 
 
 
 
 
 
 
 
 
22
 
23
  # Try to import reportlab
24
  try:
@@ -36,13 +45,14 @@ except ImportError:
36
  logging.info("Preloading Hugging Face model...")
37
  try:
38
  device = 0 if torch.cuda.is_available() else -1
 
39
  summarizer = pipeline(
40
  "summarization",
41
  model="t5-small",
42
  device=device,
43
- max_length=30,
44
  min_length=10,
45
- num_beams=1
46
  )
47
  logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
48
  except Exception as e:
@@ -108,40 +118,97 @@ def get_folder_id(folder_name):
108
 
109
  LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
110
 
111
- # Salesforce report creation (Disabled for now)
112
  def create_salesforce_reports(df):
113
- logging.info("Salesforce report creation skipped for optimization")
114
- return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
 
116
- # Save to Salesforce (Disabled for now)
117
  def save_to_salesforce(df, reminders_df):
118
- logging.info("Salesforce save operation skipped for optimization")
119
- return
120
-
121
- # Cache summarization results
122
- def cache_summary(func):
123
- @functools.wraps(func)
124
- def wrapper(df, *args, **kwargs):
125
- cache_key = f"{id(df)}_{func.__name__}"
126
- if not hasattr(wrapper, 'cache'):
127
- wrapper.cache = {}
128
- if cache_key in wrapper.cache:
129
- return wrapper.cache[cache_key]
130
- result = func(df, *args, **kwargs)
131
- wrapper.cache[cache_key] = result
132
- return result
133
- return wrapper
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
 
135
  # Summarize logs
136
- @cache_summary
137
  def summarize_logs(df):
138
- start_time = time.time()
139
  try:
140
  total_devices = df["device_id"].nunique()
141
  most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
142
  prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
143
- summary = summarizer(prompt, max_length=30, min_length=10, do_sample=False)[0]["summary_text"]
144
- logging.info(f"Summary generation took {time.time() - start_time:.2f} seconds")
145
  return summary
146
  except Exception as e:
147
  logging.error(f"Summary generation failed: {str(e)}")
@@ -149,28 +216,24 @@ def summarize_logs(df):
149
 
150
  # Anomaly detection
151
  def detect_anomalies(df):
152
- start_time = time.time()
153
  try:
154
  if "usage_hours" not in df.columns or "downtime" not in df.columns:
155
  return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
156
  features = df[["usage_hours", "downtime"]].fillna(0)
157
- if len(features) > 100:
158
- features = features.sample(n=100, random_state=42)
159
- iso_forest = IsolationForest(contamination=0.1, random_state=42, n_estimators=30)
160
  df["anomaly"] = iso_forest.fit_predict(features)
161
  anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
162
  if anomalies.empty:
163
  return "No anomalies detected.", anomalies
164
- result = "\n".join([f"- Device ID: {row['device_id']}, Usage: {row['usage_hours']}, Downtime: {row['downtime']}, Timestamp: {row['timestamp']}" for _, row in anomalies.head(5).iterrows()])
165
- logging.info(f"Anomaly detection took {time.time() - start_time:.2f} seconds")
166
- return result, anomalies
167
  except Exception as e:
168
  logging.error(f"Anomaly detection failed: {str(e)}")
169
  return f"Anomaly detection failed: {str(e)}", pd.DataFrame()
170
 
171
  # AMC reminders
172
  def check_amc_reminders(df, current_date):
173
- start_time = time.time()
174
  try:
175
  if "device_id" not in df.columns or "amc_date" not in df.columns:
176
  return "AMC reminders require 'device_id' and 'amc_date' columns.", pd.DataFrame()
@@ -180,44 +243,24 @@ def check_amc_reminders(df, current_date):
180
  reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
181
  if reminders.empty:
182
  return "No AMC reminders due within the next 30 days.", reminders
183
- result = "\n".join([f"- Device ID: {row['device_id']}, Log Type: {row['log_type']}, Status: {row['status']}, Usage: {row['usage_hours']}, Downtime: {row['downtime']}, AMC Date: {row['amc_date']}" for _, row in reminders.head(5).iterrows()])
184
- logging.info(f"AMC reminders generation took {time.time() - start_time:.2f} seconds")
185
- return result, reminders
186
  except Exception as e:
187
  logging.error(f"AMC reminder generation failed: {str(e)}")
188
  return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
189
 
190
  # Dashboard insights
191
- @cache_summary
192
  def generate_dashboard_insights(df):
193
- start_time = time.time()
194
  try:
195
  total_devices = df["device_id"].nunique()
196
  avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
197
  prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
198
- insights = summarizer(prompt, max_length=30, min_length=10, do_sample=False)[0]["summary_text"]
199
- logging.info(f"Insights generation took {time.time() - start_time:.2f} seconds")
200
  return insights
201
  except Exception as e:
202
  logging.error(f"Dashboard insights generation failed: {str(e)}")
203
  return f"Dashboard insights generation failed: {str(e)}"
204
 
205
- # Cache DataFrame processing
206
- def cache_dataframe(func):
207
- @functools.wraps(func)
208
- def wrapper(df, *args, **kwargs):
209
- cache_key = f"{id(df)}_{func.__name__}"
210
- if not hasattr(wrapper, 'cache'):
211
- wrapper.cache = {}
212
- if cache_key in wrapper.cache:
213
- return wrapper.cache[cache_key]
214
- result = func(df, *args, **kwargs)
215
- wrapper.cache[cache_key] = result
216
- return result
217
- return wrapper
218
-
219
  # Create usage chart
220
- @cache_dataframe
221
  def create_usage_chart(df):
222
  try:
223
  if df.empty:
@@ -238,12 +281,9 @@ def create_usage_chart(df):
238
  logging.error(f"Failed to create usage chart: {str(e)}")
239
  return None
240
 
241
- # Create downtime chart (Re-enabled with optimization)
242
- @cache_dataframe
243
  def create_downtime_chart(df):
244
  try:
245
- if df.empty:
246
- return None
247
  downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
248
  if len(downtime_data) > 5:
249
  downtime_data = downtime_data.nlargest(5, "downtime")
@@ -260,16 +300,11 @@ def create_downtime_chart(df):
260
  logging.error(f"Failed to create downtime chart: {str(e)}")
261
  return None
262
 
263
- # Create daily log trends chart (Re-enabled with optimization)
264
- @cache_dataframe
265
  def create_daily_log_trends_chart(df):
266
  try:
267
- if df.empty:
268
- return None
269
  df['date'] = df['timestamp'].dt.date
270
  daily_logs = df.groupby('date').size().reset_index(name='log_count')
271
- if len(daily_logs) > 30: # Limit to 30 days for faster plotting
272
- daily_logs = daily_logs.tail(30)
273
  fig = px.line(
274
  daily_logs,
275
  x='date',
@@ -283,20 +318,15 @@ def create_daily_log_trends_chart(df):
283
  logging.error(f"Failed to create daily log trends chart: {str(e)}")
284
  return None
285
 
286
- # Create weekly uptime chart (Re-enabled with optimization)
287
- @cache_dataframe
288
  def create_weekly_uptime_chart(df):
289
  try:
290
- if df.empty:
291
- return None
292
  df['week'] = df['timestamp'].dt.isocalendar().week
293
  df['year'] = df['timestamp'].dt.year
294
  weekly_data = df.groupby(['year', 'week']).agg({
295
  'usage_hours': 'sum',
296
  'downtime': 'sum'
297
  }).reset_index()
298
- if len(weekly_data) > 12: # Limit to 12 weeks for faster plotting
299
- weekly_data = weekly_data.tail(12)
300
  weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
301
  weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
302
  fig = px.bar(
@@ -312,16 +342,13 @@ def create_weekly_uptime_chart(df):
312
  logging.error(f"Failed to create weekly uptime chart: {str(e)}")
313
  return None
314
 
315
- # Create anomaly alerts chart (Re-enabled with optimization)
316
- @cache_dataframe
317
  def create_anomaly_alerts_chart(anomalies_df):
318
  try:
319
  if anomalies_df.empty:
320
  return None
321
  anomalies_df['date'] = anomalies_df['timestamp'].dt.date
322
  anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
323
- if len(anomaly_counts) > 30: # Limit to 30 days for faster plotting
324
- anomaly_counts = anomaly_counts.tail(30)
325
  fig = px.scatter(
326
  anomaly_counts,
327
  x='date',
@@ -368,12 +395,32 @@ def generate_device_cards(df):
368
  logging.error(f"Failed to generate device cards: {str(e)}")
369
  return f'<p>Error generating device cards: {str(e)}</p>'
370
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
371
  # Generate PDF content
372
- def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html):
373
  if not reportlab_available:
374
  return None
375
  try:
376
- pdf_path = f"status_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
377
  doc = SimpleDocTemplate(pdf_path, pagesize=letter)
378
  styles = getSampleStyleSheet()
379
  story = []
@@ -381,10 +428,16 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
381
  def safe_paragraph(text, style):
382
  return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
383
 
384
- story.append(Paragraph("LabOps Status Report", styles['Title']))
385
  story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
386
  story.append(Spacer(1, 12))
387
 
 
 
 
 
 
 
388
  story.append(Paragraph("Summary Report", styles['Heading2']))
389
  story.append(safe_paragraph(summary, styles['Normal']))
390
  story.append(Spacer(1, 12))
@@ -428,6 +481,9 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
428
  story.append(safe_paragraph(insights, styles['Normal']))
429
  story.append(Spacer(1, 12))
430
 
 
 
 
431
  doc.build(story)
432
  logging.info(f"PDF generated at {pdf_path}")
433
  return pdf_path
@@ -436,12 +492,11 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
436
  return None
437
 
438
  # Main processing function
439
- async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, progress=gr.Progress()):
440
  start_time = time.time()
441
- progress(0, desc="Starting processing...")
442
  try:
443
  if not file_obj:
444
- return "No file uploaded.", pd.DataFrame(), None, '<p>No device cardsPEM available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
445
 
446
  file_path = file_obj.name
447
  current_modified_time = os.path.getmtime(file_path)
@@ -452,7 +507,6 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
452
  if not file_path.endswith(".csv"):
453
  return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
454
 
455
- progress(0.1, desc="Loading CSV file...")
456
  required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
457
  dtypes = {
458
  "device_id": "string",
@@ -462,16 +516,11 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
462
  "downtime": "float32",
463
  "amc_date": "string"
464
  }
465
- df = pd.read_csv(file_path, dtype=dtypes, usecols=required_columns)
466
- if len(df) > 5000:
467
- df = df.sample(n=5000, random_state=42)
468
- logging.warning("Dataset too large, sampled to 5,000 rows")
469
-
470
  missing_columns = [col for col in required_columns if col not in df.columns]
471
  if missing_columns:
472
  return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
473
 
474
- progress(0.2, desc="Processing timestamps...")
475
  df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
476
  df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
477
  if df["timestamp"].dt.tz is None:
@@ -480,7 +529,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
480
  return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
481
 
482
  # Apply filters
483
- filtered_df = df
484
  if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
485
  filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
486
  if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
@@ -491,18 +540,22 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
491
  start_date = today + pd.Timedelta(days=days_start)
492
  end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
493
  filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
 
 
 
 
 
 
494
 
495
  if filtered_df.empty:
496
  return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
497
 
498
  # Generate table for preview
499
- progress(0.3, desc="Generating log preview...")
500
  preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
501
  preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
502
 
503
  # Run tasks concurrently
504
- progress(0.4, desc="Running analysis tasks...")
505
- with ThreadPoolExecutor(max_workers=4) as executor:
506
  future_summary = executor.submit(summarize_logs, filtered_df)
507
  future_anomalies = executor.submit(detect_anomalies, filtered_df)
508
  future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
@@ -511,35 +564,31 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
511
  future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
512
  future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
513
  future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
 
514
  future_device_cards = executor.submit(generate_device_cards, filtered_df)
 
515
 
516
- progress(0.5, desc="Collecting summary results...")
517
  summary = f"Step 1: Summary Report\n{future_summary.result()}"
518
- progress(0.6, desc="Collecting anomaly detection results...")
519
  anomalies, anomalies_df = future_anomalies.result()
520
  anomalies = f"Anomaly Detection\n{anomalies}"
521
- progress(0.7, desc="Collecting AMC reminders...")
522
  amc_reminders, reminders_df = future_amc.result()
523
  amc_reminders = f"AMC Reminders\n{amc_reminders}"
524
- progress(0.8, desc="Collecting insights...")
525
  insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
526
- progress(0.9, desc="Generating charts...")
527
  usage_chart = future_usage_chart.result()
528
  downtime_chart = future_downtime_chart.result()
529
  daily_log_chart = future_daily_log_chart.result()
530
  weekly_uptime_chart = future_weekly_uptime_chart.result()
531
- anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
532
  device_cards = future_device_cards.result()
533
 
534
- progress(0.95, desc="Generating PDF...")
535
- pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards)
536
 
537
  elapsed_time = time.time() - start_time
538
  logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
539
- if elapsed_time > 30:
540
- logging.warning(f"Processing time exceeded 30 seconds: {elapsed_time:.2f} seconds")
541
 
542
- progress(1.0, desc="Processing complete!")
543
  return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
544
  except Exception as e:
545
  logging.error(f"Failed to process file: {str(e)}")
@@ -548,7 +597,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
548
  # Update filters
549
  def update_filters(file_obj):
550
  if not file_obj:
551
- return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
552
  try:
553
  with open(file_obj.name, 'rb') as f:
554
  csv_content = f.read().decode('utf-8')
@@ -557,11 +606,12 @@ def update_filters(file_obj):
557
 
558
  lab_site_options = ['All'] + [site for site in df['lab_site'].dropna().astype(str).unique().tolist() if site.strip()] if 'lab_site' in df.columns else ['All']
559
  equipment_type_options = ['All'] + [equip for equip in df['equipment_type'].dropna().astype(str).unique().tolist() if equip.strip()] if 'equipment_type' in df.columns else ['All']
 
560
 
561
- return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All')
562
  except Exception as e:
563
  logging.error(f"Failed to update filters: {str(e)}")
564
- return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
565
 
566
  # Gradio Interface
567
  try:
@@ -591,6 +641,7 @@ try:
591
  lab_site_filter = gr.Dropdown(label="Lab Site", choices=['All'], value='All', interactive=True)
592
  equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
593
  date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
 
594
  submit_button = gr.Button("Analyze", variant="primary")
595
 
596
  with gr.Column(scale=2):
@@ -628,18 +679,18 @@ try:
628
  insights_output = gr.Markdown()
629
  with gr.Group(elem_classes="dashboard-section"):
630
  gr.Markdown("### Export Report")
631
- pdf_output = gr.File(label="Download Status Report as PDF")
632
 
633
  file_input.change(
634
  fn=update_filters,
635
  inputs=[file_input],
636
- outputs=[lab_site_filter, equipment_type_filter],
637
  queue=False
638
  )
639
 
640
  submit_button.click(
641
  fn=process_logs,
642
- inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state],
643
  outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
644
  )
645
 
 
9
  from concurrent.futures import ThreadPoolExecutor
10
  from simple_salesforce import Salesforce
11
  import os
12
+ import json
13
  import io
14
  import time
 
 
15
 
16
  # Configure logging
17
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
18
 
19
+ # Salesforce configuration
20
+ try:
21
+ sf = Salesforce(
22
+ username='multi-devicelabopsdashboard@sathkrutha.com',
23
+ password='Team@1234',
24
+ security_token=os.getenv('SF_SECURITY_TOKEN', ''),
25
+ domain='login'
26
+ )
27
+ logging.info("Salesforce connection established")
28
+ except Exception as e:
29
+ logging.error(f"Failed to connect to Salesforce: {str(e)}")
30
+ sf = None
31
 
32
  # Try to import reportlab
33
  try:
 
45
  logging.info("Preloading Hugging Face model...")
46
  try:
47
  device = 0 if torch.cuda.is_available() else -1
48
+ # Use a smaller model for faster inference
49
  summarizer = pipeline(
50
  "summarization",
51
  model="t5-small",
52
  device=device,
53
+ max_length=50,
54
  min_length=10,
55
+ num_beams=2
56
  )
57
  logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
58
  except Exception as e:
 
118
 
119
  LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
120
 
121
+ # Salesforce report creation
122
  def create_salesforce_reports(df):
123
+ if sf is None or not LABOPS_REPORTS_FOLDER_ID:
124
+ return
125
+ try:
126
+ timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
127
+ reports = [
128
+ {
129
+ "reportMetadata": {
130
+ "name": f"SmartLog_Usage_Report_{timestamp}",
131
+ "developerName": f"SmartLog_Usage_Report_{timestamp}",
132
+ "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
133
+ "reportFormat": "TABULAR",
134
+ "reportBooleanFilter": None,
135
+ "reportFilters": [],
136
+ "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.Usage_Hours__c"],
137
+ "folderId": LABOPS_REPORTS_FOLDER_ID
138
+ }
139
+ },
140
+ {
141
+ "reportMetadata": {
142
+ "name": f"SmartLog_AMC_Reminders_{timestamp}",
143
+ "developerName": f"SmartLog_AMC_Reminders_{timestamp}",
144
+ "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
145
+ "reportFormat": "TABULAR",
146
+ "reportBooleanFilter": None,
147
+ "reportFilters": [],
148
+ "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.AMC_Date__c"],
149
+ "folderId": LABOPS_REPORTS_FOLDER_ID
150
+ }
151
+ }
152
+ ]
153
+ for report in reports:
154
+ sf.restful('analytics/reports', method='POST', json=report)
155
+ logging.info("Salesforce reports created")
156
+ except Exception as e:
157
+ logging.error(f"Failed to create Salesforce reports: {str(e)}")
158
 
159
+ # Save to Salesforce
160
  def save_to_salesforce(df, reminders_df):
161
+ if sf is None:
162
+ return
163
+ try:
164
+ current_date = datetime.now()
165
+ next_30_days = current_date + timedelta(days=30)
166
+ records = []
167
+ reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
168
+
169
+ for _, row in df.iterrows():
170
+ status = str(row['status'])
171
+ log_type = str(row['log_type'])
172
+ status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
173
+ log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
174
+ if status is None or log_type is None:
175
+ continue
176
+
177
+ amc_date_str = None
178
+ if pd.notna(row['amc_date']):
179
+ try:
180
+ amc_date = pd.to_datetime(row['amc_date']).strftime('%Y-%m-%d')
181
+ amc_date_dt = datetime.strptime(amc_date, '%Y-%m-%d')
182
+ if status == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
183
+ logging.info(f"AMC Reminder for Device ID {row['device_id']}")
184
+ except:
185
+ amc_date_str = None
186
+
187
+ record = {
188
+ 'Device_Id__c': str(row['device_id'])[:50],
189
+ 'Log_Type__c': log_type,
190
+ 'Status__c': status,
191
+ 'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
192
+ 'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
193
+ 'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
194
+ 'AMC_Date__c': amc_date_str
195
+ }
196
+ if row['device_id'] not in reminder_device_ids:
197
+ records.append(record)
198
+
199
+ if records:
200
+ sf.bulk.SmartLog__c.insert(records)
201
+ logging.info(f"Saved {len(records)} records to Salesforce")
202
+ except Exception as e:
203
+ logging.error(f"Failed to save to Salesforce: {str(e)}")
204
 
205
  # Summarize logs
 
206
  def summarize_logs(df):
 
207
  try:
208
  total_devices = df["device_id"].nunique()
209
  most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
210
  prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
211
+ summary = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
 
212
  return summary
213
  except Exception as e:
214
  logging.error(f"Summary generation failed: {str(e)}")
 
216
 
217
  # Anomaly detection
218
  def detect_anomalies(df):
 
219
  try:
220
  if "usage_hours" not in df.columns or "downtime" not in df.columns:
221
  return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
222
  features = df[["usage_hours", "downtime"]].fillna(0)
223
+ if len(features) > 500:
224
+ features = features.sample(n=500, random_state=42)
225
+ iso_forest = IsolationForest(contamination=0.1, random_state=42)
226
  df["anomaly"] = iso_forest.fit_predict(features)
227
  anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
228
  if anomalies.empty:
229
  return "No anomalies detected.", anomalies
230
+ return "\n".join([f"- Device ID: {row['device_id']}, Usage: {row['usage_hours']}, Downtime: {row['downtime']}, Timestamp: {row['timestamp']}" for _, row in anomalies.head(5).iterrows()]), anomalies
 
 
231
  except Exception as e:
232
  logging.error(f"Anomaly detection failed: {str(e)}")
233
  return f"Anomaly detection failed: {str(e)}", pd.DataFrame()
234
 
235
  # AMC reminders
236
  def check_amc_reminders(df, current_date):
 
237
  try:
238
  if "device_id" not in df.columns or "amc_date" not in df.columns:
239
  return "AMC reminders require 'device_id' and 'amc_date' columns.", pd.DataFrame()
 
243
  reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
244
  if reminders.empty:
245
  return "No AMC reminders due within the next 30 days.", reminders
246
+ return "\n".join([f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}" for _, row in reminders.head(5).iterrows()]), reminders
 
 
247
  except Exception as e:
248
  logging.error(f"AMC reminder generation failed: {str(e)}")
249
  return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
250
 
251
  # Dashboard insights
 
252
  def generate_dashboard_insights(df):
 
253
  try:
254
  total_devices = df["device_id"].nunique()
255
  avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
256
  prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
257
+ insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
 
258
  return insights
259
  except Exception as e:
260
  logging.error(f"Dashboard insights generation failed: {str(e)}")
261
  return f"Dashboard insights generation failed: {str(e)}"
262
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
263
  # Create usage chart
 
264
  def create_usage_chart(df):
265
  try:
266
  if df.empty:
 
281
  logging.error(f"Failed to create usage chart: {str(e)}")
282
  return None
283
 
284
+ # Create downtime chart
 
285
  def create_downtime_chart(df):
286
  try:
 
 
287
  downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
288
  if len(downtime_data) > 5:
289
  downtime_data = downtime_data.nlargest(5, "downtime")
 
300
  logging.error(f"Failed to create downtime chart: {str(e)}")
301
  return None
302
 
303
+ # Create daily log trends chart
 
304
  def create_daily_log_trends_chart(df):
305
  try:
 
 
306
  df['date'] = df['timestamp'].dt.date
307
  daily_logs = df.groupby('date').size().reset_index(name='log_count')
 
 
308
  fig = px.line(
309
  daily_logs,
310
  x='date',
 
318
  logging.error(f"Failed to create daily log trends chart: {str(e)}")
319
  return None
320
 
321
+ # Create weekly uptime chart
 
322
  def create_weekly_uptime_chart(df):
323
  try:
 
 
324
  df['week'] = df['timestamp'].dt.isocalendar().week
325
  df['year'] = df['timestamp'].dt.year
326
  weekly_data = df.groupby(['year', 'week']).agg({
327
  'usage_hours': 'sum',
328
  'downtime': 'sum'
329
  }).reset_index()
 
 
330
  weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
331
  weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
332
  fig = px.bar(
 
342
  logging.error(f"Failed to create weekly uptime chart: {str(e)}")
343
  return None
344
 
345
+ # Create anomaly alerts chart
 
346
  def create_anomaly_alerts_chart(anomalies_df):
347
  try:
348
  if anomalies_df.empty:
349
  return None
350
  anomalies_df['date'] = anomalies_df['timestamp'].dt.date
351
  anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
 
 
352
  fig = px.scatter(
353
  anomaly_counts,
354
  x='date',
 
395
  logging.error(f"Failed to generate device cards: {str(e)}")
396
  return f'<p>Error generating device cards: {str(e)}</p>'
397
 
398
+ # Generate monthly status
399
+ def generate_monthly_status(df, selected_month):
400
+ try:
401
+ total_devices = df['device_id'].nunique()
402
+ total_usage_hours = df['usage_hours'].sum()
403
+ total_downtime = df['downtime'].sum()
404
+ avg_usage = total_usage_hours / total_devices if total_devices > 0 else 0
405
+ avg_downtime = total_downtime / total_devices if total_devices > 0 else 0
406
+ return f"""
407
+ Monthly Status for {selected_month}:
408
+ - Total Devices: {total_devices}
409
+ - Total Usage Hours: {total_usage_hours:.2f}
410
+ - Total Downtime Hours: {total_downtime:.2f}
411
+ - Average Usage per Device: {avg_usage:.2f} hours
412
+ - Average Downtime per Device: {avg_downtime:.2f} hours
413
+ """
414
+ except Exception as e:
415
+ logging.error(f"Failed to generate monthly status: {str(e)}")
416
+ return f"Failed to generate monthly status: {str(e)}"
417
+
418
  # Generate PDF content
419
+ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, df, selected_month):
420
  if not reportlab_available:
421
  return None
422
  try:
423
+ pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
424
  doc = SimpleDocTemplate(pdf_path, pagesize=letter)
425
  styles = getSampleStyleSheet()
426
  story = []
 
428
  def safe_paragraph(text, style):
429
  return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
430
 
431
+ story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
432
  story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
433
  story.append(Spacer(1, 12))
434
 
435
+ if selected_month != "All":
436
+ monthly_status = generate_monthly_status(df, selected_month)
437
+ story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
438
+ story.append(safe_paragraph(monthly_status, styles['Normal']))
439
+ story.append(Spacer(1, 12))
440
+
441
  story.append(Paragraph("Summary Report", styles['Heading2']))
442
  story.append(safe_paragraph(summary, styles['Normal']))
443
  story.append(Spacer(1, 12))
 
481
  story.append(safe_paragraph(insights, styles['Normal']))
482
  story.append(Spacer(1, 12))
483
 
484
+ story.append(Paragraph("Charts", styles['Heading2']))
485
+ story.append(Paragraph("[Chart placeholders - see dashboard for visuals]", styles['Normal']))
486
+
487
  doc.build(story)
488
  logging.info(f"PDF generated at {pdf_path}")
489
  return pdf_path
 
492
  return None
493
 
494
  # Main processing function
495
+ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
496
  start_time = time.time()
 
497
  try:
498
  if not file_obj:
499
+ return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
500
 
501
  file_path = file_obj.name
502
  current_modified_time = os.path.getmtime(file_path)
 
507
  if not file_path.endswith(".csv"):
508
  return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
509
 
 
510
  required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
511
  dtypes = {
512
  "device_id": "string",
 
516
  "downtime": "float32",
517
  "amc_date": "string"
518
  }
519
+ df = pd.read_csv(file_path, dtype=dtypes)
 
 
 
 
520
  missing_columns = [col for col in required_columns if col not in df.columns]
521
  if missing_columns:
522
  return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
523
 
 
524
  df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
525
  df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
526
  if df["timestamp"].dt.tz is None:
 
529
  return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
530
 
531
  # Apply filters
532
+ filtered_df = df.copy()
533
  if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
534
  filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
535
  if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
 
540
  start_date = today + pd.Timedelta(days=days_start)
541
  end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
542
  filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
543
+ if month_filter and month_filter != "All":
544
+ selected_date = pd.to_datetime(month_filter, format="%B %Y")
545
+ filtered_df = filtered_df[
546
+ (filtered_df['timestamp'].dt.year == selected_date.year) &
547
+ (filtered_df['timestamp'].dt.month == selected_date.month)
548
+ ]
549
 
550
  if filtered_df.empty:
551
  return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
552
 
553
  # Generate table for preview
 
554
  preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
555
  preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
556
 
557
  # Run tasks concurrently
558
+ with ThreadPoolExecutor(max_workers=6) as executor:
 
559
  future_summary = executor.submit(summarize_logs, filtered_df)
560
  future_anomalies = executor.submit(detect_anomalies, filtered_df)
561
  future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
 
564
  future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
565
  future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
566
  future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
567
+ future_anomaly_alerts_chart = executor.submit(create_anomaly_alerts_chart, pd.DataFrame())
568
  future_device_cards = executor.submit(generate_device_cards, filtered_df)
569
+ future_reports = executor.submit(create_salesforce_reports, filtered_df)
570
 
 
571
  summary = f"Step 1: Summary Report\n{future_summary.result()}"
 
572
  anomalies, anomalies_df = future_anomalies.result()
573
  anomalies = f"Anomaly Detection\n{anomalies}"
 
574
  amc_reminders, reminders_df = future_amc.result()
575
  amc_reminders = f"AMC Reminders\n{amc_reminders}"
 
576
  insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
 
577
  usage_chart = future_usage_chart.result()
578
  downtime_chart = future_downtime_chart.result()
579
  daily_log_chart = future_daily_log_chart.result()
580
  weekly_uptime_chart = future_weekly_uptime_chart.result()
581
+ anomaly_alerts_chart = future_anomaly_alerts_chart.result()
582
  device_cards = future_device_cards.result()
583
 
584
+ save_to_salesforce(filtered_df, reminders_df)
585
+ pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, filtered_df, month_filter)
586
 
587
  elapsed_time = time.time() - start_time
588
  logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
589
+ if elapsed_time > 10:
590
+ logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
591
 
 
592
  return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
593
  except Exception as e:
594
  logging.error(f"Failed to process file: {str(e)}")
 
597
  # Update filters
598
  def update_filters(file_obj):
599
  if not file_obj:
600
+ return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
601
  try:
602
  with open(file_obj.name, 'rb') as f:
603
  csv_content = f.read().decode('utf-8')
 
606
 
607
  lab_site_options = ['All'] + [site for site in df['lab_site'].dropna().astype(str).unique().tolist() if site.strip()] if 'lab_site' in df.columns else ['All']
608
  equipment_type_options = ['All'] + [equip for equip in df['equipment_type'].dropna().astype(str).unique().tolist() if equip.strip()] if 'equipment_type' in df.columns else ['All']
609
+ month_options = ['All'] + sorted(df['timestamp'].dt.strftime('%B %Y').dropna().unique().tolist()) if 'timestamp' in df.columns else ['All']
610
 
611
+ return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
612
  except Exception as e:
613
  logging.error(f"Failed to update filters: {str(e)}")
614
+ return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
615
 
616
  # Gradio Interface
617
  try:
 
641
  lab_site_filter = gr.Dropdown(label="Lab Site", choices=['All'], value='All', interactive=True)
642
  equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
643
  date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
644
+ month_filter = gr.Dropdown(label="Select Month for Report", choices=['All'], value='All', interactive=True)
645
  submit_button = gr.Button("Analyze", variant="primary")
646
 
647
  with gr.Column(scale=2):
 
679
  insights_output = gr.Markdown()
680
  with gr.Group(elem_classes="dashboard-section"):
681
  gr.Markdown("### Export Report")
682
+ pdf_output = gr.File(label="Download Monthly Status Report as PDF")
683
 
684
  file_input.change(
685
  fn=update_filters,
686
  inputs=[file_input],
687
+ outputs=[lab_site_filter, equipment_type_filter, month_filter],
688
  queue=False
689
  )
690
 
691
  submit_button.click(
692
  fn=process_logs,
693
+ inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, month_filter, last_modified_state],
694
  outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
695
  )
696