Update app.py

app.py

@@ -12,35 +12,36 @@ import torch
 from concurrent.futures import ThreadPoolExecutor
 from simple_salesforce import Salesforce
 import os
-
-# Try to import reportlab for PDF generation
-try:
-    from reportlab.lib.pagesizes import letter
-    from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
-    from reportlab.lib.styles import getSampleStyleSheet
-    reportlab_available = True
-    logging.info("reportlab module successfully imported")
-except ImportError:
-    logging.warning("reportlab module not found. PDF generation will be disabled.")
-    reportlab_available = False
+import json
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Salesforce configuration (environment variables for security)
+# Salesforce configuration
 try:
     sf = Salesforce(
         username=os.getenv('SF_USERNAME'),
         password=os.getenv('SF_PASSWORD'),
         security_token=os.getenv('SF_SECURITY_TOKEN'),
-        domain='login'  # Use 'test' for sandbox
+        domain='login'
     )
     logging.info("Salesforce connection established")
 except Exception as e:
     logging.error(f"Failed to connect to Salesforce: {str(e)}")
     sf = None
 
-# Preload Hugging Face summarization model
+# Try to import reportlab
+try:
+    from reportlab.lib.pagesizes import letter
+    from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
+    from reportlab.lib.styles import getSampleStyleSheet
+    reportlab_available = True
+    logging.info("reportlab module successfully imported")
+except ImportError:
+    logging.warning("reportlab module not found. PDF generation disabled.")
+    reportlab_available = False
+
+# Preload Hugging Face model
 logging.info("Preloading Hugging Face model...")
 try:
     device = 0 if torch.cuda.is_available() else -1
@@ -57,30 +58,125 @@ except Exception as e:
     logging.error(f"Failed to preload model: {str(e)}")
     raise e
 
-# Save results to Salesforce SmartLog__c object
+# Fetch valid picklist values from Salesforce
+def get_picklist_values(field_name):
+    if sf is None:
+        return []
+    try:
+        obj_desc = sf.SmartLog__c.describe()
+        for field in obj_desc['fields']:
+            if field['name'] == field_name:
+                return [value['value'] for value in field['picklistValues'] if value['active']]
+        return []
+    except Exception as e:
+        logging.error(f"Failed to fetch picklist values for {field_name}: {str(e)}")
+        return []
+
+# Cache picklist values at startup
+status_values = get_picklist_values('Status__c') or ["Active", "Inactive", "Pending"]
+log_type_values = get_picklist_values('Log_Type__c') or ["Smart Log", "Cell Analysis", "UV Verification"]
+logging.info(f"Valid Status__c values: {status_values}")
+logging.info(f"Valid Log_Type__c values: {log_type_values}")
+
+# Map invalid picklist values to valid ones
+picklist_mapping = {
+    'Status__c': {
+        'normal': 'Active',
+        'error': 'Inactive',
+        'warning': 'Pending',
+        'ok': 'Active',
+        'failed': 'Inactive'
+    },
+    'Log_Type__c': {
+        'maint': 'Smart Log',
+        'error': 'Cell Analysis',
+        'ops': 'UV Verification',
+        'maintenance': 'Smart Log',
+        'cell': 'Cell Analysis',
+        'uv': 'UV Verification'
+    }
+}
+
+# Create Salesforce report
+def create_salesforce_report(df):
+    if sf is None:
+        return "Salesforce connection not available."
+    try:
+        report_metadata = {
+            "reportMetadata": {
+                "name": f"SmartLog_Usage_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
+                "reportType": {"type": "SmartLog__c"},
+                "reportFormat": "SUMMARY",
+                "reportBooleanFilter": "",
+                "reportFilters": [
+                    {"column": "Status__c", "operator": "equals", "value": "Active"}
+                ],
+                "reportColumns": [
+                    {"column": "Device_Id__c"},
+                    {"column": "Log_Type__c"},
+                    {"column": "Status__c"},
+                    {"column": "Timestamp__c"},
+                    {"column": "Usage_Hours__c", "aggregate": "Sum"},
+                    {"column": "Downtime__c", "aggregate": "Sum"},
+                    {"column": "AMC_Date__c"}
+                ],
+                "groupingsDown": [{"name": "Device_Id__c", "sortOrder": "Asc"}],
+                "folderName": "LabOps Reports"
+            }
+        }
+        result = sf.restful('analytics/reports', method='POST', json=report_metadata)
+        logging.info(f"Report created: {result['id']}")
+        return result['id']
+    except Exception as e:
+        logging.error(f"Failed to create Salesforce report: {str(e)}")
+        return None
+
+# Save results to Salesforce SmartLog__c
 def save_to_salesforce(df, summary, anomalies, amc_reminders, insights):
     if sf is None:
-        logging.error("Salesforce connection not available")
         return "Salesforce connection not available."
     try:
-        for _, row in df.head(100).iterrows():  # Limit to 100 rows to avoid governor limits
+        records = []
+        for _, row in df.head(100).iterrows():
+            # Validate and map picklist values
+            status = str(row['status'])
+            log_type = str(row['log_type'])
+            
+            # Map Status__c
+            if status not in status_values:
+                status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
+                if status is None:
+                    logging.warning(f"Skipping record with invalid Status__c: {row['status']}")
+                    continue
+            
+            # Map Log_Type__c
+            if log_type not in log_type_values:
+                log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
+                if log_type is None:
+                    logging.warning(f"Skipping record with invalid Log_Type__c: {row['log_type']}")
+                    continue
+
             record = {
                 'Device_Id__c': str(row['device_id'])[:50],
-                'Log_Type__c': str(row['log_type']),
-                'Status__c': str(row['status']),
+                'Log_Type__c': log_type,
+                'Status__c': status,
                 'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
                 'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
                 'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
                 'AMC_Date__c': row['amc_date'].strftime('%Y-%m-%d') if pd.notna(row['amc_date']) else None
             }
-            sf.SmartLog__c.create(record)
-        logging.info("Data saved to Salesforce successfully")
-        return "Data saved to Salesforce."
+            records.append(record)
+        
+        # Bulk insert to reduce API calls
+        if records:
+            sf.bulk.SmartLog__c.insert(records)
+            logging.info(f"Saved {len(records)} records to Salesforce")
+        return f"Saved {len(records)} records to Salesforce."
     except Exception as e:
         logging.error(f"Failed to save to Salesforce: {str(e)}")
         return f"Failed to save to Salesforce: {str(e)}"
 
-# Format summary prompt and generate report
+# Summarize logs
 def summarize_logs(df, progress=gr.Progress()):
     progress(0.1, "Generating summary report...")
     try:
@@ -94,13 +190,13 @@ def summarize_logs(df, progress=gr.Progress()):
         logging.error(f"Summary generation failed: {str(e)}")
         return f"Failed to generate summary: {str(e)}"
 
-# Anomaly Detection using Isolation Forest
+# Anomaly detection
 def detect_anomalies(df, progress=gr.Progress()):
     progress(0.4, "Detecting anomalies...")
     try:
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns."
-        if len(df) > 1000:  # Reduced sample size for speed
+        if len(df) > 1000:
             df = df.sample(n=1000, random_state=42)
         features = df[["usage_hours", "downtime"]].fillna(0)
         iso_forest = IsolationForest(contamination=0.1, random_state=42, n_jobs=-1)
@@ -119,7 +215,7 @@ def detect_anomalies(df, progress=gr.Progress()):
         logging.error(f"Anomaly detection failed: {str(e)}")
         return f"Anomaly detection failed: {str(e)}"
 
-# AMC Reminders
+# AMC reminders
 def check_amc_reminders(df, current_date, progress=gr.Progress()):
     progress(0.6, "Checking AMC reminders...")
     try:
@@ -139,7 +235,7 @@ def check_amc_reminders(df, current_date, progress=gr.Progress()):
         logging.error(f"AMC reminder generation failed: {str(e)}")
         return f"AMC reminder generation failed: {str(e)}"
 
-# Dashboard Insights
+# Dashboard insights
 def generate_dashboard_insights(df, progress=gr.Progress()):
     progress(0.8, "Generating dashboard insights...")
     try:
@@ -152,7 +248,7 @@ def generate_dashboard_insights(df, progress=gr.Progress()):
         logging.error(f"Dashboard insights generation failed: {str(e)}")
         return f"Dashboard insights generation failed: {str(e)}"
 
-# Create a bar chart
+# Create usage chart
 def create_usage_chart(df, progress=gr.Progress()):
     progress(0.9, "Creating usage chart...")
     try:
@@ -194,32 +290,26 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights):
         def safe_paragraph(text, style):
             return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
 
-        # Title
         story.append(Paragraph("LabOps Log Analysis Report", styles['Title']))
         story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
         story.append(Spacer(1, 12))
 
-        # Summary Report
         story.append(Paragraph("Summary Report", styles['Heading2']))
         story.append(safe_paragraph(summary or "No summary available.", styles['Normal']))
         story.append(Spacer(1, 12))
 
-        # Log Preview
         story.append(Paragraph("Log Preview", styles['Heading2']))
         story.append(safe_paragraph(preview or "No preview available.", styles['Normal']))
         story.append(Spacer(1, 12))
 
-        # Anomaly Detection
         story.append(Paragraph("Anomaly Detection", styles['Heading2']))
         story.append(safe_paragraph(anomalies or "No anomalies detected.", styles['Normal']))
         story.append(Spacer(1, 12))
 
-        # AMC Reminders
         story.append(Paragraph("AMC Reminders", styles['Heading2']))
         story.append(safe_paragraph(amc_reminders or "No AMC reminders.", styles['Normal']))
         story.append(Spacer(1, 12))
 
-        # Dashboard Insights
         story.append(Paragraph("Dashboard Insights", styles['Heading2']))
         story.append(safe_paragraph(insights or "No insights generated.", styles['Normal']))
 
@@ -231,19 +321,18 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights):
         return None
 
 # Main Gradio function
-async def process_logs(file_obj, progress=gr.Progress()):
+async def process_logs(file_obj, progress правилайлаProgress()):
     try:
         progress(0, "Starting file processing...")
         if not file_obj:
-            return "No file uploaded.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, "No Salesforce data saved."
+            return "No file uploaded.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, "No Salesforce data saved.", "No report created."
 
         file_name = file_obj.name
         logging.info(f"Processing file: {file_name}")
         
         if not file_name.endswith(".csv"):
-            return "Please upload a CSV file.", "", None, "", "", "", None, ""
+            return "Please upload a CSV file.", "", None, "", "", "", None, "", ""
 
-        # Load CSV
         required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
         dtypes = {
             "device_id": "string",
@@ -256,26 +345,27 @@ async def process_logs(file_obj, progress=gr.Progress()):
         df = pd.read_csv(file_obj, dtype=dtypes)
         missing_columns = [col for col in required_columns if col not in df.columns]
         if missing_columns:
-            return f"Missing columns: {missing_columns}", None, None, None, None, None, None, None
+            return f"Missing columns: {missing_columns}", None, None, None, None, None, None, None, None
         df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
+        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
         if df.empty:
-            return "No data available.", None, None, None, None, None, None, None
+            return "No data available.", None, None, None, None, None, None, None, None
 
-        # Parallel processing for speed
         with ThreadPoolExecutor() as executor:
             future_summary = executor.submit(summarize_logs, df)
             future_anomalies = executor.submit(detect_anomalies, df)
             future_amc = executor.submit(check_amc_reminders, df, datetime.now())
             future_insights = executor.submit(generate_dashboard_insights, df)
             future_chart = executor.submit(create_usage_chart, df)
+            future_report = executor.submit(create_salesforce_report, df)
 
             summary = f"Step 1: Summary Report\n{future_summary.result()}"
             anomalies = f"Anomaly Detection\n{future_anomalies.result()}"
             amc_reminders = f"AMC Reminders\n{future_amc.result()}"
             insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
             chart = future_chart.result()
+            report_id = future_report.result()
 
-        # Log Preview
         preview_lines = ["Step 2: Log Preview (First 5 Rows)"]
         for idx, row in df.head(5).iterrows():
             preview_lines.append(
@@ -286,17 +376,15 @@ async def process_logs(file_obj, progress=gr.Progress()):
             )
         preview = "\n".join(preview_lines)
 
-        # Save to Salesforce
         salesforce_result = save_to_salesforce(df, summary, anomalies, amc_reminders, insights)
-
-        # Generate PDF
         pdf_file = generate_pdf_content(summary, preview, anomalies, amc_reminders, insights)
+        report_result = f"Report created in Salesforce with ID: {report_id}" if report_id else "Failed to create report."
 
         progress(1.0, "Done!")
-        return summary, preview, chart, anomalies, amc_reminders, insights, pdf_file, salesforce_result
+        return summary, preview, chart, anomalies, amc_reminders, insights, pdf_file, salesforce_result, report_result
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error processing file: {str(e)}", None, None, None, None, None, None, None
+        return f"Error: {str(e)}", None, None, None, None, None, None, None, None
 
 # Gradio Interface
 try:
@@ -310,7 +398,7 @@ try:
         .dashboard-section ul {margin: 2px 0; padding-left: 20px;}
     """) as iface:
         gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
-        gr.Markdown("Upload a CSV file containing lab equipment logs to analyze.")
+        gr.Markdown("Upload a CSV file to analyze and generate Salesforce reports.")
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -319,7 +407,7 @@ try:
 
             with gr.Column(scale=2):
                 with gr.Group(elem_classes="dashboard-container"):
-                    gr.Markdown("<div class='dashboard-title'>Analysis Results (Step-by-Step)</div>")
+                    gr.Markdown("<div class='dashboard-title'>Analysis Results</div>")
                     
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Step 1: Summary Report")
@@ -349,6 +437,10 @@ try:
                         gr.Markdown("### Salesforce Integration")
                         salesforce_output = gr.Markdown()
                         
+                    with gr.Group(elem_classes="dashboard-section"):
+                        gr.Markdown("### Salesforce Report")
+                        report_output = gr.Markdown()
+                        
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Download Report")
                         pdf_output = gr.File(label="Download Analysis Report as PDF")
@@ -364,7 +456,8 @@ try:
                 amc_output,
                 insights_output,
                 pdf_output,
-                salesforce_output
+                salesforce_output,
+                report_output
             ]
         )