Update app.py

app.py

@@ -182,9 +182,7 @@ class AnomalyDetectionAgent(BaseAgent):
                 'severity_tier': self._classify_severity(anomaly_score),
                 'primary_metrics_affected': self._identify_affected_metrics(event)
             },
-            'recommendations': [
-                f"Investigate {metric} anomalies" for metric in self._identify_affected_metrics(event)
-            ]
+            'recommendations': self._generate_detection_recommendations(event, anomaly_score)
         }
     
     def _calculate_anomaly_score(self, event: ReliabilityEvent) -> float:
@@ -211,83 +209,8 @@ class AnomalyDetectionAgent(BaseAgent):
         
         return min(1.0, sum(scores))
     
-def _identify_affected_metrics(self, event: ReliabilityEvent) -> List[Dict[str, Any]]:
-    """Enhanced metric analysis with severity levels"""
-    affected = []
-    
-    # Latency analysis
-    if event.latency_p99 > 500:
-        affected.append({"metric": "latency", "value": event.latency_p99, "severity": "CRITICAL", "threshold": 150})
-    elif event.latency_p99 > 300:
-        affected.append({"metric": "latency", "value": event.latency_p99, "severity": "HIGH", "threshold": 150})
-    elif event.latency_p99 > 150:
-        affected.append({"metric": "latency", "value": event.latency_p99, "severity": "MEDIUM", "threshold": 150})
-    
-    # Error rate analysis  
-    if event.error_rate > 0.3:
-        affected.append({"metric": "error_rate", "value": event.error_rate, "severity": "CRITICAL", "threshold": 0.05})
-    elif event.error_rate > 0.15:
-        affected.append({"metric": "error_rate", "value": event.error_rate, "severity": "HIGH", "threshold": 0.05})
-    elif event.error_rate > 0.05:
-        affected.append({"metric": "error_rate", "value": event.error_rate, "severity": "MEDIUM", "threshold": 0.05})
-    
-    # Resource analysis
-    if event.cpu_util and event.cpu_util > 0.9:
-        affected.append({"metric": "cpu", "value": event.cpu_util, "severity": "CRITICAL", "threshold": 0.8})
-    elif event.cpu_util and event.cpu_util > 0.8:
-        affected.append({"metric": "cpu", "value": event.cpu_util, "severity": "HIGH", "threshold": 0.8})
-        
-    if event.memory_util and event.memory_util > 0.9:
-        affected.append({"metric": "memory", "value": event.memory_util, "severity": "CRITICAL", "threshold": 0.8})
-    elif event.memory_util and event.memory_util > 0.8:
-        affected.append({"metric": "memory", "value": event.memory_util, "severity": "HIGH", "threshold": 0.8})
-    
-    return affected
-
-def _generate_detection_recommendations(self, event: ReliabilityEvent, anomaly_score: float) -> List[str]:
-    """Generate specific, actionable recommendations"""
-    recommendations = []
-    affected_metrics = self._identify_affected_metrics(event)
-    
-    for metric in affected_metrics:
-        metric_name = metric["metric"]
-        severity = metric["severity"]
-        value = metric["value"]
-        threshold = metric["threshold"]
-        
-        if metric_name == "latency":
-            if severity == "CRITICAL":
-                recommendations.append(f"🚨 CRITICAL: Latency {value}ms (>{threshold}ms) - Check database & external dependencies")
-            elif severity == "HIGH":
-                recommendations.append(f"⚠️ HIGH: Latency {value}ms (>{threshold}ms) - Investigate service performance")
-            else:
-                recommendations.append(f"📈 Latency elevated: {value}ms (>{threshold}ms) - Monitor trend")
-                
-        elif metric_name == "error_rate":
-            if severity == "CRITICAL":
-                recommendations.append(f"🚨 CRITICAL: Error rate {value*100:.1f}% (>{threshold*100}%) - Check recent deployments")
-            elif severity == "HIGH":
-                recommendations.append(f"⚠️ HIGH: Error rate {value*100:.1f}% (>{threshold*100}%) - Review application logs")
-            else:
-                recommendations.append(f"📈 Errors increasing: {value*100:.1f}% (>{threshold*100}%)")
-                
-        elif metric_name == "cpu":
-            recommendations.append(f"🔥 CPU {severity}: {value*100:.1f}% utilization - Consider scaling")
-            
-        elif metric_name == "memory":
-            recommendations.append(f"💾 Memory {severity}: {value*100:.1f}% utilization - Check for memory leaks")
-    
-    # Add overall recommendations based on anomaly score
-    if anomaly_score > 0.8:
-        recommendations.append("🎯 IMMEDIATE ACTION REQUIRED: Multiple critical metrics affected")
-    elif anomaly_score > 0.6:
-        recommendations.append("🎯 INVESTIGATE: Significant performance degradation detected")
-    elif anomaly_score > 0.4:
-        recommendations.append("📊 MONITOR: Early warning signs detected")
-    
-    return recommendations[:4]  # Return top 4 most important recommendations
-    
     def _classify_severity(self, anomaly_score: float) -> str:
+        """Classify severity based on anomaly score"""
         if anomaly_score > 0.8:
             return "CRITICAL"
         elif anomaly_score > 0.6:
@@ -296,6 +219,82 @@ def _generate_detection_recommendations(self, event: ReliabilityEvent, anomaly_s
             return "MEDIUM"
         else:
             return "LOW"
+    
+    def _identify_affected_metrics(self, event: ReliabilityEvent) -> List[Dict[str, Any]]:
+        """Enhanced metric analysis with severity levels"""
+        affected = []
+        
+        # Latency analysis
+        if event.latency_p99 > 500:
+            affected.append({"metric": "latency", "value": event.latency_p99, "severity": "CRITICAL", "threshold": 150})
+        elif event.latency_p99 > 300:
+            affected.append({"metric": "latency", "value": event.latency_p99, "severity": "HIGH", "threshold": 150})
+        elif event.latency_p99 > 150:
+            affected.append({"metric": "latency", "value": event.latency_p99, "severity": "MEDIUM", "threshold": 150})
+        
+        # Error rate analysis  
+        if event.error_rate > 0.3:
+            affected.append({"metric": "error_rate", "value": event.error_rate, "severity": "CRITICAL", "threshold": 0.05})
+        elif event.error_rate > 0.15:
+            affected.append({"metric": "error_rate", "value": event.error_rate, "severity": "HIGH", "threshold": 0.05})
+        elif event.error_rate > 0.05:
+            affected.append({"metric": "error_rate", "value": event.error_rate, "severity": "MEDIUM", "threshold": 0.05})
+        
+        # Resource analysis
+        if event.cpu_util and event.cpu_util > 0.9:
+            affected.append({"metric": "cpu", "value": event.cpu_util, "severity": "CRITICAL", "threshold": 0.8})
+        elif event.cpu_util and event.cpu_util > 0.8:
+            affected.append({"metric": "cpu", "value": event.cpu_util, "severity": "HIGH", "threshold": 0.8})
+            
+        if event.memory_util and event.memory_util > 0.9:
+            affected.append({"metric": "memory", "value": event.memory_util, "severity": "CRITICAL", "threshold": 0.8})
+        elif event.memory_util and event.memory_util > 0.8:
+            affected.append({"metric": "memory", "value": event.memory_util, "severity": "HIGH", "threshold": 0.8})
+        
+        return affected
+    
+    def _generate_detection_recommendations(self, event: ReliabilityEvent, anomaly_score: float) -> List[str]:
+        """Generate specific, actionable recommendations"""
+        recommendations = []
+        affected_metrics = self._identify_affected_metrics(event)
+        
+        for metric in affected_metrics:
+            metric_name = metric["metric"]
+            severity = metric["severity"]
+            value = metric["value"]
+            threshold = metric["threshold"]
+            
+            if metric_name == "latency":
+                if severity == "CRITICAL":
+                    recommendations.append(f"🚨 CRITICAL: Latency {value}ms (>{threshold}ms) - Check database & external dependencies")
+                elif severity == "HIGH":
+                    recommendations.append(f"⚠️ HIGH: Latency {value}ms (>{threshold}ms) - Investigate service performance")
+                else:
+                    recommendations.append(f"📈 Latency elevated: {value}ms (>{threshold}ms) - Monitor trend")
+                    
+            elif metric_name == "error_rate":
+                if severity == "CRITICAL":
+                    recommendations.append(f"🚨 CRITICAL: Error rate {value*100:.1f}% (>{threshold*100}%) - Check recent deployments")
+                elif severity == "HIGH":
+                    recommendations.append(f"⚠️ HIGH: Error rate {value*100:.1f}% (>{threshold*100}%) - Review application logs")
+                else:
+                    recommendations.append(f"📈 Errors increasing: {value*100:.1f}% (>{threshold*100}%)")
+                    
+            elif metric_name == "cpu":
+                recommendations.append(f"🔥 CPU {severity}: {value*100:.1f}% utilization - Consider scaling")
+                
+            elif metric_name == "memory":
+                recommendations.append(f"💾 Memory {severity}: {value*100:.1f}% utilization - Check for memory leaks")
+        
+        # Add overall recommendations based on anomaly score
+        if anomaly_score > 0.8:
+            recommendations.append("🎯 IMMEDIATE ACTION REQUIRED: Multiple critical metrics affected")
+        elif anomaly_score > 0.6:
+            recommendations.append("🎯 INVESTIGATE: Significant performance degradation detected")
+        elif anomaly_score > 0.4:
+            recommendations.append("📊 MONITOR: Early warning signs detected")
+        
+        return recommendations[:4]  # Return top 4 most important recommendations
 
 class RootCauseAgent(BaseAgent):
     def __init__(self):
@@ -314,59 +313,59 @@ class RootCauseAgent(BaseAgent):
                 'investigation_priority': self._prioritize_investigation(causes)
             },
             'recommendations': [
-                f"Check {cause} for issues" for cause in causes[:2]
+                f"Check {cause['cause']} for issues" for cause in causes[:2]
             ]
         }
     
-def _analyze_potential_causes(self, event: ReliabilityEvent) -> List[Dict[str, Any]]:
-    """Enhanced root cause analysis with confidence scoring"""
-    causes = []
-    
-    # High latency + high errors pattern
-    if event.latency_p99 > 500 and event.error_rate > 0.2:
-        causes.append({
-            "cause": "Database/External Dependency Failure",
-            "confidence": 0.85,
-            "evidence": f"Extreme latency ({event.latency_p99}ms) with high errors ({event.error_rate*100:.1f}%)",
-            "investigation": "Check database connection pool, external API health"
-        })
-    
-    # Resource exhaustion pattern
-    if event.cpu_util and event.cpu_util > 0.9 and event.memory_util and event.memory_util > 0.9:
-        causes.append({
-            "cause": "Resource Exhaustion",
-            "confidence": 0.90,
-            "evidence": f"CPU ({event.cpu_util*100:.1f}%) and Memory ({event.memory_util*100:.1f}%) critically high",
-            "investigation": "Check for memory leaks, infinite loops, insufficient resources"
-        })
-    
-    # Error spike pattern
-    if event.error_rate > 0.3 and event.latency_p99 < 200:
-        causes.append({
-            "cause": "Application Bug / Configuration Issue",
-            "confidence": 0.75,
-            "evidence": f"High error rate ({event.error_rate*100:.1f}%) without latency impact",
-            "investigation": "Review recent deployments, configuration changes, application logs"
-        })
-    
-    # Gradual degradation pattern
-    if 200 <= event.latency_p99 <= 400 and 0.05 <= event.error_rate <= 0.15:
-        causes.append({
-            "cause": "Gradual Performance Degradation",
-            "confidence": 0.65,
-            "evidence": f"Moderate latency ({event.latency_p99}ms) and errors ({event.error_rate*100:.1f}%)",
-            "investigation": "Check resource trends, dependency performance, capacity planning"
-        })
-    
-    if not causes:
-        causes.append({
-            "cause": "Unknown - Requires Investigation",
-            "confidence": 0.3,
-            "evidence": "Pattern does not match known failure modes",
-            "investigation": "Complete system review needed"
-        })
-    
-    return causes
+    def _analyze_potential_causes(self, event: ReliabilityEvent) -> List[Dict[str, Any]]:
+        """Enhanced root cause analysis with confidence scoring"""
+        causes = []
+        
+        # High latency + high errors pattern
+        if event.latency_p99 > 500 and event.error_rate > 0.2:
+            causes.append({
+                "cause": "Database/External Dependency Failure",
+                "confidence": 0.85,
+                "evidence": f"Extreme latency ({event.latency_p99}ms) with high errors ({event.error_rate*100:.1f}%)",
+                "investigation": "Check database connection pool, external API health"
+            })
+        
+        # Resource exhaustion pattern
+        if event.cpu_util and event.cpu_util > 0.9 and event.memory_util and event.memory_util > 0.9:
+            causes.append({
+                "cause": "Resource Exhaustion",
+                "confidence": 0.90,
+                "evidence": f"CPU ({event.cpu_util*100:.1f}%) and Memory ({event.memory_util*100:.1f}%) critically high",
+                "investigation": "Check for memory leaks, infinite loops, insufficient resources"
+            })
+        
+        # Error spike pattern
+        if event.error_rate > 0.3 and event.latency_p99 < 200:
+            causes.append({
+                "cause": "Application Bug / Configuration Issue",
+                "confidence": 0.75,
+                "evidence": f"High error rate ({event.error_rate*100:.1f}%) without latency impact",
+                "investigation": "Review recent deployments, configuration changes, application logs"
+            })
+        
+        # Gradual degradation pattern
+        if 200 <= event.latency_p99 <= 400 and 0.05 <= event.error_rate <= 0.15:
+            causes.append({
+                "cause": "Gradual Performance Degradation",
+                "confidence": 0.65,
+                "evidence": f"Moderate latency ({event.latency_p99}ms) and errors ({event.error_rate*100:.1f}%)",
+                "investigation": "Check resource trends, dependency performance, capacity planning"
+            })
+        
+        if not causes:
+            causes.append({
+                "cause": "Unknown - Requires Investigation",
+                "confidence": 0.3,
+                "evidence": "Pattern does not match known failure modes",
+                "investigation": "Complete system review needed"
+            })
+        
+        return causes
     
     def _identify_evidence(self, event: ReliabilityEvent) -> List[str]:
         """Identify evidence patterns"""
@@ -377,13 +376,12 @@ def _analyze_potential_causes(self, event: ReliabilityEvent) -> List[Dict[str, A
             evidence.append("correlated_resource_exhaustion")
         return evidence
     
-    def _prioritize_investigation(self, causes: List[str]) -> str:
-        if "database_connection_pool" in causes:
-            return "HIGH"
-        elif "resource_exhaustion" in causes:
-            return "HIGH"
-        else:
-            return "MEDIUM"
+    def _prioritize_investigation(self, causes: List[Dict[str, Any]]) -> str:
+        """Prioritize investigation based on causes"""
+        for cause in causes:
+            if "Database" in cause["cause"] or "Resource Exhaustion" in cause["cause"]:
+                return "HIGH"
+        return "MEDIUM"
 
 class OrchestrationManager:
     def __init__(self):
@@ -422,7 +420,7 @@ class OrchestrationManager:
             'incident_summary': {
                 'severity': detective_result['findings'].get('severity_tier', 'UNKNOWN'),
                 'anomaly_confidence': detective_result['confidence'],
-                'primary_metrics_affected': detective_result['findings'].get('primary_metrics_affected', [])
+                'primary_metrics_affected': [metric["metric"] for metric in detective_result['findings'].get('primary_metrics_affected', [])]
             },
             'root_cause_insights': diagnostician_result['findings'] if diagnostician_result else {},
             'recommended_actions': self._prioritize_actions(