backend/monitoring/production_monitoring.py

"""
Production Monitoring System for AegisLM SaaS Backend.

Comprehensive monitoring, alerting, and health checks for production deployment.
"""

import time
import asyncio
import logging
import psutil
from datetime import datetime, timedelta
from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func, text

from core.database import get_db, get_redis
from core.config import settings

logger = logging.getLogger(__name__)


@dataclass
class HealthCheckResult:
    """Health check result data structure."""
    service_name: str
    status: str  # "healthy", "degraded", "unhealthy"
    response_time_ms: int
    message: str
    timestamp: datetime
    details: Dict[str, Any] = field(default_factory=dict)


@dataclass
class SystemMetrics:
    """System performance metrics."""
    cpu_percent: float
    memory_percent: float
    disk_usage_percent: float
    active_connections: int
    request_rate_per_minute: float
    error_rate_per_minute: float
    timestamp: datetime


@dataclass
class AlertConfig:
    """Alert configuration."""
    error_threshold: float = 5.0  # 5% error rate triggers alert
    response_time_threshold: int = 2000  # 2 seconds response time triggers alert
    cpu_threshold: float = 80.0  # 80% CPU triggers alert
    memory_threshold: float = 85.0  # 85% memory triggers alert


class ProductionMonitor:
    """
    Production-ready monitoring system with health checks,
    metrics collection, and alerting.
    """
    
    def __init__(self):
        self.alert_config = AlertConfig()
        self.metrics_history: List[SystemMetrics] = []
        self.health_checks: Dict[str, HealthCheckResult] = {}
        self.redis_client = None
        self.start_time = datetime.utcnow()
        
    async def _get_redis_client(self):
        """Get Redis client for monitoring."""
        if not self.redis_client:
            self.redis_client = get_redis()
        return self.redis_client
    
    async def check_database_health(self) -> HealthCheckResult:
        """Check database connectivity and performance."""
        start_time = time.time()
        
        try:
            db = get_db()
            
            # Test basic connectivity
            await db.execute(text("SELECT 1"))
            
            # Check connection pool status
            pool_status = "healthy"
            if hasattr(db.pool, 'size'):
                pool_status = f"healthy (size: {db.pool.size()})"
            
            response_time = int((time.time() - start_time) * 1000)
            
            return HealthCheckResult(
                service_name="database",
                status="healthy",
                response_time_ms=response_time,
                message="Database connection successful",
                timestamp=datetime.utcnow(),
                details={
                    "pool_status": pool_status,
                    "connection_test": "passed"
                }
            )
            
        except Exception as e:
            return HealthCheckResult(
                service_name="database",
                status="unhealthy",
                response_time_ms=int((time.time() - start_time) * 1000),
                message=f"Database health check failed: {str(e)}",
                timestamp=datetime.utcnow(),
                details={"error": str(e)}
            )
    
    async def check_redis_health(self) -> HealthCheckResult:
        """Check Redis connectivity and performance."""
        start_time = time.time()
        
        try:
            redis_client = await self._get_redis_client()
            
            # Test basic operations
            await redis_client.ping()
            
            # Check memory usage
            info = redis_client.info()
            memory_usage = info.get('used_memory', 0)
            max_memory = info.get('maxmemory', 0)
            memory_percent = (memory_usage / max_memory * 100) if max_memory > 0 else 0
            
            response_time = int((time.time() - start_time) * 1000)
            
            return HealthCheckResult(
                service_name="redis",
                status="healthy",
                response_time_ms=response_time,
                message="Redis connection successful",
                timestamp=datetime.utcnow(),
                details={
                    "memory_usage_mb": memory_usage / 1024 / 1024,
                    "memory_percent": memory_percent,
                    "connected_clients": info.get('connected_clients', 0)
                }
            )
            
        except Exception as e:
            return HealthCheckResult(
                service_name="redis",
                status="unhealthy",
                response_time_ms=int((time.time() - start_time) * 1000),
                message=f"Redis health check failed: {str(e)}",
                timestamp=datetime.utcnow(),
                details={"error": str(e)}
            )
    
    def get_system_metrics(self) -> SystemMetrics:
        """Collect current system performance metrics."""
        try:
            # CPU metrics
            cpu_percent = psutil.cpu_percent(interval=1)
            
            # Memory metrics
            memory = psutil.virtual_memory()
            memory_percent = memory.percent
            
            # Disk metrics
            disk = psutil.disk_usage('/')
            disk_usage_percent = disk.percent
            
            # Network connections (estimate)
            active_connections = len(psutil.net_connections())
            
            return SystemMetrics(
                cpu_percent=cpu_percent,
                memory_percent=memory_percent,
                disk_usage_percent=disk_usage_percent,
                active_connections=active_connections,
                request_rate_per_minute=0.0,  # Would be calculated from actual request logs
                error_rate_per_minute=0.0,   # Would be calculated from actual error logs
                timestamp=datetime.utcnow()
            )
            
        except Exception as e:
            logger.error(f"Failed to collect system metrics: {e}")
            return SystemMetrics(
                cpu_percent=0.0,
                memory_percent=0.0,
                disk_usage_percent=0.0,
                active_connections=0,
                request_rate_per_minute=0.0,
                error_rate_per_minute=0.0,
                timestamp=datetime.utcnow()
            )
    
    def check_alert_conditions(self, metrics: SystemMetrics) -> List[Dict[str, Any]]:
        """Check if any alert conditions are met."""
        alerts = []
        
        # CPU alert
        if metrics.cpu_percent > self.alert_config.cpu_threshold:
            alerts.append({
                "type": "cpu_high",
                "severity": "warning",
                "message": f"CPU usage is {metrics.cpu_percent:.1f}% (threshold: {self.alert_config.cpu_threshold}%)",
                "timestamp": datetime.utcnow().isoformat(),
                "metric_value": metrics.cpu_percent,
                "threshold": self.alert_config.cpu_threshold
            })
        
        # Memory alert
        if metrics.memory_percent > self.alert_config.memory_threshold:
            alerts.append({
                "type": "memory_high",
                "severity": "warning",
                "message": f"Memory usage is {metrics.memory_percent:.1f}% (threshold: {self.alert_config.memory_threshold}%)",
                "timestamp": datetime.utcnow().isoformat(),
                "metric_value": metrics.memory_percent,
                "threshold": self.alert_config.memory_threshold
            })
        
        # Disk alert
        if metrics.disk_usage_percent > 90:
            alerts.append({
                "type": "disk_high",
                "severity": "critical",
                "message": f"Disk usage is {metrics.disk_usage_percent:.1f}%",
                "timestamp": datetime.utcnow().isoformat(),
                "metric_value": metrics.disk_usage_percent,
                "threshold": 90
            })
        
        return alerts
    
    async def run_health_checks(self) -> Dict[str, HealthCheckResult]:
        """Run all health checks."""
        logger.info("🔍 Running production health checks...")
        
        # Run health checks in parallel
        health_tasks = [
            self.check_database_health(),
            self.check_redis_health()
        ]
        
        results = await asyncio.gather(*health_tasks, return_exceptions=True)
        
        health_status = {}
        for result in results:
            if isinstance(result, HealthCheckResult):
                health_status[result.service_name] = result
            else:
                health_status[result.service_name] = HealthCheckResult(
                    service_name="unknown",
                    status="unhealthy",
                    response_time_ms=0,
                    message=f"Health check failed: {str(result)}",
                    timestamp=datetime.utcnow()
                )
        
        logger.info(f"✅ Health checks completed: {len(health_status)} services checked")
        return health_status
    
    async def store_metrics(self, metrics: SystemMetrics):
        """Store metrics in Redis for monitoring."""
        try:
            redis_client = await self._get_redis_client()
            
            # Store current metrics
            metrics_data = {
                "cpu_percent": metrics.cpu_percent,
                "memory_percent": metrics.memory_percent,
                "disk_usage_percent": metrics.disk_usage_percent,
                "active_connections": metrics.active_connections,
                "timestamp": metrics.timestamp.isoformat()
            }
            
            # Store with TTL (1 hour)
            await redis_client.setex(
                "system_metrics:current",
                3600,  # 1 hour TTL
                str(metrics_data)
            )
            
            # Add to history (keep last 24 hours)
            await redis_client.lpush("system_metrics:history", str(metrics_data))
            await redis_client.ltrim("system_metrics:history", 0, 24)  # Keep last 24 entries
            
            logger.info("📊 System metrics stored in Redis")
            
        except Exception as e:
            logger.error(f"Failed to store metrics: {e}")
    
    async def check_alerts(self) -> List[Dict[str, Any]]:
        """Check for active alerts and return them."""
        try:
            # Get current metrics
            current_metrics = self.get_system_metrics()
            
            # Check alert conditions
            alerts = self.check_alert_conditions(current_metrics)
            
            # Store alerts in Redis
            if alerts:
                redis_client = await self._get_redis_client()
                await redis_client.setex(
                    "active_alerts",
                    3600,  # 1 hour TTL
                    str(alerts)
                )
                
                logger.warning(f"🚨 {len(alerts)} active alerts detected")
            
            return alerts
            
        except Exception as e:
            logger.error(f"Failed to check alerts: {e}")
            return []
    
    async def get_monitoring_dashboard_data(self) -> Dict[str, Any]:
        """Get comprehensive monitoring dashboard data."""
        try:
            redis_client = await self._get_redis_client()
            
            # Get current metrics
            current_metrics = self.get_system_metrics()
            
            # Get health status
            health_status = await self.run_health_checks()
            
            # Get active alerts
            active_alerts = await self.check_alerts()
            
            # Get metrics history
            history_data = await redis_client.lrange("system_metrics:history", 0, 23)
            metrics_history = [
                eval(data) if isinstance(data, str) else data
                for data in history_data
            ]
            
            # Calculate uptime
            uptime_seconds = (datetime.utcnow() - self.start_time).total_seconds()
            
            return {
                "current_metrics": current_metrics,
                "health_status": health_status,
                "active_alerts": active_alerts,
                "metrics_history": metrics_history[-10:],  # Last 10 entries
                "uptime_seconds": uptime_seconds,
                "timestamp": datetime.utcnow().isoformat()
            }
            
        except Exception as e:
            logger.error(f"Failed to get monitoring data: {e}")
            return {
                "error": str(e),
                "timestamp": datetime.utcnow().isoformat()
            }


# Global production monitor instance
production_monitor = ProductionMonitor()


async def get_production_monitor():
    """Get the global production monitor instance."""
    return production_monitor


# Health check endpoint for load balancers
async def health_check_endpoint():
    """External health check endpoint."""
    try:
        monitor = get_production_monitor()
        dashboard_data = await monitor.get_monitoring_dashboard_data()
        
        # Determine overall health status
        overall_status = "healthy"
        critical_services = [
            name for name, result in dashboard_data.get("health_status", {}).items()
            if result.status != "healthy"
        ]
        
        if critical_services:
            overall_status = "degraded"
        
        return {
            "status": overall_status,
            "timestamp": datetime.utcnow().isoformat(),
            "services": dashboard_data.get("health_status", {}),
            "critical_services": critical_services,
            "uptime_seconds": dashboard_data.get("uptime_seconds", 0)
        }
        
    except Exception as e:
        return {
            "status": "unhealthy",
            "timestamp": datetime.utcnow().isoformat(),
            "error": str(e)
        }