chat_agent/utils/logging_config.py

"""
Comprehensive logging configuration for the chat agent.

This module provides structured logging setup with different handlers,
formatters, and configuration for debugging, monitoring, and analytics.
"""

import os
import logging
import logging.handlers
import json
from datetime import datetime
from typing import Dict, Any, Optional
from pathlib import Path


class StructuredFormatter(logging.Formatter):
    """Custom formatter for structured JSON logging."""
    
    def format(self, record: logging.LogRecord) -> str:
        """Format log record as structured JSON."""
        # Base log data
        log_data = {
            'timestamp': datetime.utcnow().isoformat(),
            'level': record.levelname,
            'logger': record.name,
            'message': record.getMessage(),
            'module': record.module,
            'function': record.funcName,
            'line': record.lineno
        }
        
        # Add extra fields if present
        if hasattr(record, 'error_code'):
            log_data['error_code'] = record.error_code
        if hasattr(record, 'category'):
            log_data['category'] = record.category
        if hasattr(record, 'severity'):
            log_data['severity'] = record.severity
        if hasattr(record, 'context'):
            log_data['context'] = record.context
        if hasattr(record, 'session_id'):
            log_data['session_id'] = record.session_id
        if hasattr(record, 'user_id'):
            log_data['user_id'] = record.user_id
        if hasattr(record, 'processing_time'):
            log_data['processing_time'] = record.processing_time
        if hasattr(record, 'traceback'):
            log_data['traceback'] = record.traceback
        
        # Add exception info if present
        if record.exc_info:
            log_data['exception'] = self.formatException(record.exc_info)
        
        return json.dumps(log_data, default=str)


class ChatAgentFilter(logging.Filter):
    """Custom filter for chat agent specific logging."""
    
    def filter(self, record: logging.LogRecord) -> bool:
        """Filter log records based on chat agent criteria."""
        # Add request ID if available (from Flask context)
        try:
            from flask import g
            if hasattr(g, 'request_id'):
                record.request_id = g.request_id
        except (ImportError, RuntimeError):
            pass
        
        # Add performance metrics
        if hasattr(record, 'processing_time'):
            if record.processing_time > 5.0:  # Log slow operations
                record.performance_alert = True
        
        return True


class LoggingConfig:
    """Centralized logging configuration for the chat agent."""
    
    def __init__(self, app_name: str = "chat_agent", log_level: str = "INFO"):
        """
        Initialize logging configuration.
        
        Args:
            app_name: Application name for log identification
            log_level: Default logging level
        """
        self.app_name = app_name
        self.log_level = getattr(logging, log_level.upper(), logging.INFO)
        self.log_dir = Path("logs")
        self.log_dir.mkdir(exist_ok=True)
        
        # Create formatters
        self.structured_formatter = StructuredFormatter()
        self.console_formatter = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        )
        self.detailed_formatter = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(module)s:%(funcName)s:%(lineno)d - %(message)s'
        )
        
        # Create filter
        self.chat_filter = ChatAgentFilter()
    
    def setup_logging(self) -> Dict[str, logging.Logger]:
        """
        Set up comprehensive logging configuration.
        
        Returns:
            Dict[str, logging.Logger]: Dictionary of configured loggers
        """
        # Configure root logger
        root_logger = logging.getLogger()
        root_logger.setLevel(self.log_level)
        
        # Clear existing handlers
        root_logger.handlers.clear()
        
        # Create loggers for different components
        loggers = {
            'main': self._setup_main_logger(),
            'error': self._setup_error_logger(),
            'performance': self._setup_performance_logger(),
            'security': self._setup_security_logger(),
            'api': self._setup_api_logger(),
            'websocket': self._setup_websocket_logger(),
            'database': self._setup_database_logger()
        }
        
        # Setup console logging for development
        if os.getenv('FLASK_ENV') == 'development':
            self._setup_console_logging()
        
        return loggers
    
    def _setup_main_logger(self) -> logging.Logger:
        """Setup main application logger."""
        logger = logging.getLogger(f'{self.app_name}.main')
        logger.setLevel(self.log_level)
        
        # File handler for general application logs
        file_handler = logging.handlers.RotatingFileHandler(
            self.log_dir / 'chat_agent.log',
            maxBytes=10*1024*1024,  # 10MB
            backupCount=5
        )
        file_handler.setFormatter(self.detailed_formatter)
        file_handler.addFilter(self.chat_filter)
        logger.addHandler(file_handler)
        
        return logger
    
    def _setup_error_logger(self) -> logging.Logger:
        """Setup error-specific logger."""
        logger = logging.getLogger(f'{self.app_name}.error')
        logger.setLevel(logging.WARNING)
        
        # File handler for errors with structured format
        error_handler = logging.handlers.RotatingFileHandler(
            self.log_dir / 'errors.log',
            maxBytes=50*1024*1024,  # 50MB
            backupCount=10
        )
        error_handler.setFormatter(self.structured_formatter)
        error_handler.addFilter(self.chat_filter)
        logger.addHandler(error_handler)
        
        # Critical errors to separate file
        critical_handler = logging.FileHandler(self.log_dir / 'critical.log')
        critical_handler.setLevel(logging.CRITICAL)
        critical_handler.setFormatter(self.structured_formatter)
        logger.addHandler(critical_handler)
        
        return logger
    
    def _setup_performance_logger(self) -> logging.Logger:
        """Setup performance monitoring logger."""
        logger = logging.getLogger(f'{self.app_name}.performance')
        logger.setLevel(logging.INFO)
        
        # File handler for performance metrics
        perf_handler = logging.handlers.RotatingFileHandler(
            self.log_dir / 'performance.log',
            maxBytes=20*1024*1024,  # 20MB
            backupCount=5
        )
        perf_handler.setFormatter(self.structured_formatter)
        logger.addHandler(perf_handler)
        
        return logger
    
    def _setup_security_logger(self) -> logging.Logger:
        """Setup security-related logger."""
        logger = logging.getLogger(f'{self.app_name}.security')
        logger.setLevel(logging.WARNING)
        
        # File handler for security events
        security_handler = logging.handlers.RotatingFileHandler(
            self.log_dir / 'security.log',
            maxBytes=20*1024*1024,  # 20MB
            backupCount=10
        )
        security_handler.setFormatter(self.structured_formatter)
        logger.addHandler(security_handler)
        
        return logger
    
    def _setup_api_logger(self) -> logging.Logger:
        """Setup API request/response logger."""
        logger = logging.getLogger(f'{self.app_name}.api')
        logger.setLevel(logging.INFO)
        
        # File handler for API logs
        api_handler = logging.handlers.RotatingFileHandler(
            self.log_dir / 'api.log',
            maxBytes=30*1024*1024,  # 30MB
            backupCount=7
        )
        api_handler.setFormatter(self.structured_formatter)
        logger.addHandler(api_handler)
        
        return logger
    
    def _setup_websocket_logger(self) -> logging.Logger:
        """Setup WebSocket communication logger."""
        logger = logging.getLogger(f'{self.app_name}.websocket')
        logger.setLevel(logging.INFO)
        
        # File handler for WebSocket logs
        ws_handler = logging.handlers.RotatingFileHandler(
            self.log_dir / 'websocket.log',
            maxBytes=20*1024*1024,  # 20MB
            backupCount=5
        )
        ws_handler.setFormatter(self.structured_formatter)
        logger.addHandler(ws_handler)
        
        return logger
    
    def _setup_database_logger(self) -> logging.Logger:
        """Setup database operation logger."""
        logger = logging.getLogger(f'{self.app_name}.database')
        logger.setLevel(logging.INFO)
        
        # File handler for database logs
        db_handler = logging.handlers.RotatingFileHandler(
            self.log_dir / 'database.log',
            maxBytes=20*1024*1024,  # 20MB
            backupCount=5
        )
        db_handler.setFormatter(self.structured_formatter)
        logger.addHandler(db_handler)
        
        return logger
    
    def _setup_console_logging(self):
        """Setup console logging for development."""
        console_handler = logging.StreamHandler()
        console_handler.setLevel(logging.INFO)
        console_handler.setFormatter(self.console_formatter)
        
        # Add to root logger
        root_logger = logging.getLogger()
        root_logger.addHandler(console_handler)
    
    def get_logger(self, name: str) -> logging.Logger:
        """
        Get a logger with the specified name.
        
        Args:
            name: Logger name
            
        Returns:
            logging.Logger: Configured logger instance
        """
        return logging.getLogger(f'{self.app_name}.{name}')


class PerformanceLogger:
    """Utility class for performance logging."""
    
    def __init__(self, logger: logging.Logger):
        """
        Initialize performance logger.
        
        Args:
            logger: Logger instance to use
        """
        self.logger = logger
    
    def log_operation(self, operation: str, duration: float, 
                     context: Optional[Dict[str, Any]] = None):
        """
        Log operation performance.
        
        Args:
            operation: Operation name
            duration: Operation duration in seconds
            context: Additional context information
        """
        log_data = {
            'operation': operation,
            'duration': duration,
            'context': context or {}
        }
        
        if duration > 5.0:
            self.logger.warning(f"Slow operation: {operation}", extra={
                'processing_time': duration,
                'context': context,
                'performance_alert': True
            })
        else:
            self.logger.info(f"Operation completed: {operation}", extra={
                'processing_time': duration,
                'context': context
            })
    
    def log_api_call(self, endpoint: str, method: str, status_code: int,
                    duration: float, context: Optional[Dict[str, Any]] = None):
        """
        Log API call performance.
        
        Args:
            endpoint: API endpoint
            method: HTTP method
            status_code: Response status code
            duration: Request duration in seconds
            context: Additional context information
        """
        log_data = {
            'endpoint': endpoint,
            'method': method,
            'status_code': status_code,
            'duration': duration,
            'context': context or {}
        }
        
        level = logging.INFO
        if status_code >= 400:
            level = logging.WARNING
        if duration > 2.0:
            level = logging.WARNING
        
        self.logger.log(level, f"API call: {method} {endpoint}", extra={
            'processing_time': duration,
            'status_code': status_code,
            'context': context
        })


# Global logging configuration
_logging_config = None


def setup_logging(app_name: str = "chat_agent", log_level: str = None) -> Dict[str, logging.Logger]:
    """
    Setup global logging configuration.
    
    Args:
        app_name: Application name
        log_level: Logging level (defaults to LOG_LEVEL env var or INFO)
        
    Returns:
        Dict[str, logging.Logger]: Dictionary of configured loggers
    """
    global _logging_config
    
    if log_level is None:
        log_level = os.getenv('LOG_LEVEL', 'INFO')
    
    _logging_config = LoggingConfig(app_name, log_level)
    return _logging_config.setup_logging()


def get_logger(name: str) -> logging.Logger:
    """
    Get a logger with the specified name.
    
    Args:
        name: Logger name
        
    Returns:
        logging.Logger: Configured logger instance
    """
    global _logging_config
    if _logging_config is None:
        setup_logging()
    
    return _logging_config.get_logger(name)


def get_performance_logger(name: str) -> PerformanceLogger:
    """
    Get a performance logger with the specified name.
    
    Args:
        name: Logger name
        
    Returns:
        PerformanceLogger: Performance logger instance
    """
    logger = get_logger(f'performance.{name}')
    return PerformanceLogger(logger)