utils/logger.py

import logging
from typing import Optional
from pathlib import Path


def build_logger(
        output_dir: str,
        log_file: str,
        rank: Optional[int] = 0,
        level: int = logging.INFO,
        console_level: int = logging.INFO,
        file_level: int = logging.INFO
) -> logging.Logger:
    """Setup logging configuration for distributed training.

    Args:
        output_dir: Directory to store log files
        log_file: Name of the log file
        rank: Process rank in distributed training (default: 0)
        level: Overall logging level (default: logging.INFO)
        console_level: Console logging level (default: logging.INFO)
        file_level: File logging level (default: logging.INFO)

    Returns:
        logging.Logger: Configured logger instance

    Raises:
        OSError: If log directory creation fails
    """
    # Create log directory
    log_dir = Path(output_dir) / 'logs'
    log_dir.mkdir(parents=True, exist_ok=True)

    # Add rank to log filename for distributed training
    if rank is not None and rank >= 0:
        filename = log_dir / f"{Path(log_file).stem}_rank{rank}{Path(log_file).suffix}"
    else:
        filename = log_dir / log_file

    # Get logger instance
    logger = logging.getLogger(__name__)
    logger.setLevel(level)

    # Remove existing handlers to avoid duplicate logs
    if logger.hasHandlers():
        logger.handlers.clear()

    # Logging format
    log_format = '[%(asctime)s %(levelname)-8s] %(message)s'
    date_format = '%Y-%m-%d %H:%M:%S'
    formatter = logging.Formatter(log_format, datefmt=date_format)

    # File handler
    try:
        file_handler = logging.FileHandler(filename, mode='w', encoding='utf-8')
        file_handler.setLevel(file_level)
        file_handler.setFormatter(formatter)
        logger.addHandler(file_handler)
    except Exception as e:
        logger.error(f"Failed to create file handler: {str(e)}")
        raise

    # Console handler (only for rank 0 in distributed training)
    if rank in [0, None]:
        console_handler = logging.StreamHandler()
        console_handler.setLevel(console_level)
        console_handler.setFormatter(formatter)
        logger.addHandler(console_handler)

    # Log initial information
    logger.info(f"Logging setup completed. Log file: {filename}")
    logger.info(f"Logging level - Overall: {logging.getLevelName(level)}, "
                f"Console: {logging.getLevelName(console_level)}, "
                f"File: {logging.getLevelName(file_level)}")

    return logger