import os import pandas as pd from typing import Tuple from src.config import SystemConfig from src.monitoring.logger import get_logger logger = get_logger(__name__) class DataValidationError(Exception): """Custom exception for data validation failures.""" pass class InputValidator: """Validates input datasets against system constraints before processing.""" @staticmethod def validate_file_size(file_path: str) -> bool: """Checks if the file exceeds the maximum allowed file size constraint (5MB).""" if not os.path.exists(file_path): raise FileNotFoundError(f"Dataset not found at {file_path}") file_size_mb = os.path.getsize(file_path) / (1024 * 1024) logger.info(f"Checking dataset size: {file_size_mb:.2f} MB") if file_size_mb > SystemConfig.MAX_FILE_SIZE_MB: logger.warning( f"File size {file_size_mb:.2f}MB exceeds limit of {SystemConfig.MAX_FILE_SIZE_MB}MB. " "Enforcing limits may require chunking or sampling." ) # Depending on strictness, we might raise or just warn. # We raise to adhere STRICTLY to the constraints. raise DataValidationError( f"File size {file_size_mb:.2f}MB exceeds strict constraint of {SystemConfig.MAX_FILE_SIZE_MB}MB" ) return True @staticmethod def load_and_validate_schema(file_path: str, target_col: str) -> pd.DataFrame: """ Loads the dataset, enforces row constraints, and validates schema (target existence). """ InputValidator.validate_file_size(file_path) logger.info("Loading dataset into memory...") try: df = pd.read_csv(file_path) except Exception as e: raise DataValidationError(f"Failed to parse CSV file: {str(e)}") # Target Existence Verification if target_col not in df.columns: logger.error(f"Target column '{target_col}' missing from dataset.") raise DataValidationError(f"Target column '{target_col}' not found. Available: {list(df.columns)}") # Enforce Row Limits num_rows = len(df) if num_rows > SystemConfig.MAX_ROWS: logger.warning(f"Dataset has {num_rows} rows. Downsampling to {SystemConfig.MAX_ROWS} to respect constraints.") df = df.sample(n=SystemConfig.MAX_ROWS, random_state=42).reset_index(drop=True) logger.info(f"Dataset validated successfully. Final shape: {df.shape}") return df