Add data structures and AutoML implementations

src/mlpipeline/automl/__init__.py

@@ -0,0 +1,4 @@
+from mlpipeline.automl.autogluon_trainer import AutoGluonTrainer
+from mlpipeline.automl.flaml_trainer import FLAMLTrainer
+from mlpipeline.automl.pycaret_trainer import PyCaretTrainer
+from mlpipeline.automl.automl_factory import AutoMLFactory

src/mlpipeline/automl/autogluon_trainer.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+from typing import Dict, Any, Optional
+import pandas as pd
+from autogluon.tabular import TabularPredictor
+
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class AutoGluonTrainer:
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.predictor: Optional[TabularPredictor] = None
+    
+    def train(self, train_data: pd.DataFrame, target_column: str, model_path: Path) -> Dict[str, float]:
+        logger.info("Starting AutoGluon training")
+        
+        self.predictor = TabularPredictor(
+            label=target_column,
+            path=str(model_path),
+            eval_metric=self.config.get('eval_metric'),
+            verbosity=self.config.get('verbosity', 2),
+        )
+        
+        self.predictor.fit(
+            train_data=train_data,
+            time_limit=self.config.get('time_limit', 600),
+            presets=self.config.get('presets', 'medium_quality'),
+            num_bag_folds=self.config.get('num_bag_folds', 5),
+            num_stack_levels=self.config.get('num_stack_levels', 1),
+        )
+        
+        leaderboard = self.predictor.leaderboard(silent=True)
+        best_model = leaderboard.iloc[0]
+        
+        metrics = {
+            'score': float(best_model['score_val']),
+            'score_test': float(best_model.get('score_test', 0.0)),
+        }
+        
+        logger.info(f"AutoGluon training completed. Best score: {metrics['score']}")
+        return metrics
+    
+    def predict(self, data: pd.DataFrame) -> pd.Series:
+        if self.predictor is None:
+            raise ValueError("Model not trained. Call train() first.")
+        return self.predictor.predict(data)
+    
+    def load(self, model_path: Path):
+        logger.info(f"Loading AutoGluon model from {model_path}")
+        self.predictor = TabularPredictor.load(str(model_path))
+        return self

src/mlpipeline/automl/automl_factory.py

@@ -0,0 +1,32 @@
+from typing import Union
+from mlpipeline.automl.autogluon_trainer import AutoGluonTrainer
+from mlpipeline.automl.flaml_trainer import FLAMLTrainer
+from mlpipeline.automl.pycaret_trainer import PyCaretTrainer
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class AutoMLFactory:
+    @staticmethod
+    def create_trainer(library: str, config: dict) -> Union[AutoGluonTrainer, FLAMLTrainer, PyCaretTrainer]:
+        library = library.lower()
+        
+        if library == 'autogluon':
+            logger.info("Creating AutoGluon trainer")
+            return AutoGluonTrainer(config)
+        
+        elif library == 'flaml':
+            logger.info("Creating FLAML trainer")
+            return FLAMLTrainer(config)
+        
+        elif library == 'pycaret':
+            logger.info("Creating PyCaret trainer")
+            return PyCaretTrainer(config)
+        
+        else:
+            raise ValueError(f"Unknown AutoML library: {library}. Choose from: autogluon, flaml, pycaret")
+    
+    @staticmethod
+    def get_available_libraries():
+        return ['autogluon', 'flaml', 'pycaret']

src/mlpipeline/automl/flaml_trainer.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+from typing import Dict, Any, Optional
+import pandas as pd
+import numpy as np
+from flaml import AutoML
+from sklearn.metrics import accuracy_score, r2_score
+
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.utils.common import save_pickle, load_pickle
+
+logger = get_logger(__name__)
+
+
+class FLAMLTrainer:
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.automl: Optional[AutoML] = None
+        self.task = config.get('task', 'classification')
+    
+    def train(self, train_data: pd.DataFrame, target_column: str, model_path: Path) -> Dict[str, float]:
+        logger.info("Starting FLAML training")
+        
+        X_train = train_data.drop(columns=[target_column])
+        y_train = train_data[target_column]
+        
+        self.automl = AutoML()
+        
+        settings = {
+            'time_budget': self.config.get('time_budget', 600),
+            'metric': self.config.get('metric', 'auto'),
+            'task': self.task,
+            'estimator_list': self.config.get('estimator_list', ['lgbm', 'xgboost', 'rf']),
+            'n_jobs': self.config.get('n_jobs', -1),
+            'verbose': self.config.get('verbose', 1),
+            'early_stop': self.config.get('early_stop', True),
+        }
+        
+        self.automl.fit(X_train=X_train, y_train=y_train, **settings)
+        
+        y_pred = self.automl.predict(X_train)
+        
+        if self.task == 'classification':
+            score = accuracy_score(y_train, y_pred)
+            metric_name = 'accuracy'
+        else:
+            score = r2_score(y_train, y_pred)
+            metric_name = 'r2_score'
+        
+        save_pickle(model_path / 'model.pkl', self.automl)
+        
+        metrics = {
+            metric_name: float(score),
+            'best_loss': float(self.automl.best_loss),
+        }
+        
+        logger.info(f"FLAML training completed. Best {metric_name}: {score}")
+        return metrics
+    
+    def predict(self, data: pd.DataFrame) -> np.ndarray:
+        if self.automl is None:
+            raise ValueError("Model not trained. Call train() first.")
+        return self.automl.predict(data)
+    
+    def load(self, model_path: Path):
+        logger.info(f"Loading FLAML model from {model_path}")
+        self.automl = load_pickle(model_path / 'model.pkl')
+        return self

src/mlpipeline/automl/pycaret_trainer.py

@@ -0,0 +1,91 @@
+from pathlib import Path
+from typing import Dict, Any, Optional
+import pandas as pd
+from pycaret.classification import setup as classification_setup, compare_models as classification_compare, finalize_model as classification_finalize, save_model as classification_save, load_model as classification_load
+from pycaret.regression import setup as regression_setup, compare_models as regression_compare, finalize_model as regression_finalize, save_model as regression_save, load_model as regression_load
+
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class PyCaretTrainer:
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.model: Optional[Any] = None
+        self.is_classification = None
+    
+    def train(self, train_data: pd.DataFrame, target_column: str, model_path: Path) -> Dict[str, float]:
+        logger.info("Starting PyCaret training")
+        
+        if train_data[target_column].dtype == 'object' or train_data[target_column].nunique() < 20:
+            self.is_classification = True
+            setup_fn = classification_setup
+            compare_fn = classification_compare
+            finalize_fn = classification_finalize
+            save_fn = classification_save
+        else:
+            self.is_classification = False
+            setup_fn = regression_setup
+            compare_fn = regression_compare
+            finalize_fn = regression_finalize
+            save_fn = regression_save
+        
+        exp = setup_fn(
+            data=train_data,
+            target=target_column,
+            session_id=self.config.get('session_id', 42),
+            fold=self.config.get('fold', 5),
+            verbose=self.config.get('verbose', False),
+            use_gpu=self.config.get('use_gpu', False),
+        )
+        
+        best_model = compare_fn(
+            n_select=self.config.get('n_select', 5),
+            verbose=self.config.get('verbose', False),
+        )
+        
+        if self.config.get('tuning', {}).get('enabled', True):
+            from pycaret.classification import tune_model as classification_tune
+            from pycaret.regression import tune_model as regression_tune
+            tune_fn = classification_tune if self.is_classification else regression_tune
+            
+            best_model = tune_fn(
+                best_model,
+                n_iter=self.config.get('tuning', {}).get('n_iter', 10),
+                optimize=self.config.get('tuning', {}).get('optimize', 'Accuracy'),
+            )
+        
+        self.model = finalize_fn(best_model)
+        
+        save_fn(self.model, str(model_path / 'model'))
+        
+        from pycaret.classification import pull as classification_pull
+        from pycaret.regression import pull as regression_pull
+        pull_fn = classification_pull if self.is_classification else regression_pull
+        
+        results = pull_fn()
+        
+        metrics = {
+            'score': float(results.iloc[0]['Mean']) if not results.empty else 0.0,
+        }
+        
+        logger.info(f"PyCaret training completed. Score: {metrics['score']}")
+        return metrics
+    
+    def predict(self, data: pd.DataFrame) -> pd.Series:
+        if self.model is None:
+            raise ValueError("Model not trained. Call train() first.")
+        
+        from pycaret.classification import predict_model as classification_predict
+        from pycaret.regression import predict_model as regression_predict
+        predict_fn = classification_predict if self.is_classification else regression_predict
+        
+        predictions = predict_fn(self.model, data=data)
+        return predictions.iloc[:, -1]
+    
+    def load(self, model_path: Path):
+        logger.info(f"Loading PyCaret model from {model_path}")
+        load_fn = classification_load if self.is_classification else regression_load
+        self.model = load_fn(str(model_path / 'model'))
+        return self

src/mlpipeline/config/__init__.py

@@ -0,0 +1 @@
+from mlpipeline.config.configuration import ConfigurationManager

src/mlpipeline/config/configuration.py

@@ -0,0 +1,99 @@
+from pathlib import Path
+from mlpipeline.constants import CONFIG_FILE_PATH
+from mlpipeline.utils.common import read_yaml, create_directories
+from mlpipeline.entity.config_entity import (
+    DataIngestionConfig,
+    DataValidationConfig,
+    DataTransformationConfig,
+    FeatureEngineeringConfig,
+    ModelTrainerConfig,
+    ModelEvaluationConfig,
+    ModelPusherConfig,
+)
+
+
+class ConfigurationManager:
+    def __init__(self, config_filepath=CONFIG_FILE_PATH):
+        self.config = read_yaml(config_filepath)
+        create_directories([self.config.artifacts_root])
+
+    def get_data_ingestion_config(self) -> DataIngestionConfig:
+        config = self.config.data_ingestion
+        create_directories([config.root_dir])
+        
+        return DataIngestionConfig(
+            root_dir=Path(config.root_dir),
+            source_url=config.source_url,
+            local_data_file=Path(config.local_data_file),
+            unzip_dir=Path(config.unzip_dir),
+        )
+
+    def get_data_validation_config(self) -> DataValidationConfig:
+        config = self.config.data_validation
+        create_directories([config.root_dir])
+        
+        return DataValidationConfig(
+            root_dir=Path(config.root_dir),
+            data_dir=Path(config.data_dir),
+            status_file=Path(config.status_file),
+            schema_file=Path(config.schema_file),
+        )
+
+    def get_data_transformation_config(self) -> DataTransformationConfig:
+        config = self.config.data_transformation
+        create_directories([config.root_dir])
+        
+        return DataTransformationConfig(
+            root_dir=Path(config.root_dir),
+            data_path=Path(config.data_path),
+            train_path=Path(config.train_path),
+            test_path=Path(config.test_path),
+            test_size=config.test_size,
+            random_state=config.random_state,
+        )
+
+    def get_feature_engineering_config(self) -> FeatureEngineeringConfig:
+        config = self.config.feature_engineering
+        create_directories([config.root_dir])
+        
+        return FeatureEngineeringConfig(
+            root_dir=Path(config.root_dir),
+            train_path=Path(config.train_path),
+            test_path=Path(config.test_path),
+            output_train_path=Path(config.output_train_path),
+            output_test_path=Path(config.output_test_path),
+        )
+
+    def get_model_trainer_config(self) -> ModelTrainerConfig:
+        config = self.config.model_trainer
+        create_directories([config.root_dir])
+        
+        return ModelTrainerConfig(
+            root_dir=Path(config.root_dir),
+            train_data_path=Path(config.train_data_path),
+            test_data_path=Path(config.test_data_path),
+            model_path=Path(config.model_path),
+            target_column=config.target_column,
+        )
+
+    def get_model_evaluation_config(self) -> ModelEvaluationConfig:
+        config = self.config.model_evaluation
+        create_directories([config.root_dir])
+        
+        return ModelEvaluationConfig(
+            root_dir=Path(config.root_dir),
+            model_path=Path(config.model_path),
+            test_data_path=Path(config.test_data_path),
+            metrics_file=Path(config.metrics_file),
+            target_column=config.target_column,
+        )
+
+    def get_model_pusher_config(self) -> ModelPusherConfig:
+        config = self.config.model_pusher
+        create_directories([config.root_dir])
+        
+        return ModelPusherConfig(
+            root_dir=Path(config.root_dir),
+            model_path=Path(config.model_path),
+            model_registry_path=Path(config.model_registry_path),
+        )

src/mlpipeline/entity/__init__.py

@@ -0,0 +1,18 @@
+from mlpipeline.entity.config_entity import (
+    DataIngestionConfig,
+    DataValidationConfig,
+    DataTransformationConfig,
+    FeatureEngineeringConfig,
+    ModelTrainerConfig,
+    ModelEvaluationConfig,
+    ModelPusherConfig,
+)
+
+from mlpipeline.entity.artifact_entity import (
+    DataIngestionArtifact,
+    DataValidationArtifact,
+    DataTransformationArtifact,
+    FeatureEngineeringArtifact,
+    ModelTrainerArtifact,
+    ModelEvaluationArtifact,
+)

src/mlpipeline/entity/artifact_entity.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Any
+
+
+@dataclass
+class DataIngestionArtifact:
+    data_file_path: Path
+    is_ingested: bool
+    message: str
+
+
+@dataclass
+class DataValidationArtifact:
+    validation_status: bool
+    message: str
+    schema_file_path: Path
+
+
+@dataclass
+class DataTransformationArtifact:
+    train_file_path: Path
+    test_file_path: Path
+    is_transformed: bool
+    message: str
+
+
+@dataclass
+class FeatureEngineeringArtifact:
+    train_features_path: Path
+    test_features_path: Path
+    is_engineered: bool
+    message: str
+
+
+@dataclass
+class ModelTrainerArtifact:
+    model_path: Path
+    is_trained: bool
+    message: str
+    train_metrics: Dict[str, float]
+
+
+@dataclass
+class ModelEvaluationArtifact:
+    is_model_accepted: bool
+    evaluation_metrics: Dict[str, float]
+    message: str

src/mlpipeline/entity/config_entity.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class DataIngestionConfig:
+    root_dir: Path
+    source_url: str
+    local_data_file: Path
+    unzip_dir: Path
+
+
+@dataclass(frozen=True)
+class DataValidationConfig:
+    root_dir: Path
+    data_dir: Path
+    status_file: Path
+    schema_file: Path
+
+
+@dataclass(frozen=True)
+class DataTransformationConfig:
+    root_dir: Path
+    data_path: Path
+    train_path: Path
+    test_path: Path
+    test_size: float
+    random_state: int
+
+
+@dataclass(frozen=True)
+class FeatureEngineeringConfig:
+    root_dir: Path
+    train_path: Path
+    test_path: Path
+    output_train_path: Path
+    output_test_path: Path
+
+
+@dataclass(frozen=True)
+class ModelTrainerConfig:
+    root_dir: Path
+    train_data_path: Path
+    test_data_path: Path
+    model_path: Path
+    target_column: str
+
+
+@dataclass(frozen=True)
+class ModelEvaluationConfig:
+    root_dir: Path
+    model_path: Path
+    test_data_path: Path
+    metrics_file: Path
+    target_column: str
+
+
+@dataclass(frozen=True)
+class ModelPusherConfig:
+    root_dir: Path
+    model_path: Path
+    model_registry_path: Path