Add pipeline stages implementation

config/config.yaml

@@ -2,19 +2,19 @@ artifacts_root: artifacts
 
 data_ingestion:
   root_dir: artifacts/data_ingestion
-  source_url: null
-  local_data_file: artifacts/data_ingestion/data.csv
+  source_url: kaggle://playground-series-s6e2
+  local_data_file: artifacts/data_ingestion/train_raw.csv
   unzip_dir: artifacts/data_ingestion
 
 data_validation:
   root_dir: artifacts/data_validation
-  data_dir: artifacts/data_ingestion/data.csv
+  data_dir: artifacts/data_ingestion/train_raw.csv
   status_file: artifacts/data_validation/status.txt
   schema_file: config/schema.yaml
 
 data_transformation:
   root_dir: artifacts/data_transformation
-  data_path: artifacts/data_ingestion/data.csv
+  data_path: artifacts/data_ingestion/train_raw.csv
   train_path: artifacts/data_transformation/train.csv
   test_path: artifacts/data_transformation/test.csv
   test_size: 0.2
@@ -32,14 +32,14 @@ model_trainer:
   train_data_path: artifacts/feature_engineering/train_features.csv
   test_data_path: artifacts/feature_engineering/test_features.csv
   model_path: artifacts/model_trainer/model
-  target_column: target
+  target_column: Heart Disease
 
 model_evaluation:
   root_dir: artifacts/model_evaluation
   model_path: artifacts/model_trainer/model
   test_data_path: artifacts/feature_engineering/test_features.csv
   metrics_file: artifacts/model_evaluation/metrics.json
-  target_column: target
+  target_column: Heart Disease
 
 model_pusher:
   root_dir: artifacts/model_pusher

requirements.txt

@@ -27,4 +27,5 @@ prometheus-client
 python-json-logger
 
 httpx
-requests
+requests
+kaggle

setup.py

@@ -0,0 +1,28 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="mlpipeline",
+    version="0.1.0",
+    author="AutoML Team",
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
+    python_requires=">=3.11",
+    install_requires=[
+        "fastapi",
+        "uvicorn[standard]",
+        "pydantic",
+        "pandas",
+        "numpy",
+        "scikit-learn",
+        "autogluon.tabular",
+        "flaml",
+        "pycaret",
+        "mlflow",
+        "dvc",
+        "evidently",
+        "pyyaml",
+        "python-box",
+        "ensure",
+        "kaggle",
+    ],
+)

src/mlpipeline/automl/autogluon_trainer.py

@@ -47,7 +47,7 @@ class AutoGluonTrainer:
             raise ValueError("Model not trained. Call train() first.")
         return self.predictor.predict(data)
     
-    def load(self, model_path: Path):
+    def load_model(self, model_path: Path):
         logger.info(f"Loading AutoGluon model from {model_path}")
         self.predictor = TabularPredictor.load(str(model_path))
         return self

src/mlpipeline/automl/flaml_trainer.py

@@ -2,7 +2,7 @@ from pathlib import Path
 from typing import Dict, Any, Optional
 import pandas as pd
 import numpy as np
-from flaml import AutoML
+from flaml.automl.automl import AutoML
 from sklearn.metrics import accuracy_score, r2_score
 
 from mlpipeline.logging.logger import get_logger

src/mlpipeline/components/__init__.py

@@ -0,0 +1,9 @@
+__all__ = [
+    "DataIngestion",
+    "DataValidation",
+    "DataTransformation",
+    "FeatureEngineering",
+    "AutoMLTrainer",
+    "ModelEvaluation",
+    "ModelPusher",
+]

src/mlpipeline/components/automl_trainer.py

@@ -0,0 +1,51 @@
+import pandas as pd
+from mlpipeline.entity import ModelTrainerConfig, ModelTrainerArtifact
+from mlpipeline.automl import AutoMLFactory
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.exception import ModelTrainingException
+from mlpipeline.constants import AUTOML_CONFIG_FILE_PATH
+from mlpipeline.utils.common import read_yaml
+from pathlib import Path
+import sys
+import os
+
+logger = get_logger(__name__)
+
+
+class AutoMLTrainer:
+    def __init__(self, config: ModelTrainerConfig):
+        self.config = config
+    
+    def train(self) -> ModelTrainerArtifact:
+        try:
+            logger.info("Starting model training")
+            
+            train_df = pd.read_csv(self.config.train_data_path)
+            
+            automl_config = read_yaml(Path(AUTOML_CONFIG_FILE_PATH))
+            library_config = automl_config[self.config.automl_library]
+            
+            trainer = AutoMLFactory.create_trainer(
+                self.config.automl_library,
+                library_config
+            )
+            
+            os.makedirs(self.config.root_dir, exist_ok=True)
+            
+            if self.config.automl_library == 'autogluon':
+                metrics = trainer.train(train_df, self.config.target_column, self.config.model_path)
+            else:
+                X_train = train_df.drop(columns=[self.config.target_column])
+                y_train = train_df[self.config.target_column]
+                metrics = trainer.train(X_train, y_train, self.config.model_path)
+            
+            logger.info(f"Model trained with metrics: {metrics}")
+            
+            return ModelTrainerArtifact(
+                model_path=self.config.model_path,
+                train_metrics=metrics,
+                is_trained=True,
+                message=f"Model trained successfully with score: {metrics.get('score', 0.0):.4f}"
+            )
+        except Exception as e:
+            raise ModelTrainingException(str(e), sys)

src/mlpipeline/components/data_ingestion.py

@@ -0,0 +1,62 @@
+import os
+import shutil
+from pathlib import Path
+from kaggle.api.kaggle_api_extended import KaggleApi
+
+from mlpipeline.entity import DataIngestionConfig, DataIngestionArtifact
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.exception import DataIngestionException
+import sys
+
+logger = get_logger(__name__)
+
+
+class DataIngestion:
+    def __init__(self, config: DataIngestionConfig):
+        self.config = config
+        self.kaggle_api = KaggleApi()
+        self.kaggle_api.authenticate()
+    
+    def download_data(self) -> DataIngestionArtifact:
+        try:
+            logger.info("Starting data ingestion")
+            
+            os.makedirs(self.config.root_dir, exist_ok=True)
+            
+            competition_name = "playground-series-s6e2"
+            
+            logger.info(f"Downloading dataset from Kaggle competition: {competition_name}")
+            self.kaggle_api.competition_download_files(
+                competition_name,
+                path=self.config.root_dir
+            )
+            
+            zip_file = self.config.root_dir / f"{competition_name}.zip"
+            
+            if zip_file.exists():
+                logger.info(f"Extracting {zip_file}")
+                shutil.unpack_archive(zip_file, self.config.unzip_dir)
+                zip_file.unlink()
+            
+            train_file = self.config.unzip_dir / "train.csv"
+            test_file = self.config.unzip_dir / "test.csv"
+            
+            if train_file.exists() and test_file.exists():
+                train_raw = self.config.root_dir / "train_raw.csv"
+                test_raw = self.config.root_dir / "test_raw.csv"
+                
+                shutil.copy(train_file, train_raw)
+                shutil.copy(test_file, test_raw)
+                
+                logger.info(f"Data saved: {train_raw}, {test_raw}")
+                
+                return DataIngestionArtifact(
+                    data_file_path=train_raw,
+                    is_ingested=True,
+                    message="Data ingestion completed successfully"
+                )
+            else:
+                raise FileNotFoundError("Train or test file not found after extraction")
+                
+        except Exception as e:
+            raise DataIngestionException(str(e), sys)

src/mlpipeline/components/data_transformation.py

@@ -0,0 +1,42 @@
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from mlpipeline.entity import DataTransformationConfig, DataTransformationArtifact
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.exception import DataTransformationException
+import sys
+import os
+
+logger = get_logger(__name__)
+
+
+class DataTransformation:
+    def __init__(self, config: DataTransformationConfig):
+        self.config = config
+    
+    def transform(self) -> DataTransformationArtifact:
+        try:
+            logger.info("Starting data transformation")
+            
+            df = pd.read_csv(self.config.data_path)
+            
+            train_df, test_df = train_test_split(
+                df,
+                test_size=self.config.test_size,
+                random_state=self.config.random_state
+            )
+            
+            os.makedirs(self.config.root_dir, exist_ok=True)
+            
+            train_df.to_csv(self.config.train_path, index=False)
+            test_df.to_csv(self.config.test_path, index=False)
+            
+            logger.info(f"Train shape: {train_df.shape}, Test shape: {test_df.shape}")
+            
+            return DataTransformationArtifact(
+                train_file_path=self.config.train_path,
+                test_file_path=self.config.test_path,
+                is_transformed=True,
+                message=f"Train: {train_df.shape}, Test: {test_df.shape}"
+            )
+        except Exception as e:
+            raise DataTransformationException(str(e), sys)

src/mlpipeline/components/data_validation.py

@@ -0,0 +1,45 @@
+import pandas as pd
+from mlpipeline.entity import DataValidationConfig, DataValidationArtifact
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.exception import DataValidationException
+from mlpipeline.utils.common import read_yaml
+import sys
+
+logger = get_logger(__name__)
+
+
+class DataValidation:
+    def __init__(self, config: DataValidationConfig):
+        self.config = config
+    
+    def validate_schema(self) -> DataValidationArtifact:
+        try:
+            logger.info("Starting data validation")
+            
+            df = pd.read_csv(self.config.data_dir)
+            
+            validation_status = True
+            errors = []
+            
+            if df.empty:
+                validation_status = False
+                errors.append("Dataset is empty")
+            
+            if df.isnull().all().any():
+                validation_status = False
+                errors.append("Columns with all null values found")
+            
+            status_message = "Validation passed" if validation_status else "; ".join(errors)
+            
+            with open(self.config.status_file, "w") as f:
+                f.write(status_message)
+            
+            logger.info(f"Validation status: {status_message}")
+            
+            return DataValidationArtifact(
+                validation_status=validation_status,
+                message=status_message,
+                schema_file_path=self.config.schema_file
+            )
+        except Exception as e:
+            raise DataValidationException(str(e), sys)

src/mlpipeline/components/feature_engineering.py

@@ -0,0 +1,128 @@
+import pandas as pd
+import numpy as np
+from pathlib import Path
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.feature_selection import VarianceThreshold
+from mlpipeline.entity import FeatureEngineeringConfig, FeatureEngineeringArtifact
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.exception import FeatureEngineeringException
+from mlpipeline.utils.common import save_object
+import sys
+import os
+
+logger = get_logger(__name__)
+
+
+class FeatureEngineering:
+    def __init__(self, config: FeatureEngineeringConfig):
+        self.config = config
+        self.label_encoders = {}
+        self.scaler = None
+    
+    def engineer_features(self) -> FeatureEngineeringArtifact:
+        try:
+            logger.info("Starting feature engineering")
+            
+            train_df = pd.read_csv(self.config.train_path)
+            test_df = pd.read_csv(self.config.test_path)
+            
+            train_df = self._handle_missing_values(train_df)
+            test_df = self._handle_missing_values(test_df)
+            
+            train_df = self._encode_categorical(train_df, is_train=True)
+            test_df = self._encode_categorical(test_df, is_train=False)
+            
+            train_df = self._create_interaction_features(train_df)
+            test_df = self._create_interaction_features(test_df)
+            
+            train_df = self._remove_low_variance(train_df, is_train=True)
+            test_df = self._remove_low_variance(test_df, is_train=False)
+            
+            numeric_cols = train_df.select_dtypes(include=[np.number]).columns.tolist()
+            if 'target' in numeric_cols:
+                numeric_cols.remove('target')
+            
+            if numeric_cols:
+                self.scaler = StandardScaler()
+                train_df[numeric_cols] = self.scaler.fit_transform(train_df[numeric_cols])
+                test_df[numeric_cols] = self.scaler.transform(test_df[numeric_cols])
+            
+            os.makedirs(self.config.root_dir, exist_ok=True)
+            
+            train_df.to_csv(self.config.output_train_path, index=False)
+            test_df.to_csv(self.config.output_test_path, index=False)
+            
+            preprocessor_path = Path(self.config.root_dir) / "preprocessor.pkl"
+            save_object(preprocessor_path, {
+                'scaler': self.scaler,
+                'label_encoders': self.label_encoders
+            })
+            
+            logger.info(f"Feature engineering completed. Train shape: {train_df.shape}, Test shape: {test_df.shape}")
+            
+            return FeatureEngineeringArtifact(
+                train_features_path=self.config.output_train_path,
+                test_features_path=self.config.output_test_path,
+                is_engineered=True,
+                message=f"Features engineered: {train_df.shape[1]} features"
+            )
+        except Exception as e:
+            raise FeatureEngineeringException(str(e), sys)
+    
+    def _handle_missing_values(self, df):
+        for col in df.columns:
+            if df[col].dtype in [np.float64, np.int64]:
+                df[col].fillna(df[col].median(), inplace=True)
+            else:
+                df[col].fillna(df[col].mode()[0] if not df[col].mode().empty else 'missing', inplace=True)
+        return df
+    
+    def _encode_categorical(self, df, is_train=True):
+        categorical_cols = df.select_dtypes(include=['object']).columns
+        
+        for col in categorical_cols:
+            if is_train:
+                self.label_encoders[col] = LabelEncoder()
+                df[col] = self.label_encoders[col].fit_transform(df[col].astype(str))
+            else:
+                if col in self.label_encoders:
+                    df[col] = df[col].astype(str).map(
+                        lambda x: self.label_encoders[col].transform([x])[0] 
+                        if x in self.label_encoders[col].classes_ else -1
+                    )
+        return df
+    
+    def _create_interaction_features(self, df):
+        numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+        if 'target' in numeric_cols:
+            numeric_cols.remove('target')
+        
+        if len(numeric_cols) >= 2:
+            df[f'{numeric_cols[0]}_x_{numeric_cols[1]}'] = df[numeric_cols[0]] * df[numeric_cols[1]]
+        
+        return df
+    
+    def _remove_low_variance(self, df, is_train=True, threshold=0.01):
+        if 'target' in df.columns:
+            target = df['target']
+            features = df.drop(columns=['target'])
+        else:
+            target = None
+            features = df
+        
+        if is_train:
+            self.variance_selector = VarianceThreshold(threshold=threshold)
+            self.variance_selector.fit(features)
+        
+        if hasattr(self, 'variance_selector'):
+            features_selected = pd.DataFrame(
+                self.variance_selector.transform(features),
+                columns=features.columns[self.variance_selector.get_support()],
+                index=features.index
+            )
+            
+            if target is not None:
+                return pd.concat([features_selected, target], axis=1)
+            return features_selected
+        
+        return df

src/mlpipeline/components/model_evaluation.py

@@ -0,0 +1,53 @@
+import pandas as pd
+import json
+from sklearn.metrics import accuracy_score, f1_score, classification_report
+from mlpipeline.entity import ModelEvaluationConfig, ModelEvaluationArtifact
+from autogluon.tabular import TabularPredictor
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.exception import ModelEvaluationException
+import sys
+import os
+
+logger = get_logger(__name__)
+
+
+class ModelEvaluation:
+    def __init__(self, config: ModelEvaluationConfig):
+        self.config = config
+    
+    def evaluate(self) -> ModelEvaluationArtifact:
+        try:
+            logger.info("Starting model evaluation")
+            
+            test_df = pd.read_csv(self.config.test_data_path)
+            
+            predictor = TabularPredictor.load(str(self.config.model_path))
+            
+            predictions = predictor.predict(test_df)
+            y_test = test_df[self.config.target_column]
+            
+            predictions_binary = (predictions > 0).astype(int)
+            y_test_binary = (y_test > 0).astype(int)
+            
+            accuracy = float(accuracy_score(y_test_binary, predictions_binary))
+            f1 = float(f1_score(y_test_binary, predictions_binary, average='weighted'))
+            
+            metrics = {
+                "accuracy": accuracy,
+                "f1_score": f1
+            }
+            
+            os.makedirs(self.config.root_dir, exist_ok=True)
+            
+            with open(self.config.metrics_file, "w") as f:
+                json.dump(metrics, f, indent=2)
+            
+            logger.info(f"Evaluation metrics: {metrics}")
+            
+            return ModelEvaluationArtifact(
+                is_model_accepted=True,
+                evaluation_metrics=metrics,
+                message=f"Model evaluation completed with accuracy: {accuracy:.4f}"
+            )
+        except Exception as e:
+            raise ModelEvaluationException(str(e), sys)

src/mlpipeline/components/model_pusher.py

@@ -0,0 +1,40 @@
+import shutil
+import os
+from pathlib import Path
+from mlpipeline.entity import ModelPusherConfig, ModelPusherArtifact
+from mlpipeline.logging.logger import get_logger
+from mlpipeline.exception import ModelPusherException
+import sys
+
+logger = get_logger(__name__)
+
+
+class ModelPusher:
+    def __init__(self, config: ModelPusherConfig):
+        self.config = config
+    
+    def push_model(self) -> ModelPusherArtifact:
+        try:
+            logger.info("Starting model pusher")
+            
+            os.makedirs(self.config.model_registry_path, exist_ok=True)
+            
+            model_source = Path(self.config.model_path)
+            model_dest = Path(self.config.model_registry_path) / model_source.name
+            
+            if model_source.is_dir():
+                if model_dest.exists():
+                    shutil.rmtree(model_dest)
+                shutil.copytree(model_source, model_dest)
+            else:
+                shutil.copy2(model_source, model_dest)
+            
+            logger.info(f"Model pushed to: {model_dest}")
+            
+            return ModelPusherArtifact(
+                pushed_model_path=str(model_dest),
+                is_pushed=True,
+                message="Model pushed successfully"
+            )
+        except Exception as e:
+            raise ModelPusherException(str(e), sys)

src/mlpipeline/config/configuration.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from mlpipeline.constants import CONFIG_FILE_PATH
+from mlpipeline.constants import CONFIG_FILE_PATH, AUTOML_CONFIG_FILE_PATH
 from mlpipeline.utils.common import read_yaml, create_directories
 from mlpipeline.entity.config_entity import (
     DataIngestionConfig,
@@ -66,6 +66,7 @@ class ConfigurationManager:
 
     def get_model_trainer_config(self) -> ModelTrainerConfig:
         config = self.config.model_trainer
+        automl_config = read_yaml(Path(AUTOML_CONFIG_FILE_PATH))
         create_directories([config.root_dir])
         
         return ModelTrainerConfig(
@@ -74,10 +75,12 @@ class ConfigurationManager:
             test_data_path=Path(config.test_data_path),
             model_path=Path(config.model_path),
             target_column=config.target_column,
+            automl_library=automl_config.automl_library,
         )
 
     def get_model_evaluation_config(self) -> ModelEvaluationConfig:
         config = self.config.model_evaluation
+        automl_config = read_yaml(Path(AUTOML_CONFIG_FILE_PATH))
         create_directories([config.root_dir])
         
         return ModelEvaluationConfig(
@@ -86,6 +89,7 @@ class ConfigurationManager:
             test_data_path=Path(config.test_data_path),
             metrics_file=Path(config.metrics_file),
             target_column=config.target_column,
+            automl_library=automl_config.automl_library,
         )
 
     def get_model_pusher_config(self) -> ModelPusherConfig:

src/mlpipeline/entity/__init__.py

@@ -15,4 +15,5 @@ from mlpipeline.entity.artifact_entity import (
     FeatureEngineeringArtifact,
     ModelTrainerArtifact,
     ModelEvaluationArtifact,
+    ModelPusherArtifact,
 )

src/mlpipeline/entity/artifact_entity.py

@@ -45,4 +45,11 @@ class ModelTrainerArtifact:
 class ModelEvaluationArtifact:
     is_model_accepted: bool
     evaluation_metrics: Dict[str, float]
+    message: str
+
+
+@dataclass
+class ModelPusherArtifact:
+    pushed_model_path: str
+    is_pushed: bool
     message: str

src/mlpipeline/entity/config_entity.py

@@ -44,6 +44,7 @@ class ModelTrainerConfig:
     test_data_path: Path
     model_path: Path
     target_column: str
+    automl_library: str
 
 
 @dataclass(frozen=True)
@@ -53,6 +54,7 @@ class ModelEvaluationConfig:
     test_data_path: Path
     metrics_file: Path
     target_column: str
+    automl_library: str
 
 
 @dataclass(frozen=True)

src/mlpipeline/exception/__init__.py

@@ -3,7 +3,9 @@ from mlpipeline.exception.exception import (
     DataIngestionException,
     DataValidationException,
     DataTransformationException,
+    FeatureEngineeringException,
     ModelTrainingException,
     ModelEvaluationException,
+    ModelPusherException,
     ConfigurationException,
 )

src/mlpipeline/exception/exception.py

@@ -35,6 +35,10 @@ class DataTransformationException(MLPipelineException):
     pass
 
 
+class FeatureEngineeringException(MLPipelineException):
+    pass
+
+
 class ModelTrainingException(MLPipelineException):
     pass
 
@@ -43,5 +47,9 @@ class ModelEvaluationException(MLPipelineException):
     pass
 
 
+class ModelPusherException(MLPipelineException):
+    pass
+
+
 class ConfigurationException(MLPipelineException):
     pass

src/mlpipeline/pipelines/__init__.py

@@ -0,0 +1,9 @@
+__all__ = [
+    "DataIngestionPipeline",
+    "DataValidationPipeline",
+    "DataTransformationPipeline",
+    "FeatureEngineeringPipeline",
+    "ModelTrainerPipeline",
+    "ModelEvaluationPipeline",
+    "ModelPusherPipeline",
+]

src/mlpipeline/pipelines/data_ingestion_pipeline.py

@@ -0,0 +1,23 @@
+from mlpipeline.config.configuration import ConfigurationManager
+from mlpipeline.components.data_ingestion import DataIngestion
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class DataIngestionPipeline:
+    def __init__(self):
+        self.config_manager = ConfigurationManager()
+    
+    def run(self):
+        logger.info("Data Ingestion Pipeline started")
+        config = self.config_manager.get_data_ingestion_config()
+        data_ingestion = DataIngestion(config=config)
+        artifact = data_ingestion.download_data()
+        logger.info(f"Data Ingestion Pipeline completed: {artifact.message}")
+        return artifact
+
+
+if __name__ == "__main__":
+    pipeline = DataIngestionPipeline()
+    pipeline.run()

src/mlpipeline/pipelines/data_transformation_pipeline.py

@@ -0,0 +1,23 @@
+from mlpipeline.config.configuration import ConfigurationManager
+from mlpipeline.components.data_transformation import DataTransformation
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class DataTransformationPipeline:
+    def __init__(self):
+        self.config_manager = ConfigurationManager()
+    
+    def run(self):
+        logger.info("Data Transformation Pipeline started")
+        config = self.config_manager.get_data_transformation_config()
+        data_transformation = DataTransformation(config=config)
+        artifact = data_transformation.transform()
+        logger.info(f"Data Transformation Pipeline completed: {artifact.message}")
+        return artifact
+
+
+if __name__ == "__main__":
+    pipeline = DataTransformationPipeline()
+    pipeline.run()

src/mlpipeline/pipelines/data_validation_pipeline.py

@@ -0,0 +1,23 @@
+from mlpipeline.config.configuration import ConfigurationManager
+from mlpipeline.components.data_validation import DataValidation
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class DataValidationPipeline:
+    def __init__(self):
+        self.config_manager = ConfigurationManager()
+    
+    def run(self):
+        logger.info("Data Validation Pipeline started")
+        config = self.config_manager.get_data_validation_config()
+        data_validation = DataValidation(config=config)
+        artifact = data_validation.validate_schema()
+        logger.info(f"Data Validation Pipeline completed: {artifact.message}")
+        return artifact
+
+
+if __name__ == "__main__":
+    pipeline = DataValidationPipeline()
+    pipeline.run()

src/mlpipeline/pipelines/feature_engineering_pipeline.py

@@ -0,0 +1,23 @@
+from mlpipeline.config.configuration import ConfigurationManager
+from mlpipeline.components.feature_engineering import FeatureEngineering
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class FeatureEngineeringPipeline:
+    def __init__(self):
+        self.config_manager = ConfigurationManager()
+    
+    def run(self):
+        logger.info("Feature Engineering Pipeline started")
+        config = self.config_manager.get_feature_engineering_config()
+        feature_engineering = FeatureEngineering(config=config)
+        artifact = feature_engineering.engineer_features()
+        logger.info(f"Feature Engineering Pipeline completed: {artifact.message}")
+        return artifact
+
+
+if __name__ == "__main__":
+    pipeline = FeatureEngineeringPipeline()
+    pipeline.run()

src/mlpipeline/pipelines/model_evaluation_pipeline.py

@@ -0,0 +1,23 @@
+from mlpipeline.config.configuration import ConfigurationManager
+from mlpipeline.components.model_evaluation import ModelEvaluation
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ModelEvaluationPipeline:
+    def __init__(self):
+        self.config_manager = ConfigurationManager()
+    
+    def run(self):
+        logger.info("Model Evaluation Pipeline started")
+        config = self.config_manager.get_model_evaluation_config()
+        evaluation = ModelEvaluation(config=config)
+        artifact = evaluation.evaluate()
+        logger.info(f"Model Evaluation Pipeline completed with metrics: {artifact.evaluation_metrics}")
+        return artifact
+
+
+if __name__ == "__main__":
+    pipeline = ModelEvaluationPipeline()
+    pipeline.run()

src/mlpipeline/pipelines/model_pusher_pipeline.py

@@ -0,0 +1,23 @@
+from mlpipeline.config.configuration import ConfigurationManager
+from mlpipeline.components.model_pusher import ModelPusher
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ModelPusherPipeline:
+    def __init__(self):
+        self.config_manager = ConfigurationManager()
+    
+    def run(self):
+        logger.info("Model Pusher Pipeline started")
+        config = self.config_manager.get_model_pusher_config()
+        pusher = ModelPusher(config=config)
+        artifact = pusher.push_model()
+        logger.info(f"Model Pusher Pipeline completed: {artifact.message}")
+        return artifact
+
+
+if __name__ == "__main__":
+    pipeline = ModelPusherPipeline()
+    pipeline.run()

src/mlpipeline/pipelines/model_trainer_pipeline.py

@@ -0,0 +1,23 @@
+from mlpipeline.config.configuration import ConfigurationManager
+from mlpipeline.components.automl_trainer import AutoMLTrainer
+from mlpipeline.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ModelTrainerPipeline:
+    def __init__(self):
+        self.config_manager = ConfigurationManager()
+    
+    def run(self):
+        logger.info("Model Trainer Pipeline started")
+        config = self.config_manager.get_model_trainer_config()
+        trainer = AutoMLTrainer(config=config)
+        artifact = trainer.train()
+        logger.info(f"Model Trainer Pipeline completed with metrics: {artifact.train_metrics}")
+        return artifact
+
+
+if __name__ == "__main__":
+    pipeline = ModelTrainerPipeline()
+    pipeline.run()

src/mlpipeline/utils/common.py

@@ -76,6 +76,19 @@ def load_model(path: Path) -> Any:
     return model
 
 
+def save_object(path: Path, obj):
+    with open(path, "wb") as f:
+        pickle.dump(obj, f)
+    logger.info(f"Object saved: {path}")
+
+
+def load_object(path: Path):
+    with open(path, "rb") as f:
+        obj = pickle.load(f)
+    logger.info(f"Object loaded: {path}")
+    return obj
+
+
 @ensure_annotations
 def get_size(path: Path) -> str:
     size_in_kb = round(os.path.getsize(path) / 1024)