Model Trainer And Evaluation With Hyperparameter Tuning

main.py

@@ -1,6 +1,7 @@
 from networksecurity.components.data_ingestion import DataIngestion
 from networksecurity.components.data_validation import DataValidation
 from networksecurity.components.data_transformation import DataTransformation
+from networksecurity.components.model_trainer import ModelTrainer,ModelTrainerConfig
 
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
@@ -30,5 +31,12 @@ if __name__ == "__main__":
         logging.info(f"Data transformation completed {data_transformation_artifact}")
         print(data_transformation_artifact)
 
+        logging.info("Model Trainer Started")
+        model_trainer_config=ModelTrainerConfig(traningpipelineconfig)
+        model_trainer=ModelTrainer(model_trainer_config=model_trainer_config,
+                                   data_transformation_artifact=data_transformation_artifact)
+        model_trainer_artifact=model_trainer.initiate_model_trainer()
+        logging.info(f"Model Trainer completed {model_trainer_artifact}")
+
     except Exception as e:
         raise NetworkSecurityException(e, sys)

networksecurity/components/model_trainer.py

@@ -0,0 +1,112 @@
+import os,sys
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+
+from networksecurity.entity.config_entity import ModelTrainerConfig
+from networksecurity.entity.artifact_entity import DataTransformationArtifact,ModelTrainerArtifact
+
+from networksecurity.utils.ml_utils.model.estimator import NetworkModel
+from networksecurity.utils.main_utils.utils import save_object,load_object,load_numpy_array_data,evaluate_models
+from networksecurity.utils.ml_utils.metric.classfication_metric import get_classification_score
+
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import r2_score
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier,AdaBoostClassifier
+
+class ModelTrainer:
+    def __init__(self,model_trainer_config:ModelTrainerConfig,
+                 data_transformation_artifact:DataTransformationArtifact):
+        try:
+            logging.info(f"{'>>'*20} Model Trainer {'<<'*20}")
+            self.model_trainer_config = model_trainer_config
+            self.data_transformation_artifact = data_transformation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e,sys)
+
+    def train_model(self,X_train,X_test,y_train,y_test):
+        model = {
+            "Logistic Regression": LogisticRegression(),
+            "Decision Tree": DecisionTreeClassifier(),
+            "Random Forest": RandomForestClassifier(),
+            "Gradient Boosting": GradientBoostingClassifier(),
+            "AdaBoost": AdaBoostClassifier()
+        }
+        params = {
+            "Decision Tree": {
+                'criterion':['gini','entropy','log_loss'],
+                #'splitter':['best','random'],
+                #'max_features':['sqrt','log2']
+            },
+            "Random Forest": {
+                #'criterion':['gini','entropy','log_loss'],
+                #'max_features':['sqrt','log2'],
+                'n_estimators':[8,16,32,64,128,256]
+            },
+            "Gradient Boosting": {
+                'learning_rate':[.1,.01,.05,.001],
+                'subsample':[0.6,0.7,0.75,0.8,0.85,0.9],
+                'n_estimators':[8,16,32,64,128,256]
+            },
+            "AdaBoost": {
+                'learning_rate':[.1,.01,.05,.001],
+                'n_estimators':[8,16,32,64,128,256]
+            },
+            "Logistic Regression": {},
+        }
+
+        model_report: dict = evaluate_models(X_train=X_train,y_train=y_train,
+                                             X_test=X_test,y_test=y_test,models=model,params=params)
+        
+        ## To get the best model score from dict
+        best_model_score = max(sorted(model_report.values()))
+
+        ## To get the best model name from dict
+        best_model_name = list(model_report.keys())[
+            list(model_report.values()).index(best_model_score)]
+        best_model = model[best_model_name]
+        logging.info(f"Best model found , Model Name : {best_model_name} , R2 Score : {best_model_score}")
+
+        y_train_pred = best_model.predict(X_train)
+        y_test_pred = best_model.predict(X_test)
+
+        classification_train_metric=get_classification_score(y_true=y_train, y_pred=y_train_pred)
+        classification_test_metric=get_classification_score(y_true=y_test, y_pred=y_test_pred)
+        
+
+        ## Track with mlflow
+
+
+        preprocessor = load_object(file_path=self.data_transformation_artifact.transformed_object_file_path)
+        model_dir_path = os.path.dirname(self.model_trainer_config.trained_model_file_path)
+        os.makedirs(model_dir_path, exist_ok=True)
+
+        Network_model = NetworkModel(preprocessor=preprocessor, model=best_model)
+        save_object(file_path=self.model_trainer_config.trained_model_file_path, obj=Network_model)
+        logging.info(f"Trained model saved at : {self.model_trainer_config.trained_model_file_path}")
+    
+        model_trainer_artifact=ModelTrainerArtifact(trained_model_file_path=self.model_trainer_config.trained_model_file_path,
+                            train_metric_artifact=classification_train_metric,
+                            test_metric_artifact=classification_test_metric)
+        logging.info(f"Model Trainer Artifact : {model_trainer_artifact}")
+        return model_trainer_artifact
+
+    def initiate_model_trainer(self)->ModelTrainerArtifact:
+        try:
+            logging.info("Loading transformed training array and transformed test array")
+            train_file_path = self.data_transformation_artifact.transformed_train_file_path
+            test_file_path = self.data_transformation_artifact.transformed_test_file_path
+
+            ## Load numpy array
+            train_array = load_numpy_array_data(file_path=train_file_path)
+            test_array = load_numpy_array_data(file_path=test_file_path)
+            logging.info("Splitting training and test input and target feature")
+            X_train,y_train = train_array[:,:-1],train_array[:,-1]
+            X_test,y_test = test_array[:,:-1],test_array[:,-1]
+
+            model_trainer_artifact = self.train_model(X_train=X_train, X_test=X_test,
+                                                      y_train=y_train, y_test=y_test)
+            return model_trainer_artifact
+
+        except Exception as e:
+            raise NetworkSecurityException(e,sys)

networksecurity/constant/training_pipeline/__init__.py

@@ -16,6 +16,9 @@ TEST_FILE_NAME: str = "test.csv"
 
 SCHEMA_FILE_PATH = os.path.join("data_schema", "schema.yaml")
 
+SAVED_MODEL_DIR_NAME = os.path.join("saved_models")
+MODEL_FILE_NAME: str = "model.pkl"
+
 """
 Data Ingestion realted constant start with DATA_INGESTION VAR NAME
 """
@@ -48,4 +51,16 @@ DATA_TRANSFORMATION_IMPUTER_PARAMS: dict = {
     "n_neighbors": 3,
     "weights": "uniform",
 }
-PREPROCESSING_OBJECT_FILE_NAME: str = "preprocessing_object.pkl"
+DATA_TRANSFORMATION_TRAIN_FILE_PATH: str = "train.npy"
+DATA_TRANSFORMATION_TEST_FILE_PATH: str = "test.npy"
+PREPROCESSING_OBJECT_FILE_NAME: str = "preprocessing_object.pkl"
+
+"""
+Model trainer related constanst with MODEL TRAINER VAR NAME
+"""
+
+MODEL_TRAINER_DIR_NAME: str = "model_trainer"
+MODEL_TRAINER_TRAINED_MODEL_DIR: str = "trained_model"
+MODEL_TRAINER_TRAINED_MODEL_NAME: str = "model.pkl"
+MODEL_TRAINER_EXPECTED_SCORE: float = 0.6
+MODEL_TRAINER_OVER_FITTING_UNDER_FITTING_THRESHOLD: float = 0.05

networksecurity/entity/artifact_entity.py

@@ -18,4 +18,21 @@ class DataValidationArtifact:
 class DataTransformationArtifact:
     transformed_train_file_path: str
     transformed_test_file_path: str
-    transformed_object_file_path: str
+    transformed_object_file_path: str
+
+@dataclass
+class ClassificationMetricArtifact:
+    f1_score: float
+    precision_score: float
+    recall_score: float
+
+@dataclass
+class ModelTrainerArtifact:
+    trained_model_file_path: str
+    train_metric_artifact: ClassificationMetricArtifact
+    test_metric_artifact: ClassificationMetricArtifact
+
+@dataclass
+class ModelTrainerConfig:
+    model_trainer_dir: str
+    trained_model_file_path: str

networksecurity/entity/config_entity.py

@@ -107,4 +107,16 @@ class DataTransformationConfig:
             self.data_transformation_dir,
             training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_OBJECT_DIR,
             training_pipeline.PREPROCESSING_OBJECT_FILE_NAME
-        )
+        )
+
+class ModelTrainerConfig:
+    def __init__(self, training_pipeline_config: TraningPipelineConfig):
+        self.model_trainer_dir: str = os.path.join(
+            training_pipeline_config.artifact_dir, training_pipeline.MODEL_TRAINER_DIR_NAME
+        )
+        self.trained_model_file_path: str = os.path.join(
+            self.model_trainer_dir, training_pipeline.MODEL_TRAINER_TRAINED_MODEL_DIR,
+            training_pipeline.MODEL_FILE_NAME
+        )
+        self.expected_accuracy: float = training_pipeline.MODEL_TRAINER_EXPECTED_SCORE
+        self.overfitting_underfitting_threshold = training_pipeline.MODEL_TRAINER_OVER_FITTING_UNDER_FITTING_THRESHOLD

networksecurity/utils/main_utils/utils.py

@@ -1,3 +1,5 @@
+from sklearn.metrics import r2_score
+from sklearn.model_selection import GridSearchCV
 import yaml
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
@@ -53,5 +55,63 @@ def save_object(file_path: str, obj: object) -> None:
         with open(file_path, 'wb') as file_obj:
             pickle.dump(obj, file_obj)
         logging.info("Exited the save_object method of Main Utils")
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+    
+def load_object(file_path: str) -> object:
+    try:
+        if not os.path.exists(file_path):
+            raise Exception(f"The file: {file_path} does not exist")
+        with open(file_path, 'rb') as file_obj:
+            print(file_obj)
+            return pickle.load(file_obj)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+    
+def load_numpy_array_data(file_path: str) -> np.array:
+    """
+    Load numpy array data from file
+    file_path : str : file path to load the numpy array
+    return : np.array : numpy array data loaded
+    """
+    try:
+        with open(file_path, 'rb') as file_obj:
+            return np.load(file_obj)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+    
+def evaluate_models(X_train, y_train, X_test, y_test, models: dict, params: dict) -> dict:
+    """
+    Evaluate multiple machine learning models and return their performance scores.
+
+    Args:
+        X_train: Training feature data.
+        y_train: Training target data.
+        X_test: Testing feature data.
+        y_test: Testing target data.
+        models (dict): A dictionary of model names and their corresponding model instances.
+        params (dict): A dictionary of model names and their corresponding hyperparameter grids.
+    Returns:
+        dict: A dictionary containing model names and their corresponding performance scores.
+    """
+    try:
+        report = {}
+        for i in range(len(list(models))):
+            model = list(models.values())[i]
+            param = params[list(models.keys())[i]]
+
+            gs = GridSearchCV(model, param, cv=3)
+            gs.fit(X_train, y_train)
+
+            model.set_params(**gs.best_params_)
+            model.fit(X_train, y_train)
+            
+            y_train_pred = model.predict(X_train)
+            y_test_pred = model.predict(X_test)
+            train_model_score = r2_score(y_train, y_train_pred)
+            test_model_score = r2_score(y_test, y_test_pred)
+            report[list(models.keys())[i]] = test_model_score
+        return report
+
     except Exception as e:
         raise NetworkSecurityException(e, sys)

networksecurity/utils/ml_utils/metric/classfication_metric.py

@@ -0,0 +1,19 @@
+from networksecurity.entity.artifact_entity import ClassificationMetricArtifact
+from networksecurity.exception.exception import NetworkSecurityException
+from sklearn.metrics import precision_score, recall_score, f1_score
+import sys
+
+def get_classification_score(y_true, y_pred) -> ClassificationMetricArtifact:
+    try:
+        model_f1_score = f1_score(y_true, y_pred)
+        model_precision_score = precision_score(y_true, y_pred)
+        model_recall_score = recall_score(y_true, y_pred)
+
+        classification_metric= ClassificationMetricArtifact(
+            f1_score=model_f1_score,
+            precision_score=model_precision_score,
+            recall_score=model_recall_score
+        )
+        return classification_metric
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)

networksecurity/utils/ml_utils/model/estimator.py

@@ -0,0 +1,35 @@
+from networksecurity.exception.exception import NetworkSecurityException
+import sys,os
+from networksecurity.constant.training_pipeline import SAVED_MODEL_DIR_NAME,MODEL_FILE_NAME
+from networksecurity.logging.logger import logging
+
+class NetworkModel:
+    def __init__(self,preprocessor,model):
+        """
+        Initialize the NetworkModel with preprocessor and model.
+
+        Args:
+            preprocessor: The preprocessing object.
+            model: The trained model.
+        """
+        try:
+            self.preprocessor = preprocessor
+            self.model = model
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def predict(self, X):
+        """
+        Make predictions using the preprocessor and model.
+
+        Args:
+            X: The input data for prediction.
+        Returns:
+            The predictions made by the model.
+        """
+        try:
+            X_transform = self.preprocessor.transform(X)
+            y_hat = self.model.predict(X_transform)
+            return y_hat
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)