Data Transformation done

main.py

@@ -1,8 +1,10 @@
 from networksecurity.components.data_ingestion import DataIngestion
 from networksecurity.components.data_validation import DataValidation
+from networksecurity.components.data_transformation import DataTransformation
+
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
-from networksecurity.entity.config_entity import DataIngestionConfig,DataValidationConfig
+from networksecurity.entity.config_entity import DataIngestionConfig,DataValidationConfig,DataTransformationConfig
 from networksecurity.entity.config_entity import TraningPipelineConfig
 import sys
 
@@ -10,16 +12,23 @@ if __name__ == "__main__":
     try:
         traningpipelineconfig=TraningPipelineConfig()
         dataingestionconfig=DataIngestionConfig(traningpipelineconfig)
-        dataingestion=DataIngestion(dataingestionconfig)
+        data_ingestion=DataIngestion(dataingestionconfig)
         logging.info("Initiate the data ingestion")
-        dataingestionartifact=dataingestion.initiate_data_ingestion()
+        dataingestionartifact=data_ingestion.initiate_data_ingestion()
+        logging.info(f"Data ingestion completed {dataingestionartifact}")
         print(dataingestionartifact)
         data_validation_config=DataValidationConfig(traningpipelineconfig)
-        datavalidation=DataValidation(dataingestionartifact,data_validation_config)
+        data_validation=DataValidation(dataingestionartifact,data_validation_config)
         logging.info("Initiate the data validation")
-        data_validation_artifact=datavalidation.initiate_data_validation()
+        data_validation_artifact=data_validation.initiate_data_validation()
         logging.info(f"Data validation completed {data_validation_artifact}")
         print(data_validation_artifact)
+        data_transformation_config=DataTransformationConfig(traningpipelineconfig)
+        data_transformation=DataTransformation(data_transformation_config,data_validation_artifact)
+        logging.info("Initiate the data transformation")
+        data_transformation_artifact=data_transformation.initiate_data_transformation()
+        logging.info(f"Data transformation completed {data_transformation_artifact}")
+        print(data_transformation_artifact)
 
     except Exception as e:
         raise NetworkSecurityException(e, sys)

networksecurity/components/data_transformation.py

@@ -0,0 +1,99 @@
+import os,sys
+import numpy as np
+import pandas as pd
+from sklearn.impute import KNNImputer
+from sklearn.pipeline import Pipeline
+from networksecurity.constant.training_pipeline import TARGET_COLUMN, DATA_TRANSFORMATION_IMPUTER_PARAMS
+from networksecurity.entity.artifact_entity import DataTransformationArtifact, DataValidationArtifact
+from networksecurity.entity.config_entity import DataTransformationConfig
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+from networksecurity.utils.main_utils.utils import save_numpy_array_data, save_object
+
+class DataTransformation:
+    def __init__(self,
+                 data_transformation_config: DataTransformationConfig,
+                 data_validation_artifact: DataValidationArtifact):
+        try:
+            logging.info(f"{'>>'*20} Data Transformation {'<<'*20}")
+            self.data_transformation_config = data_transformation_config
+            self.data_validation_artifact = data_validation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    @staticmethod
+    def read_data(file_path) -> pd.DataFrame:
+        try:
+            return pd.read_csv(file_path)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def get_data_transformer_object(cls)-> Pipeline:
+        """
+        It initializes the KNNImputer object with the parameters specfied in the training_pipeline.py file
+        and returns the pipeline object with the KNNImputer object as the first step.
+
+        Args:
+        cls: DataTransformation 
+
+        Returns:
+        Pipeline: Pipeline object with the KNNImputer object as the first step.
+        """
+        logging.info("Entered the get_data_transformer_object method of Data_Transformation class")
+        try:
+            imputer = KNNImputer(**DATA_TRANSFORMATION_IMPUTER_PARAMS)
+            logging.info("Created KNNImputer object with the parameters specified in the training_pipeline.py file")
+            preprocessor = Pipeline(steps=[("imputer", imputer)])
+            return preprocessor
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+    
+    def initiate_data_transformation(self)->DataTransformationArtifact:
+        logging.info("Entered initiate_data_transformation method of DataTransformation class")
+        try:
+            logging.info("Starting data transformation")
+            train_df = DataTransformation.read_data(self.data_validation_artifact.valid_train_file_path)
+            test_df = DataTransformation.read_data(self.data_validation_artifact.valid_test_file_path)
+
+            ## Training dataframe
+            input_feature_train_df = train_df.drop(columns=[TARGET_COLUMN], axis=1)
+            target_feature_train_df = train_df[TARGET_COLUMN]
+            target_feature_train_df = target_feature_train_df.replace(-1, 0)
+
+            ## Testing dataframe
+            input_feature_test_df = test_df.drop(columns=[TARGET_COLUMN], axis=1)
+            target_feature_test_df = test_df[TARGET_COLUMN]
+            target_feature_test_df = target_feature_test_df.replace(-1, 0)
+
+            preprocessor = self.get_data_transformer_object()
+
+            preprocessor_object=preprocessor.fit(input_feature_train_df)
+            transformed_input_train_feature = preprocessor_object.transform(input_feature_train_df)
+            transformed_input_test_feature = preprocessor_object.transform(input_feature_test_df)
+
+            train_arr = np.c_[transformed_input_train_feature, np.array(target_feature_train_df)]
+            test_arr = np.c_[transformed_input_test_feature, np.array(target_feature_test_df)]
+
+            ## Save numpy array data and preprocessor object
+            save_numpy_array_data(
+                file_path=self.data_transformation_config.transformed_train_file_path,
+                array=train_arr
+            )
+            save_numpy_array_data(
+                file_path=self.data_transformation_config.transformed_test_file_path,
+                array=test_arr
+            )
+            save_object(
+                file_path=self.data_transformation_config.transformed_object_file_path,
+                obj=preprocessor_object
+            )
+
+            data_transformation_artifact = DataTransformationArtifact(
+                transformed_object_file_path=self.data_transformation_config.transformed_object_file_path,
+                transformed_train_file_path=self.data_transformation_config.transformed_train_file_path,
+                transformed_test_file_path=self.data_transformation_config.transformed_test_file_path,
+            )
+            return data_transformation_artifact
+            logging.info("Data transformation completed")
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)

networksecurity/constant/training_pipeline/__init__.py

@@ -36,3 +36,16 @@ DATA_VALIDATION_VALID_DIR: str ="validated"
 DATA_VALIDATION_INVALID_DIR: str ="invalid"
 DATA_VALIDATION_DRIFT_REPORT_DIR: str ="drift_report"
 DATA_VALIDATION_DRIFT_REPORT_FILE_NAME: str ="report.yaml"
+
+"""
+Data Transformation related constant start with DATA_TRANSFORMATION VAR NAME
+"""
+DATA_TRANSFORMATION_DIR_NAME: str ="data_transformation"
+DATA_TRANSFORMATION_TRANSFORMED_DATA_DIR: str ="transformed"
+DATA_TRANSFORMATION_TRANSFORMED_OBJECT_DIR: str ="transformed_object"
+DATA_TRANSFORMATION_IMPUTER_PARAMS: dict = {
+    "missing_values": np.nan,
+    "n_neighbors": 3,
+    "weights": "uniform",
+}
+PREPROCESSING_OBJECT_FILE_NAME: str = "preprocessing_object.pkl"

networksecurity/entity/artifact_entity.py

@@ -12,4 +12,10 @@ class DataValidationArtifact:
     valid_test_file_path: str
     invalid_train_file_path: str
     invalid_test_file_path: str
-    drift_report_file_path: str
+    drift_report_file_path: str
+
+@dataclass
+class DataTransformationArtifact:
+    transformed_train_file_path: str
+    transformed_test_file_path: str
+    transformed_object_file_path: str

networksecurity/entity/config_entity.py

@@ -83,3 +83,28 @@ class DataValidationConfig:
             training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR,
             training_pipeline.DATA_VALIDATION_DRIFT_REPORT_FILE_NAME,
         )
+
+class DataTransformationConfig:
+    def __init__(self, training_pipeline_config: TraningPipelineConfig):
+        self.data_transformation_dir: str = os.path.join(
+            training_pipeline_config.artifact_dir,
+            training_pipeline.DATA_TRANSFORMATION_DIR_NAME
+        )
+
+        self.transformed_train_file_path: str = os.path.join(
+            self.data_transformation_dir,
+            training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_DATA_DIR,
+            training_pipeline.TRAIN_FILE_NAME.replace("csv", "npy")
+        )
+
+        self.transformed_test_file_path: str = os.path.join(
+            self.data_transformation_dir,
+            training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_DATA_DIR,
+            training_pipeline.TEST_FILE_NAME.replace("csv", "npy")
+        )
+
+        self.transformed_object_file_path: str = os.path.join(
+            self.data_transformation_dir,
+            training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_OBJECT_DIR,
+            training_pipeline.PREPROCESSING_OBJECT_FILE_NAME
+        )

networksecurity/utils/main_utils/utils.py

@@ -29,5 +29,29 @@ def write_yaml_file(file_path: str, content: object, replace: bool = False) -> N
         os.makedirs(os.path.dirname(file_path), exist_ok=True)
         with open(file_path, 'w') as file:
             yaml.dump(content, file)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+
+def save_numpy_array_data(file_path: str, array: np.array):
+    """
+    Save numpy array data to file
+    file_path : str : file path to save the numpy array
+    array : np.array : numpy array data to be saved
+    """
+    try:
+        dir_path = os.path.dirname(file_path)
+        os.makedirs(dir_path, exist_ok=True)
+        with open(file_path, 'wb') as file_obj:
+            np.save(file_obj, array)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+
+def save_object(file_path: str, obj: object) -> None:
+    try:
+        logging.info("Entered the save_object method of Main Utils")
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, 'wb') as file_obj:
+            pickle.dump(obj, file_obj)
+        logging.info("Exited the save_object method of Main Utils")
     except Exception as e:
         raise NetworkSecurityException(e, sys)

requirements.txt

@@ -6,6 +6,5 @@ pymongo
 certifi
 pymongo[srv]==3.11
 scikit-learn
-dill
 pyaml
 #-e .