Data Validation done

data_schema/schema.yaml

@@ -0,0 +1,66 @@
+columns:
+  - having_IP_Address: int64
+  - URL_Length: int64
+  - Shortining_Service: int64
+  - having_At_Symbol: int64
+  - double_slash_redirecting: int64
+  - Prefix_Suffix: int64
+  - having_Sub_Domain: int64
+  - SSLfinal_State: int64
+  - Domain_registeration_length: int64
+  - Favicon: int64
+  - port: int64
+  - HTTPS_token: int64
+  - Request_URL: int64
+  - URL_of_Anchor: int64
+  - Links_in_tags: int64
+  - SFH: int64
+  - Submitting_to_email: int64
+  - Abnormal_URL: int64
+  - Redirect: int64
+  - on_mouseover: int64
+  - RightClick: int64
+  - popUpWidnow: int64
+  - Iframe: int64
+  - age_of_domain: int64
+  - DNSRecord: int64 
+  - web_traffic: int64
+  - Page_Rank: int64
+  - Google_Index: int64
+  - Links_pointing_to_page: int64
+  - Statistical_report: int64
+  - Result: int64
+
+
+numerical_columns:
+  - having_IP_Address
+  - URL_Length
+  - Shortining_Service
+  - having_At_Symbol
+  - double_slash_redirecting
+  - Prefix_Suffix
+  - having_Sub_Domain
+  - SSLfinal_State
+  - Domain_registeration_length
+  - Favicon
+  - port
+  - HTTPS_token
+  - Request_URL
+  - URL_of_Anchor
+  - Links_in_tags 
+  - SFH
+  - Submitting_to_email
+  - Abnormal_URL
+  - Redirect
+  - on_mouseover
+  - RightClick
+  - popUpWidnow
+  - Iframe
+  - age_of_domain
+  - DNSRecord 
+  - web_traffic
+  - Page_Rank
+  - Google_Index
+  - Links_pointing_to_page
+  - Statistical_report
+  - Result

main.py

@@ -1,7 +1,8 @@
 from networksecurity.components.data_ingestion import DataIngestion
+from networksecurity.components.data_validation import DataValidation
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
-from networksecurity.entity.config_entity import DataIngestionConfig
+from networksecurity.entity.config_entity import DataIngestionConfig,DataValidationConfig
 from networksecurity.entity.config_entity import TraningPipelineConfig
 import sys
 
@@ -13,6 +14,12 @@ if __name__ == "__main__":
         logging.info("Initiate the data ingestion")
         dataingestionartifact=dataingestion.initiate_data_ingestion()
         print(dataingestionartifact)
- 
+        data_validation_config=DataValidationConfig(traningpipelineconfig)
+        datavalidation=DataValidation(dataingestionartifact,data_validation_config)
+        logging.info("Initiate the data validation")
+        data_validation_artifact=datavalidation.initiate_data_validation()
+        logging.info(f"Data validation completed {data_validation_artifact}")
+        print(data_validation_artifact)
+
     except Exception as e:
         raise NetworkSecurityException(e, sys)

networksecurity/components/data_validation.py

@@ -0,0 +1,125 @@
+from networksecurity.entity.artifact_entity import DataIngestionArtifact, DataValidationArtifact
+from networksecurity.entity.config_entity import DataValidationConfig
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.constant.training_pipeline import SCHEMA_FILE_PATH
+from networksecurity.utils.main_utils.utils import read_yaml_file
+from networksecurity.utils.main_utils.utils import write_yaml_file
+from networksecurity.logging.logger import logging
+from scipy.stats import ks_2samp
+import pandas as pd
+import os,sys
+
+class DataValidation:
+    def __init__(self, data_ingestion_artifact: DataIngestionArtifact,
+                 data_validation_config: DataValidationConfig):
+        
+        try:
+            self.data_ingestion_artifact = data_ingestion_artifact
+            self.data_validation_config = data_validation_config
+            self._schema_config = read_yaml_file(SCHEMA_FILE_PATH)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    @staticmethod
+    def read_data(file_path)->pd.DataFrame:
+        try:
+            return pd.read_csv(file_path)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def validate_number_of_columns(self, dataframe: pd.DataFrame)->bool:
+        try:
+            number_of_columns = len(self._schema_config)
+            logging.info(f"Required number of columns: {number_of_columns}")
+            logging.info(f"DataFrame columns count: {len(dataframe.columns)}")
+
+            if len(dataframe.columns)==number_of_columns:
+                return True
+            return False
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def validate_number_of_numerical_columns(self, dataframe: pd.DataFrame) -> bool:
+        try:
+            # Required numerical columns from schema
+            required_numerical_columns = self._schema_config["numerical_columns"]
+            required_count = len(required_numerical_columns)
+
+            # Actual numerical columns in dataframe
+            numerical_df = dataframe.select_dtypes(include=["int64"])
+            actual_count = len(numerical_df.columns)
+
+            logging.info(f"Required number of numerical columns: {required_count}")
+            logging.info(f"Dataframe numerical columns count: {actual_count}")
+
+            if actual_count == required_count:
+                return True
+            return False
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def detect_dataset_drift(self, base_df, current_df, threshold=0.01)-> bool:
+        try:
+            status = True
+            report={}
+            for column in base_df.columns:
+                d1 = base_df[column]
+                d2 = current_df[column]
+                is_same_dist=ks_2samp(d1, d2)
+                if threshold <=is_same_dist.pvalue:
+                    is_found = False
+                else:
+                    is_found = True
+                    status = False
+                report.update({column:{
+                    "p_value":float(is_same_dist.pvalue),
+                    "drift_status":is_found
+                }})
+            drift_report_file_path = self.data_validation_config.drift_report_file_path
+            dir_path = os.path.dirname(drift_report_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+            write_yaml_file(file_path=drift_report_file_path, content=report)
+
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def initiate_data_validation(self)-> DataValidationArtifact:
+        try:
+            train_file_path = self.data_ingestion_artifact.training_file_path
+            test_file_path = self.data_ingestion_artifact.testing_file_path
+
+            ## Read data from train and test
+            train_dataframe = DataValidation.read_data(train_file_path)
+            test_dataframe = DataValidation.read_data(test_file_path)
+
+            ## Validate number of columns
+            status = self.validate_number_of_columns(dataframe=train_dataframe)
+            if not status:
+                error_message = f"Train dataframe does not contain all columns \n"
+                status = self.validate_number_of_columns(dataframe=test_dataframe)
+            if not status:
+                error_message = f"Test dataframe does not contain all columns \n"
+
+            ## Check datadrift
+            status=self.detect_dataset_drift(base_df=train_dataframe, current_df=test_dataframe)
+            dir_path = os.path.dirname(self.data_validation_config.valid_train_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+
+            train_dataframe.to_csv(self.data_validation_config.valid_train_file_path, index=False, header=True)
+            test_dataframe.to_csv(self.data_validation_config.valid_test_file_path, index=False, header=True)
+
+            data_validation_artifact = DataValidationArtifact(
+                validation_status=status,
+                valid_train_file_path=self.data_validation_config.valid_train_file_path,
+                valid_test_file_path=self.data_validation_config.valid_test_file_path,
+                invalid_train_file_path=None,
+                invalid_test_file_path=None,
+                drift_report_file_path=self.data_validation_config.drift_report_file_path
+            )
+            return data_validation_artifact
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)

networksecurity/constant/training_pipeline/__init__.py

@@ -14,6 +14,8 @@ FILE_NAME: str = "phisingkData.csv"
 TRAIN_FILE_NAME: str = "train.csv"
 TEST_FILE_NAME: str = "test.csv"
 
+SCHEMA_FILE_PATH = os.path.join("data_schema", "schema.yaml")
+
 """
 Data Ingestion realted constant start with DATA_INGESTION VAR NAME
 """
@@ -24,3 +26,13 @@ DATA_INGESTION_DIR_NAME: str = "data_ingestion"
 DATA_INGESTION_FEATURE_STORE_DIR: str = "feature_store"
 DATA_INGESTION_INGESTED_DIR: str = "ingested"
 DATA_INGESTION_TRAIN_TEST_SPLIT_RATION: float = 0.2
+
+
+"""
+Data Validation related constant start with DATA_VALIDATION VAR NAME
+"""
+DATA_VALIDATION_DIR_NAME: str ="data_validation"
+DATA_VALIDATION_VALID_DIR: str ="validated"
+DATA_VALIDATION_INVALID_DIR: str ="invalid"
+DATA_VALIDATION_DRIFT_REPORT_DIR: str ="drift_report"
+DATA_VALIDATION_DRIFT_REPORT_FILE_NAME: str ="report.yaml"

networksecurity/entity/artifact_entity.py

@@ -3,4 +3,13 @@ from dataclasses import dataclass
 @dataclass
 class DataIngestionArtifact:
     training_file_path:str
-    testing_file_path:str
+    testing_file_path:str
+
+@dataclass
+class DataValidationArtifact:
+    validation_status: bool
+    valid_train_file_path: str
+    valid_test_file_path: str
+    invalid_train_file_path: str
+    invalid_test_file_path: str
+    drift_report_file_path: str

networksecurity/entity/config_entity.py

@@ -30,4 +30,56 @@ class DataIngestionConfig:
         )
         self.train_test_split_ratio: float = training_pipeline.DATA_INGESTION_TRAIN_TEST_SPLIT_RATION
         self.collection_name: str = training_pipeline.DATA_INGESTION_COLLECTION_NAME
-        self.database_name: str = training_pipeline.DATA_INGESTION_DATABASE_NAME
+        self.database_name: str = training_pipeline.DATA_INGESTION_DATABASE_NAME
+
+
+class DataValidationConfig:
+    def __init__(self, training_pipeline_config: TraningPipelineConfig):
+        # Main data validation directory
+        self.data_validation_dir: str = os.path.join(
+            training_pipeline_config.artifact_dir,
+            training_pipeline.DATA_VALIDATION_DIR_NAME
+        )
+
+        # Valid data directory
+        self.valid_data_dir: str = os.path.join(
+            self.data_validation_dir,
+            training_pipeline.DATA_VALIDATION_VALID_DIR
+        )
+
+        # Invalid data directory
+        self.invalid_data_dir: str = os.path.join(
+            self.data_validation_dir,
+            training_pipeline.DATA_VALIDATION_INVALID_DIR
+        )
+
+        # Valid train file path
+        self.valid_train_file_path: str = os.path.join(
+            self.valid_data_dir,
+            training_pipeline.TRAIN_FILE_NAME
+        )
+
+        # Valid test file path
+        self.valid_test_file_path: str = os.path.join(
+            self.valid_data_dir,
+            training_pipeline.TEST_FILE_NAME
+        )
+
+        # Invalid train file path
+        self.invalid_train_file_path: str = os.path.join(
+            self.invalid_data_dir,
+            training_pipeline.TRAIN_FILE_NAME
+        )
+
+        # Invalid test file path
+        self.invalid_test_file_path: str = os.path.join(
+            self.invalid_data_dir,
+            training_pipeline.TEST_FILE_NAME
+        )
+
+        # Drift report file path
+        self.drift_report_file_path: str = os.path.join(
+            self.data_validation_dir,
+            training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR,
+            training_pipeline.DATA_VALIDATION_DRIFT_REPORT_FILE_NAME,
+        )

networksecurity/utils/main_utils/utils.py

@@ -0,0 +1,33 @@
+import yaml
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+import os,sys
+import numpy as np
+import dill 
+import pickle
+
+def read_yaml_file(file_path: str) -> dict:
+    """
+    Reads a YAML file and returns its contents as a dictionary.
+    
+    Args:
+        file_path (str): The path to the YAML file.
+    Returns:
+        dict: The contents of the YAML file.
+    """
+    try:
+        with open(file_path, 'rb') as yaml_file:
+            return yaml.safe_load(yaml_file)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+    
+def write_yaml_file(file_path: str, content: object, replace: bool = False) -> None:
+    try:
+        if replace:
+            if os.path.exists(file_path):
+                os.remove(file_path)
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, 'w') as file:
+            yaml.dump(content, file)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)

requirements.txt

@@ -6,5 +6,6 @@ pymongo
 certifi
 pymongo[srv]==3.11
 scikit-learn
-
+dill
+pyaml
 #-e .