Clean deployment without binary files

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.github/workflows/main.yml

@@ -0,0 +1,92 @@
+name: workflow
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - 'README.md'
+
+jobs:
+  integration:
+    name: Continuous Integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code 
+        uses: actions/checkout@v3
+
+      - name: Lint code
+        run: echo "Linting repository"
+
+      - name: Run unit tests 
+        run: echo "Running unit tests"
+
+  build-and-push-ecr-image:
+    name: Continuous Delivery
+    needs: integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Install Utilities
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y jq unzip
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to ECR
+        id: build-image
+        env: 
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}   
+          ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY_NAME }}  
+          IMAGE_TAG: latest
+        run: |
+          echo "ECR_REGISTRY: $ECR_REGISTRY"
+          echo "ECR_REPOSITORY: $ECR_REPOSITORY"
+          echo "IMAGE_TAG: $IMAGE_TAG"
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
+  Continuous-Deployment:
+    needs: build-and-push-ecr-image
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+      
+      
+      - name: Pull latest images
+        run: |
+         docker pull ${{secrets.AWS_ECR_LOGIN_URI}}/${{ secrets.ECR_REPOSITORY_NAME }}:latest
+         
+      #- name: Stop and remove  container if running
+       # run: |
+        # docker ps -q --filter "name=networksecurity" | grep -q . && docker stop networksecurity && docker rm -fv networksecurity
+       
+      - name: Run Docker Image to serve users
+        run: |
+         docker run -d -p 8080:8080 --ipc="host" --name=networksecurity -e 'AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}' -e 'AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}' -e 'AWS_REGION=${{ secrets.AWS_REGION }}'  ${{secrets.AWS_ECR_LOGIN_URI}}/${{ secrets.ECR_REPOSITORY_NAME }}:latest
+      - name: Clean previous images and containers
+        run: |
+         docker system prune -f   

.gitignore

@@ -0,0 +1,14 @@
+.env
+.venv/
+pycache/
+__pycache__/
+.ipynb_checkpoints
+.vscode/
+.DS_Store
+*.pyc
+logs/
+*.log
+Artifacts/
+s3_sync_folder/
+final_model/
+data/

.python-version

@@ -0,0 +1 @@
+3.13

Dockerfile

@@ -0,0 +1,31 @@
+# for aws 
+# FROM python:3.10-slim-buster
+# WORKDIR /app
+# COPY . /app
+# RUN apt update -y && apt install awscli -y 
+# RUN apt-get update && pip install -r requirements.txt
+# CMD ["python3", "app.py"]
+ 
+FROM python:3.13-slim
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+# Copy requirements
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy all application files
+COPY --chown=user . /app
+
+# Create necessary directories
+RUN mkdir -p /app/data /app/final_model /app/templates
+
+# Expose port 7860 (HF Space requirement)
+EXPOSE 7860
+
+# Run the application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,10 @@
+---
+title: NSS
+emoji: 🏃
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,75 @@
+import os,sys
+import certifi
+from dotenv import load_dotenv
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+from src.pipeline.training_pipeline import Trainingpipeline
+from fastapi import FastAPI, File, UploadFile, Request
+from fastapi.middleware.cors import CORSMiddleware
+from uvicorn import run as app_run
+from fastapi.responses import Response
+from starlette.responses import RedirectResponse
+import pandas as pd
+from src.utils.ml_utils.model.estimator import NetworkSecurityModel
+ca = certifi.where()
+load_dotenv()
+mongo_db_uri = os.getenv("MONGO_DB_URI")
+
+from src.constant.training_pipeline import DATA_INGESTION_COLLECTION_NAME
+from src.constant.training_pipeline import DATA_INGESTION_DATBASE_NANE
+from src.utils.main_utils.utils import load_object
+# import pymongo
+
+# client = pymongo.MongoClient(mongo_db_uri,tlsCAFile=ca)
+# database = client[DATA_INGESTION_DATBASE_NANE]
+# collection = database[DATA_INGESTION_COLLECTION_NAME]
+from fastapi.templating import Jinja2Templates
+templates = Jinja2Templates(directory="./templates")  
+app = FastAPI()
+orgin = ["*"]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=orgin,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# @app.get("/", tags = ["authentication"])
+# async def index():
+#     return RedirectResponse(url="/docs")
+
+@app.get("/train")
+async def training_route():
+    try: 
+        training_pipeline = Trainingpipeline()
+        training_pipeline.run_pipeline()
+        return Response("Training successfull !!")
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+
+@app.post("/predict") # predict route
+async def predict_route(request: Request, file: UploadFile =File(...)):
+    try: 
+        df = pd.read_csv(file.file)
+        # Remove target column if it exists
+        if 'Result' in df.columns:
+            df = df.drop(columns=['Result'])
+        preprocessor = load_object(file_path = "final_model/preprocessor.pkl") 
+        model = load_object(file_path= "final_model/model.pkl")
+        NSmodel = NetworkSecurityModel(preprocessing_object= preprocessor, trained_model_object= model)
+        print(df.iloc[0])
+        y_pred = NSmodel.predict(df)
+        print(y_pred)
+        df['predicted_column'] = y_pred
+        print(df['predicted_column'])
+        df.to_csv("final_model/predicted.csv")
+        table_html = df.to_html(classes = 'table table-striped')
+        return templates.TemplateResponse("table.html", {"request": request, "table": table_html})
+    
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+if __name__ == "__main__":
+    app_run(app, host="0.0.0.0", port=8080)
+    

data_schema/schema.yaml

@@ -0,0 +1,65 @@
+columns:
+  - having_IP_Address: int64
+  - URL_Length: int64
+  - Shortining_Service: int64
+  - having_At_Symbol: int64
+  - double_slash_redirecting: int64
+  - Prefix_Suffix: int64
+  - having_Sub_Domain: int64
+  - SSLfinal_State: int64
+  - Domain_registeration_length: int64
+  - Favicon: int64
+  - port: int64
+  - HTTPS_token: int64
+  - Request_URL: int64
+  - URL_of_Anchor: int64
+  - Links_in_tags: int64
+  - SFH: int64
+  - Submitting_to_email: int64
+  - Abnormal_URL: int64
+  - Redirect: int64
+  - on_mouseover: int64
+  - RightClick: int64
+  - popUpWidnow: int64
+  - Iframe: int64
+  - age_of_domain: int64
+  - DNSRecord: int64
+  - web_traffic: int64
+  - Page_Rank: int64
+  - Google_Index: int64
+  - Links_pointing_to_page: int64
+  - Statistical_report: int64
+  - Result: int64
+
+numerical_columns:
+  - having_IP_Address
+  - URL_Length
+  - Shortining_Service
+  - having_At_Symbol
+  - double_slash_redirecting
+  - Prefix_Suffix
+  - having_Sub_Domain
+  - SSLfinal_State
+  - Domain_registeration_length
+  - Favicon
+  - port
+  - HTTPS_token
+  - Request_URL
+  - URL_of_Anchor
+  - Links_in_tags
+  - SFH
+  - Submitting_to_email
+  - Abnormal_URL
+  - Redirect
+  - on_mouseover
+  - RightClick
+  - popUpWidnow
+  - Iframe
+  - age_of_domain
+  - DNSRecord
+  - web_traffic
+  - Page_Rank
+  - Google_Index
+  - Links_pointing_to_page
+  - Statistical_report
+  - Result

data_to_mongo.py

@@ -0,0 +1,73 @@
+# Test code
+# import os
+# from pymongo.mongo_client import MongoClient
+# from dotenv import load_dotenv
+# load_dotenv()
+# uri = os.getenv("MONGODB_URL")
+
+# # Create a new client and connect to the server
+# client = MongoClient(uri)
+
+# # Send a ping to confirm a successful connection
+# try:
+#     client.admin.command('ping')
+#     print("Pinged your deployment. You successfully connected to MongoDB!")
+# except Exception as e:
+#     print(e)
+
+
+import os
+import sys
+import json
+import certifi
+import pandas as pd 
+import numpy as np 
+# import pymongo 
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+from dotenv import load_dotenv
+load_dotenv()
+
+MONGODB_URL = os.getenv("MONGODB_URL")
+print(MONGODB_URL)
+ca = certifi.where() # ca=  certified authority 
+
+class NetworkDataExtract():
+    def __init__(self):
+        try: 
+            pass
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+    def csv_to_json_converter(self, file_path):
+        try:
+            data = pd.read_csv(file_path)
+            data.reset_index(drop=True, inplace=True)
+            records = list(json.loads(data.T.to_json()).values())
+            return records 
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def insert_data_to_mongodb(self, records, database, collection):
+        try: 
+            self.database = database 
+            self.collection = collection 
+            self.records = records 
+            
+            self.mongo_client = pymongo.MongoClient(MONGODB_URL)
+            self.database = self.mongo_client[self.database]
+            
+            self.collection = self.database[self.collection ]
+            self.collection.insert_many(self.records)
+            return(len(self.records))
+        except Exception as e: 
+            raise NetworkSecurityException(e, sys)
+        
+if __name__  == "__main__":
+    FILE_PATH = "data\phisingData.csv"
+    DATABASE = "Network_data"
+    Collection = "phising_data"
+    obj = NetworkDataExtract()
+    records = obj.csv_to_json_converter(file_path=FILE_PATH)
+    print("records converted to json")
+    noOfRecords = obj.insert_data_to_mongodb(records,DATABASE, Collection)
+    print(noOfRecords)

load_data_to_sqlite.py

@@ -0,0 +1,15 @@
+from src.data.sqlite_manager import PhishingDataManager
+
+if __name__ == "__main__":
+    FILE_PATH = "data/phisingData.csv"
+    
+    print("Initializing SQLite database...")
+    db_manager = PhishingDataManager()
+    
+    print("Loading data from CSV...")
+    count = db_manager.insert_data_from_csv(FILE_PATH)
+    
+    print(f"✅ Successfully loaded {count} records into SQLite database!")
+    print(f"Database location: data/phishing_data.db")
+    
+    db_manager.close()

main.py

@@ -0,0 +1,41 @@
+from src.components.data_ingestion import DataIngestion
+from src.components.data_validation import DataValidation
+from src.components.data_transformation import DataTransformation
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+from src.entity.config_entity import Data_ingestion_config, TrainingPipelineConfig, Data_validation_config, Data_transformation_config, Model_trainer_config
+from src.components.model_trainer import ModelTrainer
+
+import sys
+
+if __name__ == "__main__":
+    try:
+        traingning_pipeline_config = TrainingPipelineConfig()       
+        data_ingestion_config = Data_ingestion_config(traingning_pipeline_config)
+        Data_ingestion  = DataIngestion(data_ingestion_config)
+        logging.info("Data ingestion started")
+        data_ingestion_artifacts = Data_ingestion.initiate_data_ingestion()
+        logging.info("Data ingestion completed")
+        print("Data ingestion completed")
+        
+        data_validation_config = Data_validation_config(traingning_pipeline_config)
+        Data_validation = DataValidation(data_ingestion_artifacts, data_validation_config)
+        logging.info("Data validation started")
+        data_validation_artifacts = Data_validation.intiate_data_validation()
+        logging.info("Data validation completed")
+        print(data_validation_artifacts)
+        
+        data_transformation_config = Data_transformation_config(traingning_pipeline_config)
+        logging.info("data Transformation started")
+        data_transformation = DataTransformation(data_validation_artifacts, data_transformation_config)
+        data_transformation_artifact = data_transformation.initiate_data_transformation()
+        print(data_transformation_artifact)
+        logging.info("data Transformation completed")
+        
+        logging.info("Model training started")
+        model_trainer_config = Model_trainer_config(traingning_pipeline_config)
+        model_trainer = ModelTrainer(model_trainer_config=model_trainer_config, data_transformation_artifact=data_transformation_artifact)
+        model_trainer_artifact = model_trainer.initiate_model_trainer()
+        logging.info("Model training completed")
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)

pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "NetworkSecuritySystemMLProject"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "certifi>=2026.1.4",
+    "dagshub>=0.6.4",
+    "dill>=0.4.0",
+    "fastapi>=0.128.0",
+    "mlflow>=3.8.1",
+    "numpy>=2.4.1",
+    "pandas>=2.3.3",
+    "pyaml>=25.7.0",
+    "pymongo[srv]==3.6",
+    "python-dotenv>=1.2.1",
+    "python-multipart>=0.0.21",
+    "uvicorn>=0.40.0",
+]

requirements.txt

@@ -0,0 +1,14 @@
+python-dotenv
+pandas
+numpy 
+pymongo[srv]==3.6
+pymongo
+certifi
+dill
+mlflow 
+pyaml
+dagshub
+fastapi 
+uvicorn 
+python-multipart
+# -e .

setup.py

@@ -0,0 +1,26 @@
+from setuptools import find_packages, setup
+from typing import List
+
+def get_requirements()-> List[str]:
+    """This function is going to return list of requirements
+    """
+    requirement_list:List[str]=[]
+    try:
+        with open('requirements.txt', 'r') as file:
+            lines= file.readlines()
+            for line in lines:
+                requirement=line.strip()
+                if requirement and requirement!= '-e .': # ignore empty line and -e .
+                    requirement_list.append(requirement)
+                    
+    except FileNotFoundError:
+        print("Error: requirements.txt file not found.")
+        
+setup(
+    name="Network_Security_system",
+    version="0.0.1",
+    author="Kshitij",
+    author_email="kshitijk146@gmail.com",
+    packages=find_packages(),
+    install_requires=get_requirements()
+)

src/cloud/s3_syncer.py

@@ -0,0 +1,45 @@
+import os 
+import shutil
+class s3sync:
+    def __init__(self, Local_sync_folder = "s3_sync_folder"):
+        self.Local_sync_folder = Local_sync_folder
+        if not os.path.exists(self.Local_sync_folder):
+            os.makedirs(self.Local_sync_folder, exist_ok=True)
+        
+    def sync_folder_to_s3(self, folder, aws_bucket_url):
+        # command = (
+        #     f"aws s3 sync {folder} {aws_bucket_url}"
+        # )
+        # os.system(command)
+        try:
+            dest_path= aws_bucket_url.replace("s3://","").replace(aws_bucket_url.split("/")[0] + "/", "")
+            destination = os.path.join(self.Local_sync_folder, dest_path)
+
+            # copy folder 
+            if os.path.exists(destination):
+                shutil.rmtree(destination)
+            shutil.copytree(folder, destination)
+            print(f"Synced {folder} to {destination}")
+        except Exception as e:
+            print(f"Error syncing folder to S3: {e}")
+
+
+
+    def sync_folder_from_s3(self, folder, aws_bucket_url):
+        # command = (
+        #     f"aws s3 sync {aws_bucket_url} {folder}"
+        # )
+        # os.system(command)
+
+        try: 
+            source_path = aws_bucket_url.replace("s3://","").replace(aws_bucket_url.split("/")[0] + "/", "")
+            source = os.path.join(self.Local_sync_folder, source_path)
+
+            if os.path.exists(folder):
+                shutil.rmtree(folder)
+            shutil.copytree(source, folder)
+            print(f"Synced {source} to {folder}")
+        except Exception as e:
+            print(f"Error syncing folder from S3: {e}")
+
+        

src/components/data_ingestion.py

@@ -0,0 +1,80 @@
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+from src.entity.config_entity import Data_ingestion_config
+from src.entity.artifact_entity import DataIngestionArtifact
+import os, sys
+import pandas as pd
+from typing import List
+from sklearn.model_selection import train_test_split
+from dotenv import load_dotenv
+# import pymongo
+import numpy as np
+from src.data.sqlite_manager import PhishingDataManager
+
+load_dotenv()
+MONGODB_URL = os.getenv("MONGODB_URL")
+
+class DataIngestion:
+    def __init__(self, data_ingestion_config: Data_ingestion_config):
+        try:
+            self.data_ingestion_config = data_ingestion_config
+            self.db_manager = PhishingDataManager()
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+    
+    def export_collection_as_dataframe(self):
+        """Export data from SQLite instead of MongoDB"""
+        try:
+            # Get all training data
+            df = self.db_manager.get_training_data(include_new_only=False)
+            return df
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+    
+    def move_data_into_feature_store(self, dataframe: pd.DataFrame):
+        try:
+            feature_store_file = self.data_ingestion_config.feature_store_file_path
+            dir_path = os.path.dirname(feature_store_file)  
+            os.makedirs(dir_path, exist_ok=True)
+            dataframe.to_csv(feature_store_file, index=False, header=True)
+            return dataframe
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+    
+    def data_train_test_split(self,dataframe:pd.DataFrame):
+        try:
+            train_set, test_set = train_test_split(
+                dataframe, test_size=self.data_ingestion_config.train_test_split_ratio
+            )  
+            logging.info("Trained test spltting done on dataframe")
+            dir_path = os.path.dirname(self.data_ingestion_config.train_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+            logging.info("Exporting train and test file path")
+            train_set.to_csv(
+                self.data_ingestion_config.train_file_path, index = False, header = True
+            )
+            test_set.to_csv(
+                self.data_ingestion_config.test_file_path, index = False, header = True
+            )
+            logging.info("Exported train and test file path.")
+            
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def initiate_data_ingestion(self):
+        try:
+            dataframe = self.export_collection_as_dataframe()
+            dataframe = self.move_data_into_feature_store(dataframe)
+            self.data_train_test_split(dataframe)
+            
+            # Mark data as used
+            self.db_manager.mark_data_as_trained()
+            
+            data_ingestion_artifact = DataIngestionArtifact(
+                train_file_path=self.data_ingestion_config.train_file_path,
+                test_file_path=self.data_ingestion_config.test_file_path
+            )
+            return data_ingestion_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+

src/components/data_transformation.py

@@ -0,0 +1,93 @@
+import sys
+import os
+import numpy as np 
+import pandas as pd
+from sklearn.impute import KNNImputer
+from sklearn.pipeline import Pipeline
+from src.constant.training_pipeline import TARGET_COLUMN
+from src.constant.training_pipeline import DATA_TRANSFORMATION_IMPUTER_PARAMS
+from src.entity.artifact_entity import (
+    DataTransformationArtifact,
+    DataValidationArtifact,
+)
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+from src.utils.main_utils.utils import save_np_array, save_object
+from src.entity.config_entity import Data_transformation_config
+class DataTransformation:
+    def __init__(self, data_validation_artifact: DataValidationArtifact, data_transformation_config: Data_transformation_config):
+        try:
+            self.data_validation_artifact:DataValidationArtifact = data_validation_artifact
+            self.data_transformation_config:Data_transformation_config = data_transformation_config
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+        
+    @staticmethod 
+    def read_data(file_path) -> pd.DataFrame:
+        try: 
+            return pd.read_csv(file_path)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+
+    def get_data_transformer_object(self) -> Pipeline:
+        """
+        it initialises a KNNImputer object with the parameter specified in the training_pipeline.py file and returns
+        a pipeline with the KNNImputer object as the first step.
+
+        args:
+            cls: DataTransformation
+        Returns:
+            a pipeline object 
+        """
+        logging.info("Entered get_data_transformation_object methof of transformation class")
+        
+        try:
+            knn_imputer = KNNImputer(**DATA_TRANSFORMATION_IMPUTER_PARAMS)
+            logging.info(f"intialise knn imputer with {DATA_TRANSFORMATION_IMPUTER_PARAMS}")
+            pipeline = Pipeline(steps=[("imputer", knn_imputer)])
+            return pipeline
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+        
+    def initiate_data_transformation(self)-> DataTransformationArtifact:
+        try: 
+            logging.info("Started data transformation!")
+            train_df = DataTransformation.read_data(self.data_validation_artifact.valid_train_file_path)
+            test_df = DataTransformation.read_data(self.data_validation_artifact.valid_test_file_path)
+            
+            # training dataframe
+            input_feature_train_df = train_df.drop(columns=[TARGET_COLUMN],axis = 1)
+            target_feature_train_df = train_df[TARGET_COLUMN]
+            target_feature_train_df = target_feature_train_df.replace(-1,0)
+            # testing dataframe 
+            input_feature_test_df = test_df.drop(columns=[TARGET_COLUMN],axis = 1)
+            target_feature_test_df = test_df[TARGET_COLUMN]
+            target_feature_test_df = target_feature_test_df.replace(-1,0)
+            
+            preprocessor = self.get_data_transformer_object()
+            preprocessor_obj = preprocessor.fit(input_feature_train_df)
+            logging.info("Preprocessor object created and fitted on training data")
+            
+            transformed_input_train_feature = preprocessor_obj.transform(input_feature_train_df)
+            transformed_input_test_feature = preprocessor_obj.transform(input_feature_test_df)
+            
+            # combining transformed input features with target feature
+            train_arr = np.c_[transformed_input_train_feature, np.array(target_feature_train_df)]
+            test_arr = np.c_[transformed_input_test_feature, np.array(target_feature_test_df)]
+            
+            # save numpy array data
+            save_np_array(self.data_transformation_config.transformed_train_file_path, array=train_arr)
+            save_np_array(self.data_transformation_config.transformed_test_file_path,array = test_arr )
+            save_object(self.data_transformation_config.transformed_object_file_path,preprocessor_obj)
+            save_object("final_model/preprocessor.pkl", preprocessor_obj)
+            
+            # preparing artifacts 
+            Data_transformation_artifact = DataTransformationArtifact(
+            transformed_object_file_path=self.data_transformation_config.transformed_object_file_path,
+            transformed_train_file_path=self.data_transformation_config.transformed_train_file_path,
+            transformed_test_file_path=self.data_transformation_config.transformed_test_file_path,
+            )
+            return Data_transformation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+        

src/components/data_validation.py

@@ -0,0 +1,104 @@
+from src.entity.artifact_entity import DataIngestionArtifact,DataValidationArtifact
+from src.entity.config_entity import Data_validation_config
+from src.exception.exception import NetworkSecurityException
+from src.constant.training_pipeline import SCHEMA_FILE_PATH
+from src.logging.logger import logging
+from scipy.stats import ks_2samp
+import pandas as pd
+import os, sys
+from src.utils.main_utils.utils import read_yaml_file, write_yaml_file
+
+class DataValidation:
+    def __init__(self, data_ingestion_artifact: DataIngestionArtifact, data_validation_config: Data_validation_config):
+        try:
+            self.data_ingestion_artifact = data_ingestion_artifact
+            self.data_validation_config = data_validation_config
+            self._schema_config = read_yaml_file(file_path=SCHEMA_FILE_PATH)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+
+    @staticmethod 
+    def read_data(file_path) -> pd.DataFrame:
+        try:
+            return pd.read_csv(file_path)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+       
+    def validate_number_of_columns(self, dataframe: pd.DataFrame)->  bool:
+        try: 
+            number_of_columns = len(self._schema_config)
+            logging.info(f"Required number of columns: {number_of_columns}")
+            logging.info(f"Data frame has columns: {len(dataframe.columns)}")
+            if len(dataframe.columns)==number_of_columns:
+                return True 
+            else:
+                return False
+            
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    def detect_drift(self,base_df, current_df, threshold=0.05)->bool:
+        try:
+            status = True
+            report = {}
+            for column in base_df:
+                d1 = base_df[column]
+                d2 = current_df[column]
+                is_sample_dist = ks_2samp(d1, d2)
+                if threshold <= is_sample_dist.pvalue:
+                    is_found = False
+                else:
+                    is_found = True
+                    status = False
+                report.update({column: {
+                    "p_value": float(is_sample_dist.pvalue),
+                    "drift_status": is_found
+                }})
+            drift_report_file_path = self.data_validation_config.drift_report_file_path
+            # Create directory
+            dir_path = os.path.dirname(drift_report_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+            write_yaml_file(file_path=drift_report_file_path, content=report)
+            return status
+        except Exception as e:
+            raise NetworkSecurityException(e, sys )  
+        
+    def intiate_data_validation(self)-> DataValidationArtifact:
+        try:
+            train_file_path = self.data_ingestion_artifact.train_file_path
+            test_file_path = self.data_ingestion_artifact.test_file_path
+            
+            # read the data from train and test csv
+            train_df = DataValidation.read_data(train_file_path)
+            test_df = DataValidation.read_data(test_file_path)
+            
+            # validating no. of columns 
+            status = self.validate_number_of_columns(dataframe=train_df)
+            if not status:
+                error_message = f"{train_file_path} does not match schema"  
+                
+                
+            status = self.validate_number_of_columns(dataframe=test_df)
+            if not status:
+                error_message = f"{test_file_path} does not match schema"
+        
+            # check data drift
+            status = self.detect_drift(base_df=train_df, current_df=test_df)   
+            dir_path = os.path.dirname(self.data_validation_config.valid_train_file_path) 
+            os.makedirs(dir_path, exist_ok=True)
+            
+            train_df.to_csv(self.data_validation_config.valid_train_file_path, index=False, header = True)
+            test_df.to_csv(self.data_validation_config.valid_test_file_path, index=False, header = True)
+            
+            
+            data_validation_artifacts = DataValidationArtifact(
+                validation_status=status,
+                valid_train_file_path=self.data_ingestion_artifact.train_file_path,
+                valid_test_file_path=self.data_ingestion_artifact.test_file_path,
+                invalid_train_file_path=None,
+                invalid_test_file_path=None,
+                drift_report_file_path=self.data_validation_config.drift_report_file_path
+            )
+            return data_validation_artifacts
+            
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)   

src/components/model_trainer.py

@@ -0,0 +1,140 @@
+import os, sys
+from src.utils.ml_utils.model.estimator import NetworkSecurityModel
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+from src.entity.artifact_entity import DataTransformationArtifact, ModelTrainerArtifact
+from src.entity.config_entity import Model_trainer_config
+from src.utils.main_utils.utils import save_object, load_object
+from src.utils.main_utils.utils import load_numpy_array_data, evaluate_models
+from src.utils.ml_utils.metric.classification_metric import classification_score
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.ensemble import (
+    RandomForestClassifier,
+    AdaBoostClassifier,
+    GradientBoostingClassifier, 
+)
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import r2_score
+import mlflow 
+import dagshub
+dagshub.init(repo_owner='kshitijk146', repo_name='MLOPS_project_network_Security_system', mlflow=True)
+class ModelTrainer:
+    def __init__(self, model_trainer_config: Model_trainer_config, data_transformation_artifact: DataTransformationArtifact):
+        try:
+            self.model_trainer_config = model_trainer_config
+            self.data_transformation_artifact = data_transformation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def track_mlflow(self,best_model, classificationmetric):
+        with mlflow.start_run():
+            f1_score = classificationmetric.f1_score
+            precision_score = classificationmetric.precision_score
+            recall_score = classificationmetric.recall_score
+        
+            mlflow.log_metric("f1_score", f1_score)
+            mlflow.log_metric("precision_score", precision_score)
+            mlflow.log_metric("recall_score", recall_score)
+            mlflow.sklearn.log_model(best_model, "model")
+        
+    def train_model(self, x_train, y_train,x_test, y_test):
+        models = {
+            "KNN": KNeighborsClassifier(),
+            "Decision Tree": DecisionTreeClassifier(),
+            "Random Forest": RandomForestClassifier(verbose=True),
+            "AdaBoost": AdaBoostClassifier(),
+            "Gradient Boosting": GradientBoostingClassifier(verbose=True),
+            "logistic regression": LogisticRegression(verbose=True)
+        }
+        params = {
+            "KNN": {
+                'n_neighbors': [3, 5, 7],  
+                'weights': ['uniform', 'distance'],
+                'metric': ['euclidean']  
+            },
+            "Decision Tree": {
+                'criterion': ['gini', 'entropy'], 
+                'max_depth': [None, 5, 10],  
+                'min_samples_split': [2, 5], 
+                'min_samples_leaf': [1, 2]  
+            },
+            "Random Forest": {
+                'n_estimators': [50, 100],  
+                'max_depth': [None, 5],  
+                'min_samples_split': [2, 5],  
+                'min_samples_leaf': [1, 2],  
+                'max_features': ['sqrt'] 
+            },
+            "AdaBoost": {
+                'n_estimators': [50, 100],  
+                'learning_rate': [0.1, 1.0],  
+                # 'algorithm': ['SAMME.R']  
+            },
+            "Gradient Boosting": {
+                'n_estimators': [50, 100],
+                'learning_rate': [0.1],  
+                'max_depth': [3, 5], 
+                'min_samples_split': [2],  
+                'min_samples_leaf': [1],  
+                'max_features': ['sqrt'] 
+            },
+            "logistic regression": {
+                'C': [1.0, 10.0],  
+                'penalty': ['l2'],  
+                'solver': ['liblinear']  
+            }
+        }
+
+        model_report:dict = evaluate_models(
+            x_train = x_train,y_train = y_train,x_test = x_test,y_test = y_test,models = models,params = params)
+        
+        # to get the best model score from the dict
+        best_model_score = max(sorted(model_report.values()))
+        
+        # to get best model name from dict 
+        best_model_name = list(model_report.keys())[
+            list(model_report.values()).index(best_model_score)
+        ]
+        logging.info(f"best model name: {best_model_name}")
+        best_model = models[best_model_name]
+        y_train_pred = best_model.predict(x_train)
+        classification_train_metric= classification_score(y_true = y_train, y_pred=y_train_pred)
+        
+        # track mlfow 
+        self.track_mlflow(best_model, classification_train_metric)
+        
+        
+        
+        y_test_pred = best_model.predict(x_test)
+        classification_test_metric = classification_score(y_true = y_test, y_pred=y_test_pred)
+        
+        preprocessor  = load_object(file_path=self.data_transformation_artifact.transformed_object_file_path)
+        model_dir_path = os.path.dirname(self.model_trainer_config.trained_model_file_path)
+        os.makedirs(model_dir_path, exist_ok=True)
+        
+        NetwerkModel= NetworkSecurityModel(preprocessing_object=preprocessor, trained_model_object=best_model)
+        save_object(self.model_trainer_config.trained_model_file_path, obj=NetwerkModel)
+        save_object("final_model/model.pkl", best_model)
+        
+        model_trainer_artifact = ModelTrainerArtifact(trained_model_file_path=self.model_trainer_config.trained_model_file_path, train_metric_artifact=classification_train_metric, test_metric_artifact=classification_test_metric)
+        logging.info(f"Model trainer artifact: {model_trainer_artifact}")
+        return model_trainer_artifact
+    
+    def initiate_model_trainer(self)-> ModelTrainerArtifact:
+        try:
+            train_file_path = self.data_transformation_artifact.transformed_train_file_path
+            test_file_path = self.data_transformation_artifact.transformed_test_file_path
+            
+            # loading training array and testing array
+            train_array = load_numpy_array_data(train_file_path)
+            test_array = load_numpy_array_data(test_file_path)
+            x_train, y_train, x_test, y_test = (
+                train_array[:, :-1],
+                train_array[:, -1],
+                test_array[:, :-1],
+                test_array[:, -1],
+            )
+            model = self.train_model(x_train, y_train, x_test=x_test, y_test=y_test)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e 

src/constant/training_pipeline/__init__.py

@@ -0,0 +1,64 @@
+import os
+import sys 
+import numpy as np 
+
+"""
+Common constant variable 
+"""
+TARGET_COLUMN = "Result"
+PIPELINE_NAME: str= "NetworkSecurity"
+ARTIFACT_DIR: str = "Artifacts"
+FILE_NAME: str  = "phisingData.csv"
+
+TRAIN_FILE_NAME: str = "train.csv"
+TEST_FILE_NAME: str = "test.csv"
+SCHEMA_FILE_PATH = os.path.join("data_schema", "schema.yaml")
+
+SAVED_MODEL_DIR = os.path.join("saved_models")
+MODEL_FILE_NAME = "model.pkl"
+"""
+Data ingestion variable 
+"""
+DATA_INGESTION_COLLECTION_NAME: str= "phising_data"
+DATA_INGESTION_DATBASE_NANE: str= "Network_data"
+DATA_INGESTION_DIR_NAME:str = "data_ingestion"
+DATA_INGESTION_FEATURE_STORE_DIR: str = "feature_store"
+DATA_INGESTION_INGESTED_DIR: str = "ingested"
+DATA_INGESTION_TRAIN_TEST_SPLIT_RATION: float = 0.2
+
+
+"""
+Data validation realated constant start with DATA_VALIDATION VAR NAME
+    
+"""
+DATA_VALIDATION_DIR_NAMR:str = "data_validation"
+DATA_VALIDATION_VALID_DIR: str = "validated"
+DATA_VALIDATION_INVALID_DIR: str = "invalid"
+DATA_VALIDATION_DRIFT_REPORT_DIR: str = "drift_report"
+DATA_VALIDATION_DRIFT_REPORT_FILE_NAME: str = "report.yaml"
+
+
+"""
+Data transformation realated constant start with DATA_TRANSFORMATION VAR NAME
+"""
+DATA_TRANSFORMATION_DIR_NAME: str = "data_transformation"
+DATA_TRANSFORMATION_TRANSFORMED_DIR_NAME: str = "transformed"
+DATA_TRANSFORMATION_TRANSFORMED_OBJECT_DIR:str = "transformed_object"
+PREPROCESSING_OBJECT_FILE_NAME:str = "preprocessing.pkl"
+# using knn imputer 
+DATA_TRANSFORMATION_IMPUTER_PARAMS: dict = {
+    "missing_values": np.nan, 
+    "n_neighbors" : 3, 
+    "weights" : "uniform"
+}
+
+"""
+Model trainer realated constant start with DATA_TRANSFORMATION VAR NAME
+"""
+MODEL_TRAINER_DIR_NAME: str = "model_trainer"
+MODEL_TRAINER_MODEL_DIR:str = "trained_model"
+MODEL_TRAINER_MODEL_NAME:str = "model.pkl"
+MODEL_TRAINER_EXPECTED_SCORE: float = 0.6
+MODEL_TRAINER_OVERFITTING_UNDERFITTING_THRESHOLD: float = 0.05
+
+TRAINING_BUCKET_NAME = "networksecuritymodelbucket"

src/data/sqlite_manager.py

@@ -0,0 +1,162 @@
+import sqlite3
+import pandas as pd
+import os
+from datetime import datetime
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+import sys
+
+class PhishingDataManager:
+    def __init__(self, db_path="data/phishing_data.db"):
+        """Initialize SQLite database for phishing data"""
+        try:
+            self.db_path = db_path
+            os.makedirs(os.path.dirname(db_path), exist_ok=True)
+            self.conn = sqlite3.connect(db_path, check_same_thread=False)
+            self._create_tables()
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def _create_tables(self):
+        """Create phishing data table and metadata table"""
+        try:
+            cursor = self.conn.cursor()
+            
+            # Main data table
+            cursor.execute("""
+                CREATE TABLE IF NOT EXISTS phishing_data (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    having_IP_Address INTEGER,
+                    URL_Length INTEGER,
+                    Shortining_Service INTEGER,
+                    having_At_Symbol INTEGER,
+                    double_slash_redirecting INTEGER,
+                    Prefix_Suffix INTEGER,
+                    having_Sub_Domain INTEGER,
+                    SSLfinal_State INTEGER,
+                    Domain_registeration_length INTEGER,
+                    Favicon INTEGER,
+                    port INTEGER,
+                    HTTPS_token INTEGER,
+                    Request_URL INTEGER,
+                    URL_of_Anchor INTEGER,
+                    Links_in_tags INTEGER,
+                    SFH INTEGER,
+                    Submitting_to_email INTEGER,
+                    Abnormal_URL INTEGER,
+                    Redirect INTEGER,
+                    on_mouseover INTEGER,
+                    RightClick INTEGER,
+                    popUpWidnow INTEGER,
+                    Iframe INTEGER,
+                    age_of_domain INTEGER,
+                    DNSRecord INTEGER,
+                    web_traffic INTEGER,
+                    Page_Rank INTEGER,
+                    Google_Index INTEGER,
+                    Links_pointing_to_page INTEGER,
+                    Statistical_report INTEGER,
+                    Result INTEGER,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    used_in_training BOOLEAN DEFAULT 0
+                )
+            """)
+            
+            # Training metadata table
+            cursor.execute("""
+                CREATE TABLE IF NOT EXISTS training_metadata (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    training_timestamp TIMESTAMP,
+                    data_count INTEGER,
+                    model_accuracy REAL,
+                    model_version TEXT,
+                    artifact_path TEXT
+                )
+            """)
+            
+            self.conn.commit()
+            logging.info("Database tables created successfully")
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def insert_data_from_csv(self, csv_path):
+        """Bulk insert from CSV (initial load)"""
+        try:
+            df = pd.read_csv(csv_path)
+            df.replace({"na": None}, inplace=True)
+            
+            # Insert only new records (avoid duplicates)
+            df.to_sql('phishing_data', self.conn, if_exists='append', index=False)
+            logging.info(f"Inserted {len(df)} records from CSV")
+            return len(df)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def add_new_samples(self, data_dict_list):
+        """Add new phishing samples incrementally"""
+        try:
+            df = pd.DataFrame(data_dict_list)
+            df.to_sql('phishing_data', self.conn, if_exists='append', index=False)
+            logging.info(f"Added {len(df)} new samples")
+            return len(df)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def get_training_data(self, include_new_only=False):
+        """Fetch data for training"""
+        try:
+            if include_new_only:
+                # Only get data not used in training yet
+                query = "SELECT * FROM phishing_data WHERE used_in_training = 0"
+            else:
+                # Get all data
+                query = "SELECT * FROM phishing_data"
+            
+            df = pd.read_sql_query(query, self.conn)
+            
+            # Drop metadata columns
+            df = df.drop(['id', 'created_at', 'used_in_training'], axis=1, errors='ignore')
+            
+            logging.info(f"Fetched {len(df)} records for training")
+            return df
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def mark_data_as_trained(self):
+        """Mark all data as used in training"""
+        try:
+            cursor = self.conn.cursor()
+            cursor.execute("UPDATE phishing_data SET used_in_training = 1 WHERE used_in_training = 0")
+            self.conn.commit()
+            logging.info("Marked data as trained")
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def get_new_data_count(self):
+        """Count untrained samples"""
+        try:
+            cursor = self.conn.cursor()
+            result = cursor.execute("SELECT COUNT(*) FROM phishing_data WHERE used_in_training = 0").fetchone()
+            return result[0]
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def log_training_run(self, data_count, accuracy, version, artifact_path):
+        """Log training metadata"""
+        try:
+            cursor = self.conn.cursor()
+            cursor.execute("""
+                INSERT INTO training_metadata (training_timestamp, data_count, model_accuracy, model_version, artifact_path)
+                VALUES (?, ?, ?, ?, ?)
+            """, (datetime.now(), data_count, accuracy, version, artifact_path))
+            self.conn.commit()
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) from e
+    
+    def should_retrain(self, threshold=100):
+        """Check if retraining is needed based on new data"""
+        new_count = self.get_new_data_count()
+        return new_count >= threshold
+    
+    def close(self):
+        self.conn.close()

src/entity/artifact_entity.py

@@ -0,0 +1,34 @@
+from dataclasses import dataclass
+
+@dataclass
+class DataIngestionArtifact:
+    train_file_path: str
+    test_file_path: str
+    
+@dataclass 
+class DataValidationArtifact:
+    validation_status: bool
+    valid_train_file_path: str
+    valid_test_file_path: str
+    invalid_train_file_path: str
+    invalid_test_file_path: str
+    drift_report_file_path: str
+    
+    
+@dataclass 
+class DataTransformationArtifact:
+    transformed_object_file_path: str
+    transformed_train_file_path: str
+    transformed_test_file_path: str
+
+@dataclass
+class ClassificationMetricArtifact:
+    f1_score: float
+    precision_score: float
+    recall_score: float
+    
+@dataclass
+class ModelTrainerArtifact:
+    trained_model_file_path: str
+    train_metric_artifact: ClassificationMetricArtifact
+    test_metric_artifact: ClassificationMetricArtifact

src/entity/config_entity.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+import os
+from src.constant import training_pipeline
+
+
+class TrainingPipelineConfig:
+    def __init__(self, timestamp = datetime.now()):
+        timestamp = timestamp.strftime("%m_%d_%Y_%H_%M_%S")
+        self.pipeline_name = training_pipeline.PIPELINE_NAME
+        self.artifact_name = training_pipeline.ARTIFACT_DIR
+        self.artifact_dir = os.path.join(self.artifact_name, timestamp)
+        self.model_dir=os.path.join("final_model")
+        self.timestamp:str = timestamp
+        
+class Data_ingestion_config:
+    def __init__(self, training_pipeline_config:TrainingPipelineConfig):
+        self.data_ingestion_dir = os.path.join(training_pipeline_config.artifact_dir,training_pipeline.DATA_INGESTION_DIR_NAME)
+        self.feature_store_file_path = os.path.join(self.data_ingestion_dir, training_pipeline.DATA_INGESTION_FEATURE_STORE_DIR, training_pipeline.FILE_NAME)
+        self.train_file_path = os.path.join(self.data_ingestion_dir, training_pipeline.DATA_INGESTION_INGESTED_DIR, training_pipeline.TRAIN_FILE_NAME)
+        self.test_file_path = os.path.join(self.data_ingestion_dir, training_pipeline.DATA_INGESTION_INGESTED_DIR, training_pipeline.TEST_FILE_NAME)
+        self.database_name = training_pipeline.DATA_INGESTION_DATBASE_NANE
+        self.collection_name = training_pipeline.DATA_INGESTION_COLLECTION_NAME
+        self.train_test_split_ratio = training_pipeline.DATA_INGESTION_TRAIN_TEST_SPLIT_RATION
+        
+class Data_validation_config:
+    def __init__(self,training_pipeline_config: TrainingPipelineConfig):
+        self.data_validation_dir:str = os.path.join(training_pipeline_config.artifact_dir, training_pipeline.DATA_VALIDATION_DIR_NAMR)
+        self.valid_data_dir:str = os.path.join(self.data_validation_dir, training_pipeline.DATA_VALIDATION_VALID_DIR)
+        self.invalid_data_dir:str = os.path.join(self.data_validation_dir, training_pipeline.DATA_VALIDATION_INVALID_DIR)
+        self.valid_train_file_path:str = os.path.join(self.valid_data_dir, training_pipeline.TRAIN_FILE_NAME)
+        self.valid_test_file_path:str = os.path.join(self.valid_data_dir, training_pipeline.TEST_FILE_NAME)
+        self.invalid_train_file_path:str = os.path.join(self.invalid_data_dir, training_pipeline.TRAIN_FILE_NAME)
+        self.invalid_test_file_path:str = os.path.join(self.invalid_data_dir, training_pipeline.TEST_FILE_NAME)
+        self.drift_report_file_path:str = os.path.join(self.data_validation_dir, training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR, training_pipeline.DATA_VALIDATION_DRIFT_REPORT_FILE_NAME)
+        
+        
+class Data_transformation_config:
+    def __init__(self, training_pipeline_config: TrainingPipelineConfig):
+        self.data_transformation_dir:str = os.path.join(training_pipeline_config.artifact_dir, training_pipeline.DATA_TRANSFORMATION_DIR_NAME)
+        
+        self.transformed_train_file_path: str = os.path.join( self.data_transformation_dir,training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_DIR_NAME,
+            training_pipeline.TRAIN_FILE_NAME.replace("csv", "npy"),)
+        
+        self.transformed_test_file_path: str = os.path.join(self.data_transformation_dir,  training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_DIR_NAME,
+            training_pipeline.TEST_FILE_NAME.replace("csv", "npy"), 
+            )
+        self.transformed_object_file_path: str = os.path.join( self.data_transformation_dir, training_pipeline.DATA_TRANSFORMATION_TRANSFORMED_OBJECT_DIR,
+            training_pipeline.PREPROCESSING_OBJECT_FILE_NAME)
+        
+class Model_trainer_config:
+    def __init__(self, training_pipeline_config: TrainingPipelineConfig):
+        self.model_trainer_dir:str = os.path.join(
+            training_pipeline_config.artifact_dir, training_pipeline.MODEL_TRAINER_DIR_NAME
+        )
+        self.trained_model_file_path:str = os.path.join(
+            self.model_trainer_dir, training_pipeline.MODEL_TRAINER_MODEL_DIR, training_pipeline.MODEL_TRAINER_MODEL_NAME
+        )
+        self.expected_accuracy:float = training_pipeline.MODEL_TRAINER_EXPECTED_SCORE
+        self.overfitting_underfitting_threshold = training_pipeline.MODEL_TRAINER_OVERFITTING_UNDERFITTING_THRESHOLD
+        

src/exception/exception.py

@@ -0,0 +1,12 @@
+import sys
+from src.logging import logger
+
+class NetworkSecurityException(Exception):
+    def __init__(self, error_message, error_details:sys):
+        self.error_message = error_message
+        _, _, exc_tb = error_details.exc_info()
+        self.line_number = exc_tb.tb_lineno
+        self.file_name = exc_tb.tb_frame.f_code.co_filename
+
+    def __str__(self):
+        return f"Error occurred in python script name [{self.file_name}] line number [{self.line_number}] error message [{self.error_message}]"

src/logging/logger.py

@@ -0,0 +1,14 @@
+import logging 
+import os
+from datetime import datetime
+
+LOG_FILE = f"{datetime.now().strftime('%m_%d_%Y_%H_%M_%S')}.log"
+logs_path= os.path.join(os.getcwd(), "logs", LOG_FILE)
+os.makedirs(logs_path, exist_ok=True)
+LOG_FILE_PATH = os.path.join(logs_path, LOG_FILE)
+
+logging.basicConfig(
+    filename = LOG_FILE_PATH,
+    format = "[%(asctime)s] %(lineno)d %(name)s - %(levelname)s - %(message)s",
+    level = logging.INFO
+)