Pipeline completed and fastapi created

app.py

@@ -0,0 +1,98 @@
+import os, certifi, sys
+
+from networksecurity.utils.ml_utils.model.estimator import NetworkModel
+
+ca = certifi.where()
+
+from dotenv import load_dotenv
+load_dotenv()
+mong_db_url = os.getenv("MONGODB_URL_KEY")
+print(mong_db_url)
+
+import pymongo
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+from networksecurity.pipeline.training_pipeline import TraningPipeline
+
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI, File, UploadFile, Request
+from uvicorn import run as app_run
+from fastapi.responses import Response
+from starlette.responses import RedirectResponse
+import pandas as pd
+
+from networksecurity.utils.main_utils.utils import load_object
+
+client = pymongo.MongoClient(mong_db_url, tlsCAFile=ca)
+
+from networksecurity.constant.training_pipeline import DATA_INGESTION_COLLECTION_NAME
+from networksecurity.constant.training_pipeline import DATA_INGESTION_DATABASE_NAME
+
+database = client[DATA_INGESTION_DATABASE_NAME]
+collection = database[DATA_INGESTION_COLLECTION_NAME]
+
+app = FastAPI(title="Network Security Prediction API")
+origins = ["*"]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+from fastapi.templating import Jinja2Templates
+templates = Jinja2Templates(directory="templates")
+
+@app.get("/")
+async def index():
+    return RedirectResponse(url="/docs")
+
+
+@app.get("/train", tags=["train"])
+async def train_route():
+    try:
+        training_pipeline = TraningPipeline()
+        training_pipeline.run_pipeline()
+        return Response(content="Training successful !!")
+
+    except Exception as e:
+        logging.error(f"Training failed: {e}")
+        raise NetworkSecurityException(e, sys)
+
+
+
+@app.post("/predict")
+async def predict_route(request: Request, file: UploadFile = File(...)):
+    try:
+        df = pd.read_csv(file.file)
+        logging.info(f"Uploaded file shape: {df.shape}")
+        logging.info(f"Uploaded columns: {df.columns.tolist()}")
+
+        preprocessor = load_object("final_model/preprocessor.pkl")
+        final_model = load_object("final_model/model.pkl")
+
+        network_model = NetworkModel(
+            preprocessor=preprocessor,
+            model=final_model
+        )
+        print(df.iloc[0])
+        y_pred = network_model.predict(df)
+        print(y_pred)
+        df["prediction_column"] = y_pred
+        print(df["prediction_column"])
+
+        os.makedirs("prediction_output", exist_ok=True)
+        df.to_csv("prediction_output/output.csv", index=False)
+        table_html = df.to_html(classes='table table-striped')
+
+        return templates.TemplateResponse("table.html", {"request": request, "table": table_html})
+
+
+    except Exception as e:
+        logging.error(f"Prediction failed: {e}")
+        raise NetworkSecurityException(e, sys)
+
+if __name__ == "__main__":
+    app_run(app, host="localhost", port=8000)

main.py

@@ -6,12 +6,12 @@ from networksecurity.components.model_trainer import ModelTrainer,ModelTrainerCo
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
 from networksecurity.entity.config_entity import DataIngestionConfig,DataValidationConfig,DataTransformationConfig
-from networksecurity.entity.config_entity import TraningPipelineConfig
+from networksecurity.entity.config_entity import TrainingPipelineConfig
 import sys
 
 if __name__ == "__main__":
     try:
-        traningpipelineconfig=TraningPipelineConfig()
+        traningpipelineconfig=TrainingPipelineConfig()
         dataingestionconfig=DataIngestionConfig(traningpipelineconfig)
         data_ingestion=DataIngestion(dataingestionconfig)
         logging.info("Initiate the data ingestion")

networksecurity/components/model_trainer.py

@@ -181,7 +181,7 @@ class ModelTrainer:
 
         # ---------- Best model selection ----------
         best_model_name = max(model_scores, key=model_scores.get)
-        best_model = models[best_model_name]
+        best_model = model_report[best_model_name]["model"]
 
         logging.info(
             f"Best Model: {best_model_name} | "

networksecurity/entity/config_entity.py

@@ -6,7 +6,7 @@ print(training_pipeline.PIPELINE_NAME)
 print(training_pipeline.ARTIFACT_DIR)
 
 
-class TraningPipelineConfig:
+class TrainingPipelineConfig:
     def __init__(self,timestamp=datetime.now()):
         timestamp=timestamp.strftime("%m_%d_%Y_%H_M_%S")
         self.pipeline_name = training_pipeline.PIPELINE_NAME
@@ -15,7 +15,7 @@ class TraningPipelineConfig:
         self.timestamp: str = timestamp
 
 class DataIngestionConfig:
-    def __init__(self,training_pipeline_config: TraningPipelineConfig):
+    def __init__(self,training_pipeline_config: TrainingPipelineConfig):
         self.data_ingestion_dir:str = os.path.join(
             training_pipeline_config.artifact_dir,training_pipeline.DATA_INGESTION_DIR_NAME
         )
@@ -34,7 +34,7 @@ class DataIngestionConfig:
 
 
 class DataValidationConfig:
-    def __init__(self, training_pipeline_config: TraningPipelineConfig):
+    def __init__(self, training_pipeline_config: TrainingPipelineConfig):
         # Main data validation directory
         self.data_validation_dir: str = os.path.join(
             training_pipeline_config.artifact_dir,
@@ -85,7 +85,7 @@ class DataValidationConfig:
         )
 
 class DataTransformationConfig:
-    def __init__(self, training_pipeline_config: TraningPipelineConfig):
+    def __init__(self, training_pipeline_config: TrainingPipelineConfig):
         self.data_transformation_dir: str = os.path.join(
             training_pipeline_config.artifact_dir,
             training_pipeline.DATA_TRANSFORMATION_DIR_NAME
@@ -110,7 +110,7 @@ class DataTransformationConfig:
         )
 
 class ModelTrainerConfig:
-    def __init__(self, training_pipeline_config: TraningPipelineConfig):
+    def __init__(self, training_pipeline_config: TrainingPipelineConfig):
         self.model_trainer_dir: str = os.path.join(
             training_pipeline_config.artifact_dir, training_pipeline.MODEL_TRAINER_DIR_NAME
         )

networksecurity/pipeline/training_pipeline.py

@@ -0,0 +1,104 @@
+import os, sys
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+
+from networksecurity.components.data_ingestion import DataIngestion
+from networksecurity.components.data_validation import DataValidation
+from networksecurity.components.data_transformation import DataTransformation
+from networksecurity.components.model_trainer import ModelTrainer
+
+from networksecurity.entity.config_entity import (
+    TrainingPipelineConfig,
+    DataIngestionConfig,
+    DataValidationConfig,
+    DataTransformationConfig,
+    ModelTrainerConfig,
+)
+
+from networksecurity.entity.artifact_entity import (
+    DataIngestionArtifact,
+    DataValidationArtifact,
+    DataTransformationArtifact,
+    ModelTrainerArtifact,
+)
+
+class TraningPipeline:
+    def __init__(self):
+        try:
+            self.training_pipeline_config = TrainingPipelineConfig()
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def start_data_ingestion(self):
+        try:
+            logging.info("Starting data ingestion")
+            self.data_ingestion_config = DataIngestionConfig(
+                training_pipeline_config=self.training_pipeline_config
+            )
+            data_ingestion = DataIngestion(
+                data_ingestion_config=self.data_ingestion_config
+            )
+            data_ingestion_artifact = data_ingestion.initiate_data_ingestion()
+            logging.info(f"Data ingestion completed and artifact {data_ingestion_artifact}")
+            return data_ingestion_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def start_data_validation(self, data_ingestion_artifact: DataIngestionArtifact):
+        try:
+            logging.info("Starting data validation")
+            self.data_validation_config = DataValidationConfig(
+                training_pipeline_config=self.training_pipeline_config
+            )
+            data_validation = DataValidation(
+                data_validation_config=self.data_validation_config,
+                data_ingestion_artifact=data_ingestion_artifact
+            )
+            data_validation_artifact = data_validation.initiate_data_validation()
+            logging.info(f"Data validation completed and artifact {data_validation_artifact}")
+            return data_validation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def start_data_transformation(self, data_validation_artifact: DataValidationArtifact):
+        try:
+            logging.info("Starting data transformation")
+            self.data_transformation_config = DataTransformationConfig(
+                training_pipeline_config=self.training_pipeline_config
+            )
+            data_transformation = DataTransformation(
+                data_transformation_config=self.data_transformation_config,
+                data_validation_artifact=data_validation_artifact
+            )
+            data_transformation_artifact = data_transformation.initiate_data_transformation()
+            logging.info(f"Data transformation completed and artifact {data_transformation_artifact}")
+            return data_transformation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def start_model_trainer(self, data_transformation_artifact: DataTransformationArtifact) -> ModelTrainerArtifact:
+        try:
+            logging.info("Starting model trainer")
+            self.model_trainer_config = ModelTrainerConfig(
+                training_pipeline_config=self.training_pipeline_config
+            )
+            model_trainer = ModelTrainer(
+                model_trainer_config=self.model_trainer_config,
+                data_transformation_artifact=data_transformation_artifact
+            )
+            model_trainer_artifact = model_trainer.initiate_model_trainer()
+            logging.info(f"Model trainer completed and artifact {model_trainer_artifact}")
+            return model_trainer_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def run_pipeline(self):
+        try:
+            data_ingestion_artifact = self.start_data_ingestion()
+            data_validation_artifact = self.start_data_validation(data_ingestion_artifact=data_ingestion_artifact)
+            data_transformation_artifact = self.start_data_transformation(data_validation_artifact=data_validation_artifact)
+            model_trainer_artifact = self.start_model_trainer(data_transformation_artifact=data_transformation_artifact)
+            logging.info("Training pipeline completed successfully")
+            return model_trainer_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)

prediction_output/output.csv

@@ -0,0 +1,13 @@
+having_IP_Address,URL_Length,Shortining_Service,having_At_Symbol,double_slash_redirecting,Prefix_Suffix,having_Sub_Domain,SSLfinal_State,Domain_registeration_length,Favicon,port,HTTPS_token,Request_URL,URL_of_Anchor,Links_in_tags,SFH,Submitting_to_email,Abnormal_URL,Redirect,on_mouseover,RightClick,popUpWidnow,Iframe,age_of_domain,DNSRecord,web_traffic,Page_Rank,Google_Index,Links_pointing_to_page,Statistical_report,prediction
+1,-1,1,1,1,-1,1,1,1,1,1,-1,-1,1,1,0,1,1,0,1,1,1,1,1,1,-1,-1,1,1,1,1.0
+-1,-1,-1,1,-1,1,1,1,-1,1,1,-1,1,1,0,1,1,-1,0,1,1,1,1,-1,-1,1,1,-1,1,1,1.0
+-1,-1,-1,1,-1,-1,0,0,1,1,1,-1,-1,-1,1,-1,1,-1,1,1,1,1,1,1,1,-1,-1,1,0,1,0.0
+-1,-1,1,1,1,-1,1,1,1,1,1,1,-1,0,1,1,1,1,0,1,1,1,1,1,1,1,-1,1,1,1,1.0
+1,1,1,1,1,-1,0,-1,-1,1,1,1,1,0,0,1,1,1,0,1,1,1,1,-1,1,1,-1,1,0,1,1.0
+1,-1,1,1,1,1,0,-1,-1,1,1,1,1,0,0,-1,1,1,0,1,1,1,1,1,1,1,-1,1,0,1,1.0
+1,1,1,1,1,-1,0,1,-1,-1,1,1,1,0,-1,1,1,1,0,-1,1,-1,1,-1,1,1,-1,1,0,1,1.0
+1,-1,1,1,1,-1,-1,-1,1,1,1,1,-1,-1,0,-1,1,1,0,1,1,1,1,-1,-1,-1,-1,1,0,1,0.0
+1,-1,1,-1,1,-1,-1,1,1,-1,-1,1,-1,0,0,0,-1,1,0,-1,1,-1,-1,1,1,1,-1,1,0,1,1.0
+1,1,1,1,1,-1,0,-1,-1,1,1,1,1,-1,0,1,1,1,0,1,1,1,1,-1,-1,-1,-1,1,0,1,0.0
+1,-1,1,1,1,-1,-1,1,1,1,1,1,-1,0,-1,-1,1,1,0,1,1,1,1,1,1,1,-1,1,0,1,1.0
+-1,1,-1,1,1,-1,0,1,-1,1,1,1,1,0,1,-1,1,1,0,1,1,1,1,1,1,1,-1,1,0,1,0.0

requirements.txt

@@ -10,4 +10,7 @@ pyaml
 mlflow
 dagshub
 seaborn
+fastapi
+uvicorn
+python-multipart
 #-e .

templates/predict.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Data Table</title>
+    <style>
+        table {
+            border-collapse: collapse;
+            width: 100%;
+        }
+        th, td {
+            border: 1px solid black;
+            padding: 8px;
+            text-align: left;
+        }
+    </style>
+</head>
+<body>
+    <h2>Predicted Data</h2>
+    {{ table | safe }}
+</body>
+</html>

valid_data/test.csv

@@ -0,0 +1,13 @@
+having_IP_Address,URL_Length,Shortining_Service,having_At_Symbol,double_slash_redirecting,Prefix_Suffix,having_Sub_Domain,SSLfinal_State,Domain_registeration_length,Favicon,port,HTTPS_token,Request_URL,URL_of_Anchor,Links_in_tags,SFH,Submitting_to_email,Abnormal_URL,Redirect,on_mouseover,RightClick,popUpWidnow,Iframe,age_of_domain,DNSRecord,web_traffic,Page_Rank,Google_Index,Links_pointing_to_page,Statistical_report
+1,-1,1,1,1,-1,1,1,1,1,1,-1,-1,1,1,0,1,1,0,1,1,1,1,1,1,-1,-1,1,1,1
+-1,-1,-1,1,-1,1,1,1,-1,1,1,-1,1,1,0,1,1,-1,0,1,1,1,1,-1,-1,1,1,-1,1,1
+-1,-1,-1,1,-1,-1,0,0,1,1,1,-1,-1,-1,1,-1,1,-1,1,1,1,1,1,1,1,-1,-1,1,0,1
+-1,-1,1,1,1,-1,1,1,1,1,1,1,-1,0,1,1,1,1,0,1,1,1,1,1,1,1,-1,1,1,1
+1,1,1,1,1,-1,0,-1,-1,1,1,1,1,0,0,1,1,1,0,1,1,1,1,-1,1,1,-1,1,0,1
+1,-1,1,1,1,1,0,-1,-1,1,1,1,1,0,0,-1,1,1,0,1,1,1,1,1,1,1,-1,1,0,1
+1,1,1,1,1,-1,0,1,-1,-1,1,1,1,0,-1,1,1,1,0,-1,1,-1,1,-1,1,1,-1,1,0,1
+1,-1,1,1,1,-1,-1,-1,1,1,1,1,-1,-1,0,-1,1,1,0,1,1,1,1,-1,-1,-1,-1,1,0,1
+1,-1,1,-1,1,-1,-1,1,1,-1,-1,1,-1,0,0,0,-1,1,0,-1,1,-1,-1,1,1,1,-1,1,0,1
+1,1,1,1,1,-1,0,-1,-1,1,1,1,1,-1,0,1,1,1,0,1,1,1,1,-1,-1,-1,-1,1,0,1
+1,-1,1,1,1,-1,-1,1,1,1,1,1,-1,0,-1,-1,1,1,0,1,1,1,1,1,1,1,-1,1,0,1
+-1,1,-1,1,1,-1,0,1,-1,1,1,1,1,0,1,-1,1,1,0,1,1,1,1,1,1,1,-1,1,0,1