Cloud pushed and s3 data storage implemented

app.py

@@ -4,12 +4,12 @@ from networksecurity.utils.ml_utils.model.estimator import NetworkModel
 
 ca = certifi.where()
 
-from dotenv import load_dotenv
+from dotenv import load_dotenv 
 load_dotenv()
 mong_db_url = os.getenv("MONGODB_URL_KEY")
 print(mong_db_url)
 
-import pymongo
+import pymongo 
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
 from networksecurity.pipeline.training_pipeline import TraningPipeline

networksecurity/cloud/s3_syncer.py

@@ -0,0 +1,10 @@
+import os
+
+class S3Sync:
+    def sync_folder_to_s3(self, folder, aws_bucket_url):
+        command = f"aws s3 sync {folder} {aws_bucket_url} "
+        os.system(command)
+    
+    def sync_folder_from_s3(self, folder, aws_bucket_url):
+        command = f"aws s3 sync {aws_bucket_url} {folder} "
+        os.system(command)

networksecurity/components/model_trainer.py

@@ -2,6 +2,9 @@ import os
 import sys
 import mlflow
 import dagshub
+import matplotlib
+matplotlib.use("Agg")
+
 import matplotlib.pyplot as plt
 import seaborn as sns
 
@@ -110,38 +113,12 @@ class ModelTrainer:
             "AdaBoost": AdaBoostClassifier(),
         }
 
-        params = {
-            "Decision Tree": {
-                "criterion": ["gini", "entropy", "log_loss"]
-            },
-            "Random Forest": {
-                "n_estimators": [8, 16, 32, 128, 256]
-            },
-            "Gradient Boosting": {
-                "learning_rate": [0.1, 0.01, 0.05, 0.001],
-                "subsample": [0.6, 0.7, 0.75, 0.85, 0.9],
-                "n_estimators": [8, 16, 32, 64, 128, 256],
-            },
-            "AdaBoost": {
-                "learning_rate": [0.1, 0.01, 0.001],
-                "n_estimators": [8, 16, 32, 64, 128, 256],
-            },
-            "Logistic Regression": {},
-        }
-
-        # ---------- Hyperparameter search ----------
-        model_report = evaluate_models(
-            X_train=X_train,
-            y_train=y_train,
-            X_test=X_test,
-            y_test=y_test,
-            models=models,
-            params=params,
-        )
-
         # ---------- MLflow logging ----------
-        model_scores = {}
-        run_id_map = {}
+        best_f1 = -1
+        best_model = None
+        best_model_name = None
+        best_run_id = None
+
 
         for model_name, model in models.items():
 
@@ -176,22 +153,21 @@ class ModelTrainer:
                     y_proba=y_test_proba,
                 )
 
-                model_scores[model_name] = test_metric.f1_score
-                run_id_map[model_name] = run.info.run_id
+                if test_metric.f1_score > best_f1:
+                    best_f1 = test_metric.f1_score
+                    best_model = model
+                    best_model_name = model_name
+                    best_run_id = run.info.run_id
 
-        # ---------- Best model selection ----------
-        best_model_name = max(model_scores, key=model_scores.get)
-        best_model = model_report[best_model_name]["model"]
 
         logging.info(
             f"Best Model: {best_model_name} | "
-            f"Test F1: {model_scores[best_model_name]}"
+            f"Test F1: {best_f1}"
         )
 
-        # ---------- Tag best model ----------
-        mlflow.start_run(run_id=run_id_map[best_model_name])
-        mlflow.set_tag("best_model", "true")
-        mlflow.end_run()
+
+        with mlflow.start_run(run_id=best_run_id):
+            mlflow.set_tag("best_model", "true")
 
         # ---------- Save final model for deployment ----------
         preprocessor = load_object(
@@ -210,16 +186,20 @@ class ModelTrainer:
             preprocessor,
         )
 
-        logging.info("Final model and preprocessor saved in final_model/")
+        logging.info(f"Final model and preprocessor saved in final_models")
+        y_train_pred = best_model.predict(X_train)
+        y_test_pred = best_model.predict(X_test)
+
+        best_train_metric = get_classification_score(y_train, y_train_pred)
+        best_test_metric = get_classification_score(y_test, y_test_pred)
 
         return ModelTrainerArtifact(
-            trained_model_file_path=os.path.join(
-                final_model_dir, "model.pkl"
-            ),
-            train_metric_artifact=train_metric,
-            test_metric_artifact=test_metric,
+            trained_model_file_path=os.path.join(final_model_dir, "model.pkl"),
+            train_metric_artifact=best_train_metric,
+            test_metric_artifact=best_test_metric,
         )
 
+
     def initiate_model_trainer(self) -> ModelTrainerArtifact:
         try:
             train_array = load_numpy_array_data(

networksecurity/constant/training_pipeline/__init__.py

@@ -64,3 +64,5 @@ MODEL_TRAINER_TRAINED_MODEL_DIR: str = "trained_model"
 MODEL_TRAINER_TRAINED_MODEL_NAME: str = "model.pkl"
 MODEL_TRAINER_EXPECTED_SCORE: float = 0.6
 MODEL_TRAINER_OVER_FITTING_UNDER_FITTING_THRESHOLD: float = 0.05
+
+TRAINING_BUCKET_NAME = "awsnetworksecuritybucket"

networksecurity/entity/config_entity.py

@@ -12,6 +12,7 @@ class TrainingPipelineConfig:
         self.pipeline_name = training_pipeline.PIPELINE_NAME
         self.artifact_name = training_pipeline.ARTIFACT_DIR
         self.artifact_dir = os.path.join(self.artifact_name,timestamp)
+        self.model_dir = os.path.join("final_model")
         self.timestamp: str = timestamp
 
 class DataIngestionConfig:

networksecurity/pipeline/training_pipeline.py

@@ -6,6 +6,8 @@ from networksecurity.components.data_ingestion import DataIngestion
 from networksecurity.components.data_validation import DataValidation
 from networksecurity.components.data_transformation import DataTransformation
 from networksecurity.components.model_trainer import ModelTrainer
+from networksecurity.constant.training_pipeline import TRAINING_BUCKET_NAME
+from networksecurity.cloud.s3_syncer import S3Sync
 
 from networksecurity.entity.config_entity import (
     TrainingPipelineConfig,
@@ -26,6 +28,7 @@ class TraningPipeline:
     def __init__(self):
         try:
             self.training_pipeline_config = TrainingPipelineConfig()
+            self.s3_sync = S3Sync()
         except Exception as e:
             raise NetworkSecurityException(e, sys)
         
@@ -92,6 +95,38 @@ class TraningPipeline:
         except Exception as e:
             raise NetworkSecurityException(e, sys)
         
+        ## Local artifact is pushed to S3
+    def sync_artifact_dir_to_s3(self):
+        try:
+            aws_bucket_url = (
+                f"s3://{TRAINING_BUCKET_NAME}/artifact/"
+                f"{self.training_pipeline_config.timestamp}"
+            )
+
+            self.s3_sync.sync_folder_to_s3(
+                folder=self.training_pipeline_config.artifact_dir,
+                aws_bucket_url=aws_bucket_url
+            )
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+
+    ## Local final_models is pushed to S3
+    def sync_saved_model_dir_to_s3(self):
+        try:
+            aws_bucket_url = (
+                f"s3://{TRAINING_BUCKET_NAME}/final_model/"
+                f"{self.training_pipeline_config.timestamp}"
+            )
+
+            self.s3_sync.sync_folder_to_s3(
+                folder=self.training_pipeline_config.model_dir,
+                aws_bucket_url=aws_bucket_url
+            )
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+        
     def run_pipeline(self):
         try:
             data_ingestion_artifact = self.start_data_ingestion()
@@ -99,6 +134,10 @@ class TraningPipeline:
             data_transformation_artifact = self.start_data_transformation(data_validation_artifact=data_validation_artifact)
             model_trainer_artifact = self.start_model_trainer(data_transformation_artifact=data_transformation_artifact)
             logging.info("Training pipeline completed successfully")
+
+            self.sync_artifact_dir_to_s3()
+            self.sync_saved_model_dir_to_s3()
+            
             return model_trainer_artifact
         except Exception as e:
             raise NetworkSecurityException(e, sys)

requirements.txt

@@ -1,16 +1,118 @@
-python-dotenv
-numpy 
-pandas
-setuptools
-pymongo
-certifi
-pymongo[srv]==3.11
-scikit-learn
-pyaml
-mlflow
-dagshub
-seaborn
-fastapi
-uvicorn
-python-multipart
-#-e .
+alembic==1.17.2
+annotated-doc==0.0.4
+annotated-types==0.7.0
+anyio==4.12.0
+appdirs==1.4.4
+backoff==2.2.1
+blinker==1.9.0
+boto3==1.42.18
+botocore==1.42.18
+cachetools==6.2.4
+certifi==2025.11.12
+cffi==2.0.0
+charset-normalizer==3.4.4
+click==8.3.1
+cloudpickle==3.1.2
+colorama==0.4.6
+contourpy==1.3.3
+cryptography==46.0.3
+cycler==0.12.1
+dacite==1.6.0
+dagshub==0.6.4
+dagshub-annotation-converter==0.1.15
+databricks-sdk==0.76.0
+dataclasses-json==0.6.7
+dill==0.4.0
+dnspython==1.16.0
+docker==7.1.0
+fastapi==0.128.0
+flask==3.1.2
+flask-cors==6.0.2
+fonttools==4.61.1
+gitdb==4.0.12
+gitpython==3.1.45
+google-auth==2.45.0
+gql==4.0.0
+graphene==3.4.3
+graphql-core==3.2.7
+graphql-relay==3.2.0
+greenlet==3.3.0
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+huey==2.5.5
+idna==3.11
+importlib-metadata==8.7.1
+itsdangerous==2.2.0
+jinja2==3.1.6
+jmespath==1.0.1
+joblib==1.5.3
+kiwisolver==1.4.9
+lxml==6.0.2
+mako==1.3.10
+markdown-it-py==4.0.0
+markupsafe==3.0.3
+marshmallow==3.26.2
+matplotlib==3.10.8
+mdurl==0.1.2
+mlflow==3.8.1
+mlflow-skinny==3.8.1
+mlflow-tracing==3.8.1
+multidict==6.7.0
+mypy-extensions==1.1.0
+# -e file:///D:/Coding%20Central/NetworkSecurity
+numpy==2.4.0
+opentelemetry-api==1.39.1
+opentelemetry-proto==1.39.1
+opentelemetry-sdk==1.39.1
+opentelemetry-semantic-conventions==0.60b1
+packaging==25.0
+pandas==2.3.3
+pathvalidate==3.3.1
+pillow==12.0.0
+propcache==0.4.1
+protobuf==6.33.2
+pyaml==25.7.0
+pyarrow==22.0.0
+pyasn1==0.6.1
+pyasn1-modules==0.4.2
+pycparser==2.23
+pydantic==2.12.5
+pydantic-core==2.41.5
+pygments==2.19.2
+pymongo==3.11.0
+pyparsing==3.3.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.1
+python-multipart==0.0.21
+pytz==2025.2
+pywin32==311
+pyyaml==6.0.3
+requests==2.32.5
+requests-toolbelt==1.0.0
+rich==14.2.0
+rsa==4.9.1
+s3transfer==0.16.0
+scikit-learn==1.8.0
+scipy==1.16.3
+seaborn==0.13.2
+semver==3.0.4
+setuptools==80.9.0
+six==1.17.0
+smmap==5.0.2
+sqlalchemy==2.0.45
+sqlparse==0.5.5
+starlette==0.50.0
+tenacity==9.1.2
+threadpoolctl==3.6.0
+treelib==1.8.0
+typing-extensions==4.15.0
+typing-inspect==0.9.0
+typing-inspection==0.4.2
+tzdata==2025.3
+urllib3==2.6.2
+uvicorn==0.40.0
+waitress==3.0.2
+werkzeug==3.1.4
+yarl==1.22.0
+zipp==3.23.0