Clean slate: fresh commit with current code

.gitignore

@@ -0,0 +1,194 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the enitre vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Jeevant
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,12 @@
+# AI_Summarizer
+
+## Workflows
+
+1. Update Config.yaml
+2. Update params.yaml
+3. Update entity
+4. Update the configuration manager in the src config
+5. Update the components
+6. Update the pipeline
+7. Update the main.py
+8. Update the app.py

config/config.yaml

@@ -0,0 +1,34 @@
+artifacts_root: artifacts_root
+
+
+data_ingestion:
+  root_dir: artifacts/data_ingestion
+  source_URL: https://github.com/entbappy/Branching-tutorial/raw/master/summarizer-data.zip
+  local_data_file: artifacts/data_ingestion/data.zip
+  unzip_dir : artifacts/data_ingestion
+
+
+data_validation:
+  root_dir: artifacts/data_validation
+  STATUS_FILE: artifacts/data_validation/status.txt
+  ALL_REQUIRED_FILES: ["train" , "test" , "validation"]
+
+
+data_transformation:
+  root_dir: artifacts/data_transformation
+  data_path: artifacts/data_ingestion/samsum_dataset
+  tokenizer_name: google/pegasus-cnn_dailymail
+
+
+model_trainer:
+  root_dir: artifacts/model_trainer
+  data_path: artifacts/data_transformation/samsum_dataset
+  model_ckpt: google/pegasus-cnn_dailymail
+
+
+model_evaluation:
+  root_dir: artifacts/model_evaluation
+  data_path: artifacts/data_transformation/samsum_dataset
+  model_path: artifacts/model_trainer/pegasus-samsum_model
+  tokenizer_path: artifacts/model_trainer/tokenizer_name
+  metrics_file_name: artifacts/model_evaluation/metrics.csv

main.py

@@ -0,0 +1,50 @@
+from textSummarizer.pipeline.stage_01_data_ingestion import DataIngestionTrainingPipeline
+from textSummarizer.pipeline.stage_02_data_validation import DataValidationTrainingPipeline
+from textSummarizer.pipeline.stage_03_data_transformation import DataTransformationTrainingPipeline
+from textSummarizer.pipeline.stage_04_model_trainer import ModelTrainerTrainingPipeline
+from textSummarizer.logging import logger
+
+
+STAGE_NAME = "Data Ingestion Stage"
+try:
+    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+    data_ingestion = DataIngestionTrainingPipeline()
+    data_ingestion.main()
+    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx============x")
+except Exception as e:
+    logger.exception(e)
+    raise e
+
+
+STAGE_NAME = "Data Validation Stage"
+try:
+    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+    data_validation = DataValidationTrainingPipeline()
+    data_validation.main()
+    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx============x")
+except Exception as e:
+    logger.exception(e)
+    raise e
+
+
+STAGE_NAME = "Data Transformation Stage"
+try:
+    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+    data_transformation = DataTransformationTrainingPipeline()
+    data_transformation.main()
+    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx============x")
+except Exception as e:
+    logger.exception(e)
+    raise e
+
+
+STAGE_NAME = "Model Trainer Stage"
+try:
+    logger.info(f"********************************")
+    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+    model_trainer = ModelTrainerTrainingPipeline()
+    model_trainer.main()
+    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx============x")
+except Exception as e:
+    logger.exception(e)
+    raise e

params.yaml

@@ -0,0 +1,10 @@
+TrainingArguments:
+  num_train_epochs: 1
+  warmup_steps: 500
+  per_device_train_batch_size: 1
+  weight_decay: 0.01
+  logging_steps: 10
+  evaluation_strategy: steps
+  eval_steps: 500
+  save_steps: 1e6
+  gradient_accumulation_steps: 16

requirements.txt

@@ -0,0 +1,21 @@
+transformers
+transformers[sentencepiece]
+dataset
+sacrebleu
+py7zr
+pandas
+nltk
+tqdm
+PyYAML
+matplotlib
+torch
+notebook
+boto3
+mypy-boto3-s3
+python-box==6.0.2
+ensure==1.0.2
+fastapi==0.78.0
+uvicorn==0.18.3
+Jinja2==3.1.2
+rouge-score
+-e .

research/01_data_ingestion.ipynb

@@ -0,0 +1,217 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "66881b4b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ef1f89d3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pwd\n",
+    "os.chdir(\"../\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "58bce27a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\order\\\\Desktop\\\\AI_Summarizer'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "375cb3da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataIngestionConfig:\n",
+    "    root_dir : Path\n",
+    "    source_URL : str\n",
+    "    local_data_file : Path\n",
+    "    unzip_dir : Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "486d78f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from textSummarizer.constants import *\n",
+    "from textSummarizer.utils.common import read_yaml, create_directories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e4d43738",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath: str = CONFIG_FILE_PATH,\n",
+    "        params_filepath: str = PARAMS_FILE_PATH):\n",
+    "        \n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "        \n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "        \n",
+    "    def get_data_ingestion_config(self) -> DataIngestionConfig:\n",
+    "        config =self.config.data_ingestion\n",
+    "        \n",
+    "        create_directories([config.root_dir])\n",
+    "        \n",
+    "        data_ingestion_config = DataIngestionConfig(\n",
+    "            root_dir = config.root_dir,\n",
+    "            source_URL = config.source_URL,\n",
+    "            local_data_file = config.local_data_file,\n",
+    "            unzip_dir = config.unzip_dir,\n",
+    "        )\n",
+    "        \n",
+    "        return data_ingestion_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "9606fbff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import urllib.request as request\n",
+    "import zipfile\n",
+    "from textSummarizer.logging import logger\n",
+    "from textSummarizer.utils.common import get_size"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "67b8be5c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DataIngestion:\n",
+    "    def __init__(self, config: DataIngestionConfig):\n",
+    "        self.config = config\n",
+    "        \n",
+    "        \n",
+    "    def download_data(self):\n",
+    "        if not os.path.exists(self.config.local_data_file):\n",
+    "            filename, headers = request.urlretrieve(\n",
+    "            url = self.config.source_URL,\n",
+    "            filename = self.config.local_data_file\n",
+    "            )\n",
+    "            logger.info(f\"{filename} download! with following info : \\n{headers}\")\n",
+    "        else:\n",
+    "            logger.info(f\"File already exists of size : {get_size(Path(self.config.local_data_file))}\")\n",
+    "            \n",
+    "    def extract_zip_file(self):\n",
+    "        \"\"\"\n",
+    "        zip_file_path : str\n",
+    "        EXtracts the zip file to the given directory.\n",
+    "        Functions returns None.\n",
+    "        \"\"\"\n",
+    "        unzip_path = self.config.unzip_dir\n",
+    "        os.makedirs(unzip_path, exist_ok=True)\n",
+    "        with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref:\n",
+    "            zip_ref.extractall(unzip_path)\n",
+    "            "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "369293bc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-06-28 23:11:14,985: INFO: common : yaml. file: config\\config.yaml loaded successfully. ]\n",
+      "[2025-06-28 23:11:14,987: INFO: common : yaml. file: params.yaml loaded successfully. ]\n",
+      "[2025-06-28 23:11:14,988: INFO: common : created directory at path : artifacts_root ]\n",
+      "[2025-06-28 23:11:14,989: INFO: common : created directory at path : artifacts/data_ingestion ]\n",
+      "[2025-06-28 23:11:14,990: INFO: 2056919966 : File already exists of size : ~7718 KB ]\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_ingestion_config = config.get_data_ingestion_config()\n",
+    "    data_ingestion = DataIngestion(data_ingestion_config)\n",
+    "    data_ingestion.download_data()\n",
+    "    data_ingestion.extract_zip_file()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0f2ce6e5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textSummarizer",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

research/02_data_validation.ipynb

@@ -0,0 +1,202 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3ef222e5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "186dcb71",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pwd\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3f4eb483",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\order\\\\Desktop\\\\AI_Summarizer'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "b36d364e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataValidationConfig:\n",
+    "    root_dir : Path\n",
+    "    STATUS_FILE : str\n",
+    "    ALL_REQUIRED_FILES : list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "05be5b22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from textSummarizer.constants import *\n",
+    "from textSummarizer.utils.common import read_yaml, create_directories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "5d9f3fbe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath: str = CONFIG_FILE_PATH,\n",
+    "        params_filepath: str = PARAMS_FILE_PATH):\n",
+    "        \n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "        \n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "    \n",
+    "    def get_data_validation_config(self) -> DataValidationConfig:\n",
+    "        config = self.config.data_validation\n",
+    "        \n",
+    "        create_directories([config.root_dir])\n",
+    "        \n",
+    "        data_validation_config = DataValidationConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            STATUS_FILE=config.STATUS_FILE,\n",
+    "            ALL_REQUIRED_FILES=config.ALL_REQUIRED_FILES,\n",
+    "        )\n",
+    "        \n",
+    "        return data_validation_config\n",
+    "            "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "7cca78e6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from textSummarizer.logging import logger"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "6cd0042a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DataValidation:\n",
+    "    def __init__(self, config: DataValidationConfig):\n",
+    "        self.config = config\n",
+    "        \n",
+    "        \n",
+    "        \n",
+    "    def validate_all_files_exists(self) -> bool:\n",
+    "        try:\n",
+    "            validation_status = None\n",
+    "            \n",
+    "            all_files = os.listdir(os.path.join(\"artifacts\" , \"data_ingestion\" , \"samsum_dataset\"))\n",
+    "            \n",
+    "            for file in all_files:\n",
+    "                if file not in self.config.ALL_REQUIRED_FILES:\n",
+    "                    validation_status = False\n",
+    "                    with open(self.config.STATUS_FILE, 'w') as f:\n",
+    "                        f.write(f\"Validation status: {validation_status}\")\n",
+    "                else:\n",
+    "                    validation_status = True\n",
+    "                    with open(self.config.STATUS_FILE, 'w') as f:\n",
+    "                        f.write(f\"Validation status: {validation_status}\")\n",
+    "                        \n",
+    "            return validation_status\n",
+    "        \n",
+    "        except Exception as e:\n",
+    "            raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "38fa96fd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-06-26 05:05:39,842: INFO: common : yaml. file: config\\config.yaml loaded successfully. ]\n",
+      "[2025-06-26 05:05:39,847: INFO: common : yaml. file: params.yaml loaded successfully. ]\n",
+      "[2025-06-26 05:05:39,848: INFO: common : created directory at path : artifacts_root ]\n",
+      "[2025-06-26 05:05:39,849: INFO: common : created directory at path : artifacts/data_validation ]\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_validation_config = config.get_data_validation_config()\n",
+    "    data_validation = DataValidation(data_validation_config)\n",
+    "    data_validation.validate_all_files_exists()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textSummarizer",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

research/03_data_transformation.ipynb

@@ -0,0 +1,224 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "aef5ef85",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "13cc3394",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pwd\n",
+    "os.chdir(\"../\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "14c37e99",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\order\\\\Desktop\\\\AI_Summarizer'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f421926f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataTransformationConfig:\n",
+    "    root_dir: Path\n",
+    "    data_path: Path\n",
+    "    tokenizer_name: Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "0731278f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from textSummarizer.constants import *\n",
+    "from textSummarizer.utils.common import read_yaml, create_directories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "f2a0eff4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath: str = CONFIG_FILE_PATH,\n",
+    "        params_filepath: str = PARAMS_FILE_PATH):\n",
+    "        \n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "        \n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "    def get_data_transformation_config(self) -> DataTransformationConfig:\n",
+    "        config= self.config.data_transformation\n",
+    "        \n",
+    "        create_directories([config.root_dir])\n",
+    "        \n",
+    "        data_transformation_config = DataTransformationConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            data_path=config.data_path,\n",
+    "            tokenizer_name=config.tokenizer_name,\n",
+    "        )\n",
+    "        \n",
+    "        return data_transformation_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "adad3d61",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "from textSummarizer.logging import logger\n",
+    "from transformers import AutoTokenizer\n",
+    "from datasets import load_dataset, load_from_disk"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "7369d543",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DataTransformation:\n",
+    "    def __init__(self, config: DataTransformationConfig):\n",
+    "        self.config = config\n",
+    "        self.tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer_name)\n",
+    "        \n",
+    "        \n",
+    "    def convert_examples_to_features(self, example_batch):\n",
+    "        input_encodings = self.tokenizer(example_batch['dialogue'] , max_length=1024, truncation=True)\n",
+    "        \n",
+    "        with self.tokenizer.as_target_tokenizer():\n",
+    "            target_encodings = self.tokenizer(example_batch['summary'], max_length=128, truncation=True)\n",
+    "        \n",
+    "        return {\n",
+    "            'input_ids': input_encodings['input_ids'],\n",
+    "            'attention_mask': input_encodings['attention_mask'],\n",
+    "            'labels': target_encodings['input_ids'],\n",
+    "        }\n",
+    "        \n",
+    "        \n",
+    "    def convert(self):\n",
+    "        dataset_samsum = load_from_disk(self.config.data_path)\n",
+    "        dataset_samsum_pt = dataset_samsum.map(self.convert_examples_to_features, batched=True)\n",
+    "        dataset_samsum_pt.save_to_disk(os.path.join(self.config.root_dir, \"samsum_dataset\"))\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "78cb118a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-06-26 22:06:59,994: INFO: common : yaml. file: config\\config.yaml loaded successfully. ]\n",
+      "[2025-06-26 22:06:59,995: INFO: common : yaml. file: params.yaml loaded successfully. ]\n",
+      "[2025-06-26 22:06:59,997: INFO: common : created directory at path : artifacts_root ]\n",
+      "[2025-06-26 22:06:59,997: INFO: common : created directory at path : artifacts/data_transformation ]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Map:   0%|          | 0/14732 [00:00<?, ? examples/s]c:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:3951: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n",
+      "  warnings.warn(\n",
+      "Map: 100%|██████████| 14732/14732 [00:01<00:00, 8932.08 examples/s]\n",
+      "Map: 100%|██████████| 819/819 [00:00<00:00, 9059.66 examples/s]\n",
+      "Map: 100%|██████████| 818/818 [00:00<00:00, 9570.16 examples/s]\n",
+      "Saving the dataset (1/1 shards): 100%|██████████| 14732/14732 [00:00<00:00, 669372.95 examples/s]\n",
+      "Saving the dataset (1/1 shards): 100%|██████████| 819/819 [00:00<00:00, 180853.69 examples/s]\n",
+      "Saving the dataset (1/1 shards): 100%|██████████| 818/818 [00:00<00:00, 136245.76 examples/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_transformation_config = config.get_data_transformation_config()\n",
+    "    data_transformation = DataTransformation(config=data_transformation_config)\n",
+    "    data_transformation.convert()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textSummarizer",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

research/04_model_trainer.ipynb

@@ -0,0 +1,642 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "83ad55ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "e692ac4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pwd\n",
+    "os.chdir(\"../\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "7c1a28ab",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\order\\\\Desktop\\\\AI_Summarizer'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "2303b7df",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "4319d974",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@dataclass(frozen=True)\n",
+    "class ModelTrainerConfig:\n",
+    "    root_dir: Path\n",
+    "    data_path: Path\n",
+    "    model_ckpt: Path\n",
+    "    num_train_epochs: int\n",
+    "    warmup_steps: int\n",
+    "    per_device_train_batch_size: int\n",
+    "    weight_decay: float\n",
+    "    logging_steps: int\n",
+    "    evaluation_strategy: str\n",
+    "    eval_steps: int\n",
+    "    save_steps: float\n",
+    "    gradient_accumulation_steps: int   "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dfd0b798",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from textSummarizer.constants import *\n",
+    "from textSummarizer.utils.common import read_yaml, create_directories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2b9b5473",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath = CONFIG_FILE_PATH,\n",
+    "        params_filepath = PARAMS_FILE_PATH):\n",
+    "        \n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "        \n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "    def get_model_trainer_config(self) -> ModelTrainerConfig:\n",
+    "        config = self.config.model_trainer\n",
+    "        params = self.params.TrainingArguments\n",
+    "        \n",
+    "        create_directories([config.root_dir])\n",
+    "        \n",
+    "        model_trainer_config = ModelTrainerConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            data_path=config.data_path,\n",
+    "            model_ckpt=config.model_ckpt,\n",
+    "            num_train_epochs=params.num_train_epochs,\n",
+    "            warmup_steps=params.warmup_steps,\n",
+    "            per_device_train_batch_size=params.per_device_train_batch_size,\n",
+    "            weight_decay=params.weight_decay,\n",
+    "            logging_steps=params.logging_steps,\n",
+    "            evaluation_strategy=params.evaluation_strategy,\n",
+    "            eval_steps=params.evaluation_strategy,\n",
+    "            save_steps=params.save_steps,\n",
+    "            gradient_accumulation_steps=params.gradient_accumulation_steps\n",
+    "        )\n",
+    "        \n",
+    "        return model_trainer_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4f5f9714",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-06-29 01:03:26,836: INFO: config : PyTorch version 2.5.1+cu121 available. ]\n",
+      "2.5.1+cu121\n",
+      "True\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import TrainingArguments, Trainer\n",
+    "from transformers import DataCollatorForSeq2Seq\n",
+    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
+    "from datasets import load_dataset, load_from_disk\n",
+    "import torch  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "dda95b61",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ModelTrainer:\n",
+    "    def __init__(self, config: ModelTrainerConfig):\n",
+    "        self.config = config\n",
+    "        \n",
+    "    \n",
+    "    def train(self):\n",
+    "        device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "        tokenizer = AutoTokenizer.from_pretrained(self.config.model_ckpt)\n",
+    "        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_ckpt, use_safetensors=True).to(device)\n",
+    "        seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model_pegasus)\n",
+    "        \n",
+    "        dataset_samsum_pt = load_from_disk(self.config.data_path)\n",
+    "        \n",
+    "        # trainer_args = TrainingArguments(\n",
+    "        #     output_dir=self.config.root_dir,\n",
+    "        #     num_train_epochs=self.config.num_train_epochs,\n",
+    "        #     warmup_steps=self.config.warmup_steps,\n",
+    "        #     per_device_train_batch_size=self.config.per_device_train_batch_size,\n",
+    "        #     per_device_eval_batch_size=self.config.per_device_train_batch_size,\n",
+    "        #     weight_decay=self.config.weight_decay,\n",
+    "        #     logging_steps=self.config.logging_steps,\n",
+    "        #     evaluation_strategy=self.config.evaluation_strategy,\n",
+    "        #     eval_steps=self.config.eval_steps,\n",
+    "        #     save_steps=1e6,\n",
+    "        #     gradient_accumulation_steps=self.config.gradient_accumulation_steps\n",
+    "        # )\n",
+    "        \n",
+    "        \n",
+    "        trainer_args = TrainingArguments(\n",
+    "            output_dir=self.config.root_dir,\n",
+    "            num_train_epochs=1,\n",
+    "            warmup_steps=500,\n",
+    "            per_device_train_batch_size=1,\n",
+    "            per_device_eval_batch_size=1,\n",
+    "            weight_decay=0.01,\n",
+    "            logging_steps=10\n",
+    "        )\n",
+    "\n",
+    "        \n",
+    "        \n",
+    "        trainer = Trainer(model=model_pegasus, args=trainer_args,\n",
+    "                  tokenizer=tokenizer, data_collator=seq2seq_data_collator,\n",
+    "                  train_dataset=dataset_samsum_pt[\"test\"],\n",
+    "                  eval_dataset=dataset_samsum_pt[\"validation\"])\n",
+    "        \n",
+    "        trainer.train()\n",
+    "        \n",
+    "        model_pegasus.save_pretrained(os.path.join(self.config.root_dir, \"pegasus-samsum-model\"))\n",
+    "        \n",
+    "        tokenizer.save_pretrained(os.path.join(self.config.root_dir, \"tokenizer\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "0eba62ba",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-06-29 01:03:38,152: INFO: common : yaml. file: config\\config.yaml loaded successfully. ]\n",
+      "[2025-06-29 01:03:38,154: INFO: common : yaml. file: params.yaml loaded successfully. ]\n",
+      "[2025-06-29 01:03:38,155: INFO: common : created directory at path : artifacts_root ]\n",
+      "[2025-06-29 01:03:38,156: INFO: common : created directory at path : artifacts/model_trainer ]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of PegasusForConditionalGeneration were not initialized from the model checkpoint at google/pegasus-cnn_dailymail and are newly initialized: ['model.decoder.embed_positions.weight', 'model.encoder.embed_positions.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "C:\\Users\\jeeva\\AppData\\Local\\Temp\\ipykernel_19788\\3518460819.py:41: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+      "  trainer = Trainer(model=model_pegasus, args=trainer_args,\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='819' max='819' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [819/819 44:07, Epoch 1/1]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>2.932800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>3.312200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>2.888900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>2.671100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>50</td>\n",
+       "      <td>3.116900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>60</td>\n",
+       "      <td>2.466100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>70</td>\n",
+       "      <td>3.369200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>80</td>\n",
+       "      <td>3.053100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>90</td>\n",
+       "      <td>3.115300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>100</td>\n",
+       "      <td>2.541900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>110</td>\n",
+       "      <td>2.536800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>120</td>\n",
+       "      <td>2.558000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>130</td>\n",
+       "      <td>3.056200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>140</td>\n",
+       "      <td>2.083200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>150</td>\n",
+       "      <td>2.065100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>160</td>\n",
+       "      <td>2.338200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>170</td>\n",
+       "      <td>2.271100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>180</td>\n",
+       "      <td>2.053200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>190</td>\n",
+       "      <td>2.143000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>200</td>\n",
+       "      <td>1.947000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>210</td>\n",
+       "      <td>1.823900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>220</td>\n",
+       "      <td>2.339900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>230</td>\n",
+       "      <td>1.958000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>240</td>\n",
+       "      <td>2.743100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>250</td>\n",
+       "      <td>2.370100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>260</td>\n",
+       "      <td>1.970100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>270</td>\n",
+       "      <td>2.276200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>280</td>\n",
+       "      <td>2.156200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>290</td>\n",
+       "      <td>1.602900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>300</td>\n",
+       "      <td>2.405400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>310</td>\n",
+       "      <td>2.053100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>320</td>\n",
+       "      <td>2.130500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>330</td>\n",
+       "      <td>1.777200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>340</td>\n",
+       "      <td>2.240200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>350</td>\n",
+       "      <td>1.661200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>360</td>\n",
+       "      <td>1.875600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>370</td>\n",
+       "      <td>2.144000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>380</td>\n",
+       "      <td>2.140400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>390</td>\n",
+       "      <td>1.908100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>400</td>\n",
+       "      <td>1.828400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>410</td>\n",
+       "      <td>1.890400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>420</td>\n",
+       "      <td>1.912100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>430</td>\n",
+       "      <td>1.972800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>440</td>\n",
+       "      <td>1.950800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>450</td>\n",
+       "      <td>1.688000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>460</td>\n",
+       "      <td>2.048800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>470</td>\n",
+       "      <td>1.665500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>480</td>\n",
+       "      <td>1.779100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>490</td>\n",
+       "      <td>1.541600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>500</td>\n",
+       "      <td>2.011600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>510</td>\n",
+       "      <td>2.258400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>520</td>\n",
+       "      <td>1.945900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>530</td>\n",
+       "      <td>1.740100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>540</td>\n",
+       "      <td>1.871500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>550</td>\n",
+       "      <td>1.490400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>560</td>\n",
+       "      <td>2.026400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>570</td>\n",
+       "      <td>1.529800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>580</td>\n",
+       "      <td>2.545900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>590</td>\n",
+       "      <td>1.941400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>600</td>\n",
+       "      <td>1.714600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>610</td>\n",
+       "      <td>1.986300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>620</td>\n",
+       "      <td>1.499800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>630</td>\n",
+       "      <td>1.948500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>640</td>\n",
+       "      <td>1.451600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>650</td>\n",
+       "      <td>1.522000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>660</td>\n",
+       "      <td>1.699000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>670</td>\n",
+       "      <td>1.506800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>680</td>\n",
+       "      <td>1.235000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>690</td>\n",
+       "      <td>1.657100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>700</td>\n",
+       "      <td>1.871200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>710</td>\n",
+       "      <td>2.028800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>720</td>\n",
+       "      <td>1.872100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>730</td>\n",
+       "      <td>1.893200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>740</td>\n",
+       "      <td>1.719900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>750</td>\n",
+       "      <td>1.739000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>760</td>\n",
+       "      <td>1.705900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>770</td>\n",
+       "      <td>2.103700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>780</td>\n",
+       "      <td>1.915000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>790</td>\n",
+       "      <td>2.152000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>800</td>\n",
+       "      <td>1.793000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>810</td>\n",
+       "      <td>1.500600</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\modeling_utils.py:3685: UserWarning: Moving the following attributes in the config to the generation config: {'max_length': 128, 'min_length': 32, 'num_beams': 8, 'length_penalty': 0.8}. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    model_trainer_config = config.get_model_trainer_config()\n",
+    "    model_trainer_config = ModelTrainer(config=model_trainer_config)\n",
+    "    model_trainer_config.train()\n",
+    "\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textSummarizer",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

research/05_Model_evaluation.ipynb

@@ -0,0 +1,210 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "d83f232a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7f3a1697",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pwd\n",
+    "os.chdir(\"../\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "70433f0a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\order\\\\Desktop\\\\AI_Summarizer'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4b141e83",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ffb2ded1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@dataclass(frozen=True)\n",
+    "class ModelEvaluationConfig:\n",
+    "    root_dir: Path\n",
+    "    data_path: Path\n",
+    "    model_path: Path\n",
+    "    tokenizer_path: Path\n",
+    "    metric_file_name: Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4cbe6e7f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from textSummarizer.constants import *\n",
+    "from textSummarizer.utils.common import read_yaml, create_directories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "6ef66517",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath = CONFIG_FILE_PATH,\n",
+    "        params_filepath = PARAMS_FILE_PATH):\n",
+    "        \n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "        \n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "        \n",
+    "    def  get_model_evaluation_config(self) -> ModelEvaluationConfig:\n",
+    "        config = self.config.model_evaluation\n",
+    "        \n",
+    "        create_directories([config.root_dir])\n",
+    "        \n",
+    "        model_evaluation_config = ModelEvaluationConfig(\n",
+    "            root_dir = config.root_dir,\n",
+    "            data_path= config.data_path,\n",
+    "            model_path = config.model_path,\n",
+    "            tokenizer_path= config.tokenizer_path,\n",
+    "            metric_file_name = config.metric_file_name\n",
+    "        )\n",
+    "        \n",
+    "        return model_evaluation_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "98b6adaf",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-07-02 02:05:06,043: INFO: config : PyTorch version 2.5.1+cu121 available. ]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
+    "from datasets import load_dataset, load_from_disk\n",
+    "import torch\n",
+    "import pandas as pd\n",
+    "from tqdm import tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ac50b978",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ModelEvaluation:\n",
+    "    def __init__(self, config: ModelEvaluationConfig):\n",
+    "        self.config = config\n",
+    "    \n",
+    "    \n",
+    "    def generate_batch_sized_chunks(self, List_of_elements, batch_size):\n",
+    "        \"\"\"split the dataset into smaller batches that we can process simulaneously\n",
+    "        yield successive batch-sized chunks from list_of_elements.\"\"\"\n",
+    "        \n",
+    "        for i in range(0, len(List_of_elements), batch_size):\n",
+    "            yield List_of_elements[i : i + batch_size]\n",
+    "            \n",
+    "    def calculate_metric_on_test_ds(self, dataset, metric, model, tokenizer,\n",
+    "                                    batch_size=16, device=\"cuda\" if torch.cuda.is_available() else \"cpu\",\n",
+    "                                    column_text=\"article\",                \n",
+    "                                    column_summary=\"highlights\"):\n",
+    "        article_batches = list(self.generate_batch_sized_chunks(dataset[column_text], batch_size))\n",
+    "        target_batches = list(self.generate_batch_sized_chunks(dataset[column_summary], batch_size))\n",
+    "        \n",
+    "        for article_batch, target_batch in tqdm(\n",
+    "            zip(article_batches, target_batches), total=len(article_batches)):\n",
+    "            \n",
+    "            inputs = tokenizer( article_batch, max_length=1024, truncation=True,\n",
+    "                padding=\"max_length\", return_tensors=\"pt\")\n",
+    "            \n",
+    "            s\n",
+    "        \n",
+    "        \n",
+    "        \n",
+    "        \n",
+    "        "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textSummarizer",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

research/trails.ipynb

@@ -0,0 +1,266 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "09cfed6a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "d = {\"key\" : \" value 1\", \"key2\" : \"value 2\", \"key3\" : \"value 3\"}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "e6fd4e25",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "' value 1'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "d['key']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1602f57d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "dict_keys(['key', 'key2', 'key3'])"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "d.keys()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "d16c1607",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from box import ConfigBox"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "1e392ff6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "d2 = ConfigBox(d)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e0d6f2c1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ConfigBox({'key': ' value 1', 'key2': 'value 2', 'key3': 'value 3'})"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "d2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "08765318",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "' value 1'"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "d2.key"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "851457ac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_product(x: int  , y: int) -> int:\n",
+    "    return x * y"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "ff2ee71b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "8"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "get_product(x = 2,y = 4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "d37d2718",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'44'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "get_product(x = 2, y = \"4\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "153cbd1d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ensure import ensure_annotations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "a65f4a8e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@ensure_annotations\n",
+    "def get_product(x: int  , y: int) -> int:\n",
+    "    return x * y"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "61cb9d3e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "8"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "get_product(x = 2,y = 4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "46a5e079",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "EnsureError",
+     "evalue": "Argument y of type <class 'str'> to <function get_product at 0x0000024DC30C3380> does not match annotation type <class 'int'>",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mEnsureError\u001b[39m                               Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mget_product\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m4\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\ensure\\main.py:870\u001b[39m, in \u001b[36mWrappedFunctionReturn.__call__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m    868\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(value, templ):\n\u001b[32m    869\u001b[39m         msg = \u001b[33m\"\u001b[39m\u001b[33mArgument \u001b[39m\u001b[38;5;132;01m{arg}\u001b[39;00m\u001b[33m of type \u001b[39m\u001b[38;5;132;01m{valt}\u001b[39;00m\u001b[33m to \u001b[39m\u001b[38;5;132;01m{f}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdoes not match annotation type \u001b[39m\u001b[38;5;132;01m{t}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m--> \u001b[39m\u001b[32m870\u001b[39m         \u001b[38;5;28;01mraise\u001b[39;00m EnsureError(msg.format(arg=arg, f=\u001b[38;5;28mself\u001b[39m.f, t=templ, valt=\u001b[38;5;28mtype\u001b[39m(value)))\n\u001b[32m    872\u001b[39m return_val = \u001b[38;5;28mself\u001b[39m.f(*args, **kwargs)\n\u001b[32m    873\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_val, \u001b[38;5;28mself\u001b[39m.return_templ):\n",
+      "\u001b[31mEnsureError\u001b[39m: Argument y of type <class 'str'> to <function get_product at 0x0000024DC30C3380> does not match annotation type <class 'int'>"
+     ]
+    }
+   ],
+   "source": [
+    "get_product(x = 2, y = \"4\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b9e1c167",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textSummarizer",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

setup.py

@@ -0,0 +1,27 @@
+import setuptools
+
+with open("README.md" , "r" , encoding="utf-8") as f:
+    long_description = f.read()
+    
+__version__ = "0.0.0"
+
+REPO_NAME = "AI_Summarizer"
+AUTHOR_USER_NAME = "jeevant"
+SRC_REPO = "textSummarizer"
+AUTHOR_EMAIL = "jeevantmudgil10@gmail.com"
+
+setuptools.setup(
+    name=SRC_REPO,
+    version=__version__,
+    author=AUTHOR_USER_NAME,
+    author_email=AUTHOR_EMAIL,
+    description="A Small python package for Text NLP App",
+    long_description=long_description,
+    long_description_content="text/markdown",
+    url=f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}",
+    project_urls={
+        "Bug Tracker": f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}/issues",
+    },
+    package_dir={"": "src"},
+    packages=setuptools.find_packages(where="src")
+)

src/textSummarizer/components/data_ingestion.py

@@ -0,0 +1,34 @@
+import os
+import urllib.request as request
+import zipfile
+from textSummarizer.logging import logger
+from textSummarizer.utils.common import get_size
+from pathlib import Path
+from textSummarizer.entity import DataIngestionConfig
+
+class DataIngestion:
+    def __init__(self, config: DataIngestionConfig):
+        self.config = config
+        
+        
+    def download_data(self):
+        if not os.path.exists(self.config.local_data_file):
+            filename, headers = request.urlretrieve(
+            url = self.config.source_URL,
+            filename = self.config.local_data_file
+            )
+            logger.info(f"{filename} download! with following info : \n{headers}")
+        else:
+            logger.info(f"File already exists of size : {get_size(Path(self.config.local_data_file))}")
+            
+    def extract_zip_file(self):
+        """
+        zip_file_path : str
+        EXtracts the zip file to the given directory.
+        Functions returns None.
+        """
+        unzip_path = self.config.unzip_dir
+        os.makedirs(unzip_path, exist_ok=True)
+        with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref:
+            zip_ref.extractall(unzip_path)
+            

src/textSummarizer/components/data_transformation.py

@@ -0,0 +1,31 @@
+import os
+from textSummarizer.logging import logger
+from transformers import AutoTokenizer
+from datasets import load_dataset, load_from_disk
+from textSummarizer.entity import DataTransformationConfig
+
+
+class DataTransformation:
+    def __init__(self, config: DataTransformationConfig):
+        self.config = config
+        self.tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer_name)
+        
+        
+    def convert_examples_to_features(self, example_batch):
+        input_encodings = self.tokenizer(example_batch['dialogue'] , max_length=1024, truncation=True)
+        
+        with self.tokenizer.as_target_tokenizer():
+            target_encodings = self.tokenizer(example_batch['summary'], max_length=128, truncation=True)
+        
+        return {
+            'input_ids': input_encodings['input_ids'],
+            'attention_mask': input_encodings['attention_mask'],
+            'labels': target_encodings['input_ids'],
+        }
+        
+        
+    def convert(self):
+        dataset_samsum = load_from_disk(self.config.data_path)
+        dataset_samsum_pt = dataset_samsum.map(self.convert_examples_to_features, batched=True)
+        dataset_samsum_pt.save_to_disk(os.path.join(self.config.root_dir, "samsum_dataset"))
+        

src/textSummarizer/components/data_validation.py

@@ -0,0 +1,30 @@
+import os
+from textSummarizer.logging import logger
+from textSummarizer.entity import DataValidationConfig
+
+class DataValidation:
+    def __init__(self, config: DataValidationConfig):
+        self.config = config
+        
+        
+        
+    def validate_all_files_exists(self) -> bool:
+        try:
+            validation_status = None
+            
+            all_files = os.listdir(os.path.join("artifacts" , "data_ingestion" , "samsum_dataset"))
+            
+            for file in all_files:
+                if file not in self.config.ALL_REQUIRED_FILES:
+                    validation_status = False
+                    with open(self.config.STATUS_FILE, 'w') as f:
+                        f.write(f"Validation status: {validation_status}")
+                else:
+                    validation_status = True
+                    with open(self.config.STATUS_FILE, 'w') as f:
+                        f.write(f"Validation status: {validation_status}")
+                        
+            return validation_status
+        
+        except Exception as e:
+            raise e

src/textSummarizer/components/model_trainer.py

@@ -0,0 +1,57 @@
+from transformers import TrainingArguments, Trainer
+from transformers import DataCollatorForSeq2Seq
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from datasets import load_dataset, load_from_disk
+from textSummarizer.entity import ModelTrainerConfig
+import torch  
+
+class ModelTrainer:
+    def __init__(self, config: ModelTrainerConfig):
+        self.config = config
+        
+    
+    def train(self):
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        tokenizer = AutoTokenizer.from_pretrained(self.config.model_ckpt)
+        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_ckpt, use_safetensors=True).to(device)
+        seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model_pegasus)
+        
+        dataset_samsum_pt = load_from_disk(self.config.data_path)
+        
+        # trainer_args = TrainingArguments(
+        #     output_dir=self.config.root_dir,
+        #     num_train_epochs=self.config.num_train_epochs,
+        #     warmup_steps=self.config.warmup_steps,
+        #     per_device_train_batch_size=self.config.per_device_train_batch_size,
+        #     per_device_eval_batch_size=self.config.per_device_train_batch_size,
+        #     weight_decay=self.config.weight_decay,
+        #     logging_steps=self.config.logging_steps,
+        #     evaluation_strategy=self.config.evaluation_strategy,
+        #     eval_steps=self.config.eval_steps,
+        #     save_steps=1e6,
+        #     gradient_accumulation_steps=self.config.gradient_accumulation_steps
+        # )
+        
+        
+        trainer_args = TrainingArguments(
+            output_dir=self.config.root_dir,
+            num_train_epochs=1,
+            warmup_steps=500,
+            per_device_train_batch_size=1,
+            per_device_eval_batch_size=1,
+            weight_decay=0.01,
+            logging_steps=10
+        )
+
+        
+        
+        trainer = Trainer(model=model_pegasus, args=trainer_args,
+                  tokenizer=tokenizer, data_collator=seq2seq_data_collator,
+                  train_dataset=dataset_samsum_pt["test"],
+                  eval_dataset=dataset_samsum_pt["validation"])
+        
+        trainer.train()
+        
+        model_pegasus.save_pretrained(os.path.join(self.config.root_dir, "pegasus-samsum-model"))
+        
+        tokenizer.save_pretrained(os.path.join(self.config.root_dir, "tokenizer"))

src/textSummarizer/config/configuration.py

@@ -0,0 +1,85 @@
+from textSummarizer.constants import *
+from textSummarizer.utils.common import read_yaml, create_directories
+from textSummarizer.entity import (DataIngestionConfig,
+                                   DataValidationConfig,
+                                   DataTransformationConfig,
+                                   ModelTrainerConfig)
+
+
+class ConfigurationManager:
+    def __init__(
+        self,
+        config_filepath: str = CONFIG_FILE_PATH,
+        params_filepath: str = PARAMS_FILE_PATH):
+        
+        self.config = read_yaml(config_filepath)
+        self.params = read_yaml(params_filepath)
+        
+        create_directories([self.config.artifacts_root])
+        
+        
+    def get_data_ingestion_config(self) -> DataIngestionConfig:
+        config =self.config.data_ingestion
+        
+        create_directories([config.root_dir])
+        
+        data_ingestion_config = DataIngestionConfig(
+            root_dir = config.root_dir,
+            source_URL = config.source_URL,
+            local_data_file = config.local_data_file,
+            unzip_dir = config.unzip_dir,
+        )
+        
+        return data_ingestion_config
+    
+    
+    def get_data_validation_config(self) -> DataValidationConfig:
+        config = self.config.data_validation
+        
+        create_directories([config.root_dir])
+        
+        data_validation_config = DataValidationConfig(
+            root_dir=config.root_dir,
+            STATUS_FILE=config.STATUS_FILE,
+            ALL_REQUIRED_FILES=config.ALL_REQUIRED_FILES,
+        )
+        
+        return data_validation_config
+    
+    
+    def get_data_transformation_config(self) -> DataTransformationConfig:
+        config= self.config.data_transformation
+        
+        create_directories([config.root_dir])
+        
+        data_transformation_config = DataTransformationConfig(
+            root_dir=config.root_dir,
+            data_path=config.data_path,
+            tokenizer_name=config.tokenizer_name,
+        )
+        
+        return data_transformation_config
+    
+    
+    def get_model_trainer_config(self) -> ModelTrainerConfig:
+        config = self.config.model_trainer
+        params = self.params.TrainingArguments
+        
+        create_directories([config.root_dir])
+        
+        model_trainer_config = ModelTrainerConfig(
+            root_dir=config.root_dir,
+            data_path=config.data_path,
+            model_ckpt=config.model_ckpt,
+            num_train_epochs=params.num_train_epochs,
+            warmup_steps=params.warmup_steps,
+            per_device_train_batch_size=params.per_device_train_batch_size,
+            weight_decay=params.weight_decay,
+            logging_steps=params.logging_steps,
+            evaluation_strategy=params.evaluation_strategy,
+            eval_steps=params.evaluation_strategy,
+            save_steps=params.save_steps,
+            gradient_accumulation_steps=params.gradient_accumulation_steps
+        )
+        
+        return model_trainer_config

src/textSummarizer/constants/__init__.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+CONFIG_FILE_PATH = Path("config/config.yaml")
+PARAMS_FILE_PATH = Path("params.yaml")

src/textSummarizer/entity/__init__.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+@dataclass(frozen=True)
+class DataIngestionConfig:
+    root_dir : Path
+    source_URL : str
+    local_data_file : Path
+    unzip_dir : Path
+    
+    
+@dataclass(frozen=True)
+class DataValidationConfig:
+    root_dir : Path
+    STATUS_FILE : str
+    ALL_REQUIRED_FILES : list
+    
+    
+@dataclass(frozen=True)
+class DataTransformationConfig:
+    root_dir: Path
+    data_path: Path
+    tokenizer_name: Path
+    
+    
+    
+@dataclass(frozen=True)
+class ModelTrainerConfig:
+    root_dir: Path
+    data_path: Path
+    model_ckpt: Path
+    num_train_epochs: int
+    warmup_steps: int
+    per_device_train_batch_size: int
+    weight_decay: float
+    logging_steps: int
+    evaluation_strategy: str
+    eval_steps: int
+    save_steps: float
+    gradient_accumulation_steps: int   

src/textSummarizer/logging/__init__.py

@@ -0,0 +1,21 @@
+import os
+import sys
+import logging
+
+logging_str = "[%(asctime)s: %(levelname)s: %(module)s : %(message)s ]"
+log_dir = "logs"
+log_filepath = os.path.join(log_dir, "running_logs.log")
+os.makedirs(log_dir, exist_ok=True)
+
+
+logging.basicConfig(
+    level= logging.INFO,
+    format= logging_str,
+    
+    handlers=[
+        logging.FileHandler(log_filepath),
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+
+logger = logging.getLogger("textSummarizerLogger")

src/textSummarizer/pipeline/stage_01_data_ingestion.py

@@ -0,0 +1,16 @@
+from textSummarizer.config.configuration import ConfigurationManager
+from textSummarizer.components.data_ingestion import DataIngestion
+from textSummarizer.logging import logger
+
+class DataIngestionTrainingPipeline:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        data_ingestion_config = config.get_data_ingestion_config()
+        data_ingestion = DataIngestion(data_ingestion_config)
+        data_ingestion.download_data()
+        data_ingestion.extract_zip_file()
+        
+        

src/textSummarizer/pipeline/stage_02_data_validation.py

@@ -0,0 +1,15 @@
+from textSummarizer.config.configuration import ConfigurationManager
+from textSummarizer.components.data_validation import DataValidation
+from textSummarizer.logging import logger
+
+class DataValidationTrainingPipeline:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        data_validation_config = config.get_data_validation_config()
+        data_validation = DataValidation(data_validation_config)
+        data_validation.validate_all_files_exists()
+        
+        

src/textSummarizer/pipeline/stage_03_data_transformation.py

@@ -0,0 +1,15 @@
+from textSummarizer.config.configuration import ConfigurationManager
+from textSummarizer.components.data_transformation import DataTransformation
+from textSummarizer.logging import logger
+
+class DataTransformationTrainingPipeline:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        data_transformation_config = config.get_data_transformation_config()
+        data_transformation = DataTransformation(config=data_transformation_config)
+        data_transformation.convert()
+        
+        

src/textSummarizer/pipeline/stage_04_model_trainer.py

@@ -0,0 +1,15 @@
+from textSummarizer.config.configuration import ConfigurationManager
+from textSummarizer.components.model_trainer import ModelTrainer
+from textSummarizer.logging import logger
+
+class ModelTrainerTrainingPipeline:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        model_trainer_config = config.get_model_trainer_config()
+        model_trainer_config = ModelTrainer(config=model_trainer_config)
+        model_trainer_config.train()
+        
+        

src/textSummarizer/utils/common.py

@@ -0,0 +1,64 @@
+import os
+from box.exceptions import BoxValueError
+import yaml
+from textSummarizer.logging import logger
+from ensure import ensure_annotations
+from box import ConfigBox
+from pathlib import Path
+from typing import Any
+
+@ensure_annotations
+def read_yaml(path_to_yaml: Path) -> ConfigBox:
+    """_summary_
+    Read YAML file and convert it into a ConfigBox object.
+
+    Args:
+        path_to_yaml (Path): _description_
+
+    
+    Raises:
+        BoxValueError: If the YAML file cannot be parsed.
+        ValueError: If the YAML file does not contain a dictionary.
+        e: empty file error
+        
+    Returns:
+        ConfigBox : ConfigBox type
+    """
+    
+    try:
+        with open(path_to_yaml) as yaml_file:
+            content = yaml.safe_load(yaml_file)
+            logger.info(f"yaml. file: {path_to_yaml} loaded successfully.")
+            return ConfigBox(content)
+    except BoxValueError:
+        raise ValueError(f"YAML file is empty with path : {path_to_yaml}")
+    except Exception as e:
+        raise e
+    
+
+@ensure_annotations
+def create_directories(path_to_directories: list, verbose=True):
+    """
+    create list of directories 
+    
+    Args:
+        path_to_directory (list): list of directories.
+        ignore_log(bool, optional) : ignore if multiple directories are created. Defaults to False.
+    """ 
+    for path in path_to_directories:
+        os.makedirs(path, exist_ok=True)
+        if verbose:
+            logger.info(f"created directory at path : {path}")
+            
+@ensure_annotations
+def get_size(path: Path) ->str:
+    """get size of file in kbs
+
+    Args:
+        path (Path): Path of the files
+
+    Returns:
+        str: =size in KB
+    """
+    size_in_kb = round(os.path.getsize(path)/1024)
+    return f"~{size_in_kb} KB"

template.py

@@ -0,0 +1,46 @@
+import os
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO, format='[%(asctime)s]:(message)s:')
+
+project_name = "testSummarizer"
+
+list_of_files = [
+    ".github/workflows/.gitkeep",
+    f"src/{project_name}/__init__.py",
+    f"src/{project_name}/components/__init__.py",
+    f"src/{project_name}/utils/__init__.py",
+    f"src/{project_name}/utils/common.py",
+    f"src/{project_name}/logging/__init__.py",
+    f"src/{project_name}/config/__init__.py",
+    f"src/{project_name}/config/configuration.py",
+    f"src/{project_name}/pipeline/__init__.py",
+    f"src/{project_name}/entity/__init__.py",
+    f"src/{project_name}/constants/__init__.py",
+    "config/config.yaml",
+    "params.yaml",
+    "app.py",
+    "main.py",
+    "Dockerfile",
+    "requirements.txt",
+    "setup.py",
+    "research/trails.ipynb"
+    
+]
+
+for filepath in list_of_files:
+    filepath = Path(filepath)
+    filedir, filename = os.path.split(filepath)
+    
+    if filedir != "":
+        os.makedirs(filedir, exist_ok=True)
+        logging.info(f"Creating directry:{filepath} for the file {filepath}")
+        
+    if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0):
+        with open(filepath, 'w') as f:
+            pass
+            logging.info(f"Creating empty file: {filepath}")
+            
+    else :
+        logging.info(f"{filename} is already exists.")