running it on newer techs

app.py

@@ -1,15 +1,12 @@
 from fastapi import FastAPI
 import uvicorn
+import subprocess
 import sys
-import os
-from fastapi.templating import Jinja2Templates
 from starlette.responses import RedirectResponse
 from fastapi.responses import Response
 from textSummarizer.pipeline.prediction import PredictionPipeline
 
-txt:str = "What is Text Summarization?"
-
-app = FastAPI()
+app = FastAPI(title="AI Text Summarizer", version="1.0.0")
 
 @app.get("/", tags=["authentication"] )
 async def index():
@@ -19,10 +16,17 @@ async def index():
 @app.get("/train")
 async def training():
     try:
-        os.system("python main.py")
+        result = subprocess.run(
+            [sys.executable, "main.py"],
+            capture_output=True, text=True, timeout=7200
+        )
+        if result.returncode != 0:
+            return Response(f"Training failed!\n{result.stderr}", status_code=500)
         return Response("Training completed successfully!")
+    except subprocess.TimeoutExpired:
+        return Response("Training timed out!", status_code=504)
     except Exception as e:
-        return Response(f"Error Occurred! {e}")
+        return Response(f"Error Occurred! {e}", status_code=500)
     
     
 @app.post("/predict")

config/config.yaml

@@ -1,4 +1,4 @@
-artifacts_root: artifacts_root
+artifacts_root: artifacts
 
 
 data_ingestion:
@@ -12,18 +12,26 @@ data_validation:
   root_dir: artifacts/data_validation
   STATUS_FILE: artifacts/data_validation/status.txt
   ALL_REQUIRED_FILES: ["train" , "test" , "validation"]
+  data_dir: artifacts/data_ingestion/samsum_dataset
 
 
 data_transformation:
   root_dir: artifacts/data_transformation
   data_path: artifacts/data_ingestion/samsum_dataset
   tokenizer_name: google/pegasus-cnn_dailymail
+  # Dev quick-run tokenizer override (use same model as trainer to avoid token mismatch)
+  dev_run: true
+  dev_model: t5-small
 
 
 model_trainer:
   root_dir: artifacts/model_trainer
   data_path: artifacts/data_transformation/samsum_dataset
   model_ckpt: google/pegasus-cnn_dailymail
+  # Development quick-train options
+  dev_run: true
+  dev_model: t5-small
+  dev_subset: 64
 
 
 model_evaluation:

params.yaml

@@ -4,7 +4,7 @@ TrainingArguments:
   per_device_train_batch_size: 1
   weight_decay: 0.01
   logging_steps: 10
-  evaluation_strategy: steps
+  eval_strategy: steps
   eval_steps: 500
   save_steps: 1e6
   gradient_accumulation_steps: 16

requirements.txt

@@ -1,21 +1,26 @@
-transformers
-transformers[sentencepiece]
-dataset
-sacrebleu
-py7zr
-pandas
-nltk
-tqdm
-PyYAML
-matplotlib
-torch
-notebook
-boto3
-mypy-boto3-s3
+transformers>=5.0.0
+transformers[torch]>=5.0.0
+sentencepiece>=0.1.98
+datasets>=2.5.0
+evaluate>=0.4.0
+rouge-score>=0.1.2
+sacrebleu>=2.0.0
+py7zr>=0.19.0
+pandas>=1.5.0
+nltk>=3.8.0
+tqdm>=4.60.0
+PyYAML>=6.0
+matplotlib>=3.6.0
+# NOTE: pin `torch` per target machine; this is a minimum
+torch>=2.0.0
+accelerate>=0.26.0
+notebook>=6.4.0
+boto3>=1.26.0
+mypy-boto3-s3>=1.25.0
 python-box==6.0.2
-ensure==1.0.2
-fastapi==0.78.0
-uvicorn==0.18.3
-Jinja2==3.1.2
-rouge-score
+fastapi>=0.95.0
+uvicorn>=0.22.0
+Jinja2>=3.1.2
+python-multipart>=0.0.6
+# Editable install of local package
 -e .

research/01_data_ingestion.ipynb

@@ -195,7 +195,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "textSummarizer",
+   "display_name": "model",
    "language": "python",
    "name": "python3"
   },
@@ -209,7 +209,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.13"
+   "version": "3.13.5"
   }
  },
  "nbformat": 4,

setup.py

@@ -1,8 +1,8 @@
 import setuptools
 
-with open("README.md" , "r" , encoding="utf-8") as f:
+with open("README.md", "r", encoding="utf-8") as f:
     long_description = f.read()
-    
+
 __version__ = "0.0.0"
 
 REPO_NAME = "AI_Summarizer"
@@ -17,11 +17,18 @@ setuptools.setup(
     author_email=AUTHOR_EMAIL,
     description="A Small python package for Text NLP App",
     long_description=long_description,
-    long_description_content="text/markdown",
+    long_description_content_type="text/markdown",
     url=f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}",
     project_urls={
         "Bug Tracker": f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}/issues",
     },
     package_dir={"": "src"},
-    packages=setuptools.find_packages(where="src")
+    packages=setuptools.find_packages(where="src"),
+    python_requires=">=3.11, <3.14",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3 :: Only",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
 )

src/textSummarizer/components/data_transformation.py

@@ -7,7 +7,12 @@ from textSummarizer.entity import DataTransformationConfig
 class DataTransformation:
     def __init__(self, config: DataTransformationConfig):
         self.config = config
-        self.tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer_name)
+        # choose tokenizer checkpoint: prefer dev_model when dev_run is enabled
+        tokenizer_checkpoint = self.config.tokenizer_name
+        if getattr(self.config, 'dev_run', False) and getattr(self.config, 'dev_model', None):
+            tokenizer_checkpoint = self.config.dev_model
+
+        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_checkpoint)
         
     def convert_examples_to_features(self, example_batch):
         input_encodings = self.tokenizer(
@@ -31,6 +36,13 @@ class DataTransformation:
         dataset_samsum = load_from_disk(str(self.config.data_path))
         logger.info("Tokenizing dataset...")
         dataset_samsum_pt = dataset_samsum.map(self.convert_examples_to_features, batched=True)
+        
+        # ensure labels are correctly formatted for seq2seq Trainer
+        def rename_for_trainer(batch):
+            # already returns 'labels' in convert_examples_to_features; keep
+            return batch
+
+        dataset_samsum_pt = dataset_samsum_pt.map(rename_for_trainer, batched=True)
         save_path = self.config.root_dir / "samsum_dataset"
         os.makedirs(save_path, exist_ok=True)
         logger.info(f"Saving processed dataset to {save_path}")

src/textSummarizer/components/data_validation.py

@@ -9,7 +9,7 @@ class DataValidation:
     def validate_all_files_exists(self) -> bool:
         try:
             # Use config for dataset directory
-            dataset_dir = self.config.root_dir
+            dataset_dir = self.config.data_dir
             all_files = set(os.listdir(dataset_dir))
 
             required_files = set(self.config.ALL_REQUIRED_FILES)

src/textSummarizer/components/model_trainer.py

@@ -13,43 +13,45 @@ class ModelTrainer:
     
     def train(self):
         device = "cuda" if torch.cuda.is_available() else "cpu"
-        tokenizer = AutoTokenizer.from_pretrained(self.config.model_ckpt)
-        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_ckpt, use_safetensors=True).to(device)
+        # allow dev-run override for a small quick model
+        model_checkpoint = self.config.model_ckpt
+        if getattr(self.config, 'dev_run', False) and getattr(self.config, 'dev_model', None):
+            model_checkpoint = self.config.dev_model
+
+        tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
+        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint).to(device)
         seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model_pegasus)
         
-        dataset_samsum_pt = load_from_disk(self.config.data_path)
-        
-        # trainer_args = TrainingArguments(
-        #     output_dir=self.config.root_dir,
-        #     num_train_epochs=self.config.num_train_epochs,
-        #     warmup_steps=self.config.warmup_steps,
-        #     per_device_train_batch_size=self.config.per_device_train_batch_size,
-        #     per_device_eval_batch_size=self.config.per_device_train_batch_size,
-        #     weight_decay=self.config.weight_decay,
-        #     logging_steps=self.config.logging_steps,
-        #     evaluation_strategy=self.config.evaluation_strategy,
-        #     eval_steps=self.config.eval_steps,
-        #     save_steps=1e6,
-        #     gradient_accumulation_steps=self.config.gradient_accumulation_steps
-        # )
-        
+        dataset_samsum_pt = load_from_disk(str(self.config.data_path))
+
+        # If dev_run is enabled, take small subsets to speed up training
+        if getattr(self.config, 'dev_run', False) and getattr(self.config, 'dev_subset', 0) > 0:
+            subset = int(self.config.dev_subset)
+            for split in ["test", "validation"]:
+                if split in dataset_samsum_pt:
+                    n = min(subset, len(dataset_samsum_pt[split]))
+                    dataset_samsum_pt[split] = dataset_samsum_pt[split].select(range(n))
         
         trainer_args = TrainingArguments(
             output_dir=self.config.root_dir,
-            num_train_epochs=1,
-            warmup_steps=500,
-            per_device_train_batch_size=1,
-            per_device_eval_batch_size=1,
-            weight_decay=0.01,
-            logging_steps=10
+            num_train_epochs=self.config.num_train_epochs,
+            warmup_steps=self.config.warmup_steps,
+            per_device_train_batch_size=self.config.per_device_train_batch_size,
+            per_device_eval_batch_size=self.config.per_device_train_batch_size,
+            weight_decay=self.config.weight_decay,
+            logging_steps=self.config.logging_steps,
+            eval_strategy=self.config.eval_strategy,
+            eval_steps=self.config.eval_steps,
+            save_steps=1e6,
+            gradient_accumulation_steps=self.config.gradient_accumulation_steps,
         )
 
         
         
         trainer = Trainer(model=model_pegasus, args=trainer_args,
-                  tokenizer=tokenizer, data_collator=seq2seq_data_collator,
-                  train_dataset=dataset_samsum_pt["test"],
-                  eval_dataset=dataset_samsum_pt["validation"])
+              processing_class=tokenizer, data_collator=seq2seq_data_collator,
+              train_dataset=dataset_samsum_pt["test"],
+              eval_dataset=dataset_samsum_pt["validation"])
         
         trainer.train()
         

src/textSummarizer/config/configuration.py

@@ -40,9 +40,10 @@ class ConfigurationManager:
         create_directories([config.root_dir])
         
         data_validation_config = DataValidationConfig(
-            root_dir=config.root_dir,
+            root_dir=Path(config.root_dir),
             STATUS_FILE=config.STATUS_FILE,
             ALL_REQUIRED_FILES=config.ALL_REQUIRED_FILES,
+            data_dir=Path(config.data_dir),
         )
         
         return data_validation_config
@@ -57,6 +58,8 @@ class ConfigurationManager:
             root_dir=Path(config.root_dir),
             data_path=Path(config.data_path),
             tokenizer_name=config.tokenizer_name,  # if this is a path, use Path(); if just a model name, keep as str
+            dev_run=getattr(config, 'dev_run', False),
+            dev_model=getattr(config, 'dev_model', None),
         )
         
         return data_transformation_config
@@ -77,11 +80,33 @@ class ConfigurationManager:
             per_device_train_batch_size=params.per_device_train_batch_size,
             weight_decay=params.weight_decay,
             logging_steps=params.logging_steps,
-            evaluation_strategy=params.evaluation_strategy,
-            eval_steps=params.evaluation_strategy,
+            eval_strategy=params.eval_strategy,
+            eval_steps=params.eval_steps,
             save_steps=params.save_steps,
             gradient_accumulation_steps=params.gradient_accumulation_steps
         )
+
+        # Optional dev quick-train settings (not all configs will have these)
+        try:
+            model_trainer_config = ModelTrainerConfig(
+                root_dir=Path(config.root_dir),
+                data_path=Path(config.data_path),
+                model_ckpt=config.model_ckpt,
+                num_train_epochs=params.num_train_epochs,
+                warmup_steps=params.warmup_steps,
+                per_device_train_batch_size=params.per_device_train_batch_size,
+                weight_decay=params.weight_decay,
+                logging_steps=params.logging_steps,
+                eval_strategy=params.eval_strategy,
+                eval_steps=params.eval_steps,
+                save_steps=params.save_steps,
+                gradient_accumulation_steps=params.gradient_accumulation_steps,
+                dev_run=getattr(config, 'dev_run', False),
+                dev_model=getattr(config, 'dev_model', None),
+                dev_subset=int(getattr(config, 'dev_subset', 0))
+            )
+        except Exception:
+            pass
         
         return model_trainer_config
     

src/textSummarizer/entity/__init__.py

@@ -14,13 +14,17 @@ class DataValidationConfig:
     root_dir : Path
     STATUS_FILE : str
     ALL_REQUIRED_FILES : list
+    data_dir : Path
     
     
 @dataclass(frozen=True)
 class DataTransformationConfig:
     root_dir: Path
     data_path: Path
-    tokenizer_name: Path
+    tokenizer_name: str
+    # dev quick-run options
+    dev_run: bool = False
+    dev_model: str | None = None
     
     
     
@@ -34,10 +38,14 @@ class ModelTrainerConfig:
     per_device_train_batch_size: int
     weight_decay: float
     logging_steps: int
-    evaluation_strategy: str
+    eval_strategy: str
     eval_steps: int
     save_steps: float
     gradient_accumulation_steps: int   
+    # dev / quick-train options
+    dev_run: bool = False
+    dev_model: str | None = None
+    dev_subset: int = 0
     
     
 @dataclass(frozen=True)

src/textSummarizer/utils/common.py

@@ -2,12 +2,10 @@ import os
 from box.exceptions import BoxValueError
 import yaml
 from textSummarizer.logging import logger
-from ensure import ensure_annotations
 from box import ConfigBox
 from pathlib import Path
 from typing import Any
 
-@ensure_annotations
 def read_yaml(path_to_yaml: Path) -> ConfigBox:
     """_summary_
     Read YAML file and convert it into a ConfigBox object.
@@ -36,8 +34,7 @@ def read_yaml(path_to_yaml: Path) -> ConfigBox:
         raise e
     
 
-@ensure_annotations
-def create_directories(path_to_directories: list, verbose=True):
+def create_directories(path_to_directories: list, verbose: bool = True) -> None:
     """
     create list of directories 
     
@@ -50,8 +47,7 @@ def create_directories(path_to_directories: list, verbose=True):
         if verbose:
             logger.info(f"created directory at path : {path}")
             
-@ensure_annotations
-def get_size(path: Path) ->str:
+def get_size(path: Path) -> str:
     """get size of file in kbs
 
     Args:

template.py

@@ -1,10 +1,9 @@
-import os
 from pathlib import Path
 import logging
 
-logging.basicConfig(level=logging.INFO, format='[%(asctime)s]:(message)s:')
+logging.basicConfig(level=logging.INFO, format="[%(asctime)s]: %(message)s")
 
-project_name = "testSummarizer"
+project_name = "textSummarizer"
 
 list_of_files = [
     ".github/workflows/.gitkeep",
@@ -25,22 +24,20 @@ list_of_files = [
     "Dockerfile",
     "requirements.txt",
     "setup.py",
-    "research/trails.ipynb"
-    
+    "research/trails.ipynb",
 ]
 
 for filepath in list_of_files:
-    filepath = Path(filepath)
-    filedir, filename = os.path.split(filepath)
-    
-    if filedir != "":
-        os.makedirs(filedir, exist_ok=True)
-        logging.info(f"Creating directry:{filepath} for the file {filepath}")
-        
-    if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0):
-        with open(filepath, 'w') as f:
-            pass
-            logging.info(f"Creating empty file: {filepath}")
-            
-    else :
-        logging.info(f"{filename} is already exists.")
+    fp = Path(filepath)
+    filedir = fp.parent
+    filename = fp.name
+
+    if filedir and str(filedir) != ".":
+        filedir.mkdir(parents=True, exist_ok=True)
+        logging.info(f"Creating directory: {filedir} for the file {filename}")
+
+    if not fp.exists() or fp.stat().st_size == 0:
+        fp.touch(exist_ok=True)
+        logging.info(f"Created empty file: {fp}")
+    else:
+        logging.info(f"{filename} already exists.")