{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "os.chdir('../')" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from dataclasses import dataclass\n", "from pathlib import Path\n", "\n", "\n", "@dataclass(frozen=True)\n", "class TrainingConfig:\n", " root_dir: Path\n", " trained_model_path: Path\n", " updated_base_model_path: Path\n", " training_data: Path\n", " params_epochs: int\n", " params_batch_size: int\n", " params_is_augmentation: bool\n", " params_image_size: list\n", "\n", "\n", "\n", "@dataclass(frozen=True)\n", "class PrepareCallbacksConfig:\n", " root_dir: Path\n", " tensorboard_root_log_dir: Path\n", " checkpoint_model_filepath: Path" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "from cnnClassfier.constants import *\n", "from cnnClassfier.utils.common import read_yaml, create_directories\n", "import tensorflow as tf\n", "\n", "\n", "class ConfigurationManager:\n", " def __init__(\n", " self, \n", " config_filepath = CONFIG_FILE_PATH,\n", " params_filepath = PARAMS_FILE_PATH):\n", " self.config = read_yaml(config_filepath)\n", " self.params = read_yaml(params_filepath)\n", " create_directories([self.config.artifacts_root])\n", "\n", "\n", " \n", " def get_prepare_callback_config(self) -> PrepareCallbacksConfig:\n", " config = self.config.prepare_callbacks\n", " model_ckpt_dir = os.path.dirname(config.checkpoint_model_filepath)\n", " create_directories([\n", " Path(model_ckpt_dir),\n", " Path(config.tensorboard_root_log_dir)\n", " ])\n", "\n", " prepare_callback_config = PrepareCallbacksConfig(\n", " root_dir=Path(config.root_dir),\n", " tensorboard_root_log_dir=Path(config.tensorboard_root_log_dir),\n", " checkpoint_model_filepath=Path(config.checkpoint_model_filepath)\n", " )\n", "\n", " return prepare_callback_config\n", " \n", "\n", "\n", "\n", "\n", " def get_training_config(self) -> TrainingConfig:\n", " training = self.config.training\n", " prepare_base_model = self.config.prepare_base_model\n", " params = self.params\n", " training_data = os.path.join(self.config.data_ingestion.unzip_dir, \"Chicken-fecal-images\")\n", " create_directories([\n", " Path(training.root_dir)\n", " ])\n", "\n", " training_config = TrainingConfig(\n", " root_dir=Path(training.root_dir),\n", " trained_model_path=Path(training.trained_model_path),\n", " updated_base_model_path=Path(prepare_base_model.updated_base_model_path),\n", " training_data=Path(training_data),\n", " params_epochs=params.EPOCHS,\n", " params_batch_size=params.BATCH_SIZE,\n", " params_is_augmentation=params.AUGMENTATION,\n", " params_image_size=params.IMAZE_SIZE\n", " )\n", "\n", " return training_config" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "import time\n", "class PrepareCallback:\n", " def __init__(self, config: PrepareCallbacksConfig):\n", " self.config = config\n", " \n", " @property\n", " def _create_tb_callbacks(self):\n", " timestamp = time.strftime('%Y-%m-%d-%H-%M-%S')\n", " \n", " tb_running_log_dir = os.path.join(\n", " str(self.config.tensorboard_root_log_dir), # Convert to string\n", " f\"tb_logs_at_{timestamp}\",\n", " )\n", " return tf.keras.callbacks.TensorBoard(log_dir=tb_running_log_dir)\n", "\n", " @property\n", " def _create_ckpt_callbacks(self):\n", " return tf.keras.callbacks.ModelCheckpoint(\n", " filepath=str(self.config.checkpoint_model_filepath), # Convert to string\n", " save_best_only=True\n", " )\n", "\n", " def get_tb_callbacks(self):\n", " return [\n", " self._create_tb_callbacks,\n", " self._create_ckpt_callbacks\n", " ]\n", " " ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import os\n", "import urllib.request as request\n", "from zipfile import ZipFile\n", "import tensorflow as tf\n", "import time\n", "\n", "\n", "class Training:\n", " def __init__(self, config: TrainingConfig):\n", " self.config = config\n", " \n", " def get_base_model(self):\n", " self.model = tf.keras.models.load_model(\n", " self.config.updated_base_model_path\n", " )\n", " \n", " def train_valid_generator(self):\n", "\n", " datagenerator_kwargs = dict(\n", " rescale = 1./255,\n", " validation_split=0.20\n", " )\n", "\n", " dataflow_kwargs = dict(\n", " target_size=self.config.params_image_size[:-1],\n", " batch_size=self.config.params_batch_size,\n", " interpolation=\"bilinear\"\n", " )\n", "\n", " valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(\n", " **datagenerator_kwargs\n", " )\n", "\n", " self.valid_generator = valid_datagenerator.flow_from_directory(\n", " directory=self.config.training_data,\n", " subset=\"validation\",\n", " shuffle=False,\n", " **dataflow_kwargs\n", " )\n", "\n", " if self.config.params_is_augmentation:\n", " train_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(\n", " rotation_range=40,\n", " horizontal_flip=True,\n", " width_shift_range=0.2,\n", " height_shift_range=0.2,\n", " shear_range=0.2,\n", " zoom_range=0.2,\n", " **datagenerator_kwargs\n", " )\n", " else:\n", " train_datagenerator = valid_datagenerator\n", "\n", " self.train_generator = train_datagenerator.flow_from_directory(\n", " directory=self.config.training_data,\n", " subset=\"training\",\n", " shuffle=True,\n", " **dataflow_kwargs\n", " )\n", "\n", " @staticmethod\n", " def save_model(path: Path, model: tf.keras.Model):\n", " model.save(path)\n", "\n", "\n", " def train(self, callback_list: list):\n", " self.steps_per_epoch = self.train_generator.samples // self.train_generator.batch_size\n", " self.validation_steps = self.valid_generator.samples // self.valid_generator.batch_size\n", "\n", " self.model.fit(\n", " self.train_generator,\n", " epochs=self.config.params_epochs,\n", " steps_per_epoch=self.steps_per_epoch,\n", " validation_steps=self.validation_steps,\n", " validation_data=self.valid_generator,\n", " callbacks=callback_list\n", " )\n", "\n", " self.save_model(\n", " path=self.config.trained_model_path,\n", " model=self.model\n", " )\n", "\n", "\n", " " ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2024-07-27 23:47:55,531: INFO: common: yaml file: config\\config.yaml loaded successfully]\n", "[2024-07-27 23:47:55,534: INFO: common: yaml file: params.yaml loaded successfully]\n", "[2024-07-27 23:47:55,536: INFO: common: Created directory at: artifacts]\n", "[2024-07-27 23:47:55,537: INFO: common: Created directory at: artifacts\\prepare_callbacks\\checkpoint_dir]\n", "[2024-07-27 23:47:55,538: INFO: common: Created directory at: artifacts\\prepare_callbacks\\tensorboard_log_dir]\n", "[2024-07-27 23:47:55,539: INFO: common: Created directory at: artifacts\\training]\n", "[2024-07-27 23:47:56,129: WARNING: module_wrapper: From c:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\keras\\src\\backend.py:1398: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n", "]\n", "[2024-07-27 23:47:56,313: WARNING: module_wrapper: From c:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\keras\\src\\layers\\pooling\\max_pooling2d.py:161: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n", "]\n", "Found 78 images belonging to 2 classes.\n", "Found 312 images belonging to 2 classes.\n", "[2024-07-27 23:47:58,334: WARNING: module_wrapper: From c:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\keras\\src\\utils\\tf_utils.py:492: The name tf.ragged.RaggedTensorValue is deprecated. Please use tf.compat.v1.ragged.RaggedTensorValue instead.\n", "]\n", "19/19 [==============================] - 53s 3s/step - loss: 13.0395 - accuracy: 0.5270 - val_loss: 24.1360 - val_accuracy: 0.3906\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "c:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\keras\\src\\engine\\training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n", " saving_api.save_model(\n" ] } ], "source": [ "try:\n", " config = ConfigurationManager()\n", " prepare_callbacks_config = config.get_prepare_callback_config()\n", " prepare_callbacks = PrepareCallback(config=prepare_callbacks_config)\n", " callback_list = prepare_callbacks.get_tb_callbacks()\n", "\n", " training_config = config.get_training_config()\n", " training = Training(config=training_config)\n", " training.get_base_model()\n", " training.train_valid_generator()\n", " training.train(\n", " callback_list=callback_list\n", " )\n", " \n", "except Exception as e:\n", " raise e" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.0" } }, "nbformat": 4, "nbformat_minor": 2 }