✨ Update Modeling Notebook

README.md

@@ -12,6 +12,7 @@ Creating ENV followed by `.env.example` and then Creating your virtual environme
 
 ```
 python -m venv venv
+venv/Script/activate
 ```
 
 Install all requirements

example/.gitignore

@@ -0,0 +1 @@
+datasets/

example/1-Basic-Model-Constructor.ipynb

@@ -0,0 +1,445 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7259c012",
+   "metadata": {},
+   "source": [
+    "# TDCE Learning Model Basic Model Construction\n",
+    "This notebook display the basic construction of Time Driven Cost Estimation Learning Model step by step without using the experiment script (experiment_script.py)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "050178b5",
+   "metadata": {},
+   "source": [
+    "Import the library"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ae7385ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import importlib\n",
+    "import sys\n",
+    "import os\n",
+    "import time\n",
+    "import numpy as np\n",
+    "import requests\n",
+    "import ipywidgets as widgets\n",
+    "from IPython.display import display\n",
+    "\n",
+    "# fmt:off\n",
+    "sys.path.append('../model')\n",
+    "sys.path.append('../functions/matrix_generator')\n",
+    "sys.path.append('../functions')\n",
+    "sys.path.append('../functions/data_extractor')\n",
+    "\n",
+    "import tdce_model as tdce\n",
+    "import material_fc_layer as mfl\n",
+    "import employee_fc_layer as efl\n",
+    "import capital_fc_layer as cfl\n",
+    "import loss\n",
+    "import cost_matrix_class as cmc\n",
+    "import display_input_variation as diva\n",
+    "import viyacrab_augmentation as viya\n",
+    "import adjust_data as ajd\n",
+    "import result_display as rd\n",
+    "\n",
+    "importlib.reload(tdce)\n",
+    "importlib.reload(mfl)\n",
+    "importlib.reload(efl)\n",
+    "importlib.reload(cfl)\n",
+    "importlib.reload(loss)\n",
+    "importlib.reload(tdce)\n",
+    "importlib.reload(cmc)\n",
+    "importlib.reload(diva)\n",
+    "importlib.reload(viya)\n",
+    "importlib.reload(ajd)\n",
+    "importlib.reload(rd)\n",
+    "# fmt:on\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6b7fd72c",
+   "metadata": {},
+   "source": [
+    "## Dataset \n",
+    "We will use our project dataset for experimental, we pick the [extended-random-dataset](https://huggingface.co/datasets/theethawats98/tdce-example-extended-random) which is the dataset with high dimension but moderate variation to use as case study for out demonstation. We create the dataset in folder `datasets` and then inside it have the folder `extended-random` again. We will create the folder if it is not exist and download the datafile from the our huggingface."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa31e7f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    os.mkdir(f\"datasets\")\n",
+    "    os.mkdir(f\"datasets/extended-random\")\n",
+    "except FileExistsError:\n",
+    "    print(\"Folder is Exist\")\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ca939290",
+   "metadata": {},
+   "source": [
+    "Download Files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f98a61ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "capital_cost_link = \"https://huggingface.co/datasets/theethawats98/tdce-example-extended-random/resolve/main/generated_capital_cost.csv\"\n",
+    "capital_path = 'datasets/extended-random/generated_capital_cost.csv'\n",
+    "employee_usage_link = \"https://huggingface.co/datasets/theethawats98/tdce-example-extended-random/resolve/main/generated_employee_usage.csv\"\n",
+    "employee_path = 'datasets/extended-random/generated_employee_usage.csv'\n",
+    "material_usage_link = \"https://huggingface.co/datasets/theethawats98/tdce-example-extended-random/resolve/main/generated_material_usage.csv\"\n",
+    "material_path = 'datasets/extended-random/generated_material_usage.csv'\n",
+    "process_data_link = \"https://huggingface.co/datasets/theethawats98/tdce-example-extended-random/resolve/main/generated_process_data.csv\"\n",
+    "process_path = 'datasets/extended-random/generated_process_data.csv'\n",
+    "\n",
+    "\n",
+    "for link, path in [\n",
+    "    (capital_cost_link, capital_path),\n",
+    "    (employee_usage_link, employee_path),\n",
+    "    (material_usage_link, material_path),\n",
+    "    (process_data_link, process_path)\n",
+    "]:\n",
+    "    if not os.path.exists(path):\n",
+    "        response = requests.get(link)\n",
+    "        if response.status_code == 200:\n",
+    "            with open(path, 'wb') as file:\n",
+    "                file.write(response.content)\n",
+    "            print(f'File {path} downloaded successfully')\n",
+    "        else:\n",
+    "            print(f'Failed to download file {path}')\n",
+    "# Downloading the datasets\n",
+    "print(\"Downloading datasets...\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8b97d5b2",
+   "metadata": {},
+   "source": [
+    "## Model Setting\n",
+    "Select the correct setting for your model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3a387bbe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hour_day_employee_widget = widgets.BoundedIntText(\n",
+    "    value=8,\n",
+    "    min=1,\n",
+    "    max=24,\n",
+    "    step=1,\n",
+    "    description='Hours per Day for Employee:',\n",
+    "    disabled=False\n",
+    ")\n",
+    "\n",
+    "\n",
+    "hour_day_capital_cost_widget = widgets.BoundedIntText(\n",
+    "    value=21,\n",
+    "    min=1,\n",
+    "    max=24,\n",
+    "    step=1,\n",
+    "    description='Hours per Day for Utility / Capital Cost:',\n",
+    "    disabled=False\n",
+    ")\n",
+    "\n",
+    "use_outlier_removal_widget = widgets.Checkbox(\n",
+    "    value=False,\n",
+    "    description='Enable Outlier Removal',\n",
+    "    disabled=False,\n",
+    "    indent=False\n",
+    ")\n",
+    "\n",
+    "outlier_index_widget = widgets.Dropdown(\n",
+    "    options=['1', '1.5', '2'],\n",
+    "    value='1.5',\n",
+    "    description='Removal Idication Index:',\n",
+    "    disabled=False,\n",
+    ")\n",
+    "\n",
+    "use_augmentation_widget = widgets.Checkbox(\n",
+    "    value=False,\n",
+    "    description='Enable Data Augmentation',\n",
+    "    disabled=False,\n",
+    "    indent=False\n",
+    ")\n",
+    "\n",
+    "element_level_lr_widget = widgets.Dropdown(\n",
+    "    options=['0.001','0.05', '0.01','0.1','0.5'],\n",
+    "    value='0.01',\n",
+    "    description='Element Level Learning Rate:',\n",
+    "    disabled=False,\n",
+    ")\n",
+    "\n",
+    "model_level_lr_widget = widgets.Dropdown(\n",
+    "    options=['0.0000001','0.00000001','0.000000001'],\n",
+    "    value='0.00000001',\n",
+    "    description='Model Level Learning Rate:',\n",
+    "    disabled=False,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "display(hour_day_employee_widget)\n",
+    "display(hour_day_capital_cost_widget)\n",
+    "display(use_outlier_removal_widget)\n",
+    "display(outlier_index_widget)\n",
+    "display(use_augmentation_widget)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ac4c4fc1",
+   "metadata": {},
+   "source": [
+    "Select Learning Rate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fce6161f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "display(element_level_lr_widget)\n",
+    "display(model_level_lr_widget)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8e7c42b1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hour_day_employee = hour_day_employee_widget.value\n",
+    "hour_day_capital_cost = hour_day_capital_cost_widget.value\n",
+    "use_outlier_removal= use_outlier_removal_widget.value\n",
+    "outlier_index = float(outlier_index_widget.value)\n",
+    "use_augmentation = use_augmentation_widget.value"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "842aa666",
+   "metadata": {},
+   "source": [
+    "Generated Cost Matrix"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f7ed24d9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "folder_path = 'datasets/extended-random'\n",
+    "\n",
+    "cost_generator = cmc.CostMatrixGenerator()\n",
+    "cost_generator.change_data_directory(folder_path)\n",
+    "cost_generator.load_data()\n",
+    "input_variation = diva.display_input_variation_by_directory(folder_path)\n",
+    "input_variation.to_csv(f\"{folder_path}/data_variation.csv\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3bd65eb3",
+   "metadata": {},
+   "source": [
+    "## Pre-Processing\n",
+    "djust and filter in input datasets (Material, Employee, Capital Cost) to match the output dataset (Process Dataset) and Depend on Outlier Removal Condition"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cec6dcf7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if use_outlier_removal:\n",
+    "    cost_generator.remove_outlier_iqr(outlier_index)\n",
+    "    (\n",
+    "        new_process_df,\n",
+    "        new_employee_usage,\n",
+    "        new_material_usage,\n",
+    "        new_capital_cost_usage,\n",
+    "    ) = cost_generator.get_data()\n",
+    "    (new_capital_cost_usage, new_employee_usage, new_material_usage) = (\n",
+    "        ajd.adjust_to_match_process(\n",
+    "            capital_cost_usage=new_capital_cost_usage,\n",
+    "            employee_usage=new_employee_usage,\n",
+    "            material_usage=new_material_usage,\n",
+    "            new_process_df=new_process_df,\n",
+    "        )\n",
+    "    )\n",
+    "    new_variation = diva.display_input_variation(\n",
+    "        new_process_df,\n",
+    "        new_material_usage,\n",
+    "        new_employee_usage,\n",
+    "        new_capital_cost_usage,\n",
+    "    )\n",
+    "    new_variation.to_csv(f\"{folder_path}/data_variation_after_outlier.csv\")\n",
+    "    new_process_df.to_csv(f\"{folder_path}/process_df_after_outlier.csv\")\n",
+    "    try:\n",
+    "        print(\"Data Variation After Outlier Removed\")\n",
+    "        display(new_variation)\n",
+    "    except Exception as e:\n",
+    "        print(\"This is not Jupyter Notebook\", e)\n",
+    "else:\n",
+    "    display(input_variation)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "00976358",
+   "metadata": {},
+   "source": [
+    "### Data Splitting\n",
+    "Split training and validation dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d1904264",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "(\n",
+    "    train_process_df,\n",
+    "    train_employee_usage,\n",
+    "    train_material_usage,\n",
+    "    train_capital_cost,\n",
+    "    validate_process_df,\n",
+    "    validate_employee_usage,\n",
+    "    validate_material_usage,\n",
+    "    validate_capital_cost,\n",
+    ") = cost_generator.train_test_split_without_matrix(0.7)\n",
+    "\n",
+    "\n",
+    "# Generate Cost Matrix for Validation Set\n",
+    "validation_payload = cost_generator.get_validation_payload(\n",
+    "    validate_process_df\n",
+    ")\n",
+    "\n",
+    "\n",
+    "# Display Variation of Train Data\n",
+    "train_variation = diva.display_input_variation(\n",
+    "    train_process_df,\n",
+    "    train_material_usage,\n",
+    "    train_employee_usage,\n",
+    "    train_capital_cost,\n",
+    ")\n",
+    "train_variation.to_csv(\n",
+    "    f\"{folder_path}/train_data_variation.csv\")\n",
+    "train_process_df.to_csv(\n",
+    "    f\"{folder_path}/train_process_df.csv\")\n",
+    "# Display Variation of Validation Data\n",
+    "validate_variation = diva.display_input_variation(\n",
+    "    validate_process_df,\n",
+    "    validate_material_usage,\n",
+    "    validate_employee_usage,\n",
+    "    validate_capital_cost,\n",
+    ")\n",
+    "validate_variation.to_csv(\n",
+    "    f\"{folder_path}/validate_data_variation.csv\"\n",
+    ")\n",
+    "if use_augmentation:\n",
+    "    # TODO:  Increase the Generalization of the Model\n",
+    "    # Augmented the Imbalance Class of Training Data\n",
+    "    train_process_df.to_csv(\n",
+    "        f\"{folder_path}/train_process_df_before_augmented.csv\"\n",
+    "    )\n",
+    "    train_process_df = viya.vy_training_augmentation(\n",
+    "        train_process_df)\n",
+    "    # Display Variation of Train Data After Augmented\n",
+    "    train_variation = diva.display_input_variation(\n",
+    "        train_process_df,\n",
+    "        train_material_usage,\n",
+    "        train_employee_usage,\n",
+    "        train_capital_cost,\n",
+    "    )\n",
+    "    train_variation.to_csv(\n",
+    "        f\"{folder_path}/train_data_variation_after_augmented.csv\"\n",
+    "    )\n",
+    "    train_process_df.to_csv(\n",
+    "        f\"{folder_path}/train_process_df_after_augmented_{round}.csv\"\n",
+    "    )\n",
+    "    \n",
+    "# Generate Matrix From Training Set\n",
+    "(\n",
+    "    material_cost_matrix,\n",
+    "    material_amount_matrix,\n",
+    "    employee_cost_matrix,\n",
+    "    employee_duration_matrix,\n",
+    "    employee_day_amount_matrix,\n",
+    "    capital_cost_matrix,\n",
+    "    day_amount_matrix,\n",
+    "    capital_cost_duration_matrix,  # New On Finetune\n",
+    "    result_matrix,\n",
+    ") = cost_generator.generate_data_from_input(\n",
+    "    train_process_df,\n",
+    "    train_material_usage,\n",
+    "    train_employee_usage,\n",
+    "    train_capital_cost,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d1bdcbbc",
+   "metadata": {},
+   "source": [
+    "## Initial Model"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

functions/display_input_variation.py

@@ -155,7 +155,7 @@ def display_input_variation_by_directory(folder_name):
         f"{folder_name}/generated_material_usage.csv")
     employee_usage_df = pd.read_csv(
         f"{folder_name}/generated_employee_usage.csv")
-    capital_cost_df = pd.read_csv(f"{folder_name}/generated_captial_cost.csv")
+    capital_cost_df = pd.read_csv(f"{folder_name}/generated_capital_cost.csv")
 
     result_variation = display_input_variation(
         process_df,

requirement.txt

@@ -8,4 +8,6 @@ scipy
 cowsay
 pyfiglet
 seaborn
-statsmodels
+statsmodels
+requests
+ipywidgets