✨ Update Jupyter Notebook Visualization

README.md

@@ -85,6 +85,10 @@ We also provide the experimented scenario to choose and test which boolean param
 
 The script will load the data from the defined directory and use them to perform the experiment.
 
+## Example
+
+For the easier experiment, we provide the Jupyter Notebook file to construct the model and training. You can visit the [Basic Model Construction]('/example/1-Basic-Model-Constructor.ipynb') and if you visit this repo from Huggingface, you can click the button **Open in Collab** to re-run this experiment. Or if you open from GitHub, you can open in GitHub Codespace to run your own Jupyter Notebook.
+
 ---
 
 © 2024, Prince of Songkla University under the Inteligent Automation Engineering Center, Faculty of Engineering.

example/.gitignore

@@ -1,4 +1,5 @@
 datasets/
 tdce-basic/
 results/
-result/
+result/
+*.pkl

example/1-Basic-Model-Constructor.ipynb

@@ -6,6 +6,7 @@
    "metadata": {},
    "source": [
     "# TDCE Learning Model Basic Model Construction\n",
+    "\n",
     "This notebook display the basic construction of Time Driven Cost Estimation Learning Model step by step without using the experiment script (experiment_script.py)"
    ]
   },
@@ -51,8 +52,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "run_from_online = False\n",
-    "ignore_download_dataset = True\n",
+    "run_from_online = True\n",
+    "ignore_download_dataset = False\n",
     "\n",
     "\n",
     "if run_from_online:\n",
@@ -78,6 +79,12 @@
     "    import result_display as rd\n",
     "    import mini_plot as mp\n",
     "    # fmt:on\n",
+    "    try:\n",
+    "        # For online execution, in Google Colab\n",
+    "        !pip install tensor-sensor\n",
+    "    except:\n",
+    "        pass\n",
+    "\n",
     "else :\n",
     "    # fmt:off\n",
     "    sys.path.append('../model')\n",
@@ -731,6 +738,8 @@
    "metadata": {},
    "source": [
     "## Visualization\n",
+    "\n",
+    "### Error Behavior\n",
     "Display the model training behavior"
    ]
   },
@@ -744,6 +753,80 @@
     "mp.plotting_learning_curve(epoch_error=error_df,element_learning_rate=element_level_lr,\n",
     "                           model_learning_rate=model_level_lr)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c101da0",
+   "metadata": {},
+   "source": [
+    "### Weight Adjustment Behavior\n",
+    "Display the Weight of Each Model Element"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bbd09ca5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "importlib.reload(mp)\n",
+    "mp.plot_model_level_weight(adjustment_data=sample_payload_df,epoch_error=error_df)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3a99ea9c",
+   "metadata": {},
+   "source": [
+    "Display the weight of each material, labor, and utility cost object."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "283c58fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "material_columns = [col for col in sample_payload_df.columns if col.startswith('material_weight_')]\n",
+    "employee_columns = [col for col in sample_payload_df.columns if col.startswith('employee_weight_')]\n",
+    "capital_columns = [col for col in sample_payload_df.columns if col.startswith('capital_cost_weight_')]\n",
+    "\n",
+    "\n",
+    "importlib.reload(mp)\n",
+    "mp.plot_element_level_weight(\n",
+    "    adjustment_data=sample_payload_df,\n",
+    "    material_columns=material_columns,labor_columns= employee_columns,\n",
+    "    utility_columns= capital_columns)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d6cc2707",
+   "metadata": {},
+   "source": [
+    "## Export Model\n",
+    "Export Model to keep and use in another place"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "74b2f7e2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tdce_model.export_model(\"tdce_model.pkl\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dce151ec",
+   "metadata": {},
+   "source": [
+    "© 2025, Intelligent Automation Engineering Center, Prince of Songkla University"
+   ]
   }
  ],
  "metadata": {

functions/mini_plot.py

@@ -1,6 +1,7 @@
 import pandas as pd
 import matplotlib.pyplot as plt
 import seaborn as sns
+import numpy as np
 
 sns.set_theme(style="whitegrid", font="Noto Sans",
               font_scale=1)
@@ -49,3 +50,113 @@ def plotting_learning_curve(epoch_error, element_learning_rate, model_learning_r
     ax[0].legend(loc="lower right")
     ax[1].legend(loc="lower right")
     fig.tight_layout(pad=3.0)
+
+
+def plot_model_level_weight(adjustment_data, epoch_error):
+    fig, ax = plt.subplots(1, 3, figsize=(14, 3))
+
+    ax[0].plot(
+        adjustment_data["model_weight_1"],
+        label="Material Element Weight",
+    )
+    ax[0].plot(
+        adjustment_data["model_weight_2"],
+        label="Labor Element Weight",
+    )
+    ax[0].plot(
+        adjustment_data["model_weight_3"],
+        label="Utiltiy Cost Element Weight",
+    )
+
+    best_train_epoch = epoch_error[
+        epoch_error["error_percent"]
+        == epoch_error["error_percent"].min()
+    ]
+    best_validate_epoch = epoch_error[
+        epoch_error["validate_error_percent"]
+        == epoch_error["validate_error_percent"].min()
+    ]
+    try:
+        best_train_epoch = best_train_epoch.iloc[0]
+        best_train_epoch = best_train_epoch["epoch"]
+        best_validate_epoch = best_validate_epoch.iloc[0]
+        best_validate_epoch = best_validate_epoch["epoch"]
+    except Exception as e:
+        best_train_epoch = np.nan
+        best_validate_epoch = np.nan
+
+    ax[0].set_xlabel(
+        f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+    )
+    ax[0].set_xticks([])
+    ax[0].legend(loc="lower right")
+    ax[0].set_title(
+        "All Model Level Weight",
+        fontdict={"fontsize": 12},
+    )
+
+    ax[1].plot(
+        adjustment_data["model_weight_1"],
+        label="Material Element Weight",
+    )
+    ax[1].legend(loc="lower right")
+    ax[1].set_title(
+        "Material Element Weight",
+        fontdict={"fontsize": 12},
+    )
+
+    ax[2].plot(
+        adjustment_data["model_weight_2"],
+        label="Labor Element Weight",
+    )
+    ax[2].plot(
+        adjustment_data["model_weight_3"],
+        label="Utiltiy Cost Element Weight",
+    )
+    ax[2].legend(loc="lower right")
+    ax[2].set_title(
+        "Labor and Utility Cost Element Weight",
+        fontdict={"fontsize": 12},
+    )
+
+
+def plot_element_level_weight(adjustment_data, material_columns, labor_columns, utility_columns):
+
+    fig, ax = plt.subplots(1, 3, figsize=(14, 3))
+    for column in material_columns:
+        ax[0].plot(
+            adjustment_data[column],
+            label=column,
+        )
+
+    for column in labor_columns:
+        ax[1].plot(
+            adjustment_data[column],
+            label=column,
+        )
+
+    for column in utility_columns:
+        ax[2].plot(
+            adjustment_data[column],
+            label=column,
+        )
+
+    ax[0].legend(loc="lower right")
+    ax[2].legend(loc="lower right")
+
+    ax[0].set_title(
+        "Material Element Weight",
+        fontdict={"fontsize": 12},
+    )
+
+    ax[1].set_title(
+        "Labor Element Weight",
+        fontdict={"fontsize": 12},
+    )
+
+    ax[2].set_title(
+        "Utility Cost Element Weight",
+        fontdict={"fontsize": 12},
+    )
+
+    fig.tight_layout(pad=2.0)

model/matrix_normalization.py

@@ -151,3 +151,150 @@ def normalize_payload(
         max_data,
         min_data,
     )
+
+
+def normalize_prediction_payload(
+    max_data,
+    min_data,
+    material_cost_matrix,
+    material_amount_matrix,
+    employee_cost_matrix,
+    employee_duration_matrix,
+    employee_day_amount_matrix,
+    capital_cost_matrix,
+    day_amount_matrix,
+    capital_cost_duration_matrix,
+):
+    # Normalized Material Matrix
+    max_arr, min_arr = norm.find_max_min(material_cost_matrix)
+    normalized_material_cost = norm.normalized(
+        material_cost_matrix, max_arr, min_arr)
+    validate_material_cost_matrix = validation_payload["validate_material_cost_matrix"]
+
+    # Normalize Validate Material Cost Matrix
+    normalized_validate_material_cost = norm.normalized_2d(
+        validate_material_cost_matrix, material_cost_matrix, max_arr, min_arr
+    )
+    max_data["material_cost"] = max_arr
+    min_data["material_cost"] = min_arr
+
+    # Normalized Material Amount Matrix
+    max_arr, min_arr = norm.find_max_min(material_amount_matrix)
+    normalized_material_amount = norm.normalized(
+        material_amount_matrix, max_arr, min_arr
+    )
+    validate_material_amount_matrix = validation_payload[
+        "validate_material_amount_matrix"
+    ]
+
+    if display_log:
+        print("normalized_material_amount", normalized_material_amount)
+        print("------------")
+        print("normalized_material_cost", normalized_material_cost)
+        print("------------")
+
+    normalized_validate_material_amount = norm.normalized_2d(
+        validate_material_amount_matrix, material_amount_matrix, max_arr, min_arr
+    )
+    max_data["material_amount"] = max_arr
+    min_data["material_amount"] = min_arr
+
+    # Normalized  Employee Cost Matrix
+    max_arr, min_arr = norm.find_max_min(employee_cost_matrix)
+    normalized_employee_cost = norm.normalized(
+        employee_cost_matrix, max_arr, min_arr)
+    validate_employee_cost_matrix = validation_payload["validate_employee_cost_matrix"]
+    normalized_validate_employee_cost = norm.normalized(
+        validate_employee_cost_matrix, max_arr, min_arr
+    )
+    max_data["employee_cost"] = max_arr
+    min_data["employee_cost"] = min_arr
+
+    # Normalized  Employee Duration Matrix
+    max_arr, min_arr = norm.find_max_min(employee_duration_matrix)
+    normalized_employee_duration = norm.normalized(
+        employee_duration_matrix, max_arr, min_arr
+    )
+    validate_employee_duration_matrix = validation_payload[
+        "validate_employee_duration_matrix"
+    ]
+    normalized_validate_employee_duration = norm.normalized(
+        validate_employee_duration_matrix,
+        max_arr,
+        min_arr,
+    )
+    max_data["employee_duration"] = max_arr
+    min_data["employee_duration"] = min_arr
+
+    # Normalized Employee Day Amount
+    max_arr, min_arr = norm.find_max_min(employee_day_amount_matrix)
+    normalized_employee_day_amount = norm.normalized(
+        employee_day_amount_matrix, max_arr, min_arr
+    )
+    validate_employee_day_amount_matrix = validation_payload[
+        "validate_employee_day_amount_matrix"
+    ]
+    normalized_validate_employee_day_amount = norm.normalized(
+        validate_employee_day_amount_matrix,
+        max_arr,
+        min_arr,
+    )
+    max_data["employee_day_amount"] = max_arr
+    min_data["employee_day_amount"] = min_arr
+
+    # Normalized Capital Cost Matrix
+    max_arr, min_arr = norm.find_max_min(capital_cost_matrix)
+    normalized_capital_cost = norm.normalized(
+        capital_cost_matrix, max_arr, min_arr)
+    validate_capital_cost_matrix = validation_payload["validate_capital_cost_matrix"]
+    normalized_validate_capital_cost = norm.normalized(
+        validate_capital_cost_matrix, max_arr, min_arr
+    )
+    max_data["capital_cost"] = max_arr
+    min_data["capital_cost"] = min_arr
+
+    # Normalized Day Amount Matrix
+    max_arr, min_arr = norm.find_max_min(day_amount_matrix)
+    normalized_day_amount = norm.normalized(
+        day_amount_matrix, max_arr, min_arr)
+    validate_day_amount_matrix = validation_payload["validate_day_amount_matrix"]
+    normalized_validate_day_amount = norm.normalized(
+        validate_day_amount_matrix, max_arr, min_arr
+    )
+    max_data["day_amount"] = max_arr
+    min_data["day_amount"] = min_arr
+
+    # Normalized Capital Cost Duration Matrix
+    validate_capital_duration_matrix = validation_payload[
+        "validate_capital_duration_matrix"
+    ]
+    max_arr, min_arr = norm.find_max_min(capital_cost_duration_matrix)
+    normalized_capital_cost_duration = norm.normalized(
+        capital_cost_duration_matrix, max_arr, min_arr
+    )
+    normalized_validate_capital_duration = norm.normalized(
+        validate_capital_duration_matrix, max_arr, min_arr
+    )
+    max_data["capital_cost_duration"] = max_arr
+    min_data["capital_cost_duration"] = min_arr
+
+    return (
+        normalized_material_cost,
+        normalized_material_amount,
+        normalized_validate_material_cost,
+        normalized_validate_material_amount,
+        normalized_employee_cost,
+        normalized_employee_duration,
+        normalized_employee_day_amount,
+        normalized_validate_employee_cost,
+        normalized_validate_employee_duration,
+        normalized_validate_employee_day_amount,
+        normalized_capital_cost,
+        normalized_capital_cost_duration,
+        normalized_day_amount,
+        normalized_validate_capital_cost,
+        normalized_validate_capital_duration,
+        normalized_validate_day_amount,
+        max_data,
+        min_data,
+    )

model/tdce_model.py

@@ -3,6 +3,7 @@ import random
 import pandas as pd
 import importlib
 import time
+import pickle
 
 import material_network as mn
 import time_driven_network as tdn
@@ -539,3 +540,103 @@ class TDCEModel:
 
     def get_sample_payload(self):
         return self.sample_payload
+
+    def export_model(self, filename="tdce_model.pkl"):
+        model_data = {
+            "material_element": self.material_element,
+            "employee_element": self.employee_element,
+            "capital_cost_element": self.capital_cost_element,
+            "weights": self.weights,
+            "bias": self.bias,
+            "loss": self.loss,
+            "loss_prime": self.loss_prime,
+            "loss_percent": self.loss_percent,
+            "material_learning_rate": self.material_learning_rate,
+            "employee_learning_rate": self.employee_learning_rate,
+            "capital_cost_learning_rate": self.capital_cost_learning_rate,
+            "early_stopping": self.use_early_stopping,
+            "patience": self.patience,
+            "use_model_weight": self.use_model_weight,
+            "max_data": self.max_data,
+            "min_data": self.min_data,
+            "weight_list": self.weight_list,
+        }
+
+        pickle.dump(model_data, open(filename, "wb"))
+        print("Model Exported Successfully")
+
+    def load_model(self, filename="tdce_model.pkl"):
+        model_data = pickle.load(open(filename, "rb"))
+        self.material_element = model_data["material_element"]
+        self.employee_element = model_data["employee_element"]
+        self.capital_cost_element = model_data["capital_cost_element"]
+        self.weights = model_data["weights"]
+        self.bias = model_data["bias"]
+        self.loss = model_data["loss"]
+        self.loss_prime = model_data["loss_prime"]
+        self.loss_percent = model_data["loss_percent"]
+        self.material_learning_rate = model_data["material_learning_rate"]
+        self.employee_learning_rate = model_data["employee_learning_rate"]
+        self.capital_cost_learning_rate = model_data[
+            "capital_cost_learning_rate"
+        ]
+        self.use_early_stopping = model_data["early_stopping"]
+        self.patience = model_data["patience"]
+        self.use_model_weight = model_data["use_model_weight"]
+        self.max_data = model_data["max_data"]
+        self.min_data = model_data["min_data"]
+        self.weight_list = model_data["weight_list"]
+        print("Model Loaded Successfully")
+
+    def predict_data(self, payload):
+        # Normalize Payload
+        (
+            normalized_material_cost,
+            normalized_material_amount,
+            normalized_employee_cost,
+            normalized_employee_duration,
+            normalized_employee_day_amount,
+            normalized_capital_cost,
+            normalized_capital_cost_duration,
+            normalized_day_amount,
+        ) = mnorm.normalize_payload(
+            material_cost=payload["material_cost"],
+            material_amount=payload["material_amount"],
+            employee_cost=payload["employee_cost"],
+            employee_duration=payload["employee_duration"],
+            employee_day_amount=payload["employee_day_amount"],
+            capital_cost=payload["capital_cost"],
+            capital_cost_duration=payload["capital_cost_duration"],
+            day_amount=payload["day_amount"],
+            max_data=self.max_data,
+            min_data=self.min_data
+        )
+
+        # Predict Material Cost
+        predicted_mc = self.material_element.predict_sample(
+            cost_input=normalized_material_cost,
+            amount_input=normalized_material_amount
+        )
+
+        # Predict Employee Cost
+        predicted_ec = self.employee_element.predict_sample(
+            cost_input=normalized_employee_cost,
+            time_input=normalized_employee_duration,
+            day_amount=normalized_employee_day_amount
+        )
+
+        # Predict Capital Cost
+        predicted_cc = self.capital_cost_element.predict_sample(
+            cost_input=normalized_capital_cost,
+            time_input=normalized_capital_cost_duration,
+            day_amount=normalized_day_amount,
+        )
+
+        # Combine Result
+        result = (
+            predicted_mc * self.weights[0]
+            + predicted_ec * self.weights[1]
+            + predicted_cc * self.weights[2]
+        ) + self.bias
+
+        return result