♻️ Strcuture code

.env.example

@@ -0,0 +1,2 @@
+EMANUFAC_API_URL = 
+EMANUFAC_API_KEY = 

.gitignore

@@ -0,0 +1,3 @@
+venv/
+.env
+*.zip

.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+  "python.analysis.extraPaths": [
+  ],
+  "git.autofetch": "all",
+  "git.blame.editorDecoration.enabled": true,
+  "git.enabled": true,
+  "dataWrangler.columnInsights.displayOnOpen": false,
+  "flake8.args": [
+    "--ignore=E501"
+  ]
+}

Readme.md

@@ -16,4 +16,8 @@ Install all requirements
 pip install -r requirement.txt
 ```
 
+## Repository Structure
+
+The Model is structure into folder model, functions. The model and functions that related to **model** is located in folder model and the **function** contain the function to do the experiment.
+
 © 2024, Prince of Songkla University under Inteligent Automation Engineering Center

functions/experiment_script/batch_experiment_script.py

@@ -0,0 +1,133 @@
+import importlib
+import sys
+import getopt
+import os
+import rsg_breakpoint_experiment as rbe
+importlib.reload(rbe)
+
+learning_rate = [0.005, 0.01, 0.05, 0.1, 0.5]
+patience_round = 50
+outlier_index = 1.5
+
+
+def run_batch_experiment(model_learning_rate,
+                         augmentation=False,
+                         remove_outlier_qtr=False,
+                         round_number=1,
+                         iteration=200,
+                         use_model_weight=True,
+                         dataset_name="simple",
+                         experiment_version=2,
+                         use_early_stopping=False
+                         ):
+
+    dataset_directory = f'data/{dataset_name}'
+    output_group_directory = f'result/{dataset_name}'
+    os.makedirs(output_group_directory, exist_ok=True)
+    output_directory = f'{output_group_directory}/{dataset_name}_{experiment_version}'
+    output_directory += f"_{model_learning_rate}"
+    if remove_outlier_qtr:
+        output_directory += "_remove_outlier"
+    if augmentation:
+        output_directory += "_augmented"
+    if use_early_stopping:
+        output_directory += "_early_stopping"
+
+    try:
+        rbe.run_experiment(data_directory=dataset_directory, epoch=iteration,
+                           learning_rate_list=learning_rate, round_number=round_number,
+                           folder_name=output_directory, breakpoint=iteration / 2,
+                           early_stopping=use_early_stopping, patience_round=patience_round,
+                           augmentation=augmentation,
+                           outlier_index=outlier_index,
+                           remove_outlier_qtr=remove_outlier_qtr,
+                           use_model_weight=use_model_weight,
+                           model_learning_rate=model_learning_rate)
+    except Exception as e:
+        print("Error in Experiment", e)
+        pass
+
+
+def run_in_many_learning_rate(dataset_name="simple", experiment_version=2, round_number=1, iteration=200):
+    model_learning_rate_list = [0.0000001, 0.00000001]
+    early_stopping_list = [False, True]
+    for model_learning_rate in model_learning_rate_list:
+        for early_stopping in early_stopping_list:
+            # Normal
+            print(
+                f'LL: {model_learning_rate} / Normal / Eearly Stopping: {early_stopping}')
+            run_batch_experiment(model_learning_rate=model_learning_rate,
+                                 augmentation=False,
+                                 remove_outlier_qtr=False,
+                                 round_number=round_number,
+                                 iteration=iteration,
+                                 use_model_weight=True,
+                                 dataset_name=dataset_name,
+                                 experiment_version=experiment_version,
+                                 use_early_stopping=early_stopping
+                                 )
+            # Remove Outlier
+            print(
+                f'LL: {model_learning_rate} / Remove Outlier / Eearly Stopping: {early_stopping}')
+            run_batch_experiment(model_learning_rate=model_learning_rate,
+                                 augmentation=False,
+                                 remove_outlier_qtr=True,
+                                 round_number=round_number,
+                                 iteration=iteration,
+                                 use_model_weight=True,
+                                 dataset_name=dataset_name,
+                                 experiment_version=experiment_version,
+                                 use_early_stopping=early_stopping
+                                 )
+            # Augmentation
+            print(
+                f'LL: {model_learning_rate} / Augmentation / Eearly Stopping: {early_stopping}')
+            run_batch_experiment(model_learning_rate=model_learning_rate,
+                                 augmentation=True,
+                                 remove_outlier_qtr=False,
+                                 round_number=round_number,
+                                 iteration=iteration,
+                                 use_model_weight=True,
+                                 dataset_name=dataset_name,
+                                 experiment_version=experiment_version,
+                                 use_early_stopping=early_stopping
+                                 )
+            print(
+                f'LL: {model_learning_rate} / Remove Outlier & Augmentation / Eearly Stopping: {early_stopping}')
+            # Remove Outlier and Augmentation
+            run_batch_experiment(model_learning_rate=model_learning_rate,
+                                 augmentation=True,
+                                 remove_outlier_qtr=True,
+                                 round_number=round_number,
+                                 iteration=iteration,
+                                 use_model_weight=True,
+                                 dataset_name=dataset_name,
+                                 experiment_version=experiment_version,
+                                 use_early_stopping=early_stopping
+                                 )
+
+
+if __name__ == "__main__":
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "d:v:r:i:", [
+                                   "dataset=", "version=", 'round=', 'iteration='])
+    except getopt.GetoptError as err:
+        print(err)
+    # Process options
+
+    for opt, arg in opts:
+        if opt in ("-d", "--dataset"):
+            dataset_name = arg
+        elif opt in ("-v", "--version"):
+            experiment_version = int(arg)
+        elif opt in ("-r", "--round"):
+            round_number = int(arg)
+        elif opt in ("-i", "--iteration"):
+            iteration = int(arg)
+
+    print(
+        f"Dataset: {dataset_name} / Version: {experiment_version} / Round: {round_number} / Iteration: {iteration}")
+    run_in_many_learning_rate(dataset_name=dataset_name,
+                              experiment_version=experiment_version,
+                              round_number=round_number,
+                              iteration=iteration)

functions/experiment_script/get_data_script.py

@@ -0,0 +1,98 @@
+import pandas as pd
+import importlib
+import datetime
+import sys
+import os
+import requests
+
+# fmt:off
+sys.path.append('../14-New-Final-Model/extractor')
+
+import emanufac_tdabc_extractor_class as tdabc_extractor
+importlib.reload(tdabc_extractor)
+
+# fmt:on
+
+
+# "https://vy.autth.theduckcreator.in.th/api/v1"
+url = "http://localhost:3007/api/v1"
+api_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjY1ZmJhOTM2ZmRmNTFkMDM4MDRlODlkMyIsInVzZXJuYW1lIjoic3VwZXJ1c2VyIiwiZXhwIjoxNzUwNjYzNDM4LCJpYXQiOjE3NDI4ODc0Mzh9.b7VQbxhiJO03ionBBD5H8I-IYxO4psfenkdeb1s1bvc"
+start_date = ""  # datetime.datetime(2024, 2,7, 6,0)
+end_date = ""  # datetime.datetime(2025, 2,12,12,30)
+profile_id = "671d24916f0d9131b270c036"
+cost_place = "65fbd3398382640527b795c3"
+
+limit = 1000
+
+
+def get_data(running_start="68309336",
+             runing_end="68309858",
+             directory_name="complicate_time_series_may_2024"):
+    # Initial Class
+    extractor = tdabc_extractor.EManufacTDABCExtractor(
+        api_key=api_key, api_url=url, costed_place='65fbd3398382640527b795c3',
+        profile_id="671d24916f0d9131b270c036")
+
+    extractor.change_profile_for_original_procedure("671d2021b54378e1c72d2074")
+    extractor.set_running_no_margin(running_start, runing_end)
+
+    # Fetch Material
+    two_month_change = datetime.timedelta(days=0)
+
+    if end_date == '' and start_date == '':
+        print('On First case')
+        material_usage = extractor.fetch_material_usage(
+            start_date=start_date, end_date=end_date, limit=limit*3, page=1)
+    else:
+        material_usage = extractor.fetch_material_usage(
+            start_date=start_date - two_month_change, end_date=end_date, limit=limit*3, page=1)
+
+    # Adjust Material Usage
+    extractor.adjust_material_usage()
+
+    # Fetch Employee Usage
+    if end_date == '' and start_date == '':
+        print('on first case')
+        employee_usage = extractor.fetch_employee_usage(
+            start_date=start_date, end_date=end_date, limit=limit*2, page=1)
+    else:
+        employee_usage = extractor.fetch_employee_usage(
+            start_date=start_date - two_month_change, end_date=end_date, limit=limit*2, page=1)
+
+    # Adjust Employee Usage
+    extractor.adjust_employee_usage()
+
+    # Get Capital Cost
+    if end_date == '' and start_date == '':
+        capital_cost_usage = extractor.fetch_capital_cost_usage(
+            start_date=start_date, end_date=end_date, limit=limit*2, page=1
+        )
+    else:
+        capital_cost_usage = extractor.fetch_capital_cost_usage(
+            start_date=start_date - two_month_change, end_date=end_date, limit=limit*2, page=1
+        )
+
+    # Adjust Capital Cost
+    extractor.adjust_capital_cost()
+
+    # Get Proces
+    process_df = extractor.fetch_process_data(
+        start_date, end_date, limit=limit, page=1)
+
+    # Adjust Process
+    extractor.adjust_process_df()
+
+    # Create Folder
+    try:
+        os.mkdir(directory_name)
+    except FileExistsError:
+        print('Folder is Exist')
+        pass
+
+    # Save
+    extractor.save_material_csv(directory_name)
+    extractor.save_employee_csv(directory_name)
+    extractor.save_capital_csv(directory_name)
+    extractor.save_process_csv(directory_name)
+
+    print('Success')

functions/experiment_script/plotting_summarize.py

@@ -0,0 +1,1284 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os
+import numpy as np
+
+sns.set_theme(style="whitegrid", font="Noto Sans",
+              font_scale=1.5)
+this_graph_palette = sns.color_palette("husl", 4)
+sns.set_palette(this_graph_palette)
+dataset_names = ["1-munchkin",
+                 "3-vicheanmas", "2-chinchilla", "4-scottishfold"]
+# cademic_dataset_names = ["Dataset 1", "Dataset 3", "Dataset 2", "Dataset 4"]
+academic_dataset_names = [
+    "Simple Dataset", "Complicated Dataset", "Actual Dataset", "Extended Random Dataset"]
+
+
+learning_rate = [0.005, 0.01, 0.05, 0.1, 0.5]
+overall_accuracy_df = pd.DataFrame()
+
+model_learning_rates = ["1e-07", "1e-08"]
+iteration = 100
+modifiers = [
+    "",
+    "_remove_outlier",
+    "_augmented",
+    "_remove_outlier_augmented",
+    "_early_stopping",
+    "_remove_outlier_early_stopping",
+    "_augmented_early_stopping",
+    "_remove_outlier_augmented_early_stopping",
+]
+displayed_modifiers = [
+    "Original",
+    "Remove Outlier",
+    "Augmentation",
+    "Remove Outlier + Augmentation",
+    "Early Stopping",
+    "Remove Outlier + Early Stopping",
+    "Augmentation + Early Stopping",
+    "Remove Outlier + Augmentation + Early Stopping",
+]
+
+
+def to_kebab_case(value):
+    return "-".join(value.lower().split())
+
+
+def plotting_learning_curve(use_academic_name=False, round_no=0,
+                            model_version=1):
+    dataset_index = 0
+    for dataset_name in dataset_names:
+        modifier_index = 0
+        for modifer in modifiers:
+            accuracy_list = []
+            # Find Best Accuracy
+            # For Each Model Learning Rate
+            fig, ax = plt.subplots(2, 5, figsize=(20, 8))
+            model_learning_rate_index = 0
+            learning_rate_index = 0
+            if use_academic_name:
+                fig.suptitle(
+                    f"Learning Curve of {academic_dataset_names[dataset_index]} - {displayed_modifiers[modifier_index]}",
+                    fontdict={"fontsize": 20, "fontweight": "bold"},
+                )
+
+            else:
+                fig.suptitle(
+                    f"Learning Curve of {academic_dataset_names[dataset_index]} - {displayed_modifiers[modifier_index]}",
+                    fontdict={"fontsize": 20, "fontweight": "bold"},
+                )
+            for model_learning_rate in model_learning_rates:
+                directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
+                # For Each Learning Rate
+                for lr in learning_rate:
+                    epoch_error_file = (
+                        f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                    )
+                    epoch_error = pd.read_csv(epoch_error_file)
+                    ax[model_learning_rate_index][learning_rate_index].plot(
+                        epoch_error["epoch"],
+                        epoch_error["error_percent"],
+                        label="Training",
+                        linewidth=4
+                    )
+                    ax[model_learning_rate_index][learning_rate_index].plot(
+                        epoch_error["epoch"],
+                        epoch_error["validate_error_percent"],
+                        label="Validation", linewidth=4
+                    )
+                    ax[model_learning_rate_index][learning_rate_index].set_title(
+                        f"{lr}/{model_learning_rate}",
+                        fontdict={"fontsize": 16},
+                    )
+                    # ax[model_learning_rate_index][learning_rate_index].set_xlabel(
+                    #     "Epoch"
+                    # )
+                    # ax[model_learning_rate_index][learning_rate_index].set_ylabel(
+                    #     "Error RMSPE"
+                    # )
+                    ax[model_learning_rate_index][learning_rate_index].set_ylim(
+                        0, 100)
+                    ax[model_learning_rate_index][learning_rate_index].set_xlim(
+                        0, 100)
+                    # ax[model_learning_rate_index][learning_rate_index].legend(
+                    #     loc="lower right"
+                    # )
+
+                    learning_rate_index += 1
+                model_learning_rate_index += 1
+                learning_rate_index = 0
+            handles, labels = ax[0][0].get_legend_handles_labels()
+            fig.legend(handles, labels, loc="lower center",
+                       ncol=2, fontsize=12)
+            fig.tight_layout(pad=2.0)
+            plt.tight_layout()
+            os.makedirs(
+                f"result/learning_curve/by-modifer/{dataset_name}", exist_ok=True
+            )
+            if modifer == "":
+                modifer = "original"
+            plt.savefig(
+                f"result/learning_curve/by-modifer/{dataset_name}/{modifer}.png"
+            )
+            plt.close(fig)
+            accuracy_list = pd.DataFrame(accuracy_list)
+            modifier_index += 1
+
+        dataset_index += 1
+
+
+def plotting_learning_curve_by_rate(use_academic_name=False, round_no=0,
+                                    model_version=1):
+    dataset_index = 0
+    for dataset_name in dataset_names:
+        # Find Best Accuracy
+        # For Each Model Learning Rate
+
+        model_learning_rate_index = 0
+        learning_rate_index = 0
+        for model_learning_rate in model_learning_rates:
+            # For Each Learning Rate
+            for lr in learning_rate:
+                fig, ax = plt.subplots(2, 4, figsize=(20, 8))
+                for modifier_index, modifer in enumerate(modifiers):
+                    directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
+                    epoch_error_file = (
+                        f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                    )
+                    epoch_error = pd.read_csv(epoch_error_file)
+                    row_index = 0 if modifier_index < 4 else 1
+                    col_index = modifier_index % 4
+                    ax[row_index][col_index].plot(
+                        epoch_error["epoch"],
+                        epoch_error["error_percent"],
+                        label="Training",
+                    )
+                    ax[row_index][col_index].plot(
+                        epoch_error["epoch"],
+                        epoch_error["validate_error_percent"],
+                        label="Validation",
+                    )
+                    if use_academic_name:
+                        ax[row_index][col_index].set_title(
+                            f"{academic_dataset_names[dataset_index]}\n{displayed_modifiers[modifier_index]}\n{lr}/{model_learning_rate}",
+                            fontdict={"fontsize": 12},
+                        )
+                    else:
+                        ax[row_index][col_index].set_title(
+                            f"{dataset_name}\n{displayed_modifiers[modifier_index]}\n{lr}/{model_learning_rate}",
+                            fontdict={"fontsize": 12},
+                        )
+                    ax[row_index][col_index].set_xlabel(
+                        f'Epoch\n Lasted Training Error {epoch_error.iloc[-1]["error_percent"]:.2f} \n Lasted Validation Error {epoch_error.iloc[-1]["validate_error_percent"]:.2f}'
+                    )
+                    ax[row_index][col_index].set_ylabel("Error RMSPE")
+                    ax[row_index][col_index].set_ylim(0, 100)
+                    ax[row_index][col_index].set_xlim(0, 100)
+                    ax[row_index][col_index].legend(loc="lower right")
+                    learning_rate_index += 1
+                fig.tight_layout(pad=2.0)
+                os.makedirs(
+                    f"result/learning_curve/by-rate/{dataset_name}", exist_ok=True
+                )
+                plt.savefig(
+                    f"result/learning_curve/by-rate/{dataset_name}/{model_learning_rate}-{lr}.png"
+                )
+                plt.close(fig)
+            model_learning_rate_index += 1
+            learning_rate_index = 0
+
+        dataset_index += 1
+
+
+def plot_graph_grid(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
+    dataset_index = 0
+    early_stopping_title = "With Early Stopping" if use_early_stopping else ""
+    for dataset_name in dataset_names:
+        # Find Best Accuracy
+        # For Each Model Learning Rate
+
+        if use_early_stopping:
+            filter_modifier = modifiers[4:]
+            filter_display_modifier = displayed_modifiers[4:]
+        else:
+            filter_modifier = modifiers[:4]
+            filter_display_modifier = displayed_modifiers[:4]
+        model_learning_rate_index = 0
+        learning_rate_index = 0
+        for model_learning_rate in model_learning_rates:
+            # For Each Learning Rate
+            fig, ax = plt.subplots(5, 4, figsize=(20, 14))
+            if use_academic_name:
+                fig.suptitle(
+                    f"Learning Curve of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+
+            else:
+                fig.suptitle(
+                    f"Learning Curve of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+            learning_rate_index = 0
+            for lr in learning_rate:
+                for modifier_index, modifer in enumerate(filter_modifier):
+                    directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
+                    epoch_error_file = (
+                        f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                    )
+                    epoch_error = pd.read_csv(epoch_error_file)
+
+                    col_index = modifier_index
+                    ax[learning_rate_index][col_index].plot(
+                        epoch_error["epoch"],
+                        epoch_error["error_percent"],
+                        label="Training",
+                        linewidth=4,
+                    )
+                    ax[learning_rate_index][col_index].plot(
+                        epoch_error["epoch"],
+                        epoch_error["validate_error_percent"],
+                        label="Validation",
+                        linewidth=4,
+                    )
+                    if use_academic_name:
+                        ax[learning_rate_index][col_index].set_title(
+                            f"{filter_display_modifier[modifier_index]}\n α = {lr}",
+                            fontdict={"fontsize": 20},
+                        )
+
+                    # best_train_epoch = epoch_error[
+                    #     epoch_error["error_percent"]
+                    #     == epoch_error["error_percent"].min()
+                    # ]
+                    # best_validate_epoch = epoch_error[
+                    #     epoch_error["validate_error_percent"]
+                    #     == epoch_error["validate_error_percent"].min()
+                    # ]
+                    # try:
+                    #     best_train_epoch = best_train_epoch.iloc[0]
+                    #     best_train_epoch = best_train_epoch["epoch"]
+                    #     best_validate_epoch = best_validate_epoch.iloc[0]
+                    #     best_validate_epoch = best_validate_epoch["epoch"]
+                    # except:
+                    #     best_train_epoch = np.nan
+                    #     best_validate_epoch = np.nan
+                    # ax[learning_rate_index][col_index].set_xlabel(
+                    #     f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+                    # )
+                    # ax[learning_rate_index][col_index].set_ylabel(
+                    #     'Error RMSPE')
+                    ax[learning_rate_index][col_index].set_ylim(0, 100)
+                    ax[learning_rate_index][col_index].set_xlim(0, 100)
+                    # ax[learning_rate_index][col_index].legend(
+                    #     loc='lower right')
+                learning_rate_index += 1
+            handles, labels = ax[0][0].get_legend_handles_labels()
+            fig.legend(handles, labels, loc="lower center",
+                       ncol=2, fontsize=12)
+            fig.tight_layout(pad=2.0)
+            early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+            if round_no > 0:
+                early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+            os.makedirs(
+                f"result/learning_curve/grid-rate-modifier/{dataset_name}{early_stopping_modifier}",
+                exist_ok=True,
+            )
+            plt.savefig(
+                f"result/learning_curve/grid-rate-modifier/{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
+            )
+            plt.close(fig)
+            model_learning_rate_index += 1
+            learning_rate_index = 0
+
+        dataset_index += 1
+
+
+# TODO: Not Implement
+
+
+def plot_model_weight(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
+    dataset_index = 0
+    early_stopping_title = "With Early Stopping" if use_early_stopping else ""
+    for dataset_name in dataset_names:
+        # Find Best Accuracy
+        # For Each Model Learning Rate
+
+        if use_early_stopping:
+            filter_modifier = modifiers[4:]
+            filter_display_modifier = displayed_modifiers[4:]
+        else:
+            filter_modifier = modifiers[:4]
+            filter_display_modifier = displayed_modifiers[:4]
+        model_learning_rate_index = 0
+        learning_rate_index = 0
+        for model_learning_rate in model_learning_rates:
+            # For Each Learning Rate
+            fig, ax = plt.subplots(5, 4, figsize=(15, 14))
+            if use_academic_name:
+                fig.suptitle(
+                    f"Weight Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+
+            else:
+                fig.suptitle(
+                    f"Weight Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+            learning_rate_index = 0
+            for lr in learning_rate:
+                for modifier_index, modifer in enumerate(filter_modifier):
+                    directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
+                    epoch_error_file = (
+                        f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                    )
+                    epoch_error = pd.read_csv(epoch_error_file)
+                    adjustment_file = (
+                        f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{lr}.csv"
+                    )
+                    adjustment_data = pd.read_csv(adjustment_file)
+
+                    col_index = modifier_index
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["model_weight_1"],
+                        label="Material Element Weight",
+                    )
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["model_weight_2"],
+                        label="Labor Element Weight",
+                    )
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["model_weight_3"],
+                        label="Utiltiy Cost Element Weight",
+                    )
+                    if use_academic_name:
+                        ax[learning_rate_index][col_index].set_title(
+                            f"{filter_display_modifier[modifier_index]}\n α = {lr}",
+                            fontdict={"fontsize": 12},
+                        )
+
+                    best_train_epoch = epoch_error[
+                        epoch_error["error_percent"]
+                        == epoch_error["error_percent"].min()
+                    ]
+                    best_validate_epoch = epoch_error[
+                        epoch_error["validate_error_percent"]
+                        == epoch_error["validate_error_percent"].min()
+                    ]
+                    try:
+                        best_train_epoch = best_train_epoch.iloc[0]
+                        best_train_epoch = best_train_epoch["epoch"]
+                        best_validate_epoch = best_validate_epoch.iloc[0]
+                        best_validate_epoch = best_validate_epoch["epoch"]
+                    except Exception as e:
+                        best_train_epoch = np.nan
+                        best_validate_epoch = np.nan
+                    ax[learning_rate_index][col_index].set_xlabel(
+                        f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+                    )
+                    ax[learning_rate_index][col_index].set_xticks([])
+
+                learning_rate_index += 1
+            handles, labels = ax[0][0].get_legend_handles_labels()
+            fig.legend(handles, labels, loc="lower center",
+                       ncol=2, fontsize=12)
+            fig.tight_layout(pad=2.0)
+            early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+            if round_no > 0:
+                early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+            os.makedirs(
+                f"result/learning_curve/hyperparameter/model-weight-{dataset_name}{early_stopping_modifier}",
+                exist_ok=True,
+            )
+            plt.savefig(
+                f"result/learning_curve/hyperparameter/model-weight-{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
+            )
+            plt.close(fig)
+            model_learning_rate_index += 1
+            learning_rate_index = 0
+
+        dataset_index += 1
+
+
+def find_amount_of_employee(sample_df):
+    column_list = sample_df.columns.tolist()
+    employee_column_list = [
+        col for col in column_list if "employee_weight_" in col]
+    number_of_employee = len(employee_column_list)
+    return number_of_employee
+
+
+def plot_element_weights(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
+    dataset_index = 0
+    early_stopping_title = "With Early Stopping" if use_early_stopping else ""
+    for dataset_name in dataset_names:
+        # Find Best Accuracy
+        # For Each Model Learning Rate
+
+        if use_early_stopping:
+            filter_modifier = modifiers[4:]
+            filter_display_modifier = displayed_modifiers[4:]
+        else:
+            filter_modifier = modifiers[:4]
+            filter_display_modifier = displayed_modifiers[:4]
+        model_learning_rate_index = 0
+        learning_rate_index = 0
+        for model_learning_rate in model_learning_rates:
+            # For Each Learning Rate
+            fig, ax = plt.subplots(5, 4, figsize=(18, 17))
+            if use_academic_name:
+                fig.suptitle(
+                    f"Bias Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+
+            else:
+                fig.suptitle(
+                    f"Model Element Weight Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+            learning_rate_index = 0
+            for lr in learning_rate:
+                for modifier_index, modifer in enumerate(filter_modifier):
+                    directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
+                    epoch_error_file = (
+                        f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                    )
+                    epoch_error = pd.read_csv(epoch_error_file)
+                    adjustment_file = (
+                        f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{lr}.csv"
+                    )
+                    adjustment_data = pd.read_csv(adjustment_file)
+
+                    col_index = modifier_index
+                    # Material Weights
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["material_weight_1"],
+                        label="A Crab", color="LightPink"
+                    )
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["material_weight_2"],
+                        label="C Crab", color="SkyBlue"
+                    )
+
+                    # In Case of Not Exist Material Weight 4 in Actual Case
+                    try:
+                        ax[learning_rate_index][col_index].plot(
+                            adjustment_data["material_weight_4"],
+                            label="Loss Crab", color="MediumPurple"
+                        )
+                        ax[learning_rate_index][col_index].plot(
+                            adjustment_data["material_weight_3"],
+                            label="Small Crab", color="LightGreen"
+                        )
+                    except:
+                        ax[learning_rate_index][col_index].plot(
+                            adjustment_data["material_weight_3"],
+                            label="Small and Loss Crab", color="MediumPurple"
+                        )
+
+                    # Employee Weights
+                    number_of_employee = find_amount_of_employee(
+                        adjustment_data)
+
+                    employee_colors = ["PaleTurquoise", "Cyan",
+                                       "LightCyan", "Turquoise", "DarkTurquoise"]
+                    emp_index = 0
+                    for emp in range(number_of_employee):
+                        ax[learning_rate_index][col_index].plot(
+                            adjustment_data[f"employee_weight_{emp + 1}"],
+                            label=f"Employee {emp}",
+                            color=employee_colors[emp % len(employee_colors)]
+                        )
+                        emp_index += 1
+
+                    # Utility Cost Weights
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["capital_cost_weight_1"],
+                        label="Electricity Cost",
+                        color="gold"
+                    )
+
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["capital_cost_weight_2"],
+                        label="Water Supply Cost",
+                        color="yellow"
+                    )
+
+                    if use_academic_name:
+                        ax[learning_rate_index][col_index].set_title(
+                            f"{filter_display_modifier[modifier_index]}\n α = {lr}",
+                            fontdict={"fontsize": 12},
+                        )
+
+                    best_train_epoch = epoch_error[
+                        epoch_error["error_percent"]
+                        == epoch_error["error_percent"].min()
+                    ]
+                    best_validate_epoch = epoch_error[
+                        epoch_error["validate_error_percent"]
+                        == epoch_error["validate_error_percent"].min()
+                    ]
+                    try:
+                        best_train_epoch = best_train_epoch.iloc[0]
+                        best_train_epoch = best_train_epoch["epoch"]
+                        best_validate_epoch = best_validate_epoch.iloc[0]
+                        best_validate_epoch = best_validate_epoch["epoch"]
+                    except Exception as e:
+                        best_train_epoch = np.nan
+                        best_validate_epoch = np.nan
+
+                    # ax[learning_rate_index][col_index].set_xlabel(
+                    #     f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+                    # )
+                    ax[learning_rate_index][col_index].set_xticks([])
+                    ax[learning_rate_index][col_index].set_ylim(-2000, 10000)
+                    # Ensure all values are positive before setting the y-axis to logarithmic scale
+                    # if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
+                    #     ax[learning_rate_index][col_index].set_yscale('log')
+                    # else:
+                    #     print(
+                    #         f"Warning: Non-positive values detected in dataset {dataset_name}, modifier {modifer}, learning rate {lr}. Skipping logarithmic scale.")
+
+                learning_rate_index += 1
+            handles, labels = ax[0][0].get_legend_handles_labels()
+            fig.legend(handles, labels, loc="lower center",
+                       ncol=8, fontsize=12)
+            fig.tight_layout(pad=3.0)
+            early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+            if round_no > 0:
+                early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+            os.makedirs(
+                f"result/learning_curve/hyperparameter/element-weight-{dataset_name}{early_stopping_modifier}",
+                exist_ok=True,
+            )
+            plt.savefig(
+                f"result/learning_curve/hyperparameter/element-weight-{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
+            )
+            plt.close(fig)
+            model_learning_rate_index += 1
+            learning_rate_index = 0
+
+        dataset_index += 1
+
+
+def plot_element_bias(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1):
+    dataset_index = 0
+    early_stopping_title = "With Early Stopping" if use_early_stopping else ""
+    for dataset_name in dataset_names:
+        # Find Best Accuracy
+        # For Each Model Learning Rate
+
+        if use_early_stopping:
+            filter_modifier = modifiers[4:]
+            filter_display_modifier = displayed_modifiers[4:]
+        else:
+            filter_modifier = modifiers[:4]
+            filter_display_modifier = displayed_modifiers[:4]
+        model_learning_rate_index = 0
+        learning_rate_index = 0
+        for model_learning_rate in model_learning_rates:
+            # For Each Learning Rate
+            fig, ax = plt.subplots(5, 4, figsize=(15, 14))
+            if use_academic_name:
+                fig.suptitle(
+                    f"Biases Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+
+            else:
+                fig.suptitle(
+                    f"Biases Adjustment of {academic_dataset_names[dataset_index]} - β ={model_learning_rate} {early_stopping_title}",
+                    fontdict={"fontsize": 14, "fontweight": "bold"},
+                )
+            learning_rate_index = 0
+            for lr in learning_rate:
+                for modifier_index, modifer in enumerate(filter_modifier):
+                    directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
+                    epoch_error_file = (
+                        f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                    )
+                    epoch_error = pd.read_csv(epoch_error_file)
+                    adjustment_file = (
+                        f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{lr}.csv"
+                    )
+                    adjustment_data = pd.read_csv(adjustment_file)
+
+                    col_index = modifier_index
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["material_bias_1"],
+                        label="Material Element Bias",
+                    )
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["employee_bias_1"],
+                        label="Labor Element Bias",
+                    )
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["capital_cost_bias_1"],
+                        label="Utility Cost Element Bias",
+                    )
+                    ax[learning_rate_index][col_index].plot(
+                        adjustment_data["model_bias"],
+                        label="Model Bias",
+                    )
+                    if use_academic_name:
+                        ax[learning_rate_index][col_index].set_title(
+                            f"{filter_display_modifier[modifier_index]}\n α = {lr}",
+                            fontdict={"fontsize": 12},
+                        )
+
+                    best_train_epoch = epoch_error[
+                        epoch_error["error_percent"]
+                        == epoch_error["error_percent"].min()
+                    ]
+                    best_validate_epoch = epoch_error[
+                        epoch_error["validate_error_percent"]
+                        == epoch_error["validate_error_percent"].min()
+                    ]
+                    try:
+                        best_train_epoch = best_train_epoch.iloc[0]
+                        best_train_epoch = best_train_epoch["epoch"]
+                        best_validate_epoch = best_validate_epoch.iloc[0]
+                        best_validate_epoch = best_validate_epoch["epoch"]
+                    except Exception as e:
+                        best_train_epoch = np.nan
+                        best_validate_epoch = np.nan
+                    ax[learning_rate_index][col_index].set_xlabel(
+                        f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+                    )
+                    ax[learning_rate_index][col_index].set_xticks([])
+
+                learning_rate_index += 1
+            handles, labels = ax[0][0].get_legend_handles_labels()
+            fig.legend(handles, labels, loc="lower center",
+                       ncol=2, fontsize=12)
+            fig.tight_layout(pad=2.0)
+            early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+            if round_no > 0:
+                early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+            os.makedirs(
+                f"result/learning_curve/hyperparameter/bias-{dataset_name}{early_stopping_modifier}",
+                exist_ok=True,
+            )
+            plt.savefig(
+                f"result/learning_curve/hyperparameter/bias-{dataset_name}{early_stopping_modifier}/{model_learning_rate}.png"
+            )
+            plt.close(fig)
+            model_learning_rate_index += 1
+            learning_rate_index = 0
+
+        dataset_index += 1
+
+
+def plot_each_case(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case=""):
+    modifier_index = modifiers.index(selected_case)
+    model_learning_rate_index = 0
+    for model_learning_rate in model_learning_rates:
+        dataset_index = 0
+        fig, ax = plt.subplots(4, 5, figsize=(24, 15))
+        fig.suptitle(
+            f"Learning Curve of {displayed_modifiers[modifier_index]} Dataset with Model Learning Rate (β) of {model_learning_rate}",
+            fontdict={"fontsize": 18, "fontweight": "bold"},
+        )
+
+        for dataset_name in dataset_names:
+            learning_rate_index = 0
+            # For Each Learning Rate
+
+            learning_rate_index = 0
+            for lr in (learning_rate):
+                directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
+                epoch_error_file = (
+                    f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                )
+                epoch_error = pd.read_csv(epoch_error_file)
+                ax[dataset_index][learning_rate_index].plot(
+                    epoch_error["epoch"],
+                    epoch_error["error_percent"],
+                    label="Training",
+                    linewidth=4
+                )
+                ax[dataset_index][learning_rate_index].plot(
+                    epoch_error["epoch"],
+                    epoch_error["validate_error_percent"],
+                    label="Validation",
+                    linewidth=4
+                )
+                if learning_rate_index == 0:
+
+                    ax[dataset_index][learning_rate_index].set_title(
+                        f"{academic_dataset_names[dataset_index]} \nα={lr}",
+                        fontsize=20,
+                    )
+                else:
+                    ax[dataset_index][learning_rate_index].set_title(
+                        f"\nα={lr}",
+                        fontsize=20,
+                    )
+
+                # best_train_epoch = epoch_error[
+                #     epoch_error["error_percent"]
+                #     == epoch_error["error_percent"].min()
+                # ]
+                # best_validate_epoch = epoch_error[
+                #     epoch_error["validate_error_percent"]
+                #     == epoch_error["validate_error_percent"].min()
+                # ]
+                # try:
+                #     best_train_epoch = best_train_epoch.iloc[0]
+                #     best_train_epoch = best_train_epoch["epoch"]
+                #     best_validate_epoch = best_validate_epoch.iloc[0]
+                #     best_validate_epoch = best_validate_epoch["epoch"]
+                # except:
+                #     best_train_epoch = np.nan
+                #     best_validate_epoch = np.nan
+                # ax[dataset_index][learning_rate_index].set_xlabel(
+                #     f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Val Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Lowest Point Iteration on Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+                # )
+                # ax[learning_rate_index][col_index].set_ylabel(
+                #     'Error RMSPE')
+                ax[dataset_index][learning_rate_index].set_ylim(0, 115)
+                ax[dataset_index][learning_rate_index].set_xlim(0, 100)
+                # ax[learning_rate_index][col_index].legend(
+                #     loc='lower right')
+                learning_rate_index += 1
+            dataset_index += 1
+
+        handles, labels = ax[0][0].get_legend_handles_labels()
+        fig.legend(handles, labels, loc="lower center",
+                   ncol=2, fontsize=18)
+        fig.tight_layout(pad=2.0)
+        early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+        if round_no > 0:
+            early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+        casename = to_kebab_case(displayed_modifiers[modifier_index])
+        os.makedirs(
+            f"result/learning_curve/by-case/{casename}{early_stopping_modifier}",
+            exist_ok=True,
+        )
+        plt.savefig(
+            f"result/learning_curve/by-case/{casename}{early_stopping_modifier}/{model_learning_rate}.png"
+        )
+        plt.close(fig)
+        model_learning_rate_index += 1
+
+
+def plot_each_case_reversed(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case=""):
+    modifier_index = modifiers.index(selected_case)
+    model_learning_rate_index = 0
+    for model_learning_rate in model_learning_rates:
+        dataset_index = 0
+        fig, ax = plt.subplots(5, 4, figsize=(24, 20))
+        fig.suptitle(
+            f"Learning Curve of {displayed_modifiers[modifier_index]} Dataset with Model Learning Rate (β) of {model_learning_rate}",
+            fontdict={"fontsize": 20, "fontweight": "bold"},
+        )
+
+        learning_rate_index = 0
+        for lr in (learning_rate):
+            dataset_index = 0
+            # For Each Learning Rate
+
+            for dataset_name in dataset_names:
+                directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
+                epoch_error_file = (
+                    f"{directory_name}/round{round_no + 1}/{iteration}-{lr}.csv"
+                )
+                epoch_error = pd.read_csv(epoch_error_file)
+                ax[learning_rate_index][dataset_index].plot(
+                    epoch_error["epoch"],
+                    epoch_error["error_percent"],
+                    label="Training",
+                    linewidth=4
+                )
+                ax[learning_rate_index][dataset_index].plot(
+                    epoch_error["epoch"],
+                    epoch_error["validate_error_percent"],
+                    label="Validation",
+                    linewidth=4
+                )
+                if dataset_index == 0:
+                    ax[learning_rate_index][dataset_index].set_title(
+                        f"α={lr}\n{academic_dataset_names[dataset_index]}",
+                        fontsize=24,
+                    )
+                else:
+                    ax[learning_rate_index][dataset_index].set_title(
+                        f"\n{academic_dataset_names[dataset_index]}",
+                        fontsize=24,
+                    )
+
+                # best_train_epoch = epoch_error[
+                #     epoch_error["error_percent"]
+                #     == epoch_error["error_percent"].min()
+                # ]
+                # best_validate_epoch = epoch_error[
+                #     epoch_error["validate_error_percent"]
+                #     == epoch_error["validate_error_percent"].min()
+                # ]
+                # try:
+                #     best_train_epoch = best_train_epoch.iloc[0]
+                #     best_train_epoch = best_train_epoch["epoch"]
+                #     best_validate_epoch = best_validate_epoch.iloc[0]
+                #     best_validate_epoch = best_validate_epoch["epoch"]
+                # except:
+                #     best_train_epoch = np.nan
+                #     best_validate_epoch = np.nan
+                # ax[learning_rate_index][dataset_index].set_xlabel(
+                #     f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Val Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Lowest Point Iteration on Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+                # )
+                # ax[learning_rate_index][col_index].set_ylabel(
+                #     'Error RMSPE')
+                ax[learning_rate_index][dataset_index].set_ylim(0, 115)
+                ax[learning_rate_index][dataset_index].set_xlim(0, 100)
+                # ax[learning_rate_index][col_index].legend(
+                #     loc='lower right')
+                dataset_index += 1
+            learning_rate_index += 1
+
+        handles, labels = ax[0][0].get_legend_handles_labels()
+        fig.legend(handles, labels, loc="lower center",
+                   ncol=2, fontsize=18)
+        fig.tight_layout(pad=2.0)
+        early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+        if round_no > 0:
+            early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+        casename = to_kebab_case(displayed_modifiers[modifier_index])
+        os.makedirs(
+            f"result/learning_curve/by-case/{casename}{early_stopping_modifier}",
+            exist_ok=True,
+        )
+        plt.savefig(
+            f"result/learning_curve/by-case/{casename}{early_stopping_modifier}/{model_learning_rate}-reversed.png"
+        )
+        plt.close(fig)
+        model_learning_rate_index += 1
+
+
+def plot_each_element_weights(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case="", learning_rate=0.01):
+    early_stopping_title = "With Early Stopping" if use_early_stopping else ""
+    model_learning_rate_index = 0
+    modifier_index = modifiers.index(selected_case)
+    this_graph_palette = sns.color_palette("husl", 9)
+    sns.set_palette(this_graph_palette)
+    for model_learning_rate in model_learning_rates:
+        # For Each Learning Rate
+        dataset_index = 0
+        fig, ax = plt.subplots(1, 4, figsize=(18, 5))
+        if use_academic_name:
+            fig.suptitle(
+                f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate} α = {learning_rate} {early_stopping_title}",
+                fontdict={"fontsize": 16, "fontweight": "bold"},
+            )
+        else:
+            fig.suptitle(
+                f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate}α = {learning_rate} {early_stopping_title}",
+                fontdict={"fontsize": 16, "fontweight": "bold"},
+            )
+
+        for dataset_name in dataset_names:
+            directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
+            epoch_error_file = (
+                f"{directory_name}/round{round_no + 1}/{iteration}-{learning_rate}.csv"
+            )
+            epoch_error = pd.read_csv(epoch_error_file)
+            adjustment_file = (
+                f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{learning_rate}.csv"
+            )
+            adjustment_data = pd.read_csv(adjustment_file)
+
+            # Material Weights
+            ax[dataset_index].plot(
+                adjustment_data["material_weight_1"],
+                label="A Crab"
+            )
+            ax[dataset_index].plot(
+                adjustment_data["material_weight_2"],
+                label="C Crab"
+            )
+            # In Case of Not Exist Material Weight 4 in Actual Case
+            try:
+                ax[dataset_index].plot(
+                    adjustment_data["material_weight_4"],
+                    label="Loss Crab"
+                )
+                ax[dataset_index].plot(
+                    adjustment_data["material_weight_3"],
+                    label="Small Crab"
+                )
+            except Exception as e:
+                print(e)
+                ax[dataset_index].plot(
+                    adjustment_data["material_weight_3"],
+                    label="Small and Loss Crab"
+                )
+
+            # Utility Cost Weights
+            ax[dataset_index].plot(
+                adjustment_data["capital_cost_weight_1"],
+                label="Electricity Cost",
+
+            )
+            ax[dataset_index].plot(
+                adjustment_data["capital_cost_weight_2"],
+                label="Water Supply Cost",
+
+            )
+
+            # Employee Weights
+            number_of_employee = find_amount_of_employee(
+                adjustment_data)
+            employee_colors = ["PaleTurquoise", "Cyan",
+                               "LightCyan", "Turquoise", "DarkTurquoise"]
+            emp_index = 0
+            for emp in range(number_of_employee):
+                ax[dataset_index].plot(
+                    adjustment_data[f"employee_weight_{emp + 1}"],
+                    label=f"Employee {emp}",
+                    linestyle="--",
+                    # color=employee_colors[emp % len(employee_colors)]
+                )
+                emp_index += 1
+            if use_academic_name:
+                ax[dataset_index].set_title(
+                    f"{academic_dataset_names[dataset_index]}",
+                    fontsize=18,
+                )
+            best_train_epoch = epoch_error[
+                epoch_error["error_percent"]
+                == epoch_error["error_percent"].min()
+            ]
+            best_validate_epoch = epoch_error[
+                epoch_error["validate_error_percent"]
+                == epoch_error["validate_error_percent"].min()
+            ]
+            try:
+                best_train_epoch = best_train_epoch.iloc[0]
+                best_train_epoch = best_train_epoch["epoch"]
+                best_validate_epoch = best_validate_epoch.iloc[0]
+                best_validate_epoch = best_validate_epoch["epoch"]
+            except Exception as e:
+                best_train_epoch = np.nan
+                best_validate_epoch = np.nan
+            # ax[dataset_index].set_xlabel(
+            #     f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+            # )
+            ax[dataset_index].set_xticks([])
+            ax[dataset_index].set_ylim(0, 10000)
+            # Ensure all values are positive before setting the y-axis to logarithmic scale
+            # if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
+            #     ax[dataset_index].set_yscale('log')
+            # else:
+            #     print(
+            #         f"Warning: Non-positive values detected in dataset {dataset_name}. Skipping logarithmic scale.")
+
+            dataset_index += 1
+
+        handles, labels = ax[3].get_legend_handles_labels()
+        print('labels', labels)
+        # Group employee labels together
+        employee_labels = [
+            label for label in labels if label.startswith("Employee")]
+        other_labels = [
+            label for label in labels if not label.startswith("Employee")]
+
+        # Combine employee labels into a single entry
+        if employee_labels:
+            other_labels.append("Employees")
+
+        labels = other_labels
+        fig.legend(handles, labels, loc="lower center",
+                   ncol=8, fontsize=16)
+        fig.tight_layout(pad=2.0)
+        early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+        if round_no > 0:
+            early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+        casename = to_kebab_case(displayed_modifiers[modifier_index])
+        os.makedirs(
+            f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}",
+            exist_ok=True,
+        )
+        plt.savefig(
+            f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}/{casename}-{model_learning_rate}.png"
+        )
+        plt.close(fig)
+        model_learning_rate_index += 1
+
+        dataset_index += 1
+
+    sns.set_palette(sns.color_palette())
+
+
+def plot_each_element_weights_select(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case="", learning_rate=0.01, is_tail=False, round=1000):
+    early_stopping_title = "With Early Stopping" if use_early_stopping else ""
+    model_learning_rate_index = 0
+    modifier_index = modifiers.index(selected_case)
+    this_graph_palette = sns.color_palette("husl", 9)
+    sns.set_palette(this_graph_palette)
+    prefix_modifier = is_tail and f"Last {round} Round" or f"First {round} Round"
+    for model_learning_rate in model_learning_rates:
+        # For Each Learning Rate
+        dataset_index = 0
+        fig, ax = plt.subplots(1, 4, figsize=(18, 5))
+        if use_academic_name:
+            fig.suptitle(
+                f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate} α = {learning_rate} {early_stopping_title}",
+                fontdict={"fontsize": 16, "fontweight": "bold"},
+            )
+        else:
+            fig.suptitle(
+                f"Model Element Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate}α = {learning_rate} {early_stopping_title}",
+                fontdict={"fontsize": 16, "fontweight": "bold"},
+            )
+
+        for dataset_name in dataset_names:
+            directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
+            epoch_error_file = (
+                f"{directory_name}/round{round_no + 1}/{iteration}-{learning_rate}.csv"
+            )
+            epoch_error = pd.read_csv(epoch_error_file)
+            adjustment_file = (
+                f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{learning_rate}.csv"
+            )
+            adjustment_data = pd.read_csv(adjustment_file)
+            if is_tail:
+                # Select the last 1000 rows of the DataFrame
+                adjustment_data = adjustment_data.tail(round)
+            else:
+                adjustment_data = adjustment_data.head(round)
+            # Material Weights
+            ax[dataset_index].plot(
+                adjustment_data["material_weight_1"],
+                label="A Crab"
+            )
+            ax[dataset_index].plot(
+                adjustment_data["material_weight_2"],
+                label="C Crab"
+            )
+            # In Case of Not Exist Material Weight 4 in Actual Case
+            try:
+                ax[dataset_index].plot(
+                    adjustment_data["material_weight_4"],
+                    label="Loss Crab"
+                )
+                ax[dataset_index].plot(
+                    adjustment_data["material_weight_3"],
+                    label="Small Crab"
+                )
+            except Exception as e:
+                print(e)
+                ax[dataset_index].plot(
+                    adjustment_data["material_weight_3"],
+                    label="Small and Loss Crab"
+                )
+            # Employee Weights
+            number_of_employee = find_amount_of_employee(
+                adjustment_data)
+            employee_colors = ["PaleTurquoise", "Cyan",
+                               "LightCyan", "Turquoise", "DarkTurquoise"]
+            emp_index = 0
+            for emp in range(number_of_employee):
+                ax[dataset_index].plot(
+                    adjustment_data[f"employee_weight_{emp + 1}"],
+                    label=f"Employee {emp}",
+                    # color=employee_colors[emp % len(employee_colors)]
+                )
+                emp_index += 1
+            # Utility Cost Weights
+            ax[dataset_index].plot(
+                adjustment_data["capital_cost_weight_1"],
+                label="Electricity Cost",
+
+            )
+            ax[dataset_index].plot(
+                adjustment_data["capital_cost_weight_2"],
+                label="Water Supply Cost",
+
+            )
+            if use_academic_name:
+                ax[dataset_index].set_title(
+                    f"{academic_dataset_names[dataset_index]}",
+                    fontsize=16,
+                )
+            best_train_epoch = epoch_error[
+                epoch_error["error_percent"]
+                == epoch_error["error_percent"].min()
+            ]
+            best_validate_epoch = epoch_error[
+                epoch_error["validate_error_percent"]
+                == epoch_error["validate_error_percent"].min()
+            ]
+            try:
+                best_train_epoch = best_train_epoch.iloc[0]
+                best_train_epoch = best_train_epoch["epoch"]
+                best_validate_epoch = best_validate_epoch.iloc[0]
+                best_validate_epoch = best_validate_epoch["epoch"]
+            except Exception as e:
+                best_train_epoch = np.nan
+                best_validate_epoch = np.nan
+            # ax[dataset_index].set_xlabel(
+            #     f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+            # )
+            ax[dataset_index].set_xticks([])
+            ax[dataset_index].set_ylim(0, 10000)
+            # Ensure all values are positive before setting the y-axis to logarithmic scale
+            # if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
+            #     ax[dataset_index].set_yscale('log')
+            # else:
+            #     print(
+            #         f"Warning: Non-positive values detected in dataset {dataset_name}. Skipping logarithmic scale.")
+
+            dataset_index += 1
+
+        handles, labels = ax[3].get_legend_handles_labels()
+        print('labels', labels)
+        # Group employee labels together
+        employee_labels = [
+            label for label in labels if label.startswith("Employee")]
+        other_labels = [
+            label for label in labels if not label.startswith("Employee")]
+
+        # Combine employee labels into a single entry
+        if employee_labels:
+            other_labels.append("Employees")
+
+        labels = other_labels
+        fig.legend(handles, labels, loc="lower center",
+                   ncol=8, fontsize=12)
+        fig.tight_layout(pad=2.0)
+        early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+        if round_no > 0:
+            early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+        prefix_filename = is_tail and f"tail-{round}" or f"head-{round}"
+        os.makedirs(
+            f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}",
+            exist_ok=True,
+        )
+        plt.savefig(
+            f"result/learning_curve/hyperparameter/element-weight-{learning_rate}{early_stopping_modifier}/{prefix_filename}-{model_learning_rate}.png"
+        )
+        plt.close(fig)
+        model_learning_rate_index += 1
+
+        dataset_index += 1
+
+    sns.set_palette(sns.color_palette())
+
+
+def plot_each_weights(use_academic_name=False, use_early_stopping=False, round_no=0, model_version=1, selected_case="", learning_rate=0.01):
+    early_stopping_title = "With Early Stopping" if use_early_stopping else ""
+    model_learning_rate_index = 0
+    modifier_index = modifiers.index(selected_case)
+    this_graph_palette = sns.color_palette("husl", 4)
+    sns.set_palette(this_graph_palette)
+    for model_learning_rate in model_learning_rates:
+        # For Each Learning Rate
+        dataset_index = 0
+        fig, ax = plt.subplots(1, 4, figsize=(18, 5))
+        if use_academic_name:
+            fig.suptitle(
+                f"Model Level Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate} α = {learning_rate} {early_stopping_title}",
+                fontdict={"fontsize": 16, "fontweight": "bold"},
+            )
+        else:
+            fig.suptitle(
+                f"Model Level Weight Adjustment of {displayed_modifiers[modifier_index]} {academic_dataset_names[dataset_index]} \n β ={model_learning_rate}α = {learning_rate} {early_stopping_title}",
+                fontdict={"fontsize": 16, "fontweight": "bold"},
+            )
+
+        for dataset_name in dataset_names:
+            directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{selected_case}"
+            epoch_error_file = (
+                f"{directory_name}/round{round_no + 1}/{iteration}-{learning_rate}.csv"
+            )
+            epoch_error = pd.read_csv(epoch_error_file)
+            adjustment_file = (
+                f"{directory_name}/round{round_no + 1}/sample-payload-list-{iteration}-{learning_rate}.csv"
+            )
+            adjustment_data = pd.read_csv(adjustment_file)
+
+            ax[dataset_index].plot(
+                adjustment_data["model_weight_1"],
+                label="Material Element Weight",
+            )
+            ax[dataset_index].plot(
+                adjustment_data["model_weight_2"],
+                label="Labor Element Weight", linewidth=3,
+                # linestyle='-',
+            )
+            ax[dataset_index].plot(
+                adjustment_data["model_weight_3"],
+                label="Utiltiy Cost Element Weight",
+                linewidth=3,
+                linestyle=':', color=sns.palettes.hls_palette(6)[4]
+            )
+            if use_academic_name:
+                ax[dataset_index].set_title(
+                    f"{academic_dataset_names[dataset_index]}",
+                    fontsize=20,
+                )
+            best_train_epoch = epoch_error[
+                epoch_error["error_percent"]
+                == epoch_error["error_percent"].min()
+            ]
+            best_validate_epoch = epoch_error[
+                epoch_error["validate_error_percent"]
+                == epoch_error["validate_error_percent"].min()
+            ]
+            try:
+                best_train_epoch = best_train_epoch.iloc[0]
+                best_train_epoch = best_train_epoch["epoch"]
+                best_validate_epoch = best_validate_epoch.iloc[0]
+                best_validate_epoch = best_validate_epoch["epoch"]
+            except Exception as e:
+                best_train_epoch = np.nan
+                best_validate_epoch = np.nan
+            # ax[dataset_index].set_xlabel(
+            #     f'Train Err {epoch_error.iloc[-1]["error_percent"]:.2f} , Validate Err {epoch_error.iloc[-1]["validate_error_percent"]:.2f} \n Best Epoch Train {best_train_epoch:.0f} , Validate {best_validate_epoch:.0f}'
+            # )
+            ax[dataset_index].set_xticks([])
+            ax[dataset_index].set_ylim(0, 30)
+            # Ensure all values are positive before setting the y-axis to logarithmic scale
+            # if (adjustment_data.select_dtypes(include=[np.number]) > 0).all().all():
+            #     ax[dataset_index].set_yscale('log')
+            # else:
+            #     print(
+            #         f"Warning: Non-positive values detected in dataset {dataset_name}. Skipping logarithmic scale.")
+
+            dataset_index += 1
+
+        handles, labels = ax[3].get_legend_handles_labels()
+        print('labels', labels)
+        # Group employee labels together
+        employee_labels = [
+            label for label in labels if label.startswith("Employee")]
+        other_labels = [
+            label for label in labels if not label.startswith("Employee")]
+
+        # Combine employee labels into a single entry
+        if employee_labels:
+            other_labels.append("Employees")
+
+        labels = other_labels
+        fig.legend(handles, labels, loc="lower center",
+                   ncol=8, fontsize=16)
+        fig.tight_layout(pad=2.0)
+        early_stopping_modifier = "/early-stopping" if use_early_stopping else ""
+        if round_no > 0:
+            early_stopping_modifier = f"/round-{round_no}/{early_stopping_modifier}"
+        os.makedirs(
+            f"result/learning_curve/hyperparameter/model-weight-{learning_rate}{early_stopping_modifier}",
+            exist_ok=True,
+        )
+        plt.savefig(
+            f"result/learning_curve/hyperparameter/model-weight-{learning_rate}{early_stopping_modifier}/{model_learning_rate}-{selected_case}.png"
+        )
+        plt.close(fig)
+        model_learning_rate_index += 1
+
+        dataset_index += 1
+
+    sns.set_palette(sns.color_palette())

functions/experiment_script/random_number_list.csv

@@ -0,0 +1,125 @@
+400 +- 30 x 31,300 +- 30 x 31,150 +- 30 x 31,120 +- 35 x 31,81 +- 10 x 10,70 +- 10 x 10,40 +- 10 x 10,100 +- 10 x 10,350 +- 20 x 30,300 +- 20 x 30,150 +- 20 x 30,120 +- 35 x 30,101 +- 10 x 10,82 +- 10 x 10,41 +- 10 x 10,120 +- 10 x 10,250 +- 30 x 31,200 +- 30 x 31,100 +- 30 x 31,80 +- 35 x 31,150 +- 10 x 10,80 +- 10 x 10,180 +- 10 x 10,200 +- 20 x 31,150 +- 20 x 31,100 +- 20 x 31,70 +- 20 x 31,200 +- 20 x 10,150 +- 20 x 10,120 +- 20 x 10,230 +- 20 x 10,60 +- 12 x 124,40 +- 12 x 31
+397,316,128,146,86,62,40,102,359,303,137,115,110,73,42,130,236,198,114,69,152,86,170,195,160,100,84,201,136,101,230,62,28
+405,278,136,99,73,61,45,104,338,280,170,98,99,82,47,112,261,209,70,86,158,79,178,208,134,109,50,189,150,124,248,63,46
+428,326,176,112,88,79,30,97,349,286,132,95,93,92,34,114,280,171,107,71,149,85,173,209,159,113,54,210,151,126,218,58,30
+404,311,179,154,77,71,41,108,348,302,170,155,109,88,32,112,220,225,88,64,143,84,190,196,157,91,80,214,150,104,238,71,48
+400,273,153,102,83,65,46,105,334,306,134,135,98,88,47,110,239,230,74,53,143,74,183,190,142,100,72,182,148,102,244,72,31
+384,276,153,153,71,71,45,96,369,305,144,110,108,76,41,117,256,185,88,97,140,85,174,185,140,116,69,220,163,125,234,65,34
+404,287,137,85,72,72,45,106,330,290,139,101,111,91,47,120,265,209,79,108,147,75,186,181,160,98,77,213,163,133,228,72,40
+424,311,138,102,83,76,49,102,357,285,163,136,94,91,43,118,251,218,108,108,140,89,180,189,165,90,63,197,153,119,212,63,39
+412,309,169,149,78,64,31,99,332,293,135,121,102,88,31,119,250,201,83,109,144,71,182,198,141,81,53,186,157,127,246,57,48
+426,296,136,152,91,72,30,101,348,314,143,86,106,92,49,127,277,223,80,111,146,79,190,218,165,117,75,197,169,102,248,52,48
+388,289,160,137,0,0,0,0,369,308,156,119,0,0,0,0,270,195,73,68,0,0,0,218,132,104,77,0,0,0,0,61,44
+406,279,144,98,0,0,0,0,343,289,162,87,0,0,0,0,229,214,123,57,0,0,0,195,163,81,57,0,0,0,0,71,38
+389,289,151,130,0,0,0,0,340,310,130,129,0,0,0,0,265,195,94,65,0,0,0,193,167,114,66,0,0,0,0,50,52
+395,315,130,132,0,0,0,0,338,291,135,140,0,0,0,0,258,207,94,70,0,0,0,197,142,100,64,0,0,0,0,57,37
+386,278,171,116,0,0,0,0,333,295,134,123,0,0,0,0,230,189,130,63,0,0,0,193,138,96,65,0,0,0,0,56,50
+411,298,140,151,0,0,0,0,334,301,166,152,0,0,0,0,235,207,128,87,0,0,0,191,170,104,79,0,0,0,0,52,34
+419,297,121,101,0,0,0,0,369,316,160,141,0,0,0,0,229,176,112,52,0,0,0,196,153,109,88,0,0,0,0,50,29
+387,311,177,149,0,0,0,0,350,306,139,99,0,0,0,0,250,210,87,67,0,0,0,205,135,113,79,0,0,0,0,71,50
+430,296,133,90,0,0,0,0,330,294,161,148,0,0,0,0,222,218,107,75,0,0,0,196,156,110,79,0,0,0,0,60,50
+402,291,169,134,0,0,0,0,370,282,133,93,0,0,0,0,269,214,114,58,0,0,0,195,159,107,85,0,0,0,0,65,46
+391,309,176,104,0,0,0,0,336,311,160,143,0,0,0,0,231,180,125,58,0,0,0,190,150,89,60,0,0,0,0,55,44
+376,314,156,105,0,0,0,0,355,297,154,152,0,0,0,0,225,216,120,82,0,0,0,191,145,116,76,0,0,0,0,57,43
+374,293,125,93,0,0,0,0,348,284,148,86,0,0,0,0,228,228,84,60,0,0,0,215,145,90,51,0,0,0,0,57,30
+417,311,132,102,0,0,0,0,331,301,159,130,0,0,0,0,255,213,126,62,0,0,0,216,164,114,54,0,0,0,0,67,29
+413,271,164,112,0,0,0,0,363,314,166,152,0,0,0,0,229,205,77,45,0,0,0,183,141,116,76,0,0,0,0,64,43
+413,324,166,118,0,0,0,0,344,294,162,144,0,0,0,0,230,196,93,108,0,0,0,191,133,93,81,0,0,0,0,49,39
+393,295,131,88,0,0,0,0,342,310,149,108,0,0,0,0,253,172,94,102,0,0,0,185,130,91,90,0,0,0,0,67,34
+398,330,120,123,0,0,0,0,342,292,132,134,0,0,0,0,239,177,125,99,0,0,0,205,132,114,72,0,0,0,0,71,37
+392,312,140,131,0,0,0,0,354,283,146,119,0,0,0,0,237,208,90,72,0,0,0,183,154,94,60,0,0,0,0,71,51
+398,303,148,85,0,0,0,0,349,282,157,131,0,0,0,0,242,189,70,91,0,0,0,192,150,112,69,0,0,0,0,65,37
+420,280,154,116,0,0,0,0,0,0,0,0,0,0,0,0,236,197,71,60,0,0,0,219,152,109,62,0,0,0,0,70,39
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,65,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,54,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,71,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,71,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,72,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,71,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,71,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,61,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,58,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,71,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,66,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,70,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,72,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,71,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,72,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,66,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,53,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,65,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,58,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,61,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,70,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,66,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,53,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,60,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,58,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,71,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,61,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,60,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,65,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,53,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0

functions/experiment_script/rsg_breakpoint_experiment.py

@@ -0,0 +1,457 @@
+import pandas as pd
+import numpy as np
+import importlib
+import sys
+import time
+import os
+
+# fmt:off
+sys.path.append('../14-New-Final-Model/model')
+sys.path.append('../14-New-Final-Model/matrix_generator')
+sys.path.append('../14-New-Final-Model')
+sys.path.append('../14-New-Final-Model/extractor')
+
+import tdce_model as tdce
+import material_fc_layer as mfl
+import employee_fc_layer as efl
+import captial_fc_layer as cfl
+import loss
+import cost_matrix_class as cmc
+import display_input_variation as diva
+import viyacrab_augmentation as viya
+import adjust_data as ajd
+import result_display as rd
+
+importlib.reload(tdce)
+importlib.reload(mfl)
+importlib.reload(efl)
+importlib.reload(cfl)
+importlib.reload(loss)
+importlib.reload(tdce)
+importlib.reload(cmc)
+importlib.reload(diva)
+importlib.reload(viya)
+importlib.reload(ajd)
+importlib.reload(rd)
+# fmt:on
+
+
+def inital_layer(
+    material_cost_matrix,
+    employee_cost_matrix,
+    capital_cost_matrix,
+):
+    total_col = 0
+    # Material FC Layer
+    row, high, col = material_cost_matrix.shape
+    material_layer_1 = mfl.MaterialFCLayer(col, 1)
+    # material_layer_1.annotate(material_cost_matrix, material_amount_matrix)
+    total_col += col
+
+    # Monthy Employee FC Layer
+    row, high, col = employee_cost_matrix.shape
+    employee_layer_1 = efl.EmployeeFCLayer(col, 1, 8)
+    total_col += col
+    # monthy_employee_layer_1.annotate(monthy_employee_cost_matrix, duration_matrix)
+
+    # Capital Cost  FC Layer
+    row, high, col = capital_cost_matrix.shape
+    capital_cost_layer1 = cfl.CapitalCostFCLayer(col, 1, 21)
+    total_col += col
+    # capital_cost_layer1.annotate(
+    #     capital_cost_matrix, life_time_matrix, machine_hour_matrix, duration_matrix)
+
+    return (
+        material_layer_1,
+        employee_layer_1,
+        capital_cost_layer1,
+    )
+
+
+def create_learning(
+    epoch,
+    learning_rate,
+    inside_learning_rate,
+    material_cost_matrix,
+    material_amount_matrix,
+    employee_cost_matrix,
+    employee_duration_matrix,
+    employee_day_amount_matrix,
+    capital_cost_matrix,
+    day_amount_matrix,
+    capital_duration_matrix,
+    result_matrix,
+    folder_name,
+    validation_payload,
+    breakpoint,
+    early_stopping=False,
+    patience_round=10,
+    use_model_weight=False,
+):
+    # Initial Model
+    tdce_model = tdce.TDCEModel()
+    # Initial Layer
+    (
+        material_layer_1,
+        employee_layer_1,
+        capital_cost_layer1,
+    ) = inital_layer(
+        capital_cost_matrix=capital_cost_matrix,
+        employee_cost_matrix=employee_cost_matrix,
+        material_cost_matrix=material_cost_matrix,
+    )
+
+    tdce_model.inital_inside_element(
+        material_layer=material_layer_1,
+        capital_cost_layer=capital_cost_layer1,
+        employee_layer=employee_layer_1,
+    )
+
+    tdce_model.use(loss=loss.mse, loss_prime=loss.mse_prime,
+                   loss_percent=loss.rmspe)
+
+    tdce_model.set_learning_rate(
+        inside_learning_rate[0],
+        inside_learning_rate[1],
+        inside_learning_rate[2],
+    )
+
+    start_time = time.time()
+
+    if early_stopping:
+        tdce_model.activate_early_stopping()
+        tdce_model.edit_patience_round(patience_round)
+
+    if use_model_weight:
+        tdce_model.activete_model_weight()
+
+    tdce_model.fit_with_validation(
+        epoch=epoch,
+        learning_rate=learning_rate,
+        material_amount_matrix=material_amount_matrix,
+        material_cost_matrix=material_cost_matrix,
+        employee_cost_matrix=employee_cost_matrix,
+        employee_duration_matrix=employee_duration_matrix,
+        employee_day_amount_matrix=employee_day_amount_matrix,
+        result_matrix=result_matrix,
+        capital_cost_matrix=capital_cost_matrix,
+        day_amount_matrix=day_amount_matrix,
+        validation_payload=validation_payload,
+        capital_cost_duration_matrix=capital_duration_matrix,
+    )
+
+    end_time = time.time()
+    time_usage = end_time - start_time
+    print(f"Learning Rate: {learning_rate} & {inside_learning_rate}")
+    print(f"Time Using {time_usage} Second")
+
+    error_list = tdce_model.get_epoch_error()
+    sample_error_list = tdce_model.get_sample_error()
+
+    error_df = pd.DataFrame(error_list)
+    sample_error_df = pd.DataFrame(sample_error_list)
+
+    before_breakpoint_error = error_df[error_df["epoch"] < breakpoint]
+    bb_minimum_error = before_breakpoint_error["error"].min()
+    bb_minimum_percent_error = before_breakpoint_error["error_percent"].min()
+    bb_minimum_validate_error = before_breakpoint_error["validate_error"].min()
+    bb_minimum_validate_percent_error = before_breakpoint_error[
+        "validate_error_percent"
+    ].min()
+
+    minimum_error = error_df["error"].min()
+    minimum_percent_error = error_df["error_percent"].min()
+    minimum_validate_error = error_df["validate_error"].min()
+    minimum_validate_percent_error = error_df["validate_error_percent"].min()
+
+    try:
+        os.mkdir(f"{folder_name}")
+    except FileExistsError:
+        print("Folder is Exist")
+        pass
+
+    error_df.to_csv(f"{folder_name}/{epoch}-{inside_learning_rate[0]}.csv")
+    sample_error_df.to_csv(
+        f"{folder_name}/error-list-{epoch}-{inside_learning_rate[0]}.csv"
+    )
+
+    sample_payload = tdce_model.get_sample_payload()
+    sample_payload_df = pd.DataFrame(sample_payload)
+    sample_payload_df.to_csv(
+        f"{folder_name}/sample-payload-list-{epoch}-{inside_learning_rate[0]}.csv",
+        index=False,
+    )
+
+    return (
+        minimum_error,
+        time_usage,
+        minimum_percent_error,
+        minimum_validate_error,
+        minimum_validate_percent_error,
+        bb_minimum_error,
+        bb_minimum_percent_error,
+        bb_minimum_validate_error,
+        bb_minimum_validate_percent_error,
+    )
+
+
+def run_experiment(
+    data_directory,
+    epoch,
+    folder_name,
+    round_number,
+    learning_rate_list,
+    model_learning_rate,
+    breakpoint,
+    early_stopping=False,
+    patience_round=10,
+    augmentation=False,
+    remove_outlier_qtr=False,
+    outlier_index=1.5,
+    use_model_weight=False,
+):
+    print("Run Experiment Script is initial !")
+    try:
+        os.mkdir(folder_name)
+    except FileExistsError:
+        print("Folder is Exist")
+        pass
+    cost_generator = cmc.CostMatrixGenerator()
+    cost_generator.change_data_directory(data_directory)
+    cost_generator.load_data()
+    if remove_outlier_qtr:
+        cost_generator.remove_outlier_iqr(outlier_index)
+        (
+            new_process_df,
+            new_employee_usage,
+            new_material_usage,
+            new_capital_cost_usage,
+        ) = cost_generator.get_data()
+
+        (new_capital_cost_usage, new_employee_usage, new_material_usage) = (
+            ajd.adjust_to_match_process(
+                capital_cost_usage=new_capital_cost_usage,
+                employee_usage=new_employee_usage,
+                material_usage=new_material_usage,
+                new_process_df=new_process_df,
+            )
+        )
+
+        new_variation = diva.display_input_variation(
+            new_process_df,
+            new_material_usage,
+            new_employee_usage,
+            new_capital_cost_usage,
+        )
+        new_variation.to_csv(f"{folder_name}/data_variation_after_outlier.csv")
+        new_process_df.to_csv(f"{folder_name}/process_df_after_outlier.csv")
+
+        try:
+            print("Data Variation After Outlier Removed")
+            display(new_variation)
+        except Exception as e:
+            print("This is not Jupyter Notebook", e)
+
+    # Save Variation of Overall Data
+    input_variation = diva.display_input_variation_by_directory(data_directory)
+    input_variation.to_csv(f"{folder_name}/data_variation.csv")
+
+    overall_result = []
+
+    # Iteration over the set of Learning Rates
+    for learn_r in learning_rate_list:
+        round_err_list = []
+        round_time_list = []
+        round_percent_list = []
+        round_validate_err_list = []
+        round_validate_err_percent_list = []
+        round_brakpoint_err_list = []
+        round_brakpoint_err_percent_list = []
+        round_brakpoint_val_err_list = []
+        round_brakpoint_val_err_percent_list = []
+        round_err_dict = {"learning_rate": str(learn_r)}
+        for round in range(round_number):
+            ##
+            (
+                train_process_df,
+                train_employee_usage,
+                train_material_usage,
+                train_capital_cost,
+                validate_process_df,
+                validate_employee_usage,
+                validate_material_usage,
+                validate_capital_cost,
+            ) = cost_generator.train_test_split_without_matrix(0.7)
+            # Generate Cost Matrix for Validation Set
+            validation_payload = cost_generator.get_validation_payload(
+                validate_process_df
+            )
+
+            # Display Variation of Train Data
+            train_variation = diva.display_input_variation(
+                train_process_df,
+                train_material_usage,
+                train_employee_usage,
+                train_capital_cost,
+            )
+            train_variation.to_csv(
+                f"{folder_name}/train_data_variation_{round}.csv")
+            train_process_df.to_csv(
+                f"{folder_name}/train_process_df_{round}.csv")
+
+            # Display Variation of Validation Data
+            validate_variation = diva.display_input_variation(
+                validate_process_df,
+                validate_material_usage,
+                validate_employee_usage,
+                validate_capital_cost,
+            )
+
+            validate_variation.to_csv(
+                f"{folder_name}/validate_data_variation_{round}.csv"
+            )
+            if augmentation:
+                # TODO:  Increase the Generalization of the Model
+                # Augmented the Imbalance Class of Training Data
+                train_process_df.to_csv(
+                    f"{folder_name}/train_process_df_before_augmented_{round}.csv"
+                )
+                train_process_df = viya.vy_training_augmentation(
+                    train_process_df)
+                # Display Variation of Train Data After Augmented
+                train_variation = diva.display_input_variation(
+                    train_process_df,
+                    train_material_usage,
+                    train_employee_usage,
+                    train_capital_cost,
+                )
+                train_variation.to_csv(
+                    f"{folder_name}/train_data_variation_after_augmented_{round}.csv"
+                )
+                train_process_df.to_csv(
+                    f"{folder_name}/train_process_df_after_augmented_{round}.csv"
+                )
+
+            # Generate Matrix From Training Set
+            (
+                material_cost_matrix,
+                material_amount_matrix,
+                employee_cost_matrix,
+                employee_duration_matrix,
+                employee_day_amount_matrix,
+                capital_cost_matrix,
+                day_amount_matrix,
+                capital_cost_duration_matrix,  # New On Finetune
+                result_matrix,
+            ) = cost_generator.generate_data_from_input(
+                train_process_df,
+                train_material_usage,
+                train_employee_usage,
+                train_capital_cost,
+            )
+            # Create Training
+            (
+                round_err,
+                round_time,
+                round_err_percent,
+                val_err,
+                val_err_percent,
+                brakpoint_err,
+                breakpoint_err_percent,
+                breakpoint_val_err,
+                breakpoint_val_err_percent,
+            ) = create_learning(
+                epoch=epoch,
+                learning_rate=model_learning_rate,
+                folder_name=f"{folder_name}/round{round + 1}",
+                # Material
+                material_amount_matrix=material_amount_matrix,
+                material_cost_matrix=material_cost_matrix,
+                # Employee
+                employee_cost_matrix=employee_cost_matrix,
+                employee_duration_matrix=employee_duration_matrix,
+                employee_day_amount_matrix=employee_day_amount_matrix,
+                # Capital Cost
+                capital_cost_matrix=capital_cost_matrix,
+                day_amount_matrix=day_amount_matrix,
+                capital_duration_matrix=capital_cost_duration_matrix,
+                # Result Matrix
+                result_matrix=result_matrix,
+                # Validation Payload
+                validation_payload=validation_payload,
+                inside_learning_rate=[learn_r, learn_r, learn_r],
+                # because it start with -1
+                breakpoint=breakpoint,
+                early_stopping=early_stopping,
+                patience_round=patience_round,
+                use_model_weight=use_model_weight,
+            )
+            print(
+                f"Learning Rate {learn_r} Round {round} / {round_number} Finish with Error {round_err} ({round_err_percent} %), time comsume {round_time} "
+            )
+            round_err_list.append(round_err)
+            round_time_list.append(round_time)
+            round_percent_list.append(round_err_percent)
+            round_validate_err_list.append(val_err)
+            round_validate_err_percent_list.append(val_err_percent)
+            round_brakpoint_err_list.append(brakpoint_err)
+            round_brakpoint_err_percent_list.append(breakpoint_err_percent)
+            round_brakpoint_val_err_list.append(breakpoint_val_err)
+            round_brakpoint_val_err_percent_list.append(
+                breakpoint_val_err_percent)
+            round_err_dict[f"round_{round}_err"] = round_err
+            round_err_dict[f"round_{round}_duration"] = round_time
+            round_err_dict[f"round_{round}_err_percent"] = round_err_percent
+            round_err_dict[f"round_{round}_val_err"] = val_err
+            round_err_dict[f"round_{round}_val_err_percent"] = val_err_percent
+            round_err_dict[f"round_{round}_brakpoint_err"] = brakpoint_err
+            round_err_dict[f"round_{round}_brakpoint_err_percent"] = (
+                breakpoint_err_percent
+            )
+            round_err_dict[f"round_{round}_brakpoint_val_err"] = breakpoint_val_err
+            round_err_dict[f"round_{round}_brakpoint_val_err_percent"] = (
+                breakpoint_val_err_percent
+            )
+
+        average_error = np.average(round_err_list)
+        average_time = np.average(round_time_list)
+        average_error_percent = np.average(round_percent_list)
+        average_val_err = np.average(round_validate_err_list)
+        average_val_err_percent = np.average(round_validate_err_percent_list)
+        breakpoint_average_error = np.average(round_brakpoint_err_list)
+        breakpoint_average_error_percent = np.average(
+            round_brakpoint_err_percent_list)
+        breakpoint_average_val_error = np.average(round_brakpoint_val_err_list)
+        breakpoint_average_val_error_percent = np.average(
+            round_brakpoint_val_err_percent_list
+        )
+        print(
+            f"Success for learning rate {learn_r} : Average error {average_error} ({average_error_percent}%) Average Time {average_time}"
+        )
+        round_err_dict["average_error"] = average_error
+        round_err_dict["average_err_percent"] = average_error_percent
+        round_err_dict["average_time"] = average_time
+        round_err_dict["average_validate_error"] = average_val_err
+        round_err_dict["average_validate_error_percent"] = average_val_err_percent
+        round_err_dict["average_brakpoint_error"] = breakpoint_average_error
+        round_err_dict["average_brakpoint_error_percent"] = (
+            breakpoint_average_error_percent
+        )
+        round_err_dict["average_brakpoint_val_error"] = breakpoint_average_val_error
+        round_err_dict["average_brakpoint_val_error_percent"] = (
+            breakpoint_average_val_error_percent
+        )
+        overall_result.append(round_err_dict)
+
+    overall_result_df = pd.DataFrame(overall_result)
+    overall_result_df.to_csv(f"{folder_name}/overall_result.csv")
+
+    rd.creating_error_csv(
+        primary_directory_name=folder_name,
+        learning_rate=learning_rate_list,
+        iteration_number=epoch,
+        round_number=round_number,
+        breakpoint_number=breakpoint,
+    )
+
+    return overall_result_df

functions/experiment_script/run_all_experiment.py

@@ -0,0 +1,21 @@
+import batch_experiment_script as bes
+import importlib
+importlib.reload(bes)
+
+
+def run_all_experiment():
+    experiment_version = 1
+    round_number = 3
+    iteration = 100
+
+    dataset_names = ['1-munchkin', '2-chinchilla',
+                     '3-vicheanmas', '4-scottishfold']
+
+    for dataset_name in dataset_names:
+        bes.run_in_many_learning_rate(dataset_name=dataset_name,
+                                      experiment_version=experiment_version,
+                                      round_number=round_number,
+                                      iteration=iteration)
+
+
+run_all_experiment()

functions/experiment_script/summarize_error.py

@@ -0,0 +1,165 @@
+import pandas as pd
+
+
+dataset_names = ["1-munchkin", "2-chinchilla",
+                 "3-vicheanmas", "4-scottishfold"]
+academic_dataset_names = ["Dataset 1", "Dataset 3", "Dataset 2", "Dataset 4"]
+
+
+learning_rate = [0.005, 0.01, 0.05, 0.1, 0.5]
+overall_accuracy_df = pd.DataFrame()
+
+model_learning_rates = ["1e-07", "1e-08"]
+iteration = 100
+modifiers = [
+    "",
+    "_remove_outlier",
+    "_augmented",
+    "_remove_outlier_augmented",
+    "_early_stopping",
+    "_remove_outlier_early_stopping",
+    "_augmented_early_stopping",
+    "_remove_outlier_augmented_early_stopping",
+]
+displayed_modifiers = [
+    "Original",
+    "Remove Outlier",
+    "Augmentation",
+    "Remove Outlier + Augmentation",
+    "Early Stopping",
+    "Remove Outlier + Early Stopping",
+    "Augmentation + Early Stopping",
+    "Remove Outlier + Augmentation + Early Stopping",
+]
+
+
+def find_overall_accuracy(round_number=3,
+                          model_version=1):
+    overall_accuracy_df = pd.DataFrame()
+    for dataset_name in dataset_names:
+        for round_no in range(round_number):
+            for modifer in modifiers:
+                accuracy_list = []
+                # Find Best Accuracy
+                # For Each Model Learning Rate
+                for model_learning_rate in model_learning_rates:
+                    directory_name = f"result/{dataset_name}/{dataset_name}_{model_version}_{model_learning_rate}{modifer}"
+                    # For Each Learning Rate
+                    for lr in learning_rate:
+                        datafile = f"{directory_name}/results/{lr}_round_error.csv"
+                        result = pd.read_csv(datafile)
+                        result_with_this_round = result[result["round"]
+                                                        == round_no]
+                        result_with_this_round = result_with_this_round.iloc[0]
+                        # Find the last iteration which is early stopping
+                        epoch_error_file = (
+                            f"{directory_name}/round{round_no+1}/{iteration}-{lr}.csv"
+                        )
+                        epoch_error = pd.read_csv(epoch_error_file)
+                        # Find Last Record
+                        epoch_error = epoch_error.iloc[-1]
+                        # Find last record epoch
+                        last_epoch = epoch_error["epoch"]
+                        accuracy_payload = {
+                            "type": "training",
+                            "learning_rate": lr,
+                            "model_learning_rate": model_learning_rate,
+                            "rmspe": result_with_this_round["error_percent"],
+                            "last_epoch": last_epoch,
+                        }
+                        accuracy_list.append(accuracy_payload)
+                        accuracy_payload = {
+                            "type": "validate",
+                            "learning_rate": lr,
+                            "model_learning_rate": model_learning_rate,
+                            "rmspe": result_with_this_round["validate_error_percent"],
+                            "last_epoch": last_epoch,
+                        }
+                        accuracy_list.append(accuracy_payload)
+
+                accuracy_list = pd.DataFrame(accuracy_list)
+                best_training = (
+                    accuracy_list[accuracy_list["type"] == "training"]
+                    .sort_values(by="rmspe")
+                    .iloc[0]
+                )
+
+                best_validate = (
+                    accuracy_list[accuracy_list["type"] == "validate"]
+                    .sort_values(by="rmspe")
+                    .iloc[0]
+                )
+
+                # Find Data Variation For Training
+                if "augmented" in modifer:
+                    train_data_variation = pd.read_csv(
+                        f"{directory_name}/train_data_variation_after_augmented_{round_no}.csv"
+                    )
+                else:
+                    train_data_variation = pd.read_csv(
+                        f"{directory_name}/train_data_variation_{round_no}.csv"
+                    )
+
+                train_cost_data = train_data_variation[
+                    train_data_variation["data"] == "Total Cost"
+                ]
+                train_cost_data = train_cost_data.iloc[0]
+                payload = {
+                    "dataset": dataset_name,
+                    "round_no": round_no,
+                    "modifer": modifer,
+                    "type": "training",
+                    "cv": train_cost_data["variation"],
+                    "iqr": train_cost_data["iqr"],
+                    "min": train_cost_data["min"],
+                    "max": train_cost_data["max"],
+                    "mean": train_cost_data["mean"],
+                    "minimum_error": best_training["rmspe"],
+                    "best_learning_rate": best_training["learning_rate"],
+                    "best_model_learning_rate": best_training["model_learning_rate"],
+                    "last_epoch": best_training["last_epoch"],
+                }
+                overall_accuracy_df = pd.concat(
+                    [overall_accuracy_df, pd.DataFrame(payload, index=[0])],
+                    ignore_index=True,
+                )
+                # Find Data Variation For Validate
+                validate_data_variation = pd.read_csv(
+                    f"{directory_name}/validate_data_variation_{round_no}.csv"
+                )
+                validate_cost_data = validate_data_variation[
+                    validate_data_variation["data"] == "Total Cost"
+                ]
+                validate_cost_data = validate_cost_data.iloc[0]
+                payload = {
+                    "dataset": dataset_name,
+                    "round_no": round_no,
+                    "modifer": modifer,
+                    "type": "validate",
+                    "cv": validate_cost_data["variation"],
+                    "iqr": validate_cost_data["iqr"],
+                    "min": validate_cost_data["min"],
+                    "max": validate_cost_data["max"],
+                    "mean": validate_cost_data["mean"],
+                    "minimum_error": best_validate["rmspe"],
+                    "best_learning_rate": best_training["learning_rate"],
+                    "best_model_learning_rate": best_training["model_learning_rate"],
+                    "last_epoch": best_training["last_epoch"],
+                }
+
+                overall_accuracy_df = pd.concat(
+                    [overall_accuracy_df, pd.DataFrame(payload, index=[0])],
+                    ignore_index=True,
+                )
+    # Post Processing
+    overall_accuracy_df["remove_outlier"] = overall_accuracy_df["modifer"].apply(
+        lambda x: 1 if "_remove_outlier" in x else 0
+    )
+    overall_accuracy_df["augmented"] = overall_accuracy_df["modifer"].apply(
+        lambda x: 1 if "_augmented" in x else 0
+    )
+    overall_accuracy_df["early_stopping"] = overall_accuracy_df["modifer"].apply(
+        lambda x: 1 if "_early_stopping" in x else 0
+    )
+
+    overall_accuracy_df.to_csv("result/overall_accuracy.csv", index=False)