File size: 4,904 Bytes
d4c7aae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | import os
import pandas as pd
def create_directory(primary_directory_name):
result_dir = f"{primary_directory_name}/results"
try:
os.mkdir(result_dir)
except FileExistsError:
print("Folder is Exist")
return result_dir
def creating_error_csv(
primary_directory_name,
learning_rate,
iteration_number,
round_number,
breakpoint_number,
):
result_dir = create_directory(primary_directory_name)
overall_learning_rate_err_ = []
for lr in learning_rate:
round_err_list = []
round_percent_list = []
round_validate_err_list = []
round_validate_err_percent_list = []
round_brakpoint_err_list = []
round_brakpoint_err_percent_list = []
round_brakpoint_val_err_list = []
round_brakpoint_val_err_percent_list = []
round_errors = []
learning_rate_err_dict = {"learning_rate": str(lr)}
for round_no in range(round_number):
round_err_dict = {"learning_rate": str(lr)}
directory_name = primary_directory_name + "/round" + str(round_no + 1)
df = pd.read_csv(f"{directory_name}/{iteration_number}-{lr}.csv")
round_err_list.append(df["error"].values[-1])
round_percent_list.append(df["error_percent"].values[-1])
round_validate_err_list.append(df["validate_error"].values[-1])
round_validate_err_percent_list.append(
df["validate_error_percent"].values[-1]
)
breakpoint_df = df[df["epoch"] <= breakpoint_number]
round_brakpoint_err_list.append(breakpoint_df["error"].values[-1])
round_brakpoint_err_percent_list.append(
breakpoint_df["error_percent"].values[-1]
)
round_brakpoint_val_err_list.append(
breakpoint_df["validate_error"].values[-1]
)
round_brakpoint_val_err_percent_list.append(
breakpoint_df["validate_error_percent"].values[-1]
)
# Added to the dictionary
round_err_dict[f"round"] = round_no
round_err_dict[f"error"] = df["error"].values[-1]
round_err_dict[f"error_percent"] = df["error_percent"].values[-1]
round_err_dict[f"validate_error"] = df["validate_error"].values[-1]
round_err_dict[f"validate_error_percent"] = df[
"validate_error_percent"
].values[-1]
round_err_dict[f"brakpoint_error"] = breakpoint_df["error"].values[-1]
round_err_dict[f"brakpoint_error_percent"] = breakpoint_df[
"error_percent"
].values[-1]
round_err_dict[f"brakpoint_validate_error"] = breakpoint_df[
"validate_error"
].values[-1]
round_err_dict[f"brakpoint_validate_error_percent"] = breakpoint_df[
"validate_error_percent"
].values[-1]
round_errors.append(round_err_dict)
round_err_df = pd.DataFrame(round_errors)
round_err_df.to_csv(f"{result_dir}/{lr}_round_error.csv", index=False)
average_error = round_err_df["error"].mean()
average_error_percent = round_err_df["error_percent"].mean()
average_validate_error = round_err_df["validate_error"].mean()
average_validate_error_percent = round_err_df["validate_error_percent"].mean()
average_brakpoint_error = round_err_df["brakpoint_error"].mean()
average_brakpoint_error_percent = round_err_df["brakpoint_error_percent"].mean()
average_brakpoint_validate_error = round_err_df[
"brakpoint_validate_error"
].mean()
average_brakpoint_validate_error_percent = round_err_df[
"brakpoint_validate_error_percent"
].mean()
learning_rate_err_dict["average_error"] = average_error
learning_rate_err_dict["average_error_percent"] = average_error_percent
learning_rate_err_dict["average_validate_error"] = average_validate_error
learning_rate_err_dict["average_validate_error_percent"] = (
average_validate_error_percent
)
learning_rate_err_dict["average_brakpoint_error"] = average_brakpoint_error
learning_rate_err_dict["average_brakpoint_error_percent"] = (
average_brakpoint_error_percent
)
learning_rate_err_dict["average_brakpoint_validate_error"] = (
average_brakpoint_validate_error
)
learning_rate_err_dict["average_brakpoint_validate_error_percent"] = (
average_brakpoint_validate_error_percent
)
overall_learning_rate_err_.append(learning_rate_err_dict)
overall_learning_rate_err_df = pd.DataFrame(overall_learning_rate_err_)
overall_learning_rate_err_df.to_csv(
f"{result_dir}/overall_learning_rate_err.csv", index=False
)
|