File size: 6,218 Bytes
f71c233 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 |
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import numpy as np
import json
import os
import os.path as osp
# LOAD FINAL RESULTS:
datasets = ["x_div_y", "x_minus_y", "x_plus_y", "permutation"]
folders = os.listdir("./")
final_results = {}
results_info = {}
for folder in folders:
if folder.startswith("run") and osp.isdir(folder):
with open(osp.join(folder, "final_info.json"), "r") as f:
final_results[folder] = json.load(f)
results_dict = np.load(
osp.join(folder, "all_results.npy"), allow_pickle=True
).item()
print(results_dict.keys())
run_info = {}
for dataset in datasets:
run_info[dataset] = {}
val_losses = []
train_losses = []
val_accs = []
train_accs = []
for k in results_dict.keys():
if dataset in k and "val_info" in k:
run_info[dataset]["step"] = [
info["step"] for info in results_dict[k]
]
val_losses.append([info["val_loss"] for info in results_dict[k]])
val_accs.append([info["val_accuracy"] for info in results_dict[k]])
if dataset in k and "train_info" in k:
train_losses.append(
[info["train_loss"] for info in results_dict[k]]
)
train_accs.append(
[info["train_accuracy"] for info in results_dict[k]]
)
mean_val_losses = np.mean(val_losses, axis=0)
mean_train_losses = np.mean(train_losses, axis=0)
mean_val_accs = np.mean(val_accs, axis=0)
mean_train_accs = np.mean(train_accs, axis=0)
if len(val_losses) > 0:
sterr_val_losses = np.std(val_losses, axis=0) / np.sqrt(
len(val_losses)
)
stderr_train_losses = np.std(train_losses, axis=0) / np.sqrt(
len(train_losses)
)
sterr_val_accs = np.std(val_accs, axis=0) / np.sqrt(len(val_accs))
stderr_train_accs = np.std(train_accs, axis=0) / np.sqrt(
len(train_accs)
)
else:
sterr_val_losses = np.zeros_like(mean_val_losses)
stderr_train_losses = np.zeros_like(mean_train_losses)
sterr_val_accs = np.zeros_like(mean_val_accs)
stderr_train_accs = np.zeros_like(mean_train_accs)
run_info[dataset]["val_loss"] = mean_val_losses
run_info[dataset]["train_loss"] = mean_train_losses
run_info[dataset]["val_loss_sterr"] = sterr_val_losses
run_info[dataset]["train_loss_sterr"] = stderr_train_losses
run_info[dataset]["val_acc"] = mean_val_accs
run_info[dataset]["train_acc"] = mean_train_accs
run_info[dataset]["val_acc_sterr"] = sterr_val_accs
run_info[dataset]["train_acc_sterr"] = stderr_train_accs
results_info[folder] = run_info
# CREATE LEGEND -- ADD RUNS HERE THAT WILL BE PLOTTED
labels = {
"run_0": "Baseline",
"run_1": "Operand Reversal",
"run_2": "Negation",
"run_3": "Combined (15%)",
"run_4": "Combined (30%)",
"run_5": "Combined (20%)",
}
# Create a programmatic color palette
def generate_color_palette(n):
cmap = plt.get_cmap("tab20")
return [mcolors.rgb2hex(cmap(i)) for i in np.linspace(0, 1, n)]
# Get the list of runs and generate the color palette
runs = list(labels.keys())
colors = generate_color_palette(len(runs))
# Plot 1: Line plot of validation accuracy for each dataset across the runs
for dataset in datasets:
plt.figure(figsize=(12, 6))
for i, run in enumerate(runs):
iters = results_info[run][dataset]["step"]
mean = results_info[run][dataset]["val_acc"]
sterr = results_info[run][dataset]["val_acc_sterr"]
plt.plot(iters, mean, label=labels[run], color=colors[i])
plt.fill_between(iters, mean - sterr, mean + sterr, color=colors[i], alpha=0.2)
plt.title(f"Validation Accuracy Across Runs for {dataset} Dataset")
plt.xlabel("Update Steps")
plt.ylabel("Validation Accuracy")
plt.legend()
plt.grid(True, which="both", ls="-", alpha=0.2)
plt.ylim(0, 1.05)
plt.tight_layout()
plt.savefig(f"val_acc_{dataset}.png")
plt.close()
# Plot 2: Bar plot of steps to 99% validation accuracy for each dataset and run
step_val_acc_99 = {dataset: [] for dataset in datasets}
for dataset in datasets:
for run in runs:
step = final_results[run][dataset]["means"].get("step_val_acc_99_mean")
step_val_acc_99[dataset].append(step if step is not None else 7500)
plt.figure(figsize=(12, 6))
x = np.arange(len(datasets))
width = 0.15
for i, run in enumerate(runs):
plt.bar(x + i*width, [step_val_acc_99[dataset][i] for dataset in datasets], width, label=labels[run])
plt.xlabel('Datasets')
plt.ylabel('Steps to 99% Validation Accuracy')
plt.title('Steps to 99% Validation Accuracy Across Runs and Datasets')
plt.xticks(x + width * 2.5, datasets)
plt.legend(loc='upper left', bbox_to_anchor=(1, 1))
plt.tight_layout()
plt.savefig('steps_to_99_acc.png')
plt.close()
# Plot 3: Bar plot of final validation accuracy for each dataset and run
final_val_acc = {dataset: [] for dataset in datasets}
for dataset in datasets:
for run in runs:
acc = final_results[run][dataset]["means"]["final_val_acc_mean"]
final_val_acc[dataset].append(acc)
plt.figure(figsize=(12, 6))
x = np.arange(len(datasets))
width = 0.15
for i, run in enumerate(runs):
plt.bar(x + i*width, [final_val_acc[dataset][i] for dataset in datasets], width, label=labels[run])
plt.xlabel('Datasets')
plt.ylabel('Final Validation Accuracy')
plt.title('Final Validation Accuracy Across Runs and Datasets')
plt.xticks(x + width * 2.5, datasets)
plt.legend(loc='upper left', bbox_to_anchor=(1, 1))
plt.tight_layout()
plt.savefig('final_val_acc.png')
plt.close()
|