we should try to do this with regression on a simple dataset from the internet somewhere because I think it is hard to see how well it is doing with completley random data

main.py

@@ -19,7 +19,7 @@ def main():
     except KeyError:
         raise f"Invalid method \"{method}\". Try one of these\n{list(options.keys())}"
 
-    X, y = random_dataset(rows=10, features=2)
+    X, y = random_dataset(rows=1000, features=10)
     func(X, y)
 
 

neural_network/activation.py

@@ -1,17 +1,10 @@
 import numpy as np
 
+relu = lambda x: np.maximum(x, 0)
+relu_prime = lambda x: np.where(x > 0, 1, 0)
 
-def sigmoid(x: float) -> float:
-    return 1.0 / (1.0 + np.exp(-x))
+tanh = lambda x: np.tanh(x)
+tanh_prime = lambda x: 1 - tanh(x) ** 2
 
-
-def sigmoid_prime(x: float) -> float:
-    return sigmoid(x) / (1.0 - sigmoid(x))
-
-
-def relu(x):
-    return np.maximum(x, 0)
-
-
-def relu_prime(x):
-    return np.where(x > 0, 1, 0)
+sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x))
+sigmoid_prime = lambda x: sigmoid(x) / 1.0 - sigmoid(x)

neural_network/main.py

@@ -1,9 +1,10 @@
 from sklearn.model_selection import train_test_split
+import matplotlib.pyplot as plt
 import numpy as np
 
 from neural_network.opts import activation
 from neural_network.backprop import bp
-from neural_network.model import Model
+from neural_network.model import Network
 from neural_network.plot import loss_history_plt
 
 
@@ -52,7 +53,9 @@ def main(
     results, loss_history = bp(X_train, y_train, wb, args)
     final = results[args["epochs"] - 1]
     func = activation[args["activation_func"]]["main"]
-    fm = Model(final_wb=final, activation_func=func)
+
+    # initialize our final network
+    fm = Network(final_wb=final, activation_func=func)
 
     # predict the x test data and compare it to y test data
     pred = fm.predict(X_test)
@@ -61,4 +64,6 @@ def main(
 
     # plot predicted versus actual
     # also plot the training loss over epochs
-    loss_history_plt(loss_history)
+    animated_loss_plt = loss_history_plt(loss_history)
+    # eventually we will save this plot
+    plt.show()

neural_network/model.py

@@ -2,7 +2,7 @@ import numpy as np
 from typing import Callable
 
 
-class Model:
+class Network:
     def __init__(self, final_wb: dict[str, np.array], activation_func: Callable):
         self.func = activation_func
         self.final_wb = final_wb

neural_network/opts.py

@@ -10,4 +10,9 @@ activation = {
         "main": sigmoid,
         "prime": sigmoid_prime,
     },
+
+    "tanh": {
+        "main": tanh,
+        "prime": tanh_prime,
+    },
 }

neural_network/plot.py

@@ -1,11 +1,17 @@
 import seaborn as sns
 import matplotlib.pyplot as plt
-from matplotlib.animation import FuncAnimation
+from matplotlib.animation import FuncAnimation, FFMpegWriter
 
 sns.set()
 
+"""
+Save plots to the plots folder for when
+we would like to show results on our little
+flask application
+"""
 
-def loss_history_plt(loss_history: list) -> None:
+
+def loss_history_plt(loss_history: list) -> FuncAnimation:
     fig, ax = plt.subplots()
 
     def animate(i):
@@ -18,5 +24,12 @@ def loss_history_plt(loss_history: list) -> None:
         ax.set_xlabel("Epoch")
         ax.set_ylabel("Training Loss")
 
-    _ = FuncAnimation(fig, animate, frames=len(loss_history), interval=100)
-    plt.show()
+    return FuncAnimation(fig, animate, frames=len(loss_history), interval=100)
+
+
+def save_plt(plot, filename: str, animated: bool, fps=10):
+    if not animated:
+        plot.savefig(filename)
+        return
+    writer = FFMpegWriter(fps=fps)
+    plot.save(filename, writer=writer)