adding unit tests for activation functions (using pytest)

numpyneuron/__init__.py

@@ -1,3 +1,10 @@
 from .loss import *
 from .activation import *
 from .nn import *
+
+ACTIVATIONS: dict[str, Activation] = {
+    "Relu": Relu(),
+    "Sigmoid": Sigmoid(),
+    "TanH": TanH(),
+    "SoftMax": SoftMax(),
+}

numpyneuron/activation.py

@@ -39,18 +39,11 @@ class Sigmoid(Activation):
 
 class SoftMax(Activation):
     def forward(self, X: np.ndarray) -> np.ndarray:
+        ax = 1 if X.ndim > 1 else 0
         exps = np.exp(
-            X - np.max(X, axis=1, keepdims=True)
+            X - np.max(X, axis=ax, keepdims=True)
         )  # Avoid numerical instability
-        return exps / np.sum(exps, axis=1, keepdims=True)
+        return exps / np.sum(exps, axis=ax, keepdims=True)
 
     def backward(self, X: np.ndarray) -> np.ndarray:
         return X
-
-
-ACTIVATIONS: dict[str, Activation] = {
-    "Relu": Relu(),
-    "Sigmoid": Sigmoid(),
-    "TanH": TanH(),
-    "SoftMax": SoftMax(),
-}

requirements.txt

@@ -1,7 +1,8 @@
-gradio==4.27.0
-matplotlib==3.8.4
-numpy==1.26.4
-plotly==5.21.0
-scikit_learn==1.4.2
-setuptools==69.5.1
-tqdm==4.66.2
+gradio==4.39.0
+matplotlib==3.6.3
+numpy==2.0.1
+plotly==5.22.0
+pytest==7.4.4
+scikit_learn==1.5.1
+setuptools==68.1.2
+tqdm==4.66.4

test/test_activation.py

@@ -0,0 +1,66 @@
+import os
+import sys
+import pytest
+
+sys.path.append(os.path.abspath(".."))
+
+import random
+import numpy as np
+from numpyneuron import (
+    TanH,
+    Sigmoid,
+    Relu,
+    SoftMax,
+    Sigmoid,
+)
+
+# these functions are meant to work with np.ndarray
+# objects, but they will also work with numbers which
+# makes testing a little bit simpler
+
+
+def test_tanh() -> None:
+    """
+    tanh(1) =~ 0.76
+    tanh'(1) =~ sech^2(1) =~ 0.419
+    """
+    tanh = TanH()
+    assert tanh.forward(1) == pytest.approx(np.tanh(1))
+    assert tanh.forward(1) == pytest.approx(0.7615941559557649)
+    assert tanh.backward(1) == pytest.approx(0.41997434161402614)
+
+
+def test_sigmoid() -> None:
+    """
+    sigmoid(1) =~ 0.73105
+    sigmoid'(1) =~ 0.1966
+    """
+    sigmoid = Sigmoid()
+    assert sigmoid.forward(1) == pytest.approx(0.7310585786300049)
+    assert sigmoid.backward(1) == pytest.approx(0.4621171572600098)
+
+
+def test_relu() -> None:
+    """
+    relu(n > 0) = n
+    relu(n < 0) = 0
+    relu'(n > 0) = 1
+    relu'(n < 0) = 0
+    """
+    relu = Relu()
+    random_n = random.randint(1, 100)
+    assert relu.forward(random_n) == random_n
+    assert relu.backward(random_n) == 1
+
+
+def test_softmax() -> None:
+    """
+    softmax([1, 2, 3]) = [0.090031, 0.244728, 0.665241]
+    """
+    softmax = SoftMax()
+    vec = np.array([1, 2, 3])
+    assert np.allclose(
+        softmax.forward(vec),
+        np.array([0.090031, 0.244728, 0.665241]),
+    )
+    assert np.allclose(softmax.backward(vec), vec)