Upload 5 files

activations.py

@@ -0,0 +1,46 @@
+import numpy as np
+import math
+
+
+class ReLU:
+    def __init__(self):
+        self.mask = None
+
+    def forward(self, x):
+        self.mask = (x > 0)
+        return x * self.mask
+
+    def backward(self, grad):
+        return grad * self.mask
+
+
+class GELU:
+    def __init__(self, approximate='none'):
+        self.approximate = approximate
+        self.x = None
+
+    def forward(self, x):
+        self.x = x
+        if self.approximate == 'tanh':
+            inner = np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))
+            return 0.5 * x * (1 + np.tanh(inner))
+        else:
+            erf_vec = np.vectorize(math.erf)
+            return 0.5 * x * (1 + erf_vec(x / np.sqrt(2)))
+
+    def backward(self, grad):
+        x = self.x
+        if self.approximate == 'tanh':
+            c = np.sqrt(2 / np.pi)
+            inner = c * (x + 0.044715 * np.power(x, 3))
+            tanh_inner = np.tanh(inner)
+            d_inner = c * (1.0 + 3.0 * 0.044715 * np.square(x))
+            dx = 0.5 * (1.0 + tanh_inner) + 0.5 * x * (1.0 - np.square(tanh_inner)) * d_inner
+            return grad * dx
+        else:
+            erf_vec = np.vectorize(math.erf)
+            cdf = 0.5 * (1.0 + erf_vec(x / np.sqrt(2)))
+            pdf = (1.0 / np.sqrt(2.0 * np.pi)) * np.exp(-0.5 * np.square(x))
+            dx = cdf + x * pdf
+            return grad * dx
+

layers.py

@@ -0,0 +1,139 @@
+import numpy as np
+
+
+class Dense:
+    def __init__(self, input_size, output_size):
+        # small weights no gradient explosion
+        self.weights = np.random.randn(input_size, output_size) * 0.01
+        self.biases = np.zeros((1, output_size))
+        self.dweights = None
+        self.dbiases = None
+
+    def forward(self, input_data):
+        self.input = input_data  # falttened 1D vector
+        return np.dot(self.input, self.weights) + self.biases
+
+    def backward(self, output_gradient, learning_rate=None):
+        # calculating gradient
+        self.dweights = self.input.T @ output_gradient  # @ is matrix product
+        self.dbiases = np.sum(output_gradient, axis=0, keepdims=True)
+        input_gradient = output_gradient @ self.weights.T
+
+        # updating parameters if learning_rate is provided
+        if learning_rate is not None:
+            self.weights -= learning_rate * self.dweights
+            self.biases -= learning_rate * self.dbiases
+
+        return input_gradient
+
+
+class Conv:
+    def __init__(self, input_shape, kernel_size, num_kernels):
+        input_depth, input_height, input_width = input_shape
+
+        self.input_shape = input_shape
+        self.input_depth = input_depth
+        self.num_kernels = num_kernels
+
+        # output shape depends on num kernels
+        # ie. num kernels = num of output feature maps
+        # no input_depth in output shape because each kernel process all input channels
+        self.output_shape = (
+            num_kernels,
+            input_height - kernel_size + 1,
+            input_width - kernel_size + 1,
+        )
+
+        self.kernel_shape = (
+            num_kernels,
+            input_depth,  # color channels
+            kernel_size,  # k_height
+            kernel_size,  # k_width
+        )
+        self.kernels = np.random.randn(*self.kernel_shape) * 0.1
+        self.biases = np.zeros((num_kernels, 1, 1))
+        self.dkernels = None
+        self.dbiases = None
+
+    def forward(self, input_data):
+        self.input = input_data
+        from numpy.lib.stride_tricks import sliding_window_view
+        patches = sliding_window_view(input_data, (self.kernel_shape[2], self.kernel_shape[3]), axis=(1, 2))
+        self.output = np.einsum('jyxkl,ijkl->iyx', patches, self.kernels) + self.biases
+        return self.output
+
+    def backward(self, output_gradient, learning_rate=None):
+        from numpy.lib.stride_tricks import sliding_window_view
+        patches = sliding_window_view(self.input, (self.kernel_shape[2], self.kernel_shape[3]), axis=(1, 2))
+        
+        self.dkernels = np.einsum('iyx,jyxkl->ijkl', output_gradient, patches)
+        self.dbiases = np.sum(output_gradient, axis=(1, 2), keepdims=True)
+
+        input_gradient = np.zeros(self.input_shape)
+        for y in range(self.output_shape[1]):
+            for x in range(self.output_shape[2]):
+                input_gradient[:, y : y + self.kernel_shape[2], x : x + self.kernel_shape[3]] += np.tensordot(
+                    output_gradient[:, y, x], self.kernels, axes=(0, 0)
+                )
+
+        if learning_rate is not None:
+            self.kernels -= learning_rate * self.dkernels
+            self.biases -= learning_rate * self.dbiases
+
+        return input_gradient
+
+
+class MaxPool:
+    def __init__(self, pool_size=2, stride=2):
+        self.pool_size = pool_size
+        self.stride = stride
+
+    def forward(self, input_data):
+        self.input = input_data
+
+        depth, height, width = input_data.shape
+
+        out_height = (height - self.pool_size) // self.stride + 1
+        out_width = (width - self.pool_size) // self.stride + 1
+
+        self.output_shape = (depth, out_height, out_width)
+
+        from numpy.lib.stride_tricks import sliding_window_view
+        patches = sliding_window_view(input_data, (self.pool_size, self.pool_size), axis=(1, 2))
+        patches = patches[:, ::self.stride, ::self.stride]
+        
+        return np.max(patches, axis=(3, 4))
+
+    def backward(self, output_gradient):
+        input_gradient = np.zeros_like(self.input)
+
+        depth, out_height, out_width = self.output_shape
+
+        for row in range(out_height):
+            for col in range(out_width):
+                start_y = row * self.stride
+                start_x = col * self.stride
+                
+                patch = self.input[:, start_y : start_y + self.pool_size, start_x : start_x + self.pool_size]
+                max_val = np.max(patch, axis=(1, 2), keepdims=True)
+                mask = (patch == max_val)
+                
+                input_gradient[:, start_y : start_y + self.pool_size, start_x : start_x + self.pool_size] += (
+                    output_gradient[:, row, col][:, np.newaxis, np.newaxis] * mask
+                )
+
+        return input_gradient
+
+
+class Flatten:
+    def __init__(self):
+        self.input_shape = None
+        self.batch_size = 0
+        
+    def forward(self, x):
+        self.input_shape = x.shape
+        self.batch_size = x.shape[0]
+        return x.reshape(self.batch_size, -1)
+        
+    def backward(self, grad_input):
+        return grad_input.reshape(self.input_shape)

model_weights.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b53ca1c9c6958b961c854da485d3b925f89c071b31c6ae10defdf18ed4c2da9
+size 135812

requirements.txt

@@ -0,0 +1,2 @@
+numpy>=2.0.0
+pillow>=10.0.0