import numpy as np class Dense: def __init__(self, input_size, output_size): # small weights no gradient explosion self.weights = np.random.randn(input_size, output_size) * 0.01 self.biases = np.zeros((1, output_size)) self.dweights = None self.dbiases = None def forward(self, input_data): self.input = input_data # falttened 1D vector return np.dot(self.input, self.weights) + self.biases def backward(self, output_gradient, learning_rate=None): # calculating gradient self.dweights = self.input.T @ output_gradient # @ is matrix product self.dbiases = np.sum(output_gradient, axis=0, keepdims=True) input_gradient = output_gradient @ self.weights.T # updating parameters if learning_rate is provided if learning_rate is not None: self.weights -= learning_rate * self.dweights self.biases -= learning_rate * self.dbiases return input_gradient class Conv: def __init__(self, input_shape, kernel_size, num_kernels): input_depth, input_height, input_width = input_shape self.input_shape = input_shape self.input_depth = input_depth self.num_kernels = num_kernels # output shape depends on num kernels # ie. num kernels = num of output feature maps # no input_depth in output shape because each kernel process all input channels self.output_shape = ( num_kernels, input_height - kernel_size + 1, input_width - kernel_size + 1, ) self.kernel_shape = ( num_kernels, input_depth, # color channels kernel_size, # k_height kernel_size, # k_width ) self.kernels = np.random.randn(*self.kernel_shape) * 0.1 self.biases = np.zeros((num_kernels, 1, 1)) self.dkernels = None self.dbiases = None def forward(self, input_data): self.input = input_data from numpy.lib.stride_tricks import sliding_window_view patches = sliding_window_view(input_data, (self.kernel_shape[2], self.kernel_shape[3]), axis=(1, 2)) self.output = np.einsum('jyxkl,ijkl->iyx', patches, self.kernels) + self.biases return self.output def backward(self, output_gradient, learning_rate=None): from numpy.lib.stride_tricks import sliding_window_view patches = sliding_window_view(self.input, (self.kernel_shape[2], self.kernel_shape[3]), axis=(1, 2)) self.dkernels = np.einsum('iyx,jyxkl->ijkl', output_gradient, patches) self.dbiases = np.sum(output_gradient, axis=(1, 2), keepdims=True) input_gradient = np.zeros(self.input_shape) for y in range(self.output_shape[1]): for x in range(self.output_shape[2]): input_gradient[:, y : y + self.kernel_shape[2], x : x + self.kernel_shape[3]] += np.tensordot( output_gradient[:, y, x], self.kernels, axes=(0, 0) ) if learning_rate is not None: self.kernels -= learning_rate * self.dkernels self.biases -= learning_rate * self.dbiases return input_gradient class MaxPool: def __init__(self, pool_size=2, stride=2): self.pool_size = pool_size self.stride = stride def forward(self, input_data): self.input = input_data depth, height, width = input_data.shape out_height = (height - self.pool_size) // self.stride + 1 out_width = (width - self.pool_size) // self.stride + 1 self.output_shape = (depth, out_height, out_width) from numpy.lib.stride_tricks import sliding_window_view patches = sliding_window_view(input_data, (self.pool_size, self.pool_size), axis=(1, 2)) patches = patches[:, ::self.stride, ::self.stride] return np.max(patches, axis=(3, 4)) def backward(self, output_gradient): input_gradient = np.zeros_like(self.input) depth, out_height, out_width = self.output_shape for row in range(out_height): for col in range(out_width): start_y = row * self.stride start_x = col * self.stride patch = self.input[:, start_y : start_y + self.pool_size, start_x : start_x + self.pool_size] max_val = np.max(patch, axis=(1, 2), keepdims=True) mask = (patch == max_val) input_gradient[:, start_y : start_y + self.pool_size, start_x : start_x + self.pool_size] += ( output_gradient[:, row, col][:, np.newaxis, np.newaxis] * mask ) return input_gradient class Flatten: def __init__(self): self.input_shape = None self.batch_size = 0 def forward(self, x): self.input_shape = x.shape self.batch_size = x.shape[0] return x.reshape(self.batch_size, -1) def backward(self, grad_input): return grad_input.reshape(self.input_shape)