File size: 5,120 Bytes
21532d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | import numpy as np
class Dense:
def __init__(self, input_size, output_size):
# small weights no gradient explosion
self.weights = np.random.randn(input_size, output_size) * 0.01
self.biases = np.zeros((1, output_size))
self.dweights = None
self.dbiases = None
def forward(self, input_data):
self.input = input_data # falttened 1D vector
return np.dot(self.input, self.weights) + self.biases
def backward(self, output_gradient, learning_rate=None):
# calculating gradient
self.dweights = self.input.T @ output_gradient # @ is matrix product
self.dbiases = np.sum(output_gradient, axis=0, keepdims=True)
input_gradient = output_gradient @ self.weights.T
# updating parameters if learning_rate is provided
if learning_rate is not None:
self.weights -= learning_rate * self.dweights
self.biases -= learning_rate * self.dbiases
return input_gradient
class Conv:
def __init__(self, input_shape, kernel_size, num_kernels):
input_depth, input_height, input_width = input_shape
self.input_shape = input_shape
self.input_depth = input_depth
self.num_kernels = num_kernels
# output shape depends on num kernels
# ie. num kernels = num of output feature maps
# no input_depth in output shape because each kernel process all input channels
self.output_shape = (
num_kernels,
input_height - kernel_size + 1,
input_width - kernel_size + 1,
)
self.kernel_shape = (
num_kernels,
input_depth, # color channels
kernel_size, # k_height
kernel_size, # k_width
)
self.kernels = np.random.randn(*self.kernel_shape) * 0.1
self.biases = np.zeros((num_kernels, 1, 1))
self.dkernels = None
self.dbiases = None
def forward(self, input_data):
self.input = input_data
from numpy.lib.stride_tricks import sliding_window_view
patches = sliding_window_view(input_data, (self.kernel_shape[2], self.kernel_shape[3]), axis=(1, 2))
self.output = np.einsum('jyxkl,ijkl->iyx', patches, self.kernels) + self.biases
return self.output
def backward(self, output_gradient, learning_rate=None):
from numpy.lib.stride_tricks import sliding_window_view
patches = sliding_window_view(self.input, (self.kernel_shape[2], self.kernel_shape[3]), axis=(1, 2))
self.dkernels = np.einsum('iyx,jyxkl->ijkl', output_gradient, patches)
self.dbiases = np.sum(output_gradient, axis=(1, 2), keepdims=True)
input_gradient = np.zeros(self.input_shape)
for y in range(self.output_shape[1]):
for x in range(self.output_shape[2]):
input_gradient[:, y : y + self.kernel_shape[2], x : x + self.kernel_shape[3]] += np.tensordot(
output_gradient[:, y, x], self.kernels, axes=(0, 0)
)
if learning_rate is not None:
self.kernels -= learning_rate * self.dkernels
self.biases -= learning_rate * self.dbiases
return input_gradient
class MaxPool:
def __init__(self, pool_size=2, stride=2):
self.pool_size = pool_size
self.stride = stride
def forward(self, input_data):
self.input = input_data
depth, height, width = input_data.shape
out_height = (height - self.pool_size) // self.stride + 1
out_width = (width - self.pool_size) // self.stride + 1
self.output_shape = (depth, out_height, out_width)
from numpy.lib.stride_tricks import sliding_window_view
patches = sliding_window_view(input_data, (self.pool_size, self.pool_size), axis=(1, 2))
patches = patches[:, ::self.stride, ::self.stride]
return np.max(patches, axis=(3, 4))
def backward(self, output_gradient):
input_gradient = np.zeros_like(self.input)
depth, out_height, out_width = self.output_shape
for row in range(out_height):
for col in range(out_width):
start_y = row * self.stride
start_x = col * self.stride
patch = self.input[:, start_y : start_y + self.pool_size, start_x : start_x + self.pool_size]
max_val = np.max(patch, axis=(1, 2), keepdims=True)
mask = (patch == max_val)
input_gradient[:, start_y : start_y + self.pool_size, start_x : start_x + self.pool_size] += (
output_gradient[:, row, col][:, np.newaxis, np.newaxis] * mask
)
return input_gradient
class Flatten:
def __init__(self):
self.input_shape = None
self.batch_size = 0
def forward(self, x):
self.input_shape = x.shape
self.batch_size = x.shape[0]
return x.reshape(self.batch_size, -1)
def backward(self, grad_input):
return grad_input.reshape(self.input_shape)
|