File size: 1,392 Bytes
21532d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | import numpy as np
import math
class ReLU:
def __init__(self):
self.mask = None
def forward(self, x):
self.mask = (x > 0)
return x * self.mask
def backward(self, grad):
return grad * self.mask
class GELU:
def __init__(self, approximate='none'):
self.approximate = approximate
self.x = None
def forward(self, x):
self.x = x
if self.approximate == 'tanh':
inner = np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))
return 0.5 * x * (1 + np.tanh(inner))
else:
erf_vec = np.vectorize(math.erf)
return 0.5 * x * (1 + erf_vec(x / np.sqrt(2)))
def backward(self, grad):
x = self.x
if self.approximate == 'tanh':
c = np.sqrt(2 / np.pi)
inner = c * (x + 0.044715 * np.power(x, 3))
tanh_inner = np.tanh(inner)
d_inner = c * (1.0 + 3.0 * 0.044715 * np.square(x))
dx = 0.5 * (1.0 + tanh_inner) + 0.5 * x * (1.0 - np.square(tanh_inner)) * d_inner
return grad * dx
else:
erf_vec = np.vectorize(math.erf)
cdf = 0.5 * (1.0 + erf_vec(x / np.sqrt(2)))
pdf = (1.0 / np.sqrt(2.0 * np.pi)) * np.exp(-0.5 * np.square(x))
dx = cdf + x * pdf
return grad * dx
|