Add MLP

MLP.py

@@ -0,0 +1,111 @@
+import numpy as np
+from abc import ABC, abstractmethod
+
+class Layer(ABC):
+    def __init__(self, input_size, output_size):
+        self.input = None
+        self.output = None
+        self.input_size = input_size
+        self.output_size = output_size
+        self.weights = np.random.randn(input_size, output_size) * np.sqrt(2. / input_size)
+        self.bias = np.zeros((1, output_size))
+
+    @staticmethod
+    def activate(z, activation: str):
+        if activation == 'sigmoid':
+            return 1 / (1 + np.exp(-z))
+        elif activation == 'relu':
+            return np.maximum(0, z)
+        else:
+            print("Undefined activation type")
+            return None
+
+    @staticmethod
+    def derivative(activation: str, z):
+        if activation == 'sigmoid':
+            return z * (1 - z)
+        elif activation == 'relu':
+            return np.where(z > 0, 1, 0)
+        else:
+            print("Undefined activation type")
+            return None
+
+    @abstractmethod
+    def feedforward(self, x_train):
+        pass
+
+    @abstractmethod
+    def backpropagation(self, x_train, y_train, learning_rate):
+        pass
+
+class Dense(Layer):
+    def __init__(self, input_size, output_size, activation: str):
+        super().__init__(input_size, output_size)
+        self.activation = activation
+
+    def feedforward(self, input):
+        self.input = input
+        z = np.dot(self.input, self.weights) + self.bias
+        self.output = self.activate(z, self.activation)
+        return self.output
+
+    def backpropagation(self, error, learning_rate):
+        d = self.derivative(self.activation, self.output)
+        db = np.sum(error * d, axis=0, keepdims=True)
+        dW = np.dot(self.input.T, error * d)
+
+        input_error = np.dot(error * d, self.weights.T)
+
+        self.weights -= learning_rate * dW
+        self.bias -= learning_rate * db
+
+        return input_error
+
+class MLP:
+    def __init__(self):
+        self.layers = []
+
+    @staticmethod
+    def loss_MSE(y_train, yhat):
+        loss = np.mean(np.square(yhat - y_train))
+        return loss
+
+    @staticmethod
+    def loss_cross(y_train, yhat):
+        epsilon = 1e-9  # avoid log(0)
+        loss = -np.sum(y_train * np.log(yhat + epsilon)) / y_train.shape[0]
+        return loss
+
+    def addlayer(self, layer: Dense):
+        self.layers.append(layer)
+
+    def predict(self, input):
+        for layer in self.layers:
+            input = layer.feedforward(input)
+        return input
+
+    def fit(self, x_train, y_train, learning_rate=0.01, batch_size=8, epochs=10, loss_type: str = 'MSE'):
+        num_samples = x_train.shape[0]
+        for epoch in range(epochs):
+            indices = np.arange(num_samples)
+            np.random.shuffle(indices)
+            x_train = x_train[indices]
+            y_train = y_train[indices]
+            loss = 0
+
+            for i in range(0, num_samples, batch_size):
+                x_batch = x_train[i: i + batch_size]
+                y_batch = y_train[i: i + batch_size]
+
+                output = self.predict(input=x_batch)
+                error = output - y_batch
+
+                if loss_type == 'MSE':
+                    loss += self.loss_MSE(y_batch, output)
+                else:
+                    loss += self.loss_cross(y_batch, output)
+
+                for layer in reversed(self.layers):
+                    error = layer.backpropagation(error, learning_rate)
+
+            print(f'Epoch {epoch + 1}/{epochs}, Loss: {loss / (num_samples // batch_size)}')