Optimised loss and regularization calculations

regularization.py

@@ -42,6 +42,29 @@ def min_corresponding_entries(W1, W2, w1, tol=0.1):
 
     return np.min(values)
 
+
+def l1_norm(W):
+    return np.sum(np.abs(W), axis=-1)
+
+
+def l2_norm(W):
+    return np.linalg.norm(W, axis=-1)
+
+
+def l1_loss(W, y, X):
+    num_dots = W.shape[0]
+    y = y.reshape(1, -1)
+    preds = W.reshape(-1, 2) @ X.T
+    return np.mean(np.abs(y - preds), axis=1).reshape(num_dots, num_dots)
+
+
+def l2_loss(W, y, X):
+    num_dots = W.shape[0]
+    y = y.reshape(1, -1)
+    preds = W.reshape(-1, 2) @ X.T
+    return np.mean((y - preds) ** 2, axis=1).reshape(num_dots, num_dots)
+
+
 class Regularization:
     def __init__(self, width, height):
         # initialized in draw_plot
@@ -77,22 +100,23 @@ class Regularization:
         # Regularization strengths
         self.alphas = [0.01, 0.1, 1, 10, 100]
 
-        def l1_loss(y, pred):
-            return np.mean(abs(y - pred))
-
-        def l2_loss(y, pred):
-            return np.mean((y - pred)**2)
-
-        self.Losses = {#'l1': mean_absolute_error, # slow
-                       #'l1': lambda y, pred: np.mean(abs(y - pred)),
-                       'l1': l1_loss,
-                       #'l2': mean_squared_error, # slow
-                       #'l2': lambda y, pred: np.mean((y - pred)**2)
-                       'l2': l2_loss
-                       }
-        self.Regularizers = {'l1': lambda w: sum(abs(w)),
-                             'l2': np.linalg.norm
-                             }
+        self.losses = {
+            #'l1': mean_absolute_error, # slow
+            #'l1': lambda y, pred: np.mean(abs(y - pred)),
+            # 'l1': l1_loss,
+            #'l2': mean_squared_error, # slow
+            #'l2': lambda y, pred: np.mean((y - pred)**2)
+            # 'l2': l2_loss
+            "l1": l1_loss,
+            "l2": l2_loss,
+        }
+
+        self.regularizers = {
+            # 'l1': lambda w: sum(abs(w)),
+            # 'l2': np.linalg.norm
+            "l1": l1_norm,
+            "l2": l2_norm,
+        }
 
         #self.Model = Ridge #l2 loss + l2 reg
         #self.Model = Lasso #l2 loss + l1 reg
@@ -100,13 +124,13 @@ class Regularization:
         self.loss_type = 'l2'
         self.reg_type = 'l2'
 
-        self.Loss = self.Losses[self.loss_type]
-        self.Regularizer = self.Regularizers[self.reg_type]
+        self.loss = self.losses[self.loss_type]
+        self.regularizer = self.regularizers[self.reg_type]
 
         self.reg_levels = [10, 20, 30]
         self.w1_range = (-100, 100)
         self.w2_range = (-100, 100)
-        self.num_dots = 100
+        self.num_dots = 500
 
         self.plot_regularization_path = False
 
@@ -132,7 +156,7 @@ class Regularization:
             #model = self.Model(alpha=alpha, fit_intercept=False)  # no intercept
             #model.fit(X, y)
             #w = model.coef_
-            #loss = self.Loss(y, model.predict(X))
+            #loss = self.loss(y, model.predict(X))
             #solutions.append((alpha, w, self.Regularizer(w), loss))
 
         # Extract contour levels from solutions
@@ -147,16 +171,18 @@ class Regularization:
 
         # compute regularizer surface
         stacked = np.stack((W1, W2), axis=-1)
-        regs = np.apply_along_axis(self.Regularizer, -1, stacked)
+        # regs = np.apply_along_axis(self.regularizer, -1, stacked)
+        regs = self.regularizer(stacked)
 
-        logger.info("Computing losses " + str(self.Loss))
+        logger.info("Computing losses " + str(self.loss))
         # compute loss surface
-        losses = np.zeros_like(W1)
-        for i in range(W1.shape[0]):
-            for j in range(W1.shape[1]):
-                w = np.array([W1[i, j], W2[i, j]])
-                y_pred = X @ w
-                losses[i, j] = self.Loss(y, y_pred)
+        losses = self.loss(stacked, y, X)
+        # losses = np.zeros_like(W1)
+        # for i in range(W1.shape[0]):
+        #     for j in range(W1.shape[1]):
+        #         w = np.array([W1[i, j], W2[i, j]])
+        #         y_pred = X @ w
+        #         losses[i, j] = self.loss(y, y_pred)
 
         logger.info("Computing loss levels")
         reg_levels = self.reg_levels
@@ -224,17 +250,17 @@ class Regularization:
         plt.close(fig)
         buf.seek(0)
         img = Image.open(buf)
-        
+
         return img
 
     def update_loss(self, loss_type):
         self.loss_type = loss_type
-        self.Loss = self.Losses[loss_type]
+        self.loss = self.losses[loss_type]
         return self.plot()
 
     def update_regularizer(self, reg_type):
         self.reg_type = reg_type
-        self.Regularizer = self.Regularizers[reg_type]
+        self.regularizer = self.regularizers[reg_type]
 
         return self.plot()
 
@@ -315,7 +341,7 @@ class Regularization:
                                                      interactive=True)
 
                         # resolution
-                        slider = gr.Slider(minimum=100, maximum=1000, value=100, step=1, label="Resolution (#points)")
+                        slider = gr.Slider(minimum=100, maximum=1000, value=500, step=1, label="Resolution (#points)")
 
                         # plot path
                         path_checkbox = gr.Checkbox(label="Show regularization path", value=False)