Delete models

models/FullyConectedModels/Grid_SearchCV.ipynb

@@ -1,1213 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "accelerator": "GPU",
-    "colab": {
-      "name": "Grid_SearchCV.ipynb",
-      "provenance": [],
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "language": "python",
-      "name": "python3"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.7.4"
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "o7mJMiThKvtT"
-      },
-      "source": [
-        "# <font color=\"purple\"><b>Grid Search CV Algorithm for Fully Connected Networks</b></font>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "kUrakNvqKvtU"
-      },
-      "source": [
-        "Using the grid search CV algorithm, the hyperparameters of this model is sought.\n",
-        "<li><b> Learning Rate:</b> 0.1, 0.01</li>\n",
-        "<li><b> Regularization Penalty:</b>0.01, 0.001, 0.0001</li>\n",
-        "<li><b> Batch Size:</b> 64, 128</li>\n",
-        "<li><b> Epochs:</b> 50, 100, 150</li>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9rFQbEcDKvtV"
-      },
-      "source": [
-        "## <font color=\"blue\">Import Libraries</font>"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "6nhsKKJZ02AK"
-      },
-      "source": [
-        "import gzip\n",
-        "import pickle\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "import numpy as np\n",
-        "from sklearn.preprocessing import LabelEncoder\n",
-        "from tensorflow.keras.utils import to_categorical\n",
-        "from tensorflow.keras import backend as K\n",
-        "from itertools import product\n",
-        "from sklearn.model_selection import train_test_split\n",
-        "from sklearn.model_selection import KFold\n",
-        "\n",
-        "from tensorflow.keras.regularizers import l2\n",
-        "from tensorflow.keras.optimizers import SGD\n",
-        "import tensorflow\n",
-        "import json\n",
-        "import cv2\n",
-        "import io\n",
-        "from sklearn.metrics import accuracy_score\n",
-        "from sklearn.metrics import precision_score\n",
-        "from sklearn.metrics import recall_score\n",
-        "try:\n",
-        "    to_unicode = unicode\n",
-        "except NameError:\n",
-        "    to_unicode = str\n",
-        "from sklearn.preprocessing import LabelEncoder\n",
-        "from tensorflow.keras.utils import to_categorical"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "TdoFKvR-m04D"
-      },
-      "source": [
-        "!pip install hickle\n",
-        "import hickle as hkl"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "XqmnAXulmtWm"
-      },
-      "source": [
-        "data = hkl.load(\"data.hkl\")\n",
-        "X_train, X_test, Y_train, y_test = data['xtrain'], data['xtest'], data['ytrain'], data['ytest']\n",
-        "x_train, x_val, y_train, y_val = train_test_split(X_train, Y_train, test_size=0.1)"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "u4zwcqNAnEoE"
-      },
-      "source": [
-        "from tensorflow.data import Dataset\n",
-        "import tensorflow.keras as keras\n",
-        "from tensorflow.keras.optimizers import Adam\n",
-        "from tensorflow.keras.layers import Conv2D,Input,MaxPooling2D, Dense, Dropout, MaxPool1D, Flatten, AveragePooling1D, BatchNormalization\n",
-        "from tensorflow.keras import Model\n",
-        "import numpy as np\n",
-        "import tensorflow as tf\n",
-        "from tensorflow.keras.models import Sequential\n"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "pW_D-_Dqm5mg"
-      },
-      "source": [
-        "def model_1(weight_decay):\n",
-        "  model = Sequential()\n",
-        "  model.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(32, 32, 1), kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
-        "  model.add(BatchNormalization())\n",
-        "  model.add(Flatten())\n",
-        "  model.add(Dense(4, activation='softmax', kernel_regularizer=l2(weight_decay)))\n",
-        "  return model\n",
-        "\n",
-        "\n",
-        "def model_2(weight_decay):\n",
-        "  model = Sequential()\n",
-        "  model.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(32, 32, 1), kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
-        "  model.add(BatchNormalization())\n",
-        "  model.add(Conv2D(128, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
-        "  model.add(BatchNormalization())\n",
-        "  model.add(Flatten())\n",
-        "  model.add(Dense(4, activation='softmax', kernel_regularizer=l2(weight_decay)))\n",
-        "  return model\n",
-        "\n",
-        "\n",
-        "def model_3(weight_decay):\n",
-        "  model = Sequential()\n",
-        "  model.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(32, 32, 1),kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(Conv2D(64, kernel_size=(3, 3), activation='relu',kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
-        "  model.add(BatchNormalization())\n",
-        "  model.add(Conv2D(128, kernel_size=(3, 3), activation='relu',kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
-        "  model.add(BatchNormalization())\n",
-        "  model.add(Conv2D(256, kernel_size=(3, 3), activation='relu',kernel_regularizer=l2(weight_decay)))\n",
-        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
-        "  model.add(BatchNormalization())\n",
-        "  model.add(Flatten())\n",
-        "  model.add(Dense(4, activation='softmax',kernel_regularizer=l2(weight_decay)))\n",
-        "  return model\n"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "p9X1E2wybnZj"
-      },
-      "source": [
-        "<font color=\"blue\"> The algorithm below is that ... </font>"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "V8U0Vk9t0l3V"
-      },
-      "source": [
-        "from sklearn.metrics import confusion_matrix\n",
-        "def encoded_label(y_predict):\n",
-        "  y_list = [] \n",
-        "  for y_hat in y_predict:\n",
-        "    y_hat = np.argmax(y_hat)\n",
-        "    y_list.append(to_categorical(y_hat))\n",
-        "  return y_list\n",
-        "\n",
-        "\n",
-        "def KFold_GridSearchCV(input_dim, X, Y, X_test, y_test, combinations, filename=\"log.csv\", acc_loss_json=\"hist.json\"):\n",
-        "    \"\"\"Summary: Grid Search CV for 3 Folds Cross Validation\n",
-        "    \"\"\"\n",
-        "    res_df = pd.DataFrame(columns=['momentum','learning rate','batch size',\n",
-        "                                      'loss1', 'acc1','loss2', 'acc2','loss3', 'acc3', 'widing factor',\n",
-        "                                      'prec1', 'prec2', 'prec3', 'recall1', 'recall2', 'recall3'])\n",
-        "    generator = tensorflow.keras.preprocessing.image.ImageDataGenerator(rotation_range=10,\n",
-        "                               width_shift_range=5./32,\n",
-        "                               height_shift_range=5./32,)\n",
-        "    hist_dict_global = {}\n",
-        "\n",
-        "    for i, combination in enumerate(combinations):\n",
-        "        kf = KFold(n_splits=3, random_state=42, shuffle=False)\n",
-        "        metrics_dict = {}\n",
-        "  \n",
-        "        for j, (train_index, test_index) in enumerate(kf.split(X)):\n",
-        "            X_train, X_val = X[train_index], X[test_index]\n",
-        "            y_train, y_val = Y[train_index], Y[test_index]\n",
-        "            model = model_1(combination[2])\n",
-        "            opt = tensorflow.keras.optimizers.SGD(learning_rate=combination[0])\n",
-        "            model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
-        "            hist = model.fit(generator.flow(X_train, y_train, batch_size=combination[1]), steps_per_epoch=len(X_train) // combination[1], epochs=combination[3],\n",
-        "                                      validation_data=(X_val, y_val),\n",
-        "                                      validation_steps=len(X_val) // combination[1],)\n",
-        "            loss, acc = model.evaluate(X_test, y_test)\n",
-        "            #yhat_classes = encoded_label( model.predict(X_test))\n",
-        "            predict =  model.predict(X_test)\n",
-        "            yhat_classes = np.argmax(predict, axis=1)\n",
-        "            print(yhat_classes)\n",
-        "            print(y_test)\n",
-        "            cm = confusion_matrix(np.argmax(y_test, axis=1), yhat_classes)\n",
-        "            recall = np.diag(cm) / np.sum(cm, axis = 1)\n",
-        "            precision = np.diag(cm) / np.sum(cm, axis = 0)\n",
-        "\n",
-        "            recall_avg = np.mean(recall)\n",
-        "            precision_avg = np.mean(precision)\n",
-        "            metrics_dict[j+1] = {\"loss\": loss, \"acc\": acc, \"epoch_stopped\": combination[3], \"precision_avg\":precision_avg,\n",
-        "                                 \"avg_recall\":recall_avg}\n",
-        "            graph_loss_acc = {\"id\": i, \"com\":j+1, \"val_acc\":hist.history[\"val_accuracy\"], \"train_acc\":hist.history[\"accuracy\"],\n",
-        "                                    \"val_loss\":hist.history[\"val_loss\"], \"train_loss\":hist.history[\"loss\"], \"epoch_stopped\": combination[3], 'learning rate': combination[0],\n",
-        "                                    'batch size': combination[1], 'reg_penalty': combination[2]}\n",
-        "\n",
-        "              # Write JSON file\n",
-        "            with io.open(acc_loss_json, 'a+', encoding='utf8') as outfile:\n",
-        "                str_ = json.dumps(graph_loss_acc)\n",
-        "                outfile.write(to_unicode(str_))\n",
-        "        row = {'momentum': combination[4],'learning rate': combination[0],\n",
-        "               'batch size': combination[1],\n",
-        "               'reg_penalty': combination[2],\n",
-        "               'epoch_stopped': metrics_dict[1][\"epoch_stopped\"],\n",
-        "               'widing factor' : 1,\n",
-        "               'loss1': metrics_dict[1][\"loss\"],\n",
-        "               'acc1': metrics_dict[1][\"acc\"],\n",
-        "               'loss2': metrics_dict[2][\"loss\"],\n",
-        "               'acc2': metrics_dict[2][\"acc\"],\n",
-        "               'loss3': metrics_dict[3][\"loss\"],\n",
-        "               'acc3': metrics_dict[3][\"acc\"],\n",
-        "               'prec1':metrics_dict[1][\"precision_avg\"],\n",
-        "               'prec2':metrics_dict[2][\"precision_avg\"],\n",
-        "               'prec3':metrics_dict[3][\"precision_avg\"],\n",
-        "               'recall1':metrics_dict[1][\"avg_recall\"],\n",
-        "               'recall2':metrics_dict[2][\"avg_recall\"],\n",
-        "               'recall3':metrics_dict[3][\"avg_recall\"]}\n",
-        "        res_df = res_df.append(row , ignore_index=True)\n",
-        "        res_df.to_csv(filename, sep=\";\")"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "tgO8xJ0Fe93I"
-      },
-      "source": [
-        "\n",
-        "if __name__ == \"__main__\":\n",
-        "    learning_rate = [0.1,0.01]\n",
-        "    batch_size = [64,128]\n",
-        "    reg_penalty = [0,0.01, 0.001, 0.0001]\n",
-        "    epochs = [50,100,150]\n",
-        "    momentum = [0.9]\n",
-        "    in_dim = (32,32,1)\n",
-        "    grid_result = \"grid_model_1.csv\"\n",
-        "    acc_loss_json = \"history.json\"\n",
-        "    # create list of all different parameter combinations\n",
-        "    param_grid = dict(learning_rate = learning_rate, batch_size = batch_size, \n",
-        "                      reg_penalty = reg_penalty, epochs = epochs, momentum=momentum)\n",
-        "    combinations = list(product(*param_grid.values()))\n",
-        "    KFold_GridSearchCV(in_dim,X_train,Y_train,X_test, y_test, combinations, grid_result, acc_loss_json)"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "iVp5lMuhpAMW"
-      },
-      "source": [
-        "data  = pd.read_csv(\"grid_model_1.csv\", sep=\";\")"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 244
-        },
-        "id": "rru6HHyrsgbl",
-        "outputId": "01496643-5865-41a6-85c2-f69242b5912b"
-      },
-      "source": [
-        "data.head(5)"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/html": [
-              "<div>\n",
-              "<style scoped>\n",
-              "    .dataframe tbody tr th:only-of-type {\n",
-              "        vertical-align: middle;\n",
-              "    }\n",
-              "\n",
-              "    .dataframe tbody tr th {\n",
-              "        vertical-align: top;\n",
-              "    }\n",
-              "\n",
-              "    .dataframe thead th {\n",
-              "        text-align: right;\n",
-              "    }\n",
-              "</style>\n",
-              "<table border=\"1\" class=\"dataframe\">\n",
-              "  <thead>\n",
-              "    <tr style=\"text-align: right;\">\n",
-              "      <th></th>\n",
-              "      <th>momentum</th>\n",
-              "      <th>learning rate</th>\n",
-              "      <th>batch size</th>\n",
-              "      <th>loss1</th>\n",
-              "      <th>acc1</th>\n",
-              "      <th>loss2</th>\n",
-              "      <th>acc2</th>\n",
-              "      <th>loss3</th>\n",
-              "      <th>acc3</th>\n",
-              "      <th>widing factor</th>\n",
-              "      <th>prec1</th>\n",
-              "      <th>prec2</th>\n",
-              "      <th>prec3</th>\n",
-              "      <th>recall1</th>\n",
-              "      <th>recall2</th>\n",
-              "      <th>recall3</th>\n",
-              "      <th>epoch_stopped</th>\n",
-              "      <th>reg_penalty</th>\n",
-              "    </tr>\n",
-              "  </thead>\n",
-              "  <tbody>\n",
-              "    <tr>\n",
-              "      <th>0</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.078664</td>\n",
-              "      <td>0.541012</td>\n",
-              "      <td>1.087765</td>\n",
-              "      <td>0.560209</td>\n",
-              "      <td>1.140540</td>\n",
-              "      <td>0.495637</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.545625</td>\n",
-              "      <td>0.570303</td>\n",
-              "      <td>0.540447</td>\n",
-              "      <td>0.549296</td>\n",
-              "      <td>0.561227</td>\n",
-              "      <td>0.506039</td>\n",
-              "      <td>50.0</td>\n",
-              "      <td>0.00</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.090459</td>\n",
-              "      <td>0.544503</td>\n",
-              "      <td>1.052103</td>\n",
-              "      <td>0.568935</td>\n",
-              "      <td>0.991915</td>\n",
-              "      <td>0.586387</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.567745</td>\n",
-              "      <td>0.581882</td>\n",
-              "      <td>0.587054</td>\n",
-              "      <td>0.549632</td>\n",
-              "      <td>0.571559</td>\n",
-              "      <td>0.584749</td>\n",
-              "      <td>100.0</td>\n",
-              "      <td>0.00</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>2</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.203270</td>\n",
-              "      <td>0.549738</td>\n",
-              "      <td>1.024475</td>\n",
-              "      <td>0.607330</td>\n",
-              "      <td>1.099551</td>\n",
-              "      <td>0.542757</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.597314</td>\n",
-              "      <td>0.629707</td>\n",
-              "      <td>0.583353</td>\n",
-              "      <td>0.558664</td>\n",
-              "      <td>0.612029</td>\n",
-              "      <td>0.532126</td>\n",
-              "      <td>150.0</td>\n",
-              "      <td>0.00</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>3</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.331875</td>\n",
-              "      <td>0.450262</td>\n",
-              "      <td>1.357389</td>\n",
-              "      <td>0.425829</td>\n",
-              "      <td>1.136434</td>\n",
-              "      <td>0.542757</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.520104</td>\n",
-              "      <td>0.556441</td>\n",
-              "      <td>0.541896</td>\n",
-              "      <td>0.467984</td>\n",
-              "      <td>0.441258</td>\n",
-              "      <td>0.549730</td>\n",
-              "      <td>50.0</td>\n",
-              "      <td>0.01</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>4</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.286225</td>\n",
-              "      <td>0.490401</td>\n",
-              "      <td>1.218342</td>\n",
-              "      <td>0.539267</td>\n",
-              "      <td>1.207052</td>\n",
-              "      <td>0.537522</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.552838</td>\n",
-              "      <td>0.569478</td>\n",
-              "      <td>0.586226</td>\n",
-              "      <td>0.494587</td>\n",
-              "      <td>0.541650</td>\n",
-              "      <td>0.547532</td>\n",
-              "      <td>100.0</td>\n",
-              "      <td>0.01</td>\n",
-              "    </tr>\n",
-              "  </tbody>\n",
-              "</table>\n",
-              "</div>"
-            ],
-            "text/plain": [
-              "   momentum  learning rate  batch size  ...   recall3  epoch_stopped  reg_penalty\n",
-              "0       0.9            0.1        64.0  ...  0.506039           50.0         0.00\n",
-              "1       0.9            0.1        64.0  ...  0.584749          100.0         0.00\n",
-              "2       0.9            0.1        64.0  ...  0.532126          150.0         0.00\n",
-              "3       0.9            0.1        64.0  ...  0.549730           50.0         0.01\n",
-              "4       0.9            0.1        64.0  ...  0.547532          100.0         0.01\n",
-              "\n",
-              "[5 rows x 18 columns]"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 17
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "LeVPJJQQt36n"
-      },
-      "source": [
-        "data[\"loss_mean\"] = (data[\"loss1\"]+data[\"loss2\"]+data[\"loss3\"])/3\n",
-        "data[\"acc_mean\"] = (data[\"acc1\"]+data[\"acc2\"]+data[\"acc3\"])/3"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "XPIKRbPMuUOr"
-      },
-      "source": [
-        "data['epoch'] = data['epoch_stopped']\n",
-        "data['weight_decay'] = data['reg_penalty']"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "5_2tMeO6x9Uq"
-      },
-      "source": [
-        "data['recall_mean'] = (data['recall1']+data['recall2']+data['recall3'])/3\n",
-        "data['prec_mean'] = (data['prec1']+data['prec2']+data['prec3'])/3"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "JA0eRctYuWl-"
-      },
-      "source": [
-        "column_list = [\"momentum\", \"learning rate\", \"epoch\",\"batch size\",\"weight_decay\",\"loss_mean\", \"acc_mean\",\"recall_mean\", \"prec_mean\"]"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 143
-        },
-        "id": "McsMaiLZuZaa",
-        "outputId": "d04090a8-b295-45ad-d64f-e5965741cd80"
-      },
-      "source": [
-        "data.sort_values(axis=0, by=\"loss_mean\", ascending=True)[column_list].head(3)"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/html": [
-              "<div>\n",
-              "<style scoped>\n",
-              "    .dataframe tbody tr th:only-of-type {\n",
-              "        vertical-align: middle;\n",
-              "    }\n",
-              "\n",
-              "    .dataframe tbody tr th {\n",
-              "        vertical-align: top;\n",
-              "    }\n",
-              "\n",
-              "    .dataframe thead th {\n",
-              "        text-align: right;\n",
-              "    }\n",
-              "</style>\n",
-              "<table border=\"1\" class=\"dataframe\">\n",
-              "  <thead>\n",
-              "    <tr style=\"text-align: right;\">\n",
-              "      <th></th>\n",
-              "      <th>momentum</th>\n",
-              "      <th>learning rate</th>\n",
-              "      <th>epoch</th>\n",
-              "      <th>batch size</th>\n",
-              "      <th>weight_decay</th>\n",
-              "      <th>loss_mean</th>\n",
-              "      <th>acc_mean</th>\n",
-              "      <th>recall_mean</th>\n",
-              "      <th>prec_mean</th>\n",
-              "    </tr>\n",
-              "  </thead>\n",
-              "  <tbody>\n",
-              "    <tr>\n",
-              "      <th>24</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.01</td>\n",
-              "      <td>50.0</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>0.0000</td>\n",
-              "      <td>1.012071</td>\n",
-              "      <td>0.612565</td>\n",
-              "      <td>0.614360</td>\n",
-              "      <td>0.615845</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>34</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.01</td>\n",
-              "      <td>100.0</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>0.0001</td>\n",
-              "      <td>1.026681</td>\n",
-              "      <td>0.614311</td>\n",
-              "      <td>0.612435</td>\n",
-              "      <td>0.619908</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>8</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.10</td>\n",
-              "      <td>150.0</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>0.0010</td>\n",
-              "      <td>1.027309</td>\n",
-              "      <td>0.603258</td>\n",
-              "      <td>0.601262</td>\n",
-              "      <td>0.606730</td>\n",
-              "    </tr>\n",
-              "  </tbody>\n",
-              "</table>\n",
-              "</div>"
-            ],
-            "text/plain": [
-              "    momentum  learning rate  epoch  ...  acc_mean  recall_mean  prec_mean\n",
-              "24       0.9           0.01   50.0  ...  0.612565     0.614360   0.615845\n",
-              "34       0.9           0.01  100.0  ...  0.614311     0.612435   0.619908\n",
-              "8        0.9           0.10  150.0  ...  0.603258     0.601262   0.606730\n",
-              "\n",
-              "[3 rows x 9 columns]"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 26
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "hCgRAh43udBu"
-      },
-      "source": [
-        "data[\"loss_na\"] = data.loc[:,[\"loss1\",\"loss2\", \"loss3\"]].isnull().sum(1)"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 181
-        },
-        "id": "-ptzEmSHudkM",
-        "outputId": "72a8ca57-603a-451f-8702-962b4be4f91a"
-      },
-      "source": [
-        "data.head(3)"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/html": [
-              "<div>\n",
-              "<style scoped>\n",
-              "    .dataframe tbody tr th:only-of-type {\n",
-              "        vertical-align: middle;\n",
-              "    }\n",
-              "\n",
-              "    .dataframe tbody tr th {\n",
-              "        vertical-align: top;\n",
-              "    }\n",
-              "\n",
-              "    .dataframe thead th {\n",
-              "        text-align: right;\n",
-              "    }\n",
-              "</style>\n",
-              "<table border=\"1\" class=\"dataframe\">\n",
-              "  <thead>\n",
-              "    <tr style=\"text-align: right;\">\n",
-              "      <th></th>\n",
-              "      <th>momentum</th>\n",
-              "      <th>learning rate</th>\n",
-              "      <th>batch size</th>\n",
-              "      <th>loss1</th>\n",
-              "      <th>acc1</th>\n",
-              "      <th>loss2</th>\n",
-              "      <th>acc2</th>\n",
-              "      <th>loss3</th>\n",
-              "      <th>acc3</th>\n",
-              "      <th>widing factor</th>\n",
-              "      <th>prec1</th>\n",
-              "      <th>prec2</th>\n",
-              "      <th>prec3</th>\n",
-              "      <th>recall1</th>\n",
-              "      <th>recall2</th>\n",
-              "      <th>recall3</th>\n",
-              "      <th>epoch_stopped</th>\n",
-              "      <th>reg_penalty</th>\n",
-              "      <th>loss_mean</th>\n",
-              "      <th>acc_mean</th>\n",
-              "      <th>epoch</th>\n",
-              "      <th>weight_decay</th>\n",
-              "      <th>loss_na</th>\n",
-              "    </tr>\n",
-              "  </thead>\n",
-              "  <tbody>\n",
-              "    <tr>\n",
-              "      <th>0</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.078664</td>\n",
-              "      <td>0.541012</td>\n",
-              "      <td>1.087765</td>\n",
-              "      <td>0.560209</td>\n",
-              "      <td>1.140540</td>\n",
-              "      <td>0.495637</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.545625</td>\n",
-              "      <td>0.570303</td>\n",
-              "      <td>0.540447</td>\n",
-              "      <td>0.549296</td>\n",
-              "      <td>0.561227</td>\n",
-              "      <td>0.506039</td>\n",
-              "      <td>50.0</td>\n",
-              "      <td>0.0</td>\n",
-              "      <td>1.102323</td>\n",
-              "      <td>0.532286</td>\n",
-              "      <td>50.0</td>\n",
-              "      <td>0.0</td>\n",
-              "      <td>0</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.090459</td>\n",
-              "      <td>0.544503</td>\n",
-              "      <td>1.052103</td>\n",
-              "      <td>0.568935</td>\n",
-              "      <td>0.991915</td>\n",
-              "      <td>0.586387</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.567745</td>\n",
-              "      <td>0.581882</td>\n",
-              "      <td>0.587054</td>\n",
-              "      <td>0.549632</td>\n",
-              "      <td>0.571559</td>\n",
-              "      <td>0.584749</td>\n",
-              "      <td>100.0</td>\n",
-              "      <td>0.0</td>\n",
-              "      <td>1.044826</td>\n",
-              "      <td>0.566609</td>\n",
-              "      <td>100.0</td>\n",
-              "      <td>0.0</td>\n",
-              "      <td>0</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>2</th>\n",
-              "      <td>0.9</td>\n",
-              "      <td>0.1</td>\n",
-              "      <td>64.0</td>\n",
-              "      <td>1.203270</td>\n",
-              "      <td>0.549738</td>\n",
-              "      <td>1.024475</td>\n",
-              "      <td>0.607330</td>\n",
-              "      <td>1.099551</td>\n",
-              "      <td>0.542757</td>\n",
-              "      <td>1.0</td>\n",
-              "      <td>0.597314</td>\n",
-              "      <td>0.629707</td>\n",
-              "      <td>0.583353</td>\n",
-              "      <td>0.558664</td>\n",
-              "      <td>0.612029</td>\n",
-              "      <td>0.532126</td>\n",
-              "      <td>150.0</td>\n",
-              "      <td>0.0</td>\n",
-              "      <td>1.109098</td>\n",
-              "      <td>0.566608</td>\n",
-              "      <td>150.0</td>\n",
-              "      <td>0.0</td>\n",
-              "      <td>0</td>\n",
-              "    </tr>\n",
-              "  </tbody>\n",
-              "</table>\n",
-              "</div>"
-            ],
-            "text/plain": [
-              "   momentum  learning rate  batch size  ...  epoch  weight_decay  loss_na\n",
-              "0       0.9            0.1        64.0  ...   50.0           0.0        0\n",
-              "1       0.9            0.1        64.0  ...  100.0           0.0        0\n",
-              "2       0.9            0.1        64.0  ...  150.0           0.0        0\n",
-              "\n",
-              "[3 rows x 23 columns]"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 23
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "YBBXsz_rzQl-"
-      },
-      "source": [
-        "generator = tensorflow.keras.preprocessing.image.ImageDataGenerator(rotation_range=10,\n",
-        "                               width_shift_range=5./32,\n",
-        "                               height_shift_range=5./32,)"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "ShIJn_53mawD"
-      },
-      "source": [
-        "kf = KFold(n_splits=3, random_state=42, shuffle=False)\n",
-        "result = []\n",
-        "for j, (train_index, test_index) in enumerate(kf.split(X_train)):\n",
-        "  x_train, x_val = X_train[train_index], X_train[test_index]\n",
-        "  y_train, y_val = Y_train[train_index], Y_train[test_index]\n",
-        "  model = model_2(0)\n",
-        "  opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
-        "  model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
-        "  hist = model.fit(generator.flow(x_train, y_train, batch_size=64), steps_per_epoch=len(x_train) //64 , epochs=50,\n",
-        "                   validation_data=(x_val, y_val),\n",
-        "                  validation_steps=len(x_val) //64 ,)\n",
-        "\n",
-        "  test = model.evaluate(X_test, y_test)\n",
-        "  result.append(test)"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "Rj6hnUVRzjX-",
-        "outputId": "41b04e1e-d756-437a-afff-ca9f3fcff46c"
-      },
-      "source": [
-        "mean_acc = (result[0][1]+result[1][1]+result[2][1])/3;mean_acc"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/plain": [
-              "0.6305991808573405"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 47
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "v_AccCit1De3",
-        "outputId": "a6299f58-4389-4f39-f318-e2371e94f4e5"
-      },
-      "source": [
-        "mean_loss = (result[0][0]+result[1][0]+result[2][0])/3;mean_loss"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/plain": [
-              "0.9891296029090881"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 48
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "8gdDigrPzYM5"
-      },
-      "source": [
-        "kf = KFold(n_splits=3, random_state=42, shuffle=False)\n",
-        "result_2 = []\n",
-        "for j, (train_index, test_index) in enumerate(kf.split(X_train)):\n",
-        "  x_train, x_val = X_train[train_index], X_train[test_index]\n",
-        "  y_train, y_val = Y_train[train_index], Y_train[test_index]\n",
-        "  model = model_3(0.0001)\n",
-        "  opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
-        "  model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
-        "  hist = model.fit(generator.flow(x_train, y_train, batch_size=64), steps_per_epoch=len(x_train) //64 , epochs=100,\n",
-        "                   validation_data=(x_val, y_val),\n",
-        "                  validation_steps=len(x_val) //64 ,)\n",
-        "\n",
-        "  test = model.evaluate(X_test, y_test)\n",
-        "  result_2.append(test)"
-      ],
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "FYVQINoA2F4l",
-        "outputId": "ac29f686-9b48-4193-bf01-4a17ab589d91"
-      },
-      "source": [
-        "mean_acc = (result_2[0][1]+result_2[1][1]+result_2[2][1])/3;mean_acc"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/plain": [
-              "0.7108784119288126"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 56
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "guueg5Wo2IE8",
-        "outputId": "986e139c-1224-43a7-b8fc-59d57e72d3a8"
-      },
-      "source": [
-        "mean_loss = (result_2[0][0]+result_2[1][0]+result_2[2][0])/3;mean_loss"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/plain": [
-              "0.9208946625391642"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 57
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "luS42Hil0H5R",
-        "outputId": "91c28154-8014-42a9-fbf2-8935cb4c521a"
-      },
-      "source": [
-        "mean_acc = (result_2[0][1]+result_2[1][1]+result_2[2][1])/3;mean_acc"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/plain": [
-              "0.6783013343811035"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 50
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "F9Zg3pI61W_J",
-        "outputId": "fe5f7861-58e9-47f6-8134-8872e494e268"
-      },
-      "source": [
-        "mean_loss = (result_2[0][0]+result_2[1][0]+result_2[2][0])/3;mean_loss"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "execute_result",
-          "data": {
-            "text/plain": [
-              "0.8747362097104391"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "execution_count": 51
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "OWx9Jej11f_V",
-        "outputId": "a5b72e6c-de64-4bee-da66-d48a5e6e1870"
-      },
-      "source": [
-        "model = model_1(0)\n",
-        "opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
-        "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
-        "model.summary()"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "stream",
-          "text": [
-            "Model: \"sequential_162\"\n",
-            "_________________________________________________________________\n",
-            "Layer (type)                 Output Shape              Param #   \n",
-            "=================================================================\n",
-            "conv2d_342 (Conv2D)          (None, 30, 30, 32)        320       \n",
-            "_________________________________________________________________\n",
-            "conv2d_343 (Conv2D)          (None, 28, 28, 64)        18496     \n",
-            "_________________________________________________________________\n",
-            "max_pooling2d_183 (MaxPoolin (None, 14, 14, 64)        0         \n",
-            "_________________________________________________________________\n",
-            "batch_normalization_183 (Bat (None, 14, 14, 64)        256       \n",
-            "_________________________________________________________________\n",
-            "flatten_162 (Flatten)        (None, 12544)             0         \n",
-            "_________________________________________________________________\n",
-            "dense_162 (Dense)            (None, 4)                 50180     \n",
-            "=================================================================\n",
-            "Total params: 69,252\n",
-            "Trainable params: 69,124\n",
-            "Non-trainable params: 128\n",
-            "_________________________________________________________________\n"
-          ],
-          "name": "stdout"
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "l8mg2Bup1lPJ",
-        "outputId": "6a2348f9-f6d8-47d3-a5d7-9169d3906d76"
-      },
-      "source": [
-        "model = model_2(0)\n",
-        "opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
-        "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
-        "model.summary()"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "stream",
-          "text": [
-            "Model: \"sequential_154\"\n",
-            "_________________________________________________________________\n",
-            "Layer (type)                 Output Shape              Param #   \n",
-            "=================================================================\n",
-            "conv2d_320 (Conv2D)          (None, 30, 30, 32)        320       \n",
-            "_________________________________________________________________\n",
-            "conv2d_321 (Conv2D)          (None, 28, 28, 64)        18496     \n",
-            "_________________________________________________________________\n",
-            "max_pooling2d_166 (MaxPoolin (None, 14, 14, 64)        0         \n",
-            "_________________________________________________________________\n",
-            "batch_normalization_166 (Bat (None, 14, 14, 64)        256       \n",
-            "_________________________________________________________________\n",
-            "conv2d_322 (Conv2D)          (None, 12, 12, 128)       73856     \n",
-            "_________________________________________________________________\n",
-            "max_pooling2d_167 (MaxPoolin (None, 6, 6, 128)         0         \n",
-            "_________________________________________________________________\n",
-            "batch_normalization_167 (Bat (None, 6, 6, 128)         512       \n",
-            "_________________________________________________________________\n",
-            "flatten_154 (Flatten)        (None, 4608)              0         \n",
-            "_________________________________________________________________\n",
-            "dense_154 (Dense)            (None, 4)                 18436     \n",
-            "=================================================================\n",
-            "Total params: 111,876\n",
-            "Trainable params: 111,492\n",
-            "Non-trainable params: 384\n",
-            "_________________________________________________________________\n"
-          ],
-          "name": "stdout"
-        }
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "vZImk4zB1mxk",
-        "outputId": "805f844e-1312-423a-b309-35944f11a210"
-      },
-      "source": [
-        "model = model_3(0)\n",
-        "opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
-        "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
-        "model.summary()"
-      ],
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "stream",
-          "text": [
-            "Model: \"sequential_155\"\n",
-            "_________________________________________________________________\n",
-            "Layer (type)                 Output Shape              Param #   \n",
-            "=================================================================\n",
-            "conv2d_323 (Conv2D)          (None, 30, 30, 32)        320       \n",
-            "_________________________________________________________________\n",
-            "conv2d_324 (Conv2D)          (None, 28, 28, 64)        18496     \n",
-            "_________________________________________________________________\n",
-            "max_pooling2d_168 (MaxPoolin (None, 14, 14, 64)        0         \n",
-            "_________________________________________________________________\n",
-            "batch_normalization_168 (Bat (None, 14, 14, 64)        256       \n",
-            "_________________________________________________________________\n",
-            "conv2d_325 (Conv2D)          (None, 12, 12, 128)       73856     \n",
-            "_________________________________________________________________\n",
-            "max_pooling2d_169 (MaxPoolin (None, 6, 6, 128)         0         \n",
-            "_________________________________________________________________\n",
-            "batch_normalization_169 (Bat (None, 6, 6, 128)         512       \n",
-            "_________________________________________________________________\n",
-            "conv2d_326 (Conv2D)          (None, 4, 4, 256)         295168    \n",
-            "_________________________________________________________________\n",
-            "max_pooling2d_170 (MaxPoolin (None, 2, 2, 256)         0         \n",
-            "_________________________________________________________________\n",
-            "batch_normalization_170 (Bat (None, 2, 2, 256)         1024      \n",
-            "_________________________________________________________________\n",
-            "flatten_155 (Flatten)        (None, 1024)              0         \n",
-            "_________________________________________________________________\n",
-            "dense_155 (Dense)            (None, 4)                 4100      \n",
-            "=================================================================\n",
-            "Total params: 393,732\n",
-            "Trainable params: 392,836\n",
-            "Non-trainable params: 896\n",
-            "_________________________________________________________________\n"
-          ],
-          "name": "stdout"
-        }
-      ]
-    }
-  ]
-}

models/FullyConectedModels/gridsearchcv/grid_model_1.csv

@@ -1,49 +0,0 @@
-;momentum;learning rate;batch size;loss1;acc1;loss2;acc2;loss3;acc3;widing factor;prec1;prec2;prec3;recall1;recall2;recall3;epoch_stopped;reg_penalty
-0;0.9;0.1;64.0;1.0786635875701904;0.5410122275352478;1.087765097618103;0.5602094531059265;1.1405400037765503;0.4956369996070862;1.0;0.5456249725117649;0.5703025971923495;0.5404468631355052;0.549295511252033;0.561227419923072;0.5060386473429952;50.0;0.0
-1;0.9;0.1;64.0;1.0904589891433716;0.5445026159286499;1.0521031618118286;0.5689354538917542;0.9919149279594421;0.5863874554634094;1.0;0.5677450027120325;0.5818822696361521;0.5870538777082881;0.5496315278923974;0.5715594791681748;0.5847485847485847;100.0;0.0
-2;0.9;0.1;64.0;1.203269600868225;0.5497382283210754;1.0244745016098022;0.6073298454284668;1.0995512008666992;0.5427573919296265;1.0;0.5973140968206758;0.6297070150268066;0.5833525841963643;0.5586641619250315;0.6120292750727534;0.5321260864739126;150.0;0.0
-3;0.9;0.1;64.0;1.3318754434585571;0.45026177167892456;1.3573890924453735;0.4258289635181427;1.1364340782165527;0.5427573919296265;1.0;0.520103685420754;0.5564409752732054;0.5418956991063357;0.4679843103756147;0.44125801734497383;0.5497298595124682;50.0;0.01
-4;0.9;0.1;64.0;1.2862250804901123;0.49040138721466064;1.2183420658111572;0.5392670035362244;1.2070523500442505;0.5375218391418457;1.0;0.5528382317603805;0.5694776622208874;0.586225988700565;0.4945869348043261;0.5416503786069004;0.5475315747054877;100.0;0.01
-5;0.9;0.1;64.0;1.1847248077392578;0.5445026159286499;1.4426966905593872;0.43106457591056824;1.2306617498397827;0.47993019223213196;1.0;0.5312498290025648;0.5339512821095386;0.5311977318393986;0.544515870602827;0.4474462735332301;0.49295993861211257;150.0;0.01
-6;0.9;0.1;64.0;1.224439263343811;0.5095986127853394;1.1083790063858032;0.5584642291069031;1.042079210281372;0.584642231464386;1.0;0.5344176263987584;0.5580401772808269;0.5725199160342243;0.5072463768115942;0.564412640499597;0.5839504698200351;50.0;0.001
-7;0.9;0.1;64.0;1.1233032941818237;0.5671902298927307;1.279549241065979;0.49389180541038513;1.0785572528839111;0.5811518430709839;1.0;0.5835013620894915;0.5600592746287815;0.5944705565842157;0.5647160810204288;0.4860936165283991;0.5825973543364849;100.0;0.001
-8;0.9;0.1;64.0;1.0618174076080322;0.5898778438568115;1.0041369199752808;0.6178010702133179;1.0159738063812256;0.6020942330360413;1.0;0.5958448974789733;0.6197077143812308;0.6046378435659604;0.5848010684967206;0.6172076715554976;0.6017768463420637;150.0;0.001
-9;0.9;0.1;64.0;1.1275826692581177;0.5095986127853394;1.037006139755249;0.547993004322052;1.1576757431030273;0.4712041914463043;1.0;0.5163513023182834;0.5527921101786539;0.4780335054814982;0.5184676434676434;0.5468667805624328;0.47072553050813926;50.0;0.0001
-10;0.9;0.1;64.0;1.0660938024520874;0.5811518430709839;1.0170561075210571;0.5828970074653625;1.0196107625961304;0.5811518430709839;1.0;0.5900164662084766;0.5999640724915478;0.5950983436203666;0.5901320901320901;0.5829641373119634;0.5774551535421101;100.0;0.0001
-11;0.9;0.1;64.0;1.0177899599075317;0.5881326198577881;1.0439162254333496;0.5654450058937073;1.1021149158477783;0.5759162306785583;1.0;0.6036141210870313;0.5902555674010898;0.6140637358081388;0.5845627802149542;0.561161664422534;0.5720825068651156;150.0;0.0001
-12;0.9;0.1;128.0;1.0818946361541748;0.5253053903579712;1.0099669694900513;0.5968586206436157;1.0918453931808472;0.5410122275352478;1.0;0.521080315961989;0.6039058290508166;0.5263435769673295;0.5298210243862418;0.6008387747518182;0.5431760268716791;50.0;0.0
-13;0.9;0.1;128.0;1.164839744567871;0.45724257826805115;0.9867958426475525;0.5933682322502136;1.0679926872253418;0.554973840713501;1.0;0.5123214368077382;0.5988966761697891;0.5643833075235515;0.4695654586958935;0.597261434217956;0.5518648018648019;100.0;0.0
-14;0.9;0.1;128.0;1.113759160041809;0.5567190051078796;1.046524167060852;0.584642231464386;0.9935498833656311;0.5724258422851562;1.0;0.6122945895111208;0.5897342892147153;0.5820350700747354;0.553781966825445;0.58606007519051;0.5708771904424078;150.0;0.0
-15;0.9;0.1;128.0;1.1370242834091187;0.5619546175003052;1.1299653053283691;0.5462478399276733;1.1480666399002075;0.5322862267494202;1.0;0.5756164609818162;0.5500235261699395;0.5222666160817157;0.5605360822752127;0.5444175389827564;0.53630849826502;50.0;0.01
-16;0.9;0.1;128.0;1.1349637508392334;0.5636998414993286;1.1408828496932983;0.5410122275352478;1.3376970291137695;0.47993019223213196;1.0;0.5815890943000785;0.5523155608073429;0.5205896541198826;0.5641309173917869;0.5482174830000918;0.4919084538649756;100.0;0.01
-17;0.9;0.1;128.0;1.1322417259216309;0.5881326198577881;1.1545387506484985;0.5602094531059265;1.2242604494094849;0.5654450058937073;1.0;0.599198600670369;0.5865846668946237;0.5603340273492765;0.5861801242236024;0.5607828162175988;0.5646859179467875;150.0;0.01
-18;0.9;0.1;128.0;1.1220964193344116;0.5514833927154541;1.1653169393539429;0.5270506143569946;1.1562250852584839;0.5235602259635925;1.0;0.5715813891898507;0.5249991236833342;0.5223967978570353;0.5467980087545306;0.5330587287109027;0.5233751755490886;50.0;0.001
-19;0.9;0.1;128.0;1.153067708015442;0.5881326198577881;1.1445388793945312;0.5602094531059265;1.1189345121383667;0.5357766151428223;1.0;0.6072767352919373;0.5604236873448876;0.5574203282247268;0.5862139068660808;0.5572374485417964;0.5373183579705318;100.0;0.001
-20;0.9;0.1;128.0;1.2125698328018188;0.5410122275352478;1.1219922304153442;0.5602094531059265;1.0713446140289307;0.5828970074653625;1.0;0.5605981222253124;0.5931479015961776;0.5828110117010443;0.5311391507043681;0.5714448594883378;0.5890498390498391;150.0;0.001
-21;0.9;0.1;128.0;1.1447523832321167;0.5567190051078796;1.1324081420898438;0.518324613571167;1.1090422868728638;0.49738219380378723;1.0;0.596030303030303;0.5490280922716101;0.5121340146227983;0.5638238573021181;0.5146260744086831;0.4972123287340679;50.0;0.0001
-22;0.9;0.1;128.0;0.9825822710990906;0.584642231464386;1.108230471611023;0.5218150019645691;1.1356145143508911;0.5200698375701904;1.0;0.5948660954866036;0.5665316277437249;0.5656796449319814;0.5828724415680937;0.5286633656198874;0.5303995521386826;100.0;0.0001
-23;0.9;0.1;128.0;1.0473191738128662;0.5951134562492371;1.0678406953811646;0.518324613571167;1.0169429779052734;0.5881326198577881;1.0;0.6219409183078248;0.5693268693732438;0.593293789725479;0.5926543263499786;0.5205706129619173;0.5907293189901885;150.0;0.0001
-24;0.9;0.01;64.0;1.0598105192184448;0.5968586206436157;1.0257790088653564;0.6038394570350647;0.9506248235702515;0.6369982361793518;1.0;0.6051437947470779;0.602723696484381;0.6396683984470244;0.5915582002538524;0.609172831998919;0.6423491966970227;50.0;0.0
-25;0.9;0.01;64.0;1.1066389083862305;0.6108202338218689;1.0695348978042603;0.6195462346076965;1.1887136697769165;0.5741710066795349;1.0;0.6467146374525526;0.6245149204071099;0.5971208662939392;0.6051062464105943;0.6201624462494028;0.5657778212126039;100.0;0.0
-26;0.9;0.01;64.0;1.076154351234436;0.6404886841773987;1.1792618036270142;0.6073298454284668;1.4671285152435303;0.547993004322052;1.0;0.6529888091081401;0.6461774006977742;0.6066030792539256;0.6368685662163923;0.6003965840922363;0.537592841940668;150.0;0.0
-27;0.9;0.01;64.0;1.2690242528915405;0.5968586206436157;1.2563925981521606;0.5968586206436157;1.2163962125778198;0.5968586206436157;1.0;0.5916521579476596;0.6248306369232546;0.5985189460592925;0.5966666184057489;0.6007983562331388;0.5971992982862548;50.0;0.01
-28;0.9;0.01;64.0;1.116417646408081;0.6160558462142944;1.1469776630401611;0.6178010702133179;1.1717371940612793;0.584642231464386;1.0;0.6261048220941269;0.6335991775298133;0.610479317595543;0.6109440076831381;0.6120992534036012;0.5811808963982876;100.0;0.01
-29;0.9;0.01;64.0;1.214686393737793;0.5811518430709839;1.2714550495147705;0.5584642291069031;1.2080868482589722;0.6073298454284668;1.0;0.6001486481081634;0.5946394895073237;0.6365187860446218;0.5804340586949283;0.5500978490108924;0.6034834730486904;150.0;0.01
-30;0.9;0.01;64.0;1.1514617204666138;0.5881326198577881;1.0881938934326172;0.5881326198577881;1.053907871246338;0.6020942330360413;1.0;0.5834891320869433;0.6065228770652975;0.6088152818711143;0.5858362651840913;0.5805794447098794;0.6013455143889928;50.0;0.001
-31;0.9;0.01;64.0;1.22362220287323;0.5968586206436157;1.0609272718429565;0.6300174593925476;1.1912355422973633;0.5968586206436157;1.0;0.621725687037072;0.6327141334647053;0.6186631707154095;0.5975027388070866;0.6298701298701299;0.5883030013464796;100.0;0.001
-32;0.9;0.01;64.0;1.1824991703033447;0.5986038446426392;1.226974368095398;0.6090750694274902;1.2909026145935059;0.6038394570350647;1.0;0.6073992988417198;0.6445184510325356;0.619910049538771;0.5955710955710956;0.6017605582822975;0.5952151713021279;150.0;0.001
-33;0.9;0.01;64.0;1.1622414588928223;0.5287958383560181;1.0493906736373901;0.5794066190719604;1.2138421535491943;0.5497382283210754;1.0;0.5809122116993566;0.6019885476001634;0.5910039233578294;0.5309117211291124;0.5840572471007254;0.5480895915678524;50.0;0.0001
-34;0.9;0.01;64.0;1.0573968887329102;0.6073298454284668;1.0098820924758911;0.62129145860672;1.0127638578414917;0.614310622215271;1.0;0.6070338521099506;0.62878369303158;0.6239062815043207;0.603968495272843;0.6201449516666908;0.6131923631923631;100.0;0.0001
-35;0.9;0.01;64.0;1.0984822511672974;0.657940685749054;1.0174990892410278;0.6561954617500305;1.1236447095870972;0.5986038446426392;1.0;0.6531590453460197;0.6671137560770346;0.5932537237439376;0.6610702099832534;0.6503381883816667;0.5962919930311235;150.0;0.0001
-36;0.9;0.01;128.0;1.0621216297149658;0.584642231464386;1.0866228342056274;0.5497382283210754;1.073479413986206;0.5671902298927307;1.0;0.5977450132718186;0.5629235472659129;0.5687495323396486;0.5795900958944438;0.545400251921991;0.565356141443098;50.0;0.0
-37;0.9;0.01;128.0;1.0547319650650024;0.6125654578208923;1.1377087831497192;0.5776614546775818;1.127797245979309;0.5636998414993286;1.0;0.6235207292967124;0.5720610223175946;0.5682047407187032;0.6082697495740974;0.5770853542592673;0.5593319723754506;100.0;0.0
-38;0.9;0.01;128.0;0.9883608222007751;0.6265270709991455;1.316406488418579;0.5567190051078796;1.1691138744354248;0.5828970074653625;1.0;0.6431009815575375;0.5975566000144896;0.6062137814624874;0.624762918241179;0.5490433479563914;0.577417148069322;150.0;0.0
-39;0.9;0.01;128.0;1.443315863609314;0.5253053903579712;1.3692501783370972;0.5828970074653625;1.342460036277771;0.5933682322502136;1.0;0.5716634678107281;0.597909057945944;0.5906145139282292;0.5164768806073153;0.57604231517275;0.5995459854155506;50.0;0.01
-40;0.9;0.01;128.0;1.2021657228469849;0.6265270709991455;1.3414463996887207;0.5619546175003052;1.2604358196258545;0.5828970074653625;1.0;0.6285688058844148;0.6186485688650869;0.6176072056565729;0.6251604675517719;0.5610753980319197;0.5805716023107328;100.0;0.01
-41;0.9;0.01;128.0;1.1669517755508423;0.6003490686416626;1.195756435394287;0.6020942330360413;1.1677254438400269;0.614310622215271;1.0;0.6203690104168971;0.6203570670428672;0.6432026808004327;0.5943404421665291;0.596520025867852;0.6156590993547515;150.0;0.01
-42;0.9;0.01;128.0;1.1474894285202026;0.5497382283210754;1.1794898509979248;0.5357766151428223;1.0964933633804321;0.5759162306785583;1.0;0.5853768536604357;0.5617280741536463;0.5739665220002184;0.5477795151708195;0.5463829648612257;0.5779051866008388;50.0;0.001
-43;0.9;0.01;128.0;1.253674864768982;0.5253053903579712;1.160112977027893;0.5776614546775818;1.1979146003723145;0.5602094531059265;1.0;0.5522893772893773;0.5946065751075158;0.609541492289036;0.5178848928848929;0.5693081073515855;0.5646515320428364;100.0;0.001
-44;0.9;0.01;128.0;1.0433732271194458;0.62129145860672;1.1583247184753418;0.6055846214294434;1.2541570663452148;0.5584642291069031;1.0;0.6268407792270415;0.6419238559031413;0.5858269376442662;0.6190530484008745;0.5973609723609723;0.5483924288272114;150.0;0.001
-45;0.9;0.01;128.0;1.1987037658691406;0.5375218391418457;1.1068445444107056;0.5445026159286499;1.1949487924575806;0.518324613571167;1.0;0.5710625395113904;0.5581101609004185;0.5326604175732083;0.5442896475505171;0.5378775813558422;0.5108786141394838;50.0;0.0001
-46;0.9;0.01;128.0;1.1030073165893555;0.5863874554634094;1.0547511577606201;0.6195462346076965;1.066347599029541;0.5881326198577881;1.0;0.6079505660141936;0.6188059698281521;0.5895490996780354;0.5875187614318049;0.6249137336093857;0.5872448807231415;100.0;0.0001
-47;0.9;0.01;128.0;1.132765293121338;0.5898778438568115;1.2393956184387207;0.5340313911437988;1.0497409105300903;0.5881326198577881;1.0;0.5911296865320181;0.566333885194723;0.6075930260346379;0.5854749115618681;0.5218344457474892;0.5897791821704865;150.0;0.0001

models/FullyConectedModels/model.py

@@ -1,120 +0,0 @@
-from tensorflow.data import Dataset
-import tensorflow.keras as keras
-from tensorflow.keras.optimizers import Adam
-from tensorflow.keras.layers import (
-    Conv2D,
-    Input,
-    MaxPooling2D,
-    Dense,
-    Dropout,
-    MaxPool1D,
-    Flatten,
-    AveragePooling1D,
-    BatchNormalization,
-)
-from tensorflow.keras import Model
-import numpy as np
-import tensorflow as tf
-from tensorflow.keras.models import Sequential
-from tensorflow.keras.models import Model
-from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
-from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
-from tensorflow.keras.layers import BatchNormalization
-from tensorflow.keras.regularizers import l2
-from tensorflow.keras import backend as K
-from tensorflow.keras.optimizers import SGD
-import warnings
-
-warnings.filterwarnings("ignore")
-
-
-def basemodel(weight_decay):
-    # 2 hidden layers
-    model_input = Input(
-        shape=(
-            32,
-            32,
-            1,
-        )
-    )
-    model = Conv2D(
-        32,
-        kernel_size=(3, 3),
-        kernel_regularizer=l2(weight_decay),
-        activation="relu",
-    )(model_input)
-    model = Conv2D(
-        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Flatten()(model)
-    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
-    model = Model(inputs=model_input, outputs=model)
-    return model
-
-
-def model_2(weight_decay):
-    model_input = Input(
-        shape=(
-            32,
-            32,
-            1,
-        )
-    )
-    model = Conv2D(
-        32,
-        kernel_size=(3, 3),
-        kernel_regularizer=l2(weight_decay),
-        activation="relu",
-    )(model_input)
-    model = Conv2D(
-        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Conv2D(
-        128, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Flatten()(model)
-    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
-    model = Model(inputs=model_input, outputs=model)
-    return model
-
-
-def model_3(weight_decay):
-    # 4 hidden layers
-    model_input = Input(
-        shape=(
-            32,
-            32,
-            1,
-        )
-    )
-    model = Conv2D(
-        32,
-        kernel_size=(3, 3),
-        kernel_regularizer=l2(weight_decay),
-        activation="relu",
-    )(model_input)
-    model = Conv2D(
-        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Conv2D(
-        128, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Conv2D(
-        256, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Flatten()(model)
-    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
-    model = Model(inputs=model_input, outputs=model)
-    return model

models/FullyConectedModels/parseval.py

@@ -1,83 +0,0 @@
-from tensorflow.data import Dataset
-import tensorflow.keras as keras
-from tensorflow.keras.optimizers import Adam
-from tensorflow.keras.layers import (
-    Conv2D,
-    Input,
-    MaxPooling2D,
-    Dense,
-    Dropout,
-    MaxPool1D,
-    Flatten,
-    AveragePooling1D,
-    BatchNormalization,
-)
-from tensorflow.keras import Model
-import numpy as np
-import tensorflow as tf
-from tensorflow.keras.models import Sequential
-from tensorflow.keras.models import Model
-from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
-from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
-from tensorflow.keras.layers import BatchNormalization
-from tensorflow.keras.regularizers import l2
-from tensorflow.keras import backend as K
-from tensorflow.keras.optimizers import SGD
-import warnings
-from constraint import tight_frame
-
-warnings.filterwarnings("ignore")
-
-
-def model_parseval(weight_decay):
-
-    model_input = Input(
-        shape=(
-            32,
-            32,
-            1,
-        )
-    )
-    model = Conv2D(
-        32,
-        kernel_size=(3, 3),
-        activation="relu",
-        input_shape=(32, 32, 1),
-        kernel_regularizer=l2(weight_decay),
-        kernel_constraint=tight_frame(0.001),
-        kernel_initializer="Orthogonal",
-    )(model_input)
-    model = Conv2D(
-        64,
-        kernel_size=(3, 3),
-        activation="relu",
-        kernel_regularizer=l2(weight_decay),
-        kernel_initializer="Orthogonal",
-        kernel_constraint=tight_frame(0.001),
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Conv2D(
-        128,
-        kernel_size=(3, 3),
-        activation="relu",
-        kernel_initializer="Orthogonal",
-        kernel_regularizer=l2(weight_decay),
-        kernel_constraint=tight_frame(0.001),
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Conv2D(
-        256,
-        kernel_size=(3, 3),
-        activation="relu",
-        kernel_initializer="Orthogonal",
-        kernel_regularizer=l2(weight_decay),
-        kernel_constraint=tight_frame(0.001),
-    )(model)
-    model = MaxPooling2D(pool_size=(2, 2))(model)
-    model = BatchNormalization()(model)
-    model = Flatten()(model)
-    model = Dense(4, activation="softmax", kernel_regularizer=l2(weight_decay))(model)
-    model = Model(inputs=model_input, outputs=model)
-    return model

models/Parseval_Networks/README.md

@@ -1,3 +0,0 @@
-## ParsevalNetworks
-* Orthogonality Constraint
-* Convexity Constraint

models/Parseval_Networks/constraint.py

@@ -1,81 +0,0 @@
-from tensorflow.python.keras.constraints import Constraint
-from tensorflow.python.ops import math_ops, array_ops
-
-
-class TightFrame(Constraint):
-    """
-    Parseval (tight) frame contstraint, as introduced in https://arxiv.org/abs/1704.08847
-
-    Constraints the weight matrix to be a tight frame, so that the Lipschitz
-    constant of the layer is <= 1. This increases the robustness of the network
-    to adversarial noise.
-
-    Warning: This constraint simply performs the update step on the weight matrix
-    (or the unfolded weight matrix for convolutional layers). Thus, it does not
-    handle the necessary scalings for convolutional layers.
-
-    Args:
-        scale (float):    Retraction parameter (length of retraction step).
-        num_passes (int): Number of retraction steps.
-
-    Returns:
-        Weight matrix after applying regularizer.
-    """
-
-    def __init__(self, scale, num_passes=1):
-        """[summary]
-
-        Args:
-            scale ([type]): [description]
-            num_passes (int, optional): [description]. Defaults to 1.
-
-        Raises:
-            ValueError: [description]
-        """
-        self.scale = scale
-
-        if num_passes < 1:
-            raise ValueError(
-                "Number of passes cannot be non-positive! (got {})".format(num_passes)
-            )
-        self.num_passes = num_passes
-
-    def __call__(self, w):
-        """[summary]
-
-        Args:
-            w ([type]): weight of conv or linear layers
-
-        Returns:
-            [type]: returns new weights
-        """
-        transpose_channels = len(w.shape) == 4
-
-        # Move channels_num to the front in order to make the dimensions correct for matmul
-        if transpose_channels:
-            w_reordered = array_ops.reshape(w, (-1, w.shape[3]))
-
-        else:
-            w_reordered = w
-
-        last = w_reordered
-        for i in range(self.num_passes):
-            temp1 = math_ops.matmul(last, last, transpose_a=True)
-            temp2 = (1 + self.scale) * w_reordered - self.scale * math_ops.matmul(
-                w_reordered, temp1
-            )
-
-            last = temp2
-
-        # Move channels_num to the back again
-        if transpose_channels:
-            return array_ops.reshape(last, w.shape)
-        else:
-            return last
-
-    def get_config(self):
-        return {"scale": self.scale, "num_passes": self.num_passes}
-
-
-# Alias
-tight_frame = TightFrame

models/Parseval_Networks/convexity_constraint.py

@@ -1,53 +0,0 @@
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import variables
-from tensorflow.python.framework import dtypes
-import numpy as _np
-
-
-def convex_add(input_layer, layer_3, initial_convex_par=0.5, trainable=False):
-    """
-    Do a convex combination of input_layer and layer_3. That is, return the output of
-
-        lamda* input_layer + (1 - lamda) * layer_3
-
-
-    Args:
-            input_layer (tf.Tensor):	Input to take convex combinatio of
-            layer_3 (tf.Tensor):		Input to take convex combinatio of
-            initial_convex_par (float):	Initial value for convex parameter. Must be
-                                                                    in [0, 1].
-            trainable (bool):			Whether convex parameter should be trainable
-                                                                    or not.
-
-    Returns:
-            tf.Tensor: Result of convex combination
-    """
-    # Will implement this as sigmoid(p)*input_layer + (1-sigmoid(p))*layer_3 to ensure
-    # convex parameter to be in the unit interval without constraints during
-    # optimization
-
-    # Find value for p, also check for legal initial_convex_par
-    if initial_convex_par < 0:
-        raise ValueError("Convex parameter must be >=0")
-
-    elif initial_convex_par == 0:
-        # sigmoid(-16) is approximately a 32bit roundoff error, practically 0
-        initial_p_value = -16
-
-    elif initial_convex_par < 1:
-        # Compute inverse of sigmoid to find initial p value
-        initial_p_value = -_np.log(1 / initial_convex_par - 1)
-
-    elif initial_convex_par == 1:
-        # Same argument as for 0
-        initial_p_value = 16
-
-    else:
-        raise ValueError("Convex parameter must be <=1")
-
-    p = variables.Variable(
-        initial_value=initial_p_value, dtype=dtypes.float32, trainable=trainable
-    )
-
-    lam = math_ops.sigmoid(p)
-    return input_layer * lam + (1 - lam) * layer_3

models/Parseval_Networks/parsevalnet.py

@@ -1,328 +0,0 @@
-from tensorflow.keras.models import Model
-from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
-from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
-from tensorflow.keras.layers import BatchNormalization
-from tensorflow.keras.regularizers import l2
-from tensorflow.keras import backend as K
-from tensorflow.keras.optimizers import SGD
-import warnings
-from constraint import tight_frame
-from convexity_constraint import convex_add
-
-warnings.filterwarnings("ignore")
-
-
-class ParsevalNetwork(Model):
-    def __init__(
-        self,
-        input_dim,
-        weight_decay,
-        momentum,
-        nb_classes=4,
-        N=2,
-        k=1,
-        dropout=0.0,
-        verbose=1,
-    ):
-        """[Assign the initial parameters of the wide residual network]
-
-        Args:
-            weight_decay ([float]): [description]
-            input_dim ([tuple]): [input dimension]
-            nb_classes (int, optional): [output class]. Defaults to 4.
-            N (int, optional): [the number of blocks]. Defaults to 2.
-            k (int, optional): [network width]. Defaults to 1.
-            dropout (float, optional): [dropout value to prevent overfitting]. Defaults to 0.0.
-            verbose (int, optional): [description]. Defaults to 1.
-
-        Returns:
-            [Model]: [parsevalnetwork]
-        """
-        self.weight_decay = weight_decay
-        self.input_dim = input_dim
-        self.nb_classes = nb_classes
-        self.N = N
-        self.k = k
-        self.dropout = dropout
-        self.verbose = verbose
-
-    def initial_conv(self, input):
-        """[summary]
-
-        Args:
-            input ([type]): [description]
-
-        Returns:
-            [type]: [description]
-        """
-        x = Convolution2D(
-            16,
-            (3, 3),
-            padding="same",
-            kernel_initializer="orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(input)
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        return x
-
-    def expand_conv(self, init, base, k, strides=(1, 1)):
-        """[summary]
-
-        Args:
-            init ([type]): [description]
-            base ([type]): [description]
-            k ([type]): [description]
-            strides (tuple, optional): [description]. Defaults to (1, 1).
-
-        Returns:
-            [type]: [description]
-        """
-        x = Convolution2D(
-            base * k,
-            (3, 3),
-            padding="same",
-            strides=strides,
-            kernel_initializer="Orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(init)
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = Convolution2D(
-            base * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="Orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(x)
-
-        skip = Convolution2D(
-            base * k,
-            (1, 1),
-            padding="same",
-            strides=strides,
-            kernel_initializer="Orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(init)
-
-        m = Add()([x, skip])
-
-        return m
-
-    def conv1_block(self, input, k=1, dropout=0.0):
-        """[summary]
-
-        Args:
-            input ([type]): [description]
-            k (int, optional): [description]. Defaults to 1.
-            dropout (float, optional): [description]. Defaults to 0.0.
-
-        Returns:
-            [type]: [description]
-        """
-        init = input
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(input)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            16 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="Orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(x)
-
-        if dropout > 0.0:
-            x = Dropout(dropout)(x)
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            16 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="Orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(x)
-        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
-        return m
-
-    def conv2_block(self, input, k=1, dropout=0.0):
-        """[summary]
-
-        Args:
-            input ([type]): [description]
-            k (int, optional): [description]. Defaults to 1.
-            dropout (float, optional): [description]. Defaults to 0.0.
-
-        Returns:
-            [type]: [description]
-        """
-        init = input
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(input)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            32 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="Orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(x)
-
-        if dropout > 0.0:
-            x = Dropout(dropout)(x)
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            32 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="Orthogonal",
-            kernel_regularizer=l2(self.weight_decay),
-            kernel_constraint=tight_frame(0.001),
-            use_bias=False,
-        )(x)
-
-        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
-        return m
-
-    def conv3_block(self, input, k=1, dropout=0.0):
-        init = input
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(input)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            64 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="Orthogonal",
-            kernel_constraint=tight_frame(0.001),
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        if dropout > 0.0:
-            x = Dropout(dropout)(x)
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            64 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="Orthogonal",
-            kernel_constraint=tight_frame(0.001),
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
-        return m
-
-    def create_wide_residual_network(self):
-        """create a wide residual network model
-
-
-        Returns:
-            [Model]: [wide residual network]
-        """
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        ip = Input(shape=self.input_dim)
-
-        x = self.initial_conv(ip)
-        nb_conv = 4
-
-        x = self.expand_conv(x, 16, self.k)
-        nb_conv += 2
-
-        for i in range(self.N - 1):
-            x = self.conv1_block(x, self.k, self.dropout)
-            nb_conv += 2
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = self.expand_conv(x, 32, self.k, strides=(2, 2))
-        nb_conv += 2
-
-        for i in range(self.N - 1):
-            x = self.conv2_block(x, self.k, self.dropout)
-            nb_conv += 2
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = self.expand_conv(x, 64, self.k, strides=(2, 2))
-        nb_conv += 2
-
-        for i in range(self.N - 1):
-            x = self.conv3_block(x, self.k, self.dropout)
-            nb_conv += 2
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = AveragePooling2D((8, 8))(x)
-        x = Flatten()(x)
-
-        x = Dense(
-            self.nb_classes,
-            kernel_regularizer=l2(self.weight_decay),
-            activation="softmax",
-        )(x)
-
-        model = Model(ip, x)
-
-        if self.verbose:
-            print("Parseval  Network-%d-%d created." % (nb_conv, self.k))
-        return model

models/README.md

@@ -1,15 +0,0 @@
-## Models
-
-````
-├── Parseval_network
-│   ├── __init__.py
-│   └── Parseval_resnet.py
-├── Parseval_Networks_OC
-│   ├── constraint.py
-│   ├── parsnet_oc.py
-│   └── README.md
-├── README.md
-├── _utility.py
-└── wideresnet
-    └── wresnet.py
-````

models/_utility.py

@@ -1,109 +0,0 @@
-from tensorflow.keras.callbacks import LearningRateScheduler
-
-# Define configuration parameters
-import math
-import cleverhans
-from cleverhans.tf2.attacks.fast_gradient_method import fast_gradient_method
-import tensorflow as tf
-
-import numpy as np
-
-
-def step_decay(epoch):
-    """[summary]
-
-    Args:
-        epoch (int): epoch number
-
-    Returns:
-        lrate(float): new learning rate
-    """
-    initial_lrate = 0.1
-    factor = 0.1
-    if epoch < 10:
-        lrate = initial_lrate
-    elif epoch < 20:
-        lrate = initial_lrate * math.pow(factor, 1)
-    elif epoch < 30:
-        lrate = initial_lrate * math.pow(factor, 2)
-    elif epoch < 40:
-        lrate = initial_lrate * math.pow(factor, 3)
-    else:
-        lrate = initial_lrate * math.pow(factor, 4)
-    return lrate
-
-
-def step_decay_conv(epoch):
-    """step decay for learning rate in convolutional networks
-
-    Args:
-        epoch (int): epoch number
-
-    Returns:
-        lrate(float): new learning rate
-    """
-    initial_lrate = 0.01
-    factor = 0.1
-    if epoch < 10:
-        lrate = initial_lrate
-    elif epoch < 20:
-        lrate = initial_lrate * math.pow(factor, 1)
-    elif epoch < 30:
-        lrate = initial_lrate * math.pow(factor, 2)
-    elif epoch < 40:
-        lrate = initial_lrate * math.pow(factor, 3)
-    else:
-        lrate = initial_lrate * math.pow(factor, 4)
-    return lrate
-
-
-def print_test(model, X_adv, X_test, y_test, epsilon):
-    """
-    returns the test results and show the SNR and evaluation results
-    """
-    loss, acc = model.evaluate(X_adv, y_test)
-    print("epsilon: {} and test evaluation : {}, {}".format(epsilon, loss, acc))
-    SNR = 20 * np.log10(np.linalg.norm(X_test) / np.linalg.norm(X_test - X_adv))
-    print("SNR: {}".format(SNR))
-    return loss, acc
-
-
-def get_adversarial_examples(pretrained_model, X_true, y_true, epsilon):
-    """
-    The attack requires the model to ouput the logits
-    returns the adversarial example/s of a given image/s for epsilon value using
-    fast gradient sign method
-    """
-    logits_model = tf.keras.Model(
-        pretrained_model.input, pretrained_model.layers[-1].output
-    )
-    X_adv = []
-
-    for i in range(len(X_true)):
-
-        random_index = i
-
-        original_image = X_true[random_index]
-        original_image = tf.convert_to_tensor(
-            original_image.reshape((1, 32, 32))
-        )  # The .reshape just gives it the proper form to input into the model, a batch of 1 a.k.a a tensor
-        original_label = y_true[random_index]
-        original_label = np.reshape(np.argmax(original_label), (1,)).astype("int64")
-
-        adv_example_targeted_label = fast_gradient_method(
-            logits_model,
-            original_image,
-            epsilon,
-            np.inf,
-            y=original_label,
-            targeted=False,
-        )
-        X_adv.append(np.array(adv_example_targeted_label).reshape(32, 32, 1))
-
-    X_adv = np.array(X_adv)
-
-    return X_adv
-
-
-lrate_conv = LearningRateScheduler(step_decay_conv)
-lrate = LearningRateScheduler(step_decay)

models/wideresnet/wresnet.py

@@ -1,329 +0,0 @@
-from tensorflow.keras.models import Model
-from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
-from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
-from tensorflow.keras.layers import BatchNormalization
-from tensorflow.keras.regularizers import l2
-from tensorflow.keras import backend as K
-from tensorflow.keras.optimizers import SGD
-import warnings
-
-warnings.filterwarnings("ignore")
-
-
-class WideResidualNetwork(object):
-    def __init__(
-        self,
-        input_dim,
-        weight_decay,
-        momentum,
-        nb_classes=100,
-        N=2,
-        k=1,
-        dropout=0.0,
-        verbose=1,
-    ):
-        """[Assign the initial parameters of the wide residual network]
-
-        Args:
-            weight_decay ([float]): [description]
-            input_dim ([tuple]): [input dimension]
-            nb_classes (int, optional): [output class]. Defaults to 100.
-            N (int, optional): [the number of blocks]. Defaults to 2.
-            k (int, optional): [network width]. Defaults to 1.
-            dropout (float, optional): [dropout value to prevent overfitting]. Defaults to 0.0.
-            verbose (int, optional): [description]. Defaults to 1.
-
-        Returns:
-            [Model]: [wideresnet]
-        """
-        self.weight_decay = weight_decay
-        self.input_dim = input_dim
-        self.nb_classes = nb_classes
-        self.N = N
-        self.k = k
-        self.dropout = dropout
-        self.verbose = verbose
-
-    def initial_conv(self, input):
-        """[summary]
-
-        Args:
-            input ([type]): [description]
-
-        Returns:
-            [type]: [description]
-        """
-        x = Convolution2D(
-            16,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(input)
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        return x
-
-    def expand_conv(self, init, base, k, strides=(1, 1)):
-        """[summary]
-
-        Args:
-            init ([type]): [description]
-            base ([type]): [description]
-            k ([type]): [description]
-            strides (tuple, optional): [description]. Defaults to (1, 1).
-
-        Returns:
-            [type]: [description]
-        """
-        x = Convolution2D(
-            base * k,
-            (3, 3),
-            padding="same",
-            strides=strides,
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(init)
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = Convolution2D(
-            base * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        skip = Convolution2D(
-            base * k,
-            (1, 1),
-            padding="same",
-            strides=strides,
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(init)
-
-        m = Add()([x, skip])
-
-        return m
-
-    def conv1_block(self, input, k=1, dropout=0.0):
-        """[summary]
-
-        Args:
-            input ([type]): [description]
-            k (int, optional): [description]. Defaults to 1.
-            dropout (float, optional): [description]. Defaults to 0.0.
-
-        Returns:
-            [type]: [description]
-        """
-        init = input
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(input)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            16 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        if dropout > 0.0:
-            x = Dropout(dropout)(x)
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            16 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        m = Add()([init, x])
-        return m
-
-    def conv2_block(self, input, k=1, dropout=0.0):
-        """[summary]
-
-        Args:
-            input ([type]): [description]
-            k (int, optional): [description]. Defaults to 1.
-            dropout (float, optional): [description]. Defaults to 0.0.
-
-        Returns:
-            [type]: [description]
-        """
-        init = input
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-        print("conv2:channel:  {}".format(channel_axis))
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(input)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            32 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        if dropout > 0.0:
-            x = Dropout(dropout)(x)
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            32 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        m = Add()([init, x])
-        return m
-
-    def conv3_block(self, input, k=1, dropout=0.0):
-        """[summary]
-
-        Args:
-            input ([type]): [description]
-            k (int, optional): [description]. Defaults to 1.
-            dropout (float, optional): [description]. Defaults to 0.0.
-
-        Returns:
-            [type]: [description]
-        """
-        init = input
-
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(input)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            64 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        if dropout > 0.0:
-            x = Dropout(dropout)(x)
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-        x = Convolution2D(
-            64 * k,
-            (3, 3),
-            padding="same",
-            kernel_initializer="he_normal",
-            kernel_regularizer=l2(self.weight_decay),
-            use_bias=False,
-        )(x)
-
-        m = Add()([init, x])
-        return m
-
-    def create_wide_residual_network(self):
-        """create a wide residual network model
-
-
-        Returns:
-            [Model]: [wide residual network]
-        """
-        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
-
-        ip = Input(shape=self.input_dim)
-
-        x = self.initial_conv(ip)
-        nb_conv = 4
-
-        x = self.expand_conv(x, 16, self.k)
-        nb_conv += 2
-
-        for i in range(self.N - 1):
-            x = self.conv1_block(x, self.k, self.dropout)
-            nb_conv += 2
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = self.expand_conv(x, 32, self.k, strides=(2, 2))
-        nb_conv += 2
-
-        for i in range(self.N - 1):
-            x = self.conv2_block(x, self.k, self.dropout)
-            nb_conv += 2
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = self.expand_conv(x, 64, self.k, strides=(2, 2))
-        nb_conv += 2
-
-        for i in range(self.N - 1):
-            x = self.conv3_block(x, self.k, self.dropout)
-            nb_conv += 2
-
-        x = BatchNormalization(
-            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
-        )(x)
-        x = Activation("relu")(x)
-
-        x = AveragePooling2D((8, 8))(x)
-        x = Flatten()(x)
-
-        x = Dense(
-            self.nb_classes,
-            kernel_regularizer=l2(self.weight_decay),
-            activation="softmax",
-        )(x)
-
-        model = Model(ip, x)
-
-        if self.verbose:
-            print("Wide Residual Network-%d-%d created." % (nb_conv, self.k))
-        return model