Upload P2 - Secom Notebook - Mercury.ipynb

P2 - Secom Notebook - Mercury.ipynb

@@ -0,0 +1,1422 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "# **Classifying products in Semiconductor Industry**"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Import the data**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# import pandas for data manipulation\n",
+    "# import numpy for numerical computation\n",
+    "# import seaborn for data visualization\n",
+    "# import matplotlib for data visualization\n",
+    "# import stats for statistical analysis\n",
+    "# import train_test_split for splitting data into training and testing sets\n",
+    "\n",
+    "\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import seaborn as sns\n",
+    "import matplotlib.pyplot as plt\n",
+    "from scipy import stats\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "import mercury as mr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/mercury+json": {
+       "allow_download": true,
+       "code_uid": "App.0.40.24.1-randc1b961c9",
+       "continuous_update": false,
+       "description": "Recumpute everything dynamically",
+       "full_screen": true,
+       "model_id": "mercury-app",
+       "notify": "{}",
+       "output": "app",
+       "schedule": "",
+       "show_code": false,
+       "show_prompt": false,
+       "show_sidebar": true,
+       "static_notebook": false,
+       "title": "Secom Web App Demo",
+       "widget": "App"
+      },
+      "text/html": [
+       "<h3>Mercury Application</h3><small>This output won't appear in the web app.</small>"
+      ],
+      "text/plain": [
+       "mercury.App"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "app = mr.App(title=\"Secom Web App Demo\", description=\"Recumpute everything dynamically\", continuous_update=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Read the features data from the the url of csv into pandas dataframes and rename the columns to F1, F2, F3, etc.\n",
+    "# Read the labels data from the url of csv into pandas dataframes and rename the columns to pass/fail and date/time\n",
+    "\n",
+    "#url_data = 'https://archive.ics.uci.edu/ml/machine-learning-databases/secom/secom.data'\n",
+    "#url_labels = 'https://archive.ics.uci.edu/ml/machine-learning-databases/secom/secom_labels.data'\n",
+    "\n",
+    "url_data = '..\\Dataset\\secom_data.csv'\n",
+    "url_labels = '..\\Dataset\\secom_labels.csv'\n",
+    "\n",
+    "features = pd.read_csv(url_data,  delimiter=' ', header=None)\n",
+    "labels = pd.read_csv(url_labels,  delimiter=' ', names=['pass/fail', 'date_time'])\n",
+    "\n",
+    "prefix = 'F'\n",
+    "new_column_names = [prefix + str(i) for i in range(1, len(features.columns)+1)]\n",
+    "features.columns = new_column_names\n",
+    "\n",
+    "labels['pass/fail'] = labels['pass/fail'].replace({-1: 0, 1: 1})\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Split the data**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dropped date/time column from labels dataframe\n"
+     ]
+    }
+   ],
+   "source": [
+    "# if there is a date/time column, drop it from the features and labels dataframes, else continue\n",
+    "\n",
+    "if 'date_time' in labels.columns:\n",
+    "    labels = labels.drop(['date_time'], axis=1)\n",
+    "    print('Dropped date/time column from labels dataframe')\n",
+    "\n",
+    "\n",
+    "# Split the dataset and the labels into training and testing sets\n",
+    "# use stratify to ensure that the training and testing sets have the same percentage of pass and fail labels\n",
+    "# use random_state to ensure that the same random split is generated each time the code is run\n",
+    "\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=0.25, stratify=labels, random_state=13)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "### **Functions**"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Feature Removal**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def columns_to_drop(df,drop_duplicates='yes', missing_values_threshold=100, variance_threshold=0, \n",
+    "                   correlation_threshold=1.1):\n",
+    "   \n",
+    "   print('Shape of the dataframe is: ', df.shape)\n",
+    "\n",
+    "    # Drop duplicated columns\n",
+    "   if drop_duplicates == 'yes':\n",
+    "      new_column_names = df.columns\n",
+    "      df = df.T.drop_duplicates().T\n",
+    "      print('the number of columns to be dropped due to duplications is: ', len(new_column_names) - len(df.columns))\n",
+    "      drop_duplicated = list(set(new_column_names) - set(df.columns))\n",
+    "\n",
+    "   elif drop_duplicates == 'no':\n",
+    "      df = df.T.T\n",
+    "      print('No columns were dropped due to duplications')   \n",
+    "\n",
+    "   # Print the percentage of columns in df with missing values more than or equal to threshold\n",
+    "   print('the number of columns to be dropped due to missing values is: ', len(df.isnull().mean()[df.isnull().mean() > missing_values_threshold/100].index))\n",
+    "      \n",
+    "   # Print into a list the columns to be dropped due to missing values\n",
+    "   drop_missing = list(df.isnull().mean()[df.isnull().mean() > missing_values_threshold/100].index)\n",
+    "\n",
+    "   # Drop columns with more than or equal to threshold missing values from df\n",
+    "   df.drop(drop_missing, axis=1, inplace=True)\n",
+    "    \n",
+    "   # Print the number of columns in df with variance less than threshold\n",
+    "   print('the number of columns to be dropped due to low variance is: ', len(df.var()[df.var() <= variance_threshold].index))\n",
+    "\n",
+    "   # Print into a list the columns to be dropped due to low variance\n",
+    "   drop_variance = list(df.var()[df.var() <= variance_threshold].index)\n",
+    "\n",
+    "   # Drop columns with more than or equal to threshold variance from df\n",
+    "   df.drop(drop_variance, axis=1, inplace=True)\n",
+    "\n",
+    "   # Print the number of columns in df with more than or equal to threshold correlation\n",
+    "      \n",
+    "   # Create correlation matrix and round it to 4 decimal places\n",
+    "   corr_matrix = df.corr().abs().round(4)\n",
+    "   upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n",
+    "   to_drop = [column for column in upper.columns if any(upper[column] >= correlation_threshold)]\n",
+    "   print('the number of columns to be dropped due to high correlation is: ', len(to_drop))\n",
+    "\n",
+    "   # Print into a list the columns to be dropped due to high correlation\n",
+    "   drop_correlation = [column for column in upper.columns if any(upper[column] >= correlation_threshold)]\n",
+    "\n",
+    "   # Drop columns with more than or equal to threshold correlation from df\n",
+    "   df.drop(to_drop, axis=1, inplace=True)\n",
+    "   \n",
+    "   if drop_duplicates == 'yes':\n",
+    "      dropped = (drop_duplicated+drop_missing+drop_variance+drop_correlation)\n",
+    "\n",
+    "   elif drop_duplicates =='no':\n",
+    "      dropped = (drop_missing+drop_variance+drop_correlation)\n",
+    "   \n",
+    "   print('Total number of columns to be dropped is: ', len(dropped))\n",
+    "   print('New shape of the dataframe is: ', df.shape)\n",
+    "\n",
+    "   global drop_duplicates_var\n",
+    "   drop_duplicates_var = drop_duplicates\n",
+    "   \n",
+    "   global missing_values_threshold_var\n",
+    "   missing_values_threshold_var = missing_values_threshold\n",
+    "\n",
+    "   global variance_threshold_var\n",
+    "   variance_threshold_var = variance_threshold\n",
+    "\n",
+    "   global correlation_threshold_var\n",
+    "   correlation_threshold_var = correlation_threshold\n",
+    "   \n",
+    "   print(type(dropped))\n",
+    "   return dropped"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Outlier Removal**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def outlier_removal(z_df, z_threshold=4):\n",
+    "    \n",
+    "    global outlier_var\n",
+    "\n",
+    "    if z_threshold == 'none':\n",
+    "        print('No outliers were removed')\n",
+    "        outlier_var = 'none'\n",
+    "        return z_df\n",
+    "        \n",
+    "    else:\n",
+    "        print('The z-score threshold is:', z_threshold)\n",
+    "\n",
+    "        z_df_copy = z_df.copy()\n",
+    "\n",
+    "        z_scores = np.abs(stats.zscore(z_df_copy))\n",
+    "\n",
+    "     # Identify the outliers in the dataset using the z-score method\n",
+    "        outliers_mask = z_scores > z_threshold\n",
+    "        z_df_copy[outliers_mask] = np.nan\n",
+    "\n",
+    "        outliers_count = np.count_nonzero(outliers_mask)\n",
+    "        print('The number of outliers in the whole dataset is / was:', outliers_count)\n",
+    "\n",
+    "        outlier_var = z_threshold\n",
+    "\n",
+    "    print(type(z_df_copy))\n",
+    "    return z_df_copy"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Scaling Methods**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# define a function to scale the dataframe using different scaling models\n",
+    "\n",
+    "def scale_dataframe(scale_model,df_fit, df_transform):\n",
+    "    \n",
+    "    global scale_model_var\n",
+    "\n",
+    "    if scale_model == 'robust':\n",
+    "        from sklearn.preprocessing import RobustScaler\n",
+    "        scaler = RobustScaler()\n",
+    "        scaler.fit(df_fit)\n",
+    "        df_scaled = scaler.transform(df_transform)\n",
+    "        df_scaled = pd.DataFrame(df_scaled, columns=df_transform.columns)\n",
+    "        print('The dataframe has been scaled using the robust scaling model')\n",
+    "        scale_model_var = 'robust'\n",
+    "        return df_scaled\n",
+    "    \n",
+    "    elif scale_model == 'standard':\n",
+    "        from sklearn.preprocessing import StandardScaler\n",
+    "        scaler = StandardScaler()\n",
+    "        scaler.fit(df_fit)\n",
+    "        df_scaled = scaler.transform(df_transform)\n",
+    "        df_scaled = pd.DataFrame(df_scaled, columns=df_transform.columns)\n",
+    "        print('The dataframe has been scaled using the standard scaling model')\n",
+    "        scale_model_var = 'standard'\n",
+    "        return df_scaled\n",
+    "    \n",
+    "    elif scale_model == 'normal':\n",
+    "        from sklearn.preprocessing import Normalizer\n",
+    "        scaler = Normalizer()\n",
+    "        scaler.fit(df_fit)\n",
+    "        df_scaled = scaler.transform(df_transform)\n",
+    "        df_scaled = pd.DataFrame(df_scaled, columns=df_transform.columns)\n",
+    "        print('The dataframe has been scaled using the normal scaling model')\n",
+    "        scale_model_var = 'normal'\n",
+    "        return df_scaled\n",
+    "    \n",
+    "    elif scale_model == 'minmax':\n",
+    "        from sklearn.preprocessing import MinMaxScaler\n",
+    "        scaler = MinMaxScaler()\n",
+    "        scaler.fit(df_fit)\n",
+    "        df_scaled = scaler.transform(df_transform)\n",
+    "        df_scaled = pd.DataFrame(df_scaled, columns=df_transform.columns)\n",
+    "        print('The dataframe has been scaled using the minmax scaling model')\n",
+    "        scale_model_var = 'minmax'\n",
+    "        return df_scaled\n",
+    "    \n",
+    "    elif scale_model == 'none':\n",
+    "        print('The dataframe has not been scaled')\n",
+    "        scale_model_var = 'none'\n",
+    "        return df_transform\n",
+    "    \n",
+    "    else:\n",
+    "        print('Please choose a valid scaling model: robust, standard, normal, or minmax')\n",
+    "        return None"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Missing Value Imputation**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# define a function to impute missing values using different imputation models\n",
+    "\n",
+    "def impute_missing_values(imputation, df_fit, df_transform, n_neighbors=5):\n",
+    "\n",
+    "    print('Number of missing values before imputation: ', df_transform.isnull().sum().sum())\n",
+    "\n",
+    "    global imputation_var\n",
+    "\n",
+    "    if imputation == 'knn':\n",
+    "\n",
+    "        from sklearn.impute import KNNImputer\n",
+    "        imputer = KNNImputer(n_neighbors=n_neighbors)\n",
+    "        imputer.fit(df_fit)\n",
+    "        df_imputed = imputer.transform(df_transform)\n",
+    "        df_imputed = pd.DataFrame(df_imputed, columns=df_transform.columns)\n",
+    "        print('Number of missing values after imputation: ', df_imputed.isnull().sum().sum())\n",
+    "        imputation_var = 'knn'\n",
+    "        return df_imputed\n",
+    "    \n",
+    "    elif imputation == 'mean':\n",
+    "\n",
+    "        from sklearn.impute import SimpleImputer\n",
+    "        imputer = SimpleImputer(strategy='mean')\n",
+    "        imputer.fit(df_fit)\n",
+    "        df_imputed = imputer.transform(df_transform)\n",
+    "        df_imputed = pd.DataFrame(df_imputed, columns=df_transform.columns)\n",
+    "        print('Number of missing values after imputation: ', df_imputed.isnull().sum().sum())\n",
+    "        imputation_var = 'mean'\n",
+    "        return df_imputed\n",
+    "    \n",
+    "    elif imputation == 'median':\n",
+    "\n",
+    "        from sklearn.impute import SimpleImputer\n",
+    "        imputer = SimpleImputer(strategy='median')\n",
+    "        imputer.fit(df_fit)\n",
+    "        df_imputed = imputer.transform(df_transform)\n",
+    "        df_imputed = pd.DataFrame(df_imputed, columns=df_transform.columns)\n",
+    "        print('Number of missing values after imputation: ', df_imputed.isnull().sum().sum())\n",
+    "        imputation_var = 'median'\n",
+    "        return df_imputed\n",
+    "    \n",
+    "    elif imputation == 'most_frequent':\n",
+    "            \n",
+    "        from sklearn.impute import SimpleImputer\n",
+    "        imputer = SimpleImputer(strategy='most_frequent')\n",
+    "        imputer.fit(df_fit)\n",
+    "        df_imputed = imputer.transform(df_transform)\n",
+    "        df_imputed = pd.DataFrame(df_imputed, columns=df_transform.columns)\n",
+    "        print('Number of missing values after imputation: ', df_imputed.isnull().sum().sum())\n",
+    "        imputation_var = 'most_frequent'\n",
+    "        return df_imputed\n",
+    "    \n",
+    "    else:\n",
+    "        print('Please choose an imputation model from the following: knn, mean, median, most_frequent')\n",
+    "        df_imputed = df_transform.copy()\n",
+    "        return df_imputed\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Imbalance Treatment**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "#define a function to oversample and understamble the imbalance in the training set\n",
+    "\n",
+    "def imbalance_treatment(method, X_train, y_train):\n",
+    "\n",
+    "    global imbalance_var\n",
+    "\n",
+    "    if method == 'smote':        \n",
+    "        from imblearn.over_sampling import SMOTE\n",
+    "        sm = SMOTE(random_state=42)\n",
+    "        X_train_res, y_train_res = sm.fit_resample(X_train, y_train)\n",
+    "        print('Shape of the training set after oversampling with SMOTE: ', X_train_res.shape)\n",
+    "        print('Value counts of the target variable after oversampling with SMOTE: \\n', y_train_res.value_counts())\n",
+    "        imbalance_var = 'smote'\n",
+    "        return X_train_res, y_train_res\n",
+    "    \n",
+    "    if method == 'undersampling':\n",
+    "        from imblearn.under_sampling import RandomUnderSampler\n",
+    "        rus = RandomUnderSampler(random_state=42)\n",
+    "        X_train_res, y_train_res = rus.fit_resample(X_train, y_train)\n",
+    "        print('Shape of the training set after undersampling with RandomUnderSampler: ', X_train_res.shape)\n",
+    "        print('Value counts of the target variable after undersampling with RandomUnderSampler: \\n', y_train_res.value_counts())\n",
+    "        imbalance_var = 'random_undersampling'\n",
+    "        return X_train_res, y_train_res\n",
+    "    \n",
+    "    if method == 'rose':\n",
+    "        from imblearn.over_sampling import RandomOverSampler\n",
+    "        ros = RandomOverSampler(random_state=42)\n",
+    "        X_train_res, y_train_res = ros.fit_resample(X_train, y_train)\n",
+    "        print('Shape of the training set after oversampling with RandomOverSampler: ', X_train_res.shape)\n",
+    "        print('Value counts of the target variable after oversampling with RandomOverSampler: \\n', y_train_res.value_counts())\n",
+    "        imbalance_var = 'rose'\n",
+    "        return X_train_res, y_train_res\n",
+    "    \n",
+    "    \n",
+    "    if method == 'none':\n",
+    "        X_train_res = X_train\n",
+    "        y_train_res = y_train\n",
+    "        print('Shape of the training set after no resampling: ', X_train_res.shape)\n",
+    "        print('Value counts of the target variable after no resampling: \\n', y_train_res.value_counts())\n",
+    "        imbalance_var = 'none'\n",
+    "        return X_train_res, y_train_res\n",
+    "    \n",
+    "    else:\n",
+    "        print('Please choose a valid resampling method: smote, rose, undersampling or none')\n",
+    "        X_train_res = X_train\n",
+    "        y_train_res = y_train\n",
+    "        return X_train_res, y_train_res"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Training Models**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# define a function where you can choose the model you want to use to train the data\n",
+    "\n",
+    "def train_model(model, X_train, y_train, X_test, y_test):\n",
+    "\n",
+    "    global model_var\n",
+    "\n",
+    "    if model == 'random_forest':\n",
+    "        from sklearn.ensemble import RandomForestClassifier\n",
+    "        rfc = RandomForestClassifier(n_estimators=100, random_state=13)\n",
+    "        rfc.fit(X_train, y_train)\n",
+    "        y_pred = rfc.predict(X_test)\n",
+    "        model_var = 'random_forest'\n",
+    "        return y_pred\n",
+    "\n",
+    "    if model == 'logistic_regression':\n",
+    "        from sklearn.linear_model import LogisticRegression\n",
+    "        lr = LogisticRegression()\n",
+    "        lr.fit(X_train, y_train)\n",
+    "        y_pred = lr.predict(X_test)\n",
+    "        model_var = 'logistic_regression'\n",
+    "        return y_pred\n",
+    "    \n",
+    "    if model == 'knn':\n",
+    "        from sklearn.neighbors import KNeighborsClassifier\n",
+    "        knn = KNeighborsClassifier(n_neighbors=5)\n",
+    "        knn.fit(X_train, y_train)\n",
+    "        y_pred = knn.predict(X_test)\n",
+    "        model_var = 'knn'\n",
+    "        return y_pred\n",
+    "    \n",
+    "    if model == 'svm':\n",
+    "        from sklearn.svm import SVC\n",
+    "        svm = SVC()\n",
+    "        svm.fit(X_train, y_train)\n",
+    "        y_pred = svm.predict(X_test)\n",
+    "        model_var = 'svm'\n",
+    "        return y_pred\n",
+    "    \n",
+    "    if model == 'naive_bayes':\n",
+    "        from sklearn.naive_bayes import GaussianNB\n",
+    "        nb = GaussianNB()\n",
+    "        nb.fit(X_train, y_train)\n",
+    "        y_pred = nb.predict(X_test)\n",
+    "        model_var = 'naive_bayes'\n",
+    "        return y_pred\n",
+    "    \n",
+    "    if model == 'decision_tree':\n",
+    "        from sklearn.tree import DecisionTreeClassifier\n",
+    "        dt = DecisionTreeClassifier()\n",
+    "        dt.fit(X_train, y_train)\n",
+    "        y_pred = dt.predict(X_test)\n",
+    "        model_var = 'decision_tree'\n",
+    "        return y_pred\n",
+    "    \n",
+    "    if model == 'xgboost':\n",
+    "        from xgboost import XGBClassifier\n",
+    "        xgb = XGBClassifier()\n",
+    "        xgb.fit(X_train, y_train)\n",
+    "        y_pred = xgb.predict(X_test)\n",
+    "        model_var = 'xgboost'\n",
+    "        return y_pred\n",
+    "    \n",
+    "    else:\n",
+    "        print('Please choose a model from the following: random_forest, logistic_regression, knn, svm, naive_bayes, decision_tree, xgboost')\n",
+    "        return None"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Evaluation Function**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "#define a function that prints the strings below\n",
+    "\n",
+    "from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score\n",
+    "\n",
+    "def evaluate_models(model='all'):\n",
+    "    print('Have the duplicates been removed?', drop_duplicates_var)\n",
+    "    print('What is the missing values threshold %?', missing_values_threshold_var)\n",
+    "    print('What is the variance threshold?', variance_threshold_var)\n",
+    "    print('What is the correlation threshold?', correlation_threshold_var)\n",
+    "    print('What is the outlier removal threshold?', outlier_var)\n",
+    "    print('What is the scaling method?', scale_model_var)\n",
+    "    print('What is the imputation method?', imputation_var) \n",
+    "    print('What is the imbalance treatment?', imbalance_var)\n",
+    "\n",
+    "    all_models = ['random_forest', 'logistic_regression', 'knn', 'svm', 'naive_bayes', 'decision_tree', 'xgboost']\n",
+    "    evaluation_score_append = []\n",
+    "    evaluation_count_append = []\n",
+    "    \n",
+    "    for selected_model in all_models:\n",
+    "      \n",
+    "        if model == 'all' or model == selected_model:\n",
+    "\n",
+    "            evaluation_score = []\n",
+    "            evaluation_count = []\n",
+    "\n",
+    "            y_pred = globals()['y_pred_' + selected_model]  # Get the prediction variable dynamically\n",
+    "\n",
+    "            def namestr(obj, namespace):\n",
+    "                return [name for name in namespace if namespace[name] is obj]\n",
+    "\n",
+    "            model_name = namestr(y_pred, globals())[0]\n",
+    "            model_name = model_name.replace('y_pred_', '')            \n",
+    "\n",
+    "            cm = confusion_matrix(y_test, y_pred)\n",
+    "\n",
+    "            # create a dataframe with the results for each model\n",
+    "\n",
+    "            evaluation_score.append(model_name)\n",
+    "            evaluation_score.append(round(accuracy_score(y_test, y_pred), 2))\n",
+    "            evaluation_score.append(round(precision_score(y_test, y_pred, zero_division=0), 2))\n",
+    "            evaluation_score.append(round(recall_score(y_test, y_pred), 2))\n",
+    "            evaluation_score.append(round(f1_score(y_test, y_pred), 2))\n",
+    "            evaluation_score_append.append(evaluation_score)\n",
+    "\n",
+    "\n",
+    "            # create a dataframe with the true positives, true negatives, false positives and false negatives for each model\n",
+    "\n",
+    "            evaluation_count.append(model_name)\n",
+    "            evaluation_count.append(cm[0][0])\n",
+    "            evaluation_count.append(cm[0][1])\n",
+    "            evaluation_count.append(cm[1][0])\n",
+    "            evaluation_count.append(cm[1][1])\n",
+    "            evaluation_count_append.append(evaluation_count)\n",
+    "\n",
+    "        \n",
+    "    evaluation_score_append = pd.DataFrame(evaluation_score_append, \n",
+    "                                        columns=['Model', 'Accuracy', 'Precision', 'Recall', 'F1-score'])\n",
+    "    \n",
+    "  \n",
+    "\n",
+    "    evaluation_count_append = pd.DataFrame(evaluation_count_append,\n",
+    "                                        columns=['Model', 'True Negatives', 'False Positives', 'False Negatives', 'True Positives'])\n",
+    "    \n",
+    "   \n",
+    "    return evaluation_score_append, evaluation_count_append"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "### **Input Variables**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/mercury+json": {
+       "choices": [
+        "yes",
+        "no"
+       ],
+       "code_uid": "Select.0.40.16.25-rand87a54245",
+       "disabled": false,
+       "hidden": false,
+       "label": "Drop Duplicates",
+       "model_id": "5464c30e15a543b5901a81c32250831b",
+       "url_key": "",
+       "value": "yes",
+       "widget": "Select"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5464c30e15a543b5901a81c32250831b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Select"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "code_uid": "Text.0.40.15.28-rand6e89b88d",
+       "disabled": false,
+       "hidden": false,
+       "label": "Missing Value Threeshold",
+       "model_id": "1740f68e25d04e1cb3b7768df15d5873",
+       "rows": 1,
+       "url_key": "",
+       "value": "80",
+       "widget": "Text"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1740f68e25d04e1cb3b7768df15d5873",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Text"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "code_uid": "Text.0.40.15.31-rand74006b42",
+       "disabled": false,
+       "hidden": false,
+       "label": "Variance Threshold",
+       "model_id": "559d04e880944fd29ca3478a7a4b20ff",
+       "rows": 1,
+       "url_key": "",
+       "value": "0",
+       "widget": "Text"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "559d04e880944fd29ca3478a7a4b20ff",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Text"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "code_uid": "Text.0.40.15.34-rand05bb51ec",
+       "disabled": false,
+       "hidden": false,
+       "label": "Correlation Threshold",
+       "model_id": "896e1c9e96b04caeb3df77ad7c0c5ff2",
+       "rows": 1,
+       "url_key": "",
+       "value": "1",
+       "widget": "Text"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "896e1c9e96b04caeb3df77ad7c0c5ff2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Text"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "choices": [
+        "none",
+        3,
+        4,
+        5
+       ],
+       "code_uid": "Select.0.40.16.38-randc39f4f69",
+       "disabled": false,
+       "hidden": false,
+       "label": "Outlier Removal Threshold",
+       "model_id": "6d9dea0253ca4bfbb452c5ee16bfb821",
+       "url_key": "",
+       "value": "none",
+       "widget": "Select"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6d9dea0253ca4bfbb452c5ee16bfb821",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Select"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "choices": [
+        "none",
+        "normal",
+        "standard",
+        "minmax",
+        "robust"
+       ],
+       "code_uid": "Select.0.40.16.46-rand80d22513",
+       "disabled": false,
+       "hidden": false,
+       "label": "Scaling Variables",
+       "model_id": "7fbd91c876aa4b2591239e1cc5e8c0d1",
+       "url_key": "",
+       "value": "none",
+       "widget": "Select"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7fbd91c876aa4b2591239e1cc5e8c0d1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Select"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "choices": [
+        "mean",
+        "median",
+        "knn",
+        "most_frequent"
+       ],
+       "code_uid": "Select.0.40.16.50-rand00afecfa",
+       "disabled": false,
+       "hidden": false,
+       "label": "Imputation Methods",
+       "model_id": "15298e99f1ad40469be71f5787356f53",
+       "url_key": "",
+       "value": "mean",
+       "widget": "Select"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "15298e99f1ad40469be71f5787356f53",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Select"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "choices": [
+        "none",
+        "smote",
+        "undersampling",
+        "rose"
+       ],
+       "code_uid": "Select.0.40.16.55-rand9393c38d",
+       "disabled": false,
+       "hidden": false,
+       "label": "Imbalance Treatment",
+       "model_id": "9c2d3e6384a1481ea6dc6f7060404ef8",
+       "url_key": "",
+       "value": "none",
+       "widget": "Select"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9c2d3e6384a1481ea6dc6f7060404ef8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Select"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/mercury+json": {
+       "choices": [
+        "random_forest",
+        "logistic_regression",
+        "knn",
+        "svm",
+        "naive_bayes",
+        "decision_tree",
+        "xgboost"
+       ],
+       "code_uid": "Select.0.40.16.60-rand44169a53",
+       "disabled": false,
+       "hidden": false,
+       "label": "Model Selection",
+       "model_id": "081168c57bb84be68f9a62734a7d5520",
+       "url_key": "",
+       "value": "random_forest",
+       "widget": "Select"
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "081168c57bb84be68f9a62734a7d5520",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "mercury.Select"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "\n",
+    "evaluation_score_df = pd.DataFrame(columns=['Model', 'Accuracy', 'Precision', 'Recall', 'F1-score', 'model_variables'])\n",
+    "evaluation_count_df = pd.DataFrame(columns=['Model', 'True Negatives', 'False Positives', 'False Negatives', 'True Positives', 'model_variables'])\n",
+    "\n",
+    "#############################################################################################################\n",
+    "# reset the dataframe containing all results, evaluation_score_df and evaluation_count_df\n",
+    "\n",
+    "reset_results = 'no' # 'yes' or 'no'\n",
+    "\n",
+    "#############################################################################################################\n",
+    "\n",
+    "if reset_results == 'yes':\n",
+    "    evaluation_score_df = pd.DataFrame(columns=['Model', 'Accuracy', 'Precision', 'Recall', 'F1-score', 'model_variables'])\n",
+    "    evaluation_count_df = pd.DataFrame(columns=['Model', 'True Negatives', 'False Positives', 'False Negatives', 'True Positives', 'model_variables'])\n",
+    "    \n",
+    "\n",
+    "#############################################################################################################\n",
+    "\n",
+    "# input train and test sets\n",
+    "input_train_set = X_train\n",
+    "input_test_set = X_test\n",
+    "\n",
+    "\n",
+    "\n",
+    "# input feature removal variables\n",
+    "input_drop_duplicates = mr.Select(label=\"Drop Duplicates\", value=\"yes\", choices=[\"yes\", \"no\"])           # 'yes' or 'no'\n",
+    "input_drop_duplicates = str(input_drop_duplicates.value)\n",
+    "\n",
+    "input_missing_values_threshold = mr.Text(label=\"Missing Value Threeshold\", value='80')     # 0-100 (removes columns with more missing values than the threshold)\n",
+    "input_missing_values_threshold = int(input_missing_values_threshold.value)\n",
+    "\n",
+    "input_variance_threshold = mr.Text(label=\"Variance Threshold\", value='0')          # \n",
+    "input_variance_threshold = float(input_variance_threshold.value)\n",
+    "\n",
+    "input_correlation_threshold = mr.Text(label=\"Correlation Threshold\", value='1')       # \n",
+    "input_correlation_threshold = float(input_correlation_threshold.value)\n",
+    "\n",
+    "# input outlier removal variables\n",
+    "input_outlier_removal_threshold = mr.Select(label=\"Outlier Removal Threshold\", value=\"none\", choices=['none', 3, 4, 5])      # 'none' or zscore from 0 to 100\n",
+    "\n",
+    "if input_outlier_removal_threshold.value != 'none':\n",
+    "    input_outlier_removal_threshold = int(input_outlier_removal_threshold.value)\n",
+    "elif input_outlier_removal_threshold.value == 'none':\n",
+    "    input_outlier_removal_threshold = str(input_outlier_removal_threshold.value)\n",
+    "\n",
+    "# input scaling variables\n",
+    "input_scale_model = mr.Select(label=\"Scaling Variables\", value=\"none\", choices=['none', 'normal', 'standard', 'minmax', 'robust'])              # 'none', 'normal', 'standard', 'minmax', 'robust'\n",
+    "input_scale_model = str(input_scale_model.value)\n",
+    "\n",
+    "# input imputation variables\n",
+    "input_imputation_method = mr.Select(label=\"Imputation Methods\", value=\"mean\", choices=['mean', 'median', 'knn', 'most_frequent'])          # 'mean', 'median', 'knn', 'most_frequent'\n",
+    "input_n_neighbors = 5                  # only for knn imputation\n",
+    "input_imputation_method = str(input_imputation_method.value)\n",
+    "\n",
+    "# input imbalance treatment variables\n",
+    "input_imbalance_treatment = mr.Select(label=\"Imbalance Treatment\", value=\"none\", choices=['none', 'smote', 'undersampling', 'rose'])       # 'none', 'smote', 'undersampling', 'rose'\n",
+    "input_imbalance_treatment = str(input_imbalance_treatment.value)\n",
+    "\n",
+    "\n",
+    "# input model\n",
+    "input_model = mr.Select(label=\"Model Selection\", value=\"random_forest\", choices=['random_forest', 'logistic_regression', 'knn', 'svm', 'naive_bayes','decision_tree','xgboost'])                      # 'all', 'random_forest', 'logistic_regression', 'knn', \n",
+    "                                        # 'svm', 'naive_bayes', # 'decision_tree', 'xgboost'\n",
+    "input_model = str(input_model.value)\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "### **Transform Data**"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Remove Features**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Shape of the dataframe is:  (1175, 590)\n",
+      "the number of columns to be dropped due to duplications is:  104\n",
+      "the number of columns to be dropped due to missing values is:  8\n",
+      "the number of columns to be dropped due to low variance is:  12\n",
+      "the number of columns to be dropped due to high correlation is:  21\n",
+      "Total number of columns to be dropped is:  145\n",
+      "New shape of the dataframe is:  (1175, 445)\n",
+      "<class 'list'>\n",
+      "No outliers were removed\n",
+      "The dataframe has not been scaled\n",
+      "The dataframe has not been scaled\n",
+      "Number of missing values before imputation:  19977\n",
+      "Number of missing values after imputation:  0\n",
+      "Number of missing values before imputation:  6954\n",
+      "Number of missing values after imputation:  0\n",
+      "Shape of the training set after no resampling:  (1175, 445)\n",
+      "Value counts of the target variable after no resampling: \n",
+      " pass/fail\n",
+      "0            1097\n",
+      "1              78\n",
+      "dtype: int64\n"
+     ]
+    }
+   ],
+   "source": [
+    "# remove features using the function list_columns_to_drop\n",
+    "\n",
+    "dropped = columns_to_drop(input_train_set, \n",
+    "                               input_drop_duplicates, input_missing_values_threshold, \n",
+    "                               input_variance_threshold, input_correlation_threshold)\n",
+    "\n",
+    "# drop the columns from the training and testing sets and save the new sets as new variables\n",
+    "\n",
+    "X_train2 = input_train_set.drop(dropped, axis=1)\n",
+    "X_test2 = input_test_set.drop(dropped, axis=1)\n",
+    "\n",
+    "X_train_dropped_outliers = outlier_removal(X_train2, input_outlier_removal_threshold)\n",
+    "\n",
+    "\n",
+    "X_train_scaled = scale_dataframe(input_scale_model, X_train_dropped_outliers, X_train_dropped_outliers)\n",
+    "X_test_scaled = scale_dataframe(input_scale_model, X_train_dropped_outliers, X_test2)\n",
+    "\n",
+    "# impute the missing values in the training and testing sets using the function impute_missing_values\n",
+    "\n",
+    "X_train_imputed = impute_missing_values(input_imputation_method,X_train_scaled, X_train_scaled, input_n_neighbors)\n",
+    "X_test_imputed = impute_missing_values(input_imputation_method,X_train_scaled, X_test_scaled, input_n_neighbors)\n",
+    "\n",
+    "# treat imbalance in the training set using the function oversample\n",
+    "\n",
+    "X_train_res, y_train_res = imbalance_treatment(input_imbalance_treatment, X_train_imputed, y_train)\n",
+    "\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "### **Model Training**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# disable warnings\n",
+    "\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "# train the model using the function train_model and save the predictions as new variables\n",
+    "\n",
+    "y_pred_random_forest = train_model('random_forest', X_train_res, y_train_res, X_test_imputed, y_test)\n",
+    "y_pred_logistic_regression = train_model('logistic_regression', X_train_res, y_train_res, X_test_imputed, y_test)\n",
+    "y_pred_knn = train_model('knn', X_train_res, y_train_res, X_test_imputed, y_test)\n",
+    "y_pred_svm = train_model('svm', X_train_res, y_train_res, X_test_imputed, y_test)\n",
+    "y_pred_naive_bayes = train_model('naive_bayes', X_train_res, y_train_res, X_test_imputed, y_test)\n",
+    "y_pred_decision_tree = train_model('decision_tree', X_train_res, y_train_res, X_test_imputed, y_test)\n",
+    "y_pred_xgboost = train_model('xgboost', X_train_res, y_train_res, X_test_imputed, y_test)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "skip"
+    }
+   },
+   "source": [
+    "#### **Evaluate and Save**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "slide"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Have the duplicates been removed? yes\n",
+      "What is the missing values threshold %? 80\n",
+      "What is the variance threshold? 0.0\n",
+      "What is the correlation threshold? 1.0\n",
+      "What is the outlier removal threshold? none\n",
+      "What is the scaling method? none\n",
+      "What is the imputation method? mean\n",
+      "What is the imbalance treatment? none\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Model</th>\n",
+       "      <th>Accuracy</th>\n",
+       "      <th>Precision</th>\n",
+       "      <th>Recall</th>\n",
+       "      <th>F1-score</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>random_forest</td>\n",
+       "      <td>0.93</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           Model  Accuracy  Precision  Recall  F1-score\n",
+       "0  random_forest      0.93        0.0     0.0       0.0"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Model</th>\n",
+       "      <th>True Negatives</th>\n",
+       "      <th>False Positives</th>\n",
+       "      <th>False Negatives</th>\n",
+       "      <th>True Positives</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>random_forest</td>\n",
+       "      <td>366</td>\n",
+       "      <td>0</td>\n",
+       "      <td>26</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           Model  True Negatives  False Positives  False Negatives  \\\n",
+       "0  random_forest             366                0               26   \n",
+       "\n",
+       "   True Positives  \n",
+       "0               0  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUcAAAFMCAYAAABYjn6oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAr6ElEQVR4nO3deVxU5f4H8M8IDKDAiBAgCogbIpsIV8R9BZdUyszMfffmvucvl1tm7mFqmOWClLfUzDLzZu6KLCWCK6KmLMIgyirG5nB+f5ijI0djZDkz+Hm/XrzunWfOefjOvPDTM+d55jkyQRAEEBGRhlpSF0BEpIsYjkREIhiOREQiGI5ERCIYjkREIhiOREQiGI5ERCIMpS6gIkpLS5GWlgZzc3PIZDKpyyEiPSAIAu7fvw97e3vUqvX88aFeh2NaWhocHBykLoOI9FBKSgoaNmz43Of1OhzNzc0BAPKWIyEzkEtcDema5BNrpC6BdND9vDw0dXZQ58fz6HU4Pv4oLTOQMxypDAsLC6lLIB32T5fiOCFDRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguGow8YP6oDfdy3AndOrcef0apzYMRsB7VtqHOPibIs96yYi/dRqZISvwckds+FgZ6lxjJ+nM/63eSruRayF8tQqHPpqOkyMjarzpZCENm8KQYtmzqhrZoJ2bXwQHn5a6pL0gqHUBdDzpd7JwaINP+HP5HsAgGH9/LAneALavrMC8TfT4dzQGke3zcKOHyPw8aZfkJtfgBbOdigsKlH34efpjJ82voc123/DrJV7UPxQBc/mDVBaKkj1sqga7dm9C3Nnz8BnG0Lg3649tny1GUGv98a5C1fg6OgodXk6TSYIgt7+K8nLy4NCoYCxx3jIDORSl1MtUk+sxP+t+xE7foxE2IrRKClRYeyisOcef3LHbByNvoqPQn6pxip1Q/YfG6UuQXId2/nB27s11n++Sd3WysMV/foHYemy5RJWJp28vDzYWimQm5sLCwuL5x7Hj9V6olYtGQYF+qCOqRzRF25BJpOhVwc3XE/OwP7PJyPp6HKcCpuDfl081ee8ZmmGNp7OuJuVj+Ohs5B45BP8tmU62rVqLOEroepSXFyM2HMx6N4zQKO9e48AREVGSFSV/mA46ji3pva4e2YtcqPXYf0HgzF49le4ejMdNvXMYF7HBHNG98ThiCvo9++N2H/8PL5bOw4dfJoCAJwbWgMAPpjYB9t+iMCAySGIi0/Bwc1T0cTxNSlfFlWDe/fuQaVSwcbGVqPd1tYWd+6kS1SV/uA1Rx13LfEO/N5ZjrrmtRHUvRW++mg4AsZ9htz7BQCAAycuYsPO4wCAC9dS4efVGOPf6oDwmBuoVUsGANi6Nxxf748CAJxPuI0ubVwwcoA/Fm/YL82Lomolk8k0HguCUKaNyuLIUceVPFThZso9nLuSjMUb9uPitVRMHtIF97LzUVKiQvxNpcbxCTfT1bPVyrt5AID4m5qjhIRb6WVmtKnmsba2hoGBQZlRYkZGRpnRJJXFcNQzMshgLDdEyUMVYq4kobmT5h95MycbJCuzAQBJaZlIy8hB80Y2Gsc0dbJBsjKr2momacjlcni39sGxI4c12o8dPYy2/u0kqkp/8GO1DvtwSj/8duYKUtKzYV7HBIMCfdDJtxn6Tw4BAATvOIKvV45B+LkbOHn2GgLatUSfTu4IHP+Zuo/gHUewcFJfXLyWivMJtzGsnx9cGtni3blbpXpZVI2mzZiFsaOGo7WPL/za+mPrli+RkpyMcRMmSV2azmM46jAbK3Ns/XgE7KwtkJtfiEvXU9F/cgiORV8FAOw/fgFTl32HuWMCsHbeW7iWlIEhc7cgIu6muo+N/z0BE2MjrJo9EJaK2rh4LRWv/3sjbt2+J9XLomo06O3ByMrMxCfLPkK6Ugk3N3f8+PNBODk5SV2azpN8nWNISAhWr14NpVIJNzc3rFu3Dh07dizXua/iOkcqP65zJDF6sc5x165dmDFjBj744APExsaiY8eO6N27N5KTk6Usi4hI2nD89NNPMXbsWIwbNw6urq5Yt24dHBwcsGnTpn8+mYioCkkWjsXFxYiJiUFAgObq/YCAAEREiK/eLyoqQl5ensYPEVFVkCwcH6/et7Utu3o/PV189f7y5cuhUCjUPw4ODtVRKhG9giRf56jN6v0FCxYgNzdX/ZOSklIdJRLRK0iycHy8ev/ZUWJGRkaZ0eRjxsbGsLCw0PipCeop6iDp6HI41q8naR1uTe1x49elqG3CmX9dkZmZCUd7GyQlJkpax6WLF9GkUUM8ePBA0jqqk2ThKJfL4ePjg8OHNVfvHz58GO3avVqr9+eOCcDBUxfV31pZM3cgzuych5zoYER99365+pAbGeLT+YOQcmwF7kWsxZ51E9HApq7GMXXNTbF16Qikn1qN9FOrsXXpCCjMTNXPX76RhrOXkjB1WNdKe21UMatXLkefvv3g1KgRACA5ORkDg/rBSlEHDe2sMWvGNBQXF7+wj6KiIsycPhUN7axhpaiDt97oj9u3b2sck52djTEjh8PWSgFbKwXGjByOnJwc9fPuHh7w/VcbbPgsuLJfos6S9GP1rFmzsGXLFmzbtg3x8fGYOXMmkpOTMWnSq7N638TYCCOD/BG6L1LdJpPJEPZTFL7/7Vy5+1k9dyD6d/XEiAXb0X10MMxM5di7fpJ68wkACF0+Cp4uDTFgSggGTAmBp0tDbP14hEY/YfujMGFQR43zSBoFBQXYsX0rRo0ZBwBQqVR4s39fPHjwAEdPhCNs53f4cd9ezJ87+4X9zJ01A/t/2oewnd/h6Ilw5OfnY+CA16FSqdTHjBr+Li6cj8NPB37FTwd+xYXzcRg7arhGPyNGjsaXmzdpnFeTSfoNmcGDByMzMxMfffQRlEol3N3dcfDgq7V6P7B9SzxUqRB94Za6bfaq7wEA1pZ94N6swT/2YWFmglFB/hi7MAzHoxMAAGMWhuH6/5aim18LHImMh4uzLQLbu6HT8NX441ISAGDy0v/iZNgcNHOywfWkDADA4Yh41FPUQUefZjj5x7XKfrmkhUO//g+GhoZo6+8PADhy+DfEx1/B9YMpsLe3BwCsWLUWE8aOwodLl4leZsrNzUXo9q3YGvo1unXvAQDYtuMbNHN2wLGjR9AzIBBX4+Px26FfcTI8Cm38/AAAn3/xFbp09Me1hAQ0d3EBAPQMCERWZiZOnzqJLl27VcdbICnJJ2Tee+89JCYmoqioCDExMejUqZPUJVWrDq2b4tyVii1693Z1hNzIEEci49Vtyru5uPxnGtp6OQN4dLuEnPt/qYMRAH6/mIic+3+hrdeTzW9LHqpw8Voq2ns3qVBNVHHhp0+htY+v+nF0VCTc3NzVwQg8CqyioiLEnosR7SP2XAxKSkrQ46kNb+3t7eHm5q7e8DY6KhIKhUIdjADg17YtFAqFxqa4crkcHp5eOPOK3ING8nB81TnZ14Pybm6F+rCzskBRcQly/t7j8bGMzPuwtXo0mrC1ssDdrPwy597NyoetteaIIy0jB072VhWqiSouKSkR9es/CcI76emweWay0tLSEnK5/LnL39LT0yGXy2FpqblFnY2tLe78fc6dO+l4zcamzLmv2diU2e7MvkEDySeHqgvDUWImxnIUFj2skr5lMhme/uK82NfoZTIAz7QXFJWgtgnvTii1woICmJiYaLSJLXN7mc1rnz3nef3imXZTE1P8VfCXVr9LXzEcJZaZkw9Li9oV6iM9Mw/GciPUNTfVaH+tnhkyMh99i+hOZh5srMzLnGttaYY7mfc12iwVtXEvu+wok6qXlZU1snOy1Y9t7ezUo73HsrOzUVJS8tzlb3Z2diguLkZ2drZG+92MDPUo1NbWDhl37pQ5997du7B9ZlPc7OwsWFu/GrfYYDhK7PzV22jR2K5CfcTGJ6O45CG6t22hbrOztoBbE3tEnX800RN94RbqmteGr9uTya5/uTuhrnltRJ2/qdGfWxN7xCVoLvWg6ufl7Y2rV66oH/u19cfly5egVD7Z/f3I4d9gbGwM79Y+on14t/aBkZERjj614a1SqcTly5fUG976tfVHbm4u/vj9d/Uxv0dHIzc3t8ymuJcvX0KrVt6V8vp0HcNRYocj49GycX2NUV9jB2t4Nm8AW2sLmBobwbN5A3g2bwAjQwMAgP1rCsT9sFAddHn5hQj9MRIrZr2JLm2aw8ulIbZ9PBKXbqSp935MuHUHh85cxueLh6CNRyO08WiEzxe9i19OXlTPVAOAY/16sLdR4Pjf55F0evYMxJUrl9Wjvh49A+Dq2hJjRw1HXGwsjh87igXz52D02PHqmerU1FR4ubdQB51CocCo0WPx/rzZOH7sKOJiYzFm5DC4u3uoZ69buLoiILAXJk8aj+ioKERHRWHypPHo0/d19Uw1ACQlJiItNRVd/z6vpuNmtxK7fCMN5+KTMTCgNbbuPQMA2LR4KDr5NlMfE71rAQDApc9iJCuzYGhoABdnO5g+9U2WeWv2QqUqxTcrx8LU2AjHf0/AhOlfo7T0yfXE0f+3A2vnvYWfQyYDAH45eREzV+zRqOft3r44EnlVfasFko67hwda+/hi757dGDdhIgwMDPDD/l8wY+p76Na5PUxNTfH2O+9ixao16nMelpTgWkICCp66LrhqbTAMDA0xbMjbKCgoQNdu3fHl1lAYGBioj9kethOzZ0xDvz6PZrX7vt4fwes198Pcvetb9OgZ8MostZN8s9uKqCmb3QZ2aInlM9+Az1ufiE6aVBe5kSEu/bQYIxeEIvKZj9r6qCZsdvvr/w5iwfw5iIm7hFq1pPugV1RUBHfXZtjx9bdo1769ZHVUhvJudsuRow44FH4FTR1s0MBGgdt3ciSrw7F+PazceqhGBGNN0at3H9y4fh2pqamS7kKVnJSE+e9/oPfBqA2OHKnGqgkjR6p8enGbBCIiXcVwJCISwXAkIhLBcCQiEsFwJCISwXAkIhLBcCQiEsFwJCISwXAkIhLBcCQiEsFwJCISwXAkIhLBcCQiEsFwJCISwXAkIhLBcCQiEsFwJCISwXAkIhLBcCQiEsFwJCISUa67D+7fv7/cHfbv3/+liyEi0hXlCsegoKBydSaTyaBSqSpSDxGRTihXOJaWllZ1HUREOqVC1xwLCwsrqw4iIp2idTiqVCosXboUDRo0gJmZGW7evAkAWLRoEbZu3VrpBRIRSUHrcFy2bBlCQ0OxatUqyOVydbuHhwe2bNlSqcUREUlF63AMCwvDl19+iaFDh8LAwEDd7unpiatXr1ZqcUREUtE6HFNTU9G0adMy7aWlpSgpKamUooiIpKZ1OLq5ueH06dNl2vfs2QNvb+9KKYqISGrlWsrztCVLlmD48OFITU1FaWkpfvjhByQkJCAsLAwHDhyoihqJiKqd1iPHfv36YdeuXTh48CBkMhkWL16M+Ph4/Pzzz+jZs2dV1EhEVO20HjkCQGBgIAIDAyu7FiIinfFS4QgAZ8+eRXx8PGQyGVxdXeHj41OZdRERSUrrcLx9+zaGDBmCM2fOoG7dugCAnJwctGvXDt9++y0cHBwqu0Yiomqn9TXHMWPGoKSkBPHx8cjKykJWVhbi4+MhCALGjh1bFTUSEVU7rUeOp0+fRkREBFxcXNRtLi4u2LBhA9q3b1+pxRERSUXrkaOjo6PoYu+HDx+iQYMGlVIUEZHUtA7HVatWYerUqTh79iwEQQDwaHJm+vTpWLNmTaUXSEQkhXJ9rLa0tIRMJlM/fvDgAfz8/GBo+Oj0hw8fwtDQEGPGjCn3xrhERLqsXOG4bt26Ki6DiEi3lCscR44cWdV1EBHplJdeBA4ABQUFZSZnLCwsKlQQEZEu0HpC5sGDB5gyZQpsbGxgZmYGS0tLjR8ioppA63CcN28ejh07hpCQEBgbG2PLli348MMPYW9vj7CwsKqokYio2mn9sfrnn39GWFgYunTpgjFjxqBjx45o2rQpnJycsHPnTgwdOrQq6iQiqlZajxyzsrLg7OwM4NH1xaysLABAhw4dcOrUqcqtjohIIlqHY+PGjZGYmAgAaNmyJXbv3g3g0Yjy8UYURET6TutwHD16NM6fPw8AWLBggfra48yZMzF37txKL5CISApaX3OcOXOm+v937doVV69exdmzZ9GkSRN4eXlVanFERFKp0DpH4NFGFI6OjpVRCxGRzihXOK5fv77cHU6bNu2liyEi0hXlCsfg4OBydSaTyRiORFQjlCscb926VdV1EBHpFK1nq4mIXgUMRyIiEQxHIiIRDEciIhEMRyIiES8VjqdPn8awYcPg7++P1NRUAMDXX3+N8PDwSi2OiEgqWofj3r17ERgYCFNTU8TGxqKoqAgAcP/+fXzyySeVXiARkRS0DsePP/4YX3zxBb766isYGRmp29u1a4dz585VanFERFLROhwTEhLQqVOnMu0WFhbIycmpjJqIiCSndTjWr18fN27cKNMeHh6Oxo0bV0pRRERS0zocJ06ciOnTpyM6OhoymQxpaWnYuXMn5syZg/fee68qaiQiqnZab1k2b9485ObmomvXrigsLESnTp1gbGyMOXPmYMqUKVVRIxFRtZMJgiC8zIl//fUXrly5gtLSUrRs2RJmZmaVXds/ysvLg0KhgLHHeMgM5NX++0m3Zf+xUeoSSAfl5eXB1kqB3NxcWFhYPPe4l97stnbt2vD19X3Z04mIdJrW4di1a1fIZLLnPn/s2LEKFUREpAu0DsdWrVppPC4pKUFcXBwuXbqEkSNHVlZdRESS0jocn7cr+H/+8x/k5+dXuCAiIl1QaRtPDBs2DNu2baus7oiIJFVp4RgZGQkTE5PK6o6ISFJaf6x+8803NR4LggClUomzZ89i0aJFlVYYEZGUtA5HhUKh8bhWrVpwcXHBRx99hICAgEorjIhISlqFo0qlwqhRo+Dh4YF69epVVU1ERJLT6pqjgYEBAgMDkZubW1X1EBHpBK0nZDw8PHDz5s2qqIWISGdoHY7Lli3DnDlzcODAASiVSuTl5Wn8EBHVBFpPyPTq1QsA0L9/f42vEQqCAJlMBpVKVXnVERFJROtwPH78eFXUQUSkU7QOR2dnZzg4OJTZfEIQBKSkpFRaYUREUtL6mqOzszPu3r1bpj0rKwvOzs6VUhQRkdS0DsfH1xaflZ+fz68PElGNUe6P1bNmzQIAyGQyLFq0CLVr11Y/p1KpEB0dXWY7MyIifVXucIyNjQXwaOR48eJFyOVPbksgl8vh5eWFOXPmVH6FREQSKHc4Pp6lHj16ND777LMX3nuBiEjfaT1bvX379qqog4hIp1Tafo5ERDUJw5GISATDkYhIBMORiEgEw5GISATDkYhIBMORiEgEw5GISATDkYhIBMORiEgEw5GISATDkYhIBMORiEgEw5GISATDkYhIBMORiEgEw5GISATDkYhIBMNRz8wZE4Dwb+YiI3wNko4ux+5Px6OZk02Z41ycbbFn3USkn1qNjPA1OLljNhzsLCWomKS2eVMIWjRzRl0zE7Rr44Pw8NNSl6QXGI56pmPrpvhi1yl0HrEGr/97IwwMDHBg0xTUNnlyN0jnhtY4um0Wrt1KR+D4z9Bm8HIs/+pXFBaVSFg5SWHP7l2YO3sG5r//AaL+iEW7Dh0R9HpvJCcnS12azpMJgiBIXcTLysvLg0KhgLHHeMgM5P98Qg1kbWmGlGMr0GNsMM6c+xMAELZiNEpKVBi7KEzi6qSV/cdGqUuQXMd2fvD2bo31n29St7XycEW//kFYumy5hJVJJy8vD7ZWCuTm5r7wLqocOeo5CzMTAEB27l8AAJlMhl4d3HA9OQP7P5+MpKPLcSpsDvp18ZSyTJJAcXExYs/FoHvPAI327j0CEBUZIVFV+oPhqOdWzh6IM+du4MqfSgCATT0zmNcxwZzRPXE44gr6/Xsj9h8/j+/WjkMHn6YSV0vV6d69e1CpVLCxsdVot7W1xZ076RJVpT+0vm816Y7g99+GRzN7dB8drG6rVevRf+8OnLiIDTuPAwAuXEuFn1djjH+rA8JjbkhSK0lHJpNpPBYEoUwblcWRo576dP4gvN7ZA4Hj1yM1I0fdfi87HyUlKsTfVGocn3AznbPVrxhra2sYGBiUGSVmZGSUGU1SWQxHPRQ8fxAGdPNCr4nrkZSWqfFcyUMVYq4kobmT5h9/MycbJCuzq7NMkphcLod3ax8cO3JYo/3Y0cNo699Ooqr0Bz9W65l1C97G4N6+GDTzS+Q/KIStlTkAIDe/UL1UJ3jHEXy9cgzCz93AybPXENCuJfp0ckfg+M+kLJ0kMG3GLIwdNRytfXzh19YfW7d8iZTkZIybMEnq0nQel/LomYJY8eUp4xd/jW9+jlY/HjGgLeaOCUADm7q4lpSBj7/4BQdOXKyuMnUCl/I8snlTCD5duwrpSiXc3Nyxam0wOnTsJHVZkinvUh5Jw/HUqVNYvXo1YmJioFQqsW/fPgQFBZX7/FcxHKn8GI4kRi/WOT548ABeXl7YuJF/xESkWyS95ti7d2/07t273McXFRWhqKhI/TgvL68qyiIi0q/Z6uXLl0OhUKh/HBwcpC6JiGoovQrHBQsWIDc3V/2TkpIidUlEVEPp1VIeY2NjGBsbS10GEb0C9GrkWFPVU9RB0tHlcKxfT9I63Jra48avSzW2PyNpZWZmwtHeBkmJiZLWceniRTRp1BAPHjyQtI7qxHDUAXPHBODgqYtIVmYBANbMHYgzO+chJzoYUd+9X64+5EaG+HT+IKQcW4F7EWuxZ91ENLCpq3FMXXNTbF06AumnViP91GpsXToCCjNT9fOXb6Th7KUkTB3WtdJeG1XM6pXL0advPzg1agQASE5OxsCgfrBS1EFDO2vMmjENxcXFL+yjqKgIM6dPRUM7a1gp6uCtN/rj9u3bGsdkZ2djzMjhsLVSwNZKgTEjhyMnJ0f9vLuHB3z/1QYbPgvGq0LScMzPz0dcXBzi4uIAALdu3UJcXNwrtRGnibERRgb5I3RfpLpNJpMh7KcofP/buXL3s3ruQPTv6okRC7aj++hgmJnKsXf9JNSq9WSDgdDlo+Dp0hADpoRgwJQQeLo0xNaPR2j0E7Y/ChMGddQ4j6RRUFCAHdu3YtSYcQAAlUqFN/v3xYMHD3D0RDjCdn6HH/ftxfy5s1/Yz9xZM7D/p30I2/kdjp4IR35+PgYOeB0qlUp9zKjh7+LC+Tj8dOBX/HTgV1w4H4exo4Zr9DNi5Gh8uXmTxnk1maThePbsWXh7e8Pb2xsAMGvWLHh7e2Px4sVSllWtAtu3xEOVCtEXbqnbZq/6Hpt3n8Kt25kvOPMJCzMTjAryx/uf7sPx6AScT7iNMQvD4N7UHt38WgB4dNuEwPZueO+jnYi+cAvRF25h8tL/om9nD43bLByOiEc9RR109GlWuS+UtHbo1//B0NAQbf39AQBHDv+G+Pgr2LbjG7Ty9ka37j2wYtVabN/61XOXteXm5iJ0+1asWLUW3br3QCtvb2zb8Q0uXbqIY0ePAACuxsfjt0O/ImTzFrT190dbf398/sVXOPjLAVxLSFD31TMgEFmZmTh96mTVv3gdIGk4dunSBYIglPkJDQ2Vsqxq1aF1U5y7UrGRsrerI+RGhjgSGa9uU97NxeU/09DWyxkA4OfpjJz7f+GPS0nqY36/mIic+3+hrVdjdVvJQxUuXktFe+8mFaqJKi789Cm09vFVP46OioSbmzvs7e3VbT0DAlFUVITYczGifcSei0FJSQl6PLXhrb29Pdzc3NUb3kZHRUKhUKCNn5/6GL+2baFQKDQ2xZXL5fDw9MKZV+QeNLzmKDEn+3pQ3s2tUB92VhYoKi5Bzv0CjfaMzPuwtXr09ShbKwvczcovc+7drHzYWmt+hSotIwdO9lYVqokqLikpEfXrPwnCO+npsLHV3G3J0tIScrkc6enim9emp6dDLpfD0lJzuzobW1vc+fucO3fS8ZpN2Zu0vWZjU2a7M/sGDSSfHKouDEeJmRjLUVj0sEr6lslkePqL82Jfo5fJADzTXlBUgtomRlVSE5VfYUEBTExMNNrENql9mc1rnz3nef3imXZTE1P8VfCXVr9LXzEcJZaZkw9Li9oV6iM9Mw/GciPUNTfVaH+tnhkyMh9di7qTmQebv7c3e5q1pRnuZN7XaLNU1Ma97LKjTKpeVlbWyM55sgenrZ2derT3WHZ2NkpKSmBrK755rZ2dHYqLi5GdrbmX592MDPUo1NbWDhl37pQ5997du7B9ZlPc7OwsWFu/9lKvR98wHCV2/upttGhsV6E+YuOTUVzyEN3btlC32VlbwK2JPaLOP5roib5wC3XNa8PXzUl9zL/cnVDXvDaizt/U6M+tiT3iEjSXelD18/L2xtUrV9SP/dr64/LlS1Aqn+zyfuTwbzA2NoZ3ax/RPrxb+8DIyAhHn9rwVqlU4vLlS+oNb/3a+iM3Nxd//P67+pjfo6ORm5tbZlPcy5cvoVUr70p5fbqO4Sixw5HxaNm4vsaor7GDNTybN4CttQVMjY3g2bwBPJs3gJGhAQDA/jUF4n5YqA66vPxChP4YiRWz3kSXNs3h5dIQ2z4eiUs30nAs+ioAIOHWHRw6cxmfLx6CNh6N0MajET5f9C5+OXkR15My1L/bsX492NsocPzv80g6PXsG4sqVy+pRX4+eAXB1bYmxo4YjLjYWx48dxYL5czB67Hj11lupqanwcm+hDjqFQoFRo8fi/XmzcfzYUcTFxmLMyGFwd/dAt+49AAAtXF0RENgLkyeNR3RUFKKjojB50nj06fs6mru4qOtJSkxEWmoquv59Xk2nV18frIku30jDufhkDAxoja17zwAANi0eik6+T5bSRO9aAABw6bMYycosGBoawMXZDqZPfZNl3pq9UKlK8c3KsTA1NsLx3xMwYfrXKC19cj1x9P/twNp5b+HnkMkAgF9OXsTMFXs06nm7ty+ORF7lLRV0gLuHB1r7+GLvnt0YN2EiDAwM8MP+XzBj6nvo1rk9TE1N8fY772LFqjXqcx6WlOBaQgIKnrouuGptMAwMDTFsyNsoKChA127d8eXWUBgYGKiP2R62E7NnTEO/Po9mtfu+3h/B6zW3Ety961v06BkAJycnvAq4E7gOCOzQEstnvgGftz4RnTSpLnIjQ1z6aTFGLghF5DMftfVRTdjs9tf/HcSC+XMQE3dJfWdJKRQVFcHdtRl2fP0t2rVvL1kdlaG8m91y5KgDDoVfQVMHGzSwUeD2nRzJ6nCsXw8rtx6qEcFYU/Tq3Qc3rl9HamqqpFv0JSclYf77H+h9MGqDI0eqsWrCyJEqn17cJoGISFcxHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRDAciYhEMByJiEQwHImIRBhKXUBFCILw6H9VxRJXQrooLy9P6hJIB93/++/icX48j0z4pyN02O3bt+Hg4CB1GUSkh1JSUtCwYcPnPq/X4VhaWoq0tDSYm5tDJpNJXY7k8vLy4ODggJSUFFhYWEhdDukI/l1oEgQB9+/fh729PWrVev6VRb3+WF2rVq0XJv+rysLCgv8IqAz+XTyhUCj+8RhOyBARiWA4EhGJYDjWIMbGxliyZAmMjY2lLoV0CP8uXo5eT8gQEVUVjhyJiEQwHImIRDAciYhEMByJiEQwHGuIkJAQODs7w8TEBD4+Pjh9+rTUJZHETp06hX79+sHe3h4ymQw//vij1CXpFYZjDbBr1y7MmDEDH3zwAWJjY9GxY0f07t0bycnJUpdGEnrw4AG8vLywceNGqUvRS1zKUwP4+fmhdevW2LRpk7rN1dUVQUFBWL58uYSVka6QyWTYt28fgoKCpC5Fb3DkqOeKi4sRExODgIAAjfaAgABERERIVBWR/mM46rl79+5BpVLB1tZWo93W1hbp6ekSVUWk/xiONcSzW7YJgsBt3IgqgOGo56ytrWFgYFBmlJiRkVFmNElE5cdw1HNyuRw+Pj44fPiwRvvhw4fRrl07iaoi0n96vdktPTJr1iwMHz4cvr6+8Pf3x5dffonk5GRMmjRJ6tJIQvn5+bhx44b68a1btxAXF4d69erB0dFRwsr0A5fy1BAhISFYtWoVlEol3N3dERwcjE6dOkldFknoxIkT6Nq1a5n2kSNHIjQ0tPoL0jMMRyIiEbzmSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UiSatSoEdatW6d+LNV2/v/5z3/QqlWr5z5/4sQJyGQy5OTklLvPLl26YMaMGRWqKzQ0FHXr1q1QH/RyGI6kU5RKJXr37l2uY/8p0IgqghtPUIUVFxdDLpdXSl92dnaV0g9RRXHkSBq6dOmCKVOmYMqUKahbty6srKywcOFCPP0V/EaNGuHjjz/GqFGjoFAoMH78eABAREQEOnXqBFNTUzg4OGDatGl48OCB+ryMjAz069cPpqamcHZ2xs6dO8v8/mc/Vt++fRvvvPMO6tWrhzp16sDX1xfR0dEIDQ3Fhx9+iPPnz0Mmk0Emk6k3U8jNzcWECRNgY2MDCwsLdOvWDefPn9f4PStWrICtrS3Mzc0xduxYFBYWavU+ZWZmYsiQIWjYsCFq164NDw8PfPvtt2WOe/jw4Qvfy+LiYsybNw8NGjRAnTp14OfnhxMnTmhVC1UNhiOVsWPHDhgaGiI6Ohrr169HcHAwtmzZonHM6tWr4e7ujpiYGCxatAgXL15EYGAg3nzzTVy4cAG7du1CeHg4pkyZoj5n1KhRSExMxLFjx/D9998jJCQEGRkZz60jPz8fnTt3RlpaGvbv34/z589j3rx5KC0txeDBgzF79my4ublBqVRCqVRi8ODBEAQBffv2RXp6Og4ePIiYmBi0bt0a3bt3R1ZWFgBg9+7dWLJkCZYtW4azZ8+ifv36CAkJ0eo9KiwshI+PDw4cOIBLly5hwoQJGD58OKKjo7V6L0ePHo0zZ87gu+++w4ULFzBo0CD06tUL169f16oeqgIC0VM6d+4suLq6CqWlpeq2+fPnC66ururHTk5OQlBQkMZ5w4cPFyZMmKDRdvr0aaFWrVpCQUGBkJCQIAAQoqKi1M/Hx8cLAITg4GB1GwBh3759giAIwubNmwVzc3MhMzNTtNYlS5YIXl5eGm1Hjx4VLCwshMLCQo32Jk2aCJs3bxYEQRD8/f2FSZMmaTzv5+dXpq+nHT9+XAAgZGdnP/eYPn36CLNnz1Y//qf38saNG4JMJhNSU1M1+unevbuwYMECQRAEYfv27YJCoXju76Sqw2uOVEbbtm017j/j7++PtWvXQqVSwcDAAADg6+urcU5MTAxu3Lih8VFZEASUlpbi1q1buHbtGgwNDTXOa9GixQtnYuPi4uDt7Y169eqVu/aYmBjk5+fDyspKo72goAB//vknACA+Pr7MRsD+/v44fvx4uX+PSqXCihUrsGvXLqSmpqKoqAhFRUWoU6eOxnEvei/PnTsHQRDQvHlzjXOKiorK1E/Vj+FIL+XZECgtLcXEiRMxbdq0Msc6OjoiISEBQNkbgb2Iqamp1nWVlpaifv36otftKnNJzNq1axEcHIx169bBw8MDderUwYwZM1BcXKxVrQYGBoiJiVH/R+cxMzOzSquVXg7DkcqIiooq87hZs2Zl/gE/rXXr1rh8+TKaNm0q+ryrqysePnyIs2fPok2bNgCAhISEF64b9PT0xJYtW5CVlSU6epTL5VCpVGXqSE9Ph6GhIRo1avTcWqKiojBixAiN16iN06dPY8CAARg2bBiAR0F3/fp1uLq6ahz3ovfS29sbKpUKGRkZ6Nixo1a/n6oeJ2SojJSUFMyaNQsJCQn49ttvsWHDBkyfPv2F58yfPx+RkZGYPHky4uLicP36dezfvx9Tp04FALi4uKBXr14YP348oqOjERMTg3Hjxr1wdDhkyBDY2dkhKCgIZ86cwc2bN7F3715ERkYCeDRr/vi+KPfu3UNRURF69OgBf39/BAUF4dChQ0hMTERERAQWLlyIs2fPAgCmT5+Obdu2Ydu2bbh27RqWLFmCy5cva/UeNW3aFIcPH0ZERATi4+MxceJE0fuEv+i9bN68OYYOHYoRI0bghx9+wK1bt/DHH39g5cqVOHjwoFb1UBWQ+qIn6ZbOnTsL7733njBp0iTBwsJCsLS0FN5//32NSQUnJyeNSZTHfv/9d6Fnz56CmZmZUKdOHcHT01NYtmyZ+nmlUin07dtXMDY2FhwdHYWwsLAyfeGpCRlBEITExERh4MCBgoWFhVC7dm3B19dXiI6OFgRBEAoLC4WBAwcKdevWFQAI27dvFwRBEPLy8oSpU6cK9vb2gpGRkeDg4CAMHTpUSE5OVve7bNkywdraWjAzMxNGjhwpzJs3T6sJmczMTGHAgAGCmZmZYGNjIyxcuFAYMWKEMGDAAK3ey+LiYmHx4sVCo0aNBCMjI8HOzk544403hAsXLgiCwAkZKfEeMqShS5cuaNWqlcZX+oheRfxYTUQkguFIRCSCH6uJiERw5EhEJILhSEQkguFIRCSC4UhEJILhSEQkguFIRCSC4UhEJILhSEQk4v8B7YgKY252X9IAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 350x350 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluation_score_output, evaluation_counts_output = evaluate_models(input_model)\n",
+    "\n",
+    "# check if the model has already been evaluated and if not, append the results to the dataframe\n",
+    "\n",
+    "evaluation_score_df = pd.concat([evaluation_score_output, evaluation_score_df], ignore_index=True)    \n",
+    "display(pd.DataFrame(evaluation_score_output))\n",
+    "\n",
+    "evaluation_count_df = pd.concat([evaluation_counts_output, evaluation_count_df], ignore_index=True)    \n",
+    "display(pd.DataFrame(evaluation_counts_output))\n",
+    "\n",
+    "from mlxtend.plotting import plot_confusion_matrix\n",
+    "\n",
+    "# select the model index and filter the row from evaluation_count_df dataframe\n",
+    "model_index = 0\n",
+    "\n",
+    "selected_model = evaluation_count_df[evaluation_count_df.index == model_index]\n",
+    "\n",
+    "# create a np.array with selected_model values\n",
+    "\n",
+    "\n",
+    "conf_matrix = np.array([[selected_model['True Negatives'].values[0], selected_model['False Positives'].values[0]],\n",
+    "                [selected_model['False Negatives'].values[0], selected_model['True Positives'].values[0]]])\n",
+    "\n",
+    "#change the size of the graph\n",
+    "\n",
+    "plt.rcParams['figure.figsize'] = [3.5, 3.5]\n",
+    "\n",
+    "fig, ax = plot_confusion_matrix(\n",
+    "    conf_mat=conf_matrix,\n",
+    "    show_absolute=True,\n",
+    "    show_normed=True\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### **Plot Evaluation**"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}