{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"sencat_models.ipynb","provenance":[{"file_id":"1Wd79wkxXF6CyKVFhYCid1jpIsv0HWRH-","timestamp":1625292490447},{"file_id":"1liafFKE2IZeDAgr_AibZfKykYLvHVzD3","timestamp":1624675505095},{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/cnn_rnn_multiclass.ipynb","timestamp":1621843389117},{"file_id":"153374pBL1OZuAJMWKK_ae0D9_DUq4P0P","timestamp":1590420423516}],"collapsed_sections":["sPm_9b6Y8EPi","lrmGuYXf82E_","jbpLZfOPGGKN","fqK6yZoCHIyE","u4DuyDZMkcE3","B6CXqtIDR4jS","H-RSSqI7BDqE","KMtXKRpRYBx2","BtZBU5XNZaMi","qLkfB_XXPS3O"]},"environment":{"name":"tf2-2-2-gpu.2-2.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf2-2-2-gpu.2-2:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"cells":[{"cell_type":"markdown","metadata":{"id":"3zR8-aK009p9"},"source":["# User Parameters"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dbSiWEihH0zu","executionInfo":{"status":"ok","timestamp":1640597449549,"user_tz":-330,"elapsed":9,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"aa9e2077-d222-4e56-c83a-d2fb96e096f2"},"source":["EMBEDDING_SIZE = \"200\" #@param [50, 150, 200, 250, 300, 350, 400, 450, 500]\n","embedding_type = \"word2vec\" #@param [\"fastText\",\"word2vec\"]\n","model_type = \"GRU\" #@param [\"RNN\",\"GRU\", \"LSTM\", \"BiLSTM\" ] \n","\n","stack_modeles = \"\" #@param [\"\",\"2\",\"3\"]\n","apply_CNN = True #@param {type:\"boolean\"}\n","\n","model_name = model_type \n","if(stack_modeles == \"2\" or stack_modeles == \"3\"):\n"," model_name = \"stacked_\" + model_name + \"_\" + stack_modeles\n","if(apply_CNN):\n"," model_name = \"CNN_\" + model_name \n","\n","EMBEDDING_SIZE=int(EMBEDDING_SIZE)\n","print(model_name)"],"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["CNN_GRU\n"]}]},{"cell_type":"markdown","metadata":{"id":"p7y2rE1vNZWc"},"source":["# Folder Paths"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"IDq--mTz9k24","executionInfo":{"status":"ok","timestamp":1640608074642,"user_tz":-330,"elapsed":10625097,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"3c144aeb-f601-4145-e799-04ca5ddba59c"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","metadata":{"id":"gWPD3dgEqB5d","executionInfo":{"status":"ok","timestamp":1640608074643,"user_tz":-330,"elapsed":8,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Word embedding'\n","\n","\n","word_embedding_path = folder_path + \"/word embedding/word2vec/\"+embedding_type+str(EMBEDDING_SIZE)\n","word_embedding_keydvectors_path = folder_path +\"/word embedding/word2vec/keyed_vectors/keyed.kv\"\n","embedding_matrix_path = folder_path + '/word embedding/word2vec/'+embedding_type+str(EMBEDDING_SIZE)\n","\n","model_save_path = folder_path + \"/word embedding/\"+embedding_type+str(EMBEDDING_SIZE)+\".hdf5\""],"execution_count":3,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0BYvE-D5QAc0"},"source":["# Dependencies"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Cf2gZocJff8x","executionInfo":{"status":"ok","timestamp":1640608135719,"user_tz":-330,"elapsed":61083,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"8492d53e-8ba8-4e57-df97-6a6f8f2be672"},"source":["# import os, sys\n","# nb_path = '/content/colabnotebooks'\n","# # os.symlink('/content/drive/MyDrive/Colab Notebooks', nb_path)\n","# sys.path.insert(0,nb_path)\n","\n","# !pip uninstall --target=$nb_path keras-nightly\n","# !pip install --target=$nb_path tensorflow==1.14.0\n","# !pip install --target=$nb_path q keras==2.3.1\n","# !pip install --target=$nb_path 'h5py<3.0.0'\n","\n","!pip uninstall keras-nightly\n","!pip install tensorflow==1.14.0\n","!pip install q keras==2.3.1\n","!pip install 'h5py<3.0.0'"],"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[33mWARNING: Skipping keras-nightly as it is not installed.\u001b[0m\n","Collecting tensorflow==1.14.0\n"," Downloading tensorflow-1.14.0-cp37-cp37m-manylinux1_x86_64.whl (109.3 MB)\n","\u001b[K |████████████████████████████████| 109.3 MB 46 kB/s \n","\u001b[?25hRequirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.8.1)\n","Collecting tensorflow-estimator<1.15.0rc0,>=1.14.0rc0\n"," Downloading tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488 kB)\n","\u001b[K |████████████████████████████████| 488 kB 45.2 MB/s \n","\u001b[?25hCollecting keras-applications>=1.0.6\n"," Downloading Keras_Applications-1.0.8-py3-none-any.whl (50 kB)\n","\u001b[K |████████████████████████████████| 50 kB 6.0 MB/s \n","\u001b[?25hRequirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.2.0)\n","Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.12.0)\n","Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.4.0)\n","Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.2)\n","Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.19.5)\n","Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (3.17.3)\n","Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.15.0)\n","Collecting tensorboard<1.15.0,>=1.14.0\n"," Downloading tensorboard-1.14.0-py3-none-any.whl (3.1 MB)\n","\u001b[K |████████████████████████████████| 3.1 MB 61.8 MB/s \n","\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.0)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.37.0)\n","Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.42.0)\n","Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.13.3)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras-applications>=1.0.6->tensorflow==1.14.0) (3.1.0)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (1.0.1)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.3.6)\n","Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (57.4.0)\n","Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (4.8.2)\n","Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.10.0.2)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.6.0)\n","Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py->keras-applications>=1.0.6->tensorflow==1.14.0) (1.5.2)\n","Installing collected packages: tensorflow-estimator, tensorboard, keras-applications, tensorflow\n"," Attempting uninstall: tensorflow-estimator\n"," Found existing installation: tensorflow-estimator 2.7.0\n"," Uninstalling tensorflow-estimator-2.7.0:\n"," Successfully uninstalled tensorflow-estimator-2.7.0\n"," Attempting uninstall: tensorboard\n"," Found existing installation: tensorboard 2.7.0\n"," Uninstalling tensorboard-2.7.0:\n"," Successfully uninstalled tensorboard-2.7.0\n"," Attempting uninstall: tensorflow\n"," Found existing installation: tensorflow 2.7.0\n"," Uninstalling tensorflow-2.7.0:\n"," Successfully uninstalled tensorflow-2.7.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","kapre 0.3.6 requires tensorflow>=2.0.0, but you have tensorflow 1.14.0 which is incompatible.\u001b[0m\n","Successfully installed keras-applications-1.0.8 tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0\n","Collecting q\n"," Downloading q-2.6-py2.py3-none-any.whl (6.8 kB)\n","Collecting keras==2.3.1\n"," Downloading Keras-2.3.1-py2.py3-none-any.whl (377 kB)\n","\u001b[K |████████████████████████████████| 377 kB 29.2 MB/s \n","\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.1.2)\n","Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.15.0)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (3.1.0)\n","Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.4.1)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (3.13)\n","Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.19.5)\n","Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.0.8)\n","Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py->keras==2.3.1) (1.5.2)\n","Installing collected packages: q, keras\n"," Attempting uninstall: keras\n"," Found existing installation: keras 2.7.0\n"," Uninstalling keras-2.7.0:\n"," Successfully uninstalled keras-2.7.0\n","Successfully installed keras-2.3.1 q-2.6\n","Collecting h5py<3.0.0\n"," Downloading h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)\n","\u001b[K |████████████████████████████████| 2.9 MB 16.2 MB/s \n","\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from h5py<3.0.0) (1.15.0)\n","Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from h5py<3.0.0) (1.19.5)\n","Installing collected packages: h5py\n"," Attempting uninstall: h5py\n"," Found existing installation: h5py 3.1.0\n"," Uninstalling h5py-3.1.0:\n"," Successfully uninstalled h5py-3.1.0\n","Successfully installed h5py-2.10.0\n"]}]},{"cell_type":"code","metadata":{"id":"gZ9YDtD6qn4t","executionInfo":{"status":"ok","timestamp":1640608139255,"user_tz":-330,"elapsed":3544,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"20737151-68a1-40d7-ae8e-91bdc287b2b1"},"source":["from __future__ import print_function\n","\n","import collections\n","import pickle\n","import re\n","import random\n","import sys\n","import os \n","import time\n","\n","import gensim\n","from gensim.models.keyedvectors import KeyedVectors\n","from gensim.models.fasttext import FastText\n","from gensim.models import word2vec\n","\n","from sklearn.model_selection import train_test_split,cross_val_score, cross_val_predict, KFold, GridSearchCV\n","from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix,precision_recall_fscore_support\n","\n","import pandas as pd\n","import numpy as np\n","from numpy import array\n","from numpy import asarray\n","from numpy import zeros\n","from numpy import cumsum\n","\n","import tensorflow as tf\n","import keras\n","from keras import backend as K\n","from keras.models import Sequential,Model,load_model\n","from keras.preprocessing.text import Tokenizer\n","from keras.preprocessing.sequence import pad_sequences\n","from keras.layers import Dropout, Activation, Flatten, \\\n"," Embedding, Convolution1D, MaxPooling1D, AveragePooling1D, \\\n"," Input, Dense, merge, Add,TimeDistributed, Bidirectional,SpatialDropout1D\n","from keras.layers.recurrent import LSTM, GRU, SimpleRNN\n","from keras.regularizers import l2, l1_l2\n","from keras.constraints import maxnorm\n","from keras import callbacks\n","from keras.utils import generic_utils,plot_model\n","from keras.optimizers import Adadelta\n","from keras.callbacks import ModelCheckpoint,EarlyStopping\n","from keras.wrappers.scikit_learn import KerasClassifier\n","\n","import matplotlib.image as mpimg\n","import matplotlib.pyplot as plt"],"execution_count":5,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n","Using TensorFlow backend.\n"]}]},{"cell_type":"markdown","metadata":{"id":"xXiAlHRZ7X_p"},"source":["# Load Data"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GqqHP9rwpbdt","executionInfo":{"status":"ok","timestamp":1640608143955,"user_tz":-330,"elapsed":4704,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"6943734c-a804-45d7-ec0f-6baf4b5f36b9"},"source":["# lankadeepa_data_path = folder_path + '/data/lankadeepa_tagged_comments.csv'\n","# gossip_lanka_data_path = folder_path + '/data/gossip_lanka_tagged_comments.csv'\n","# lankadeepa_data = pd.read_csv(lankadeepa_data_path)[:9059]\n","# lankadeepa_data = lankadeepa_data[((lankadeepa_data['label']==2) | (lankadeepa_data['label']==4))]\n","# all_data = pd.concat([lankadeepa_data], ignore_index=True)\n","# all_data.info()\n","\n","facebook_data_path = folder_path + '/corpus_for_word_embedding.csv'\n","all_data = pd.read_csv(facebook_data_path)\n","all_data=all_data[:150000]\n","all_data.info()"],"execution_count":6,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","RangeIndex: 150000 entries, 0 to 149999\n","Data columns (total 3 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 Unnamed: 0 150000 non-null int64 \n"," 1 comment 150000 non-null object\n"," 2 label 150000 non-null int64 \n","dtypes: int64(2), object(1)\n","memory usage: 3.4+ MB\n"]}]},{"cell_type":"code","metadata":{"id":"IwNmM2hmhkui","executionInfo":{"status":"ok","timestamp":1640608143958,"user_tz":-330,"elapsed":25,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/","height":206},"outputId":"e4790150-cd1b-4595-967e-7d1c67214d56"},"source":["all_data.head()"],"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Unnamed: 0commentlabel
027විශ්වාසනීය වෙනසක්2
133අපේ යාලුවො හැමෝටම උදාවුන මේ නත්තල සුභ නත්තලක් ...2
234දෙව්රේ වෙලීලා සේනක පෙරේරා2
337මා මුව අගින් නාමල් උඩුගම2
438මේ ඇස් දිහා බලන් ශිහාන් මිහිරංග2
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "],"text/plain":[" Unnamed: 0 comment label\n","0 27 විශ්වාසනීය වෙනසක් 2\n","1 33 අපේ යාලුවො හැමෝටම උදාවුන මේ නත්තල සුභ නත්තලක් ... 2\n","2 34 දෙව්රේ වෙලීලා සේනක පෙරේරා 2\n","3 37 මා මුව අගින් නාමල් උඩුගම 2\n","4 38 මේ ඇස් දිහා බලන් ශිහාන් මිහිරංග 2"]},"metadata":{},"execution_count":7}]},{"cell_type":"markdown","metadata":{"id":"Zk1Up-Co0r3t"},"source":["## Count Tokens"]},{"cell_type":"code","metadata":{"id":"KS7TzP2tyhF6","executionInfo":{"status":"ok","timestamp":1640608143959,"user_tz":-330,"elapsed":12,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# data_path = folder_path + \"corpus/new/preprocess_from_isuru/gossip_lanka_all_comments.csv\"\n","# # \"corpus/new/preprocess_from_isuru/lankadeepa_comments_with_article_2.csv\"\n","# data = pd.read_csv(data_path)\n","\n","# def count_tokens(pandas_df):\n","\n","# count = 0\n","# for index, row in pandas_df.iterrows():\n","# comment_words,article_words = 0,0\n","# if (type(row['comment']) == str) :\n","# comment_words = len(row['comment'].split())\n","# # if (type(row['article']) == str) :\n","# # article_words = len(row['article'].split())\n","# count += (comment_words + article_words)\n","# return count\n","\n","# tokens = count_tokens(data)\n","# print(tokens)"],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"UTRgXvPTbzi3","executionInfo":{"status":"ok","timestamp":1640608143959,"user_tz":-330,"elapsed":11,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# all_data_path = folder_path + \n","# \"corpus/new/tagged_comments_all_with_punctuation_marks.csv\" done\n","# \"corpus/new/tagged_comments_all_with_punctuation_marks_question_only.csv\"\n","# corpus/new/tagged_comments_all_without_punctuation_marks.csv\n","\n","# all_data = pd.read_csv(all_data_path)"],"execution_count":9,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Shq4w6GcqbIq"},"source":["# Create Input"]},{"cell_type":"markdown","metadata":{"id":"IHqkf0XG8dqL"},"source":["## comment-label split"]},{"cell_type":"code","metadata":{"id":"y9ynks11qZ-e","executionInfo":{"status":"ok","timestamp":1640608143960,"user_tz":-330,"elapsed":12,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def text_preprocessing_2(data):\n"," comments = data['comment']\n"," labels = data['label']\n","\n"," comments_splitted = []\n","\n"," for comment in comments:\n"," lines = []\n"," try:\n"," words = comment.split()\n"," lines += words\n"," except:\n"," continue\n"," comments_splitted.append(lines)\n","\n"," return comments_splitted,labels"],"execution_count":10,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"LdKgzrbd8Uy8"},"source":["## Takenize and Split Data"]},{"cell_type":"code","metadata":{"id":"BZoOqtodrUY1","executionInfo":{"status":"ok","timestamp":1640608148069,"user_tz":-330,"elapsed":4120,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"d2ee5684-ad4f-4b09-fb15-ca2c1bb0c72b"},"source":["comment_texts, comment_labels = text_preprocessing_2(all_data)\n","\n","# prepare tokenizer\n","\n","t = Tokenizer()\n","t.fit_on_texts(comment_texts)\n","vocab_size = len(t.word_index) + 1\n","print(vocab_size)"],"execution_count":11,"outputs":[{"output_type":"stream","name":"stdout","text":["108176\n"]}]},{"cell_type":"code","metadata":{"id":"UMhZbzGur4kt","executionInfo":{"status":"ok","timestamp":1640608149814,"user_tz":-330,"elapsed":1761,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["encoded_docs = t.texts_to_sequences(comment_texts)"],"execution_count":12,"outputs":[]},{"cell_type":"code","metadata":{"id":"NxyzZc0I0zwJ","executionInfo":{"status":"ok","timestamp":1640608149815,"user_tz":-330,"elapsed":8,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"b13d877f-8423-4b0a-da3f-0bb93b3925fb"},"source":["encoded_docs[0][0]"],"execution_count":13,"outputs":[{"output_type":"execute_result","data":{"text/plain":["25084"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"Ho5NTzHtsKoN","executionInfo":{"status":"ok","timestamp":1640608151232,"user_tz":-330,"elapsed":1419,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["max_length = len(max(encoded_docs, key=len))\n","padded_docs = pad_sequences(encoded_docs, maxlen=max_length)\n","\n","comment_labels = np.array(comment_labels)\n","padded_docs = np.array(padded_docs)"],"execution_count":14,"outputs":[]},{"cell_type":"code","metadata":{"id":"PLem3iwp1ETL","executionInfo":{"status":"ok","timestamp":1640608151233,"user_tz":-330,"elapsed":23,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"f3d7e260-7ff5-410e-e66d-63b0d4ff67a4"},"source":["padded_docs.shape"],"execution_count":15,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(150000, 215)"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","metadata":{"id":"3spC-Z3dm1Qb","executionInfo":{"status":"ok","timestamp":1640608151234,"user_tz":-330,"elapsed":19,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"4ff513a9-dddc-45e2-b3c2-4cccfbaa2dff"},"source":["max_length"],"execution_count":16,"outputs":[{"output_type":"execute_result","data":{"text/plain":["215"]},"metadata":{},"execution_count":16}]},{"cell_type":"code","metadata":{"id":"gBRINFtdvEQS","executionInfo":{"status":"ok","timestamp":1640608151234,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"4ed6d2f0-b5f2-419a-c1ed-775abfffeffc"},"source":["comment_labels = pd.get_dummies(comment_labels).values\n","print('Shape of label tensor:', comment_labels.shape)"],"execution_count":17,"outputs":[{"output_type":"stream","name":"stdout","text":["Shape of label tensor: (150000, 2)\n"]}]},{"cell_type":"code","metadata":{"id":"0AQOleUlBtik","executionInfo":{"status":"ok","timestamp":1640608151234,"user_tz":-330,"elapsed":15,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"559ad0a4-da13-47a7-fe5b-e0d4ab7c3eff"},"source":["padded_docs.shape"],"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(150000, 215)"]},"metadata":{},"execution_count":18}]},{"cell_type":"code","metadata":{"id":"ov3AUwTrv3RU","executionInfo":{"status":"ok","timestamp":1640608151235,"user_tz":-330,"elapsed":14,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"f7b32652-96f1-4d64-fea3-9d7ed9c14c5a"},"source":["comment_labels.shape"],"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(150000, 2)"]},"metadata":{},"execution_count":19}]},{"cell_type":"code","metadata":{"id":"Drgv_J0ZUV8j","executionInfo":{"status":"ok","timestamp":1640608151235,"user_tz":-330,"elapsed":12,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["X_train, X_test, y_train, y_test = train_test_split(padded_docs, comment_labels, test_size=0.2, random_state=0)\n"],"execution_count":20,"outputs":[]},{"cell_type":"code","metadata":{"id":"qz7KhCUSBPYx","executionInfo":{"status":"ok","timestamp":1640608151235,"user_tz":-330,"elapsed":12,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"57ddf9c2-36d7-4803-ffab-10805a0d2276"},"source":["X_train.shape"],"execution_count":21,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(120000, 215)"]},"metadata":{},"execution_count":21}]},{"cell_type":"code","metadata":{"id":"49a_GcIvkmch","executionInfo":{"status":"ok","timestamp":1640608151236,"user_tz":-330,"elapsed":11,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"6c828c39-1d6d-48c0-8f18-228ab8a49e64"},"source":["(unique, counts) = np.unique(y_test, return_counts = True)\n","frequencies = np.asarray((unique, counts)).T\n","print(frequencies)"],"execution_count":22,"outputs":[{"output_type":"stream","name":"stdout","text":["[[ 0 30000]\n"," [ 1 30000]]\n"]}]},{"cell_type":"markdown","metadata":{"id":"91fqgdk67oaZ"},"source":["# Word Embedding"]},{"cell_type":"markdown","metadata":{"id":"sPm_9b6Y8EPi"},"source":["## Generate Embedding Metrix"]},{"cell_type":"code","metadata":{"id":"kOB0YKL-fNW5","executionInfo":{"status":"ok","timestamp":1640608151236,"user_tz":-330,"elapsed":8,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def generate_embedding_metrix():\n"," if (embedding_type == 'fasttext'):\n"," word_embedding_model = FastText.load(word_embedding_path)\n"," else:\n"," word_embedding_model = word2vec.Word2Vec.load(word_embedding_path)\n"," \n"," word_vectors = word_embedding_model.wv\n"," word_vectors.save(word_embedding_keydvectors_path)\n"," word_vectors = KeyedVectors.load(word_embedding_keydvectors_path, mmap='r')\n","\n"," embeddings_index = dict()\n"," \n"," for word, vocab_obj in word_vectors.vocab.items():\n"," embeddings_index[word]=word_vectors[word]\n","\n"," # create a weight matrix for words in training docs\n"," embedding_matrix = zeros((vocab_size, EMBEDDING_SIZE))\n"," for word, i in t.word_index.items():\n"," embedding_vector = embeddings_index.get(word)\n"," if embedding_vector is not None:\n"," embedding_matrix[i] = embedding_vector\n","\n"," pickle.dump(embedding_matrix, open(embedding_matrix_path, 'wb'))\n"," return embedding_matrix"],"execution_count":23,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"wVYEOwiN8H3s"},"source":["## Load Embedding Matrix"]},{"cell_type":"code","metadata":{"id":"YNSwoRM292-u","executionInfo":{"status":"ok","timestamp":1640608151236,"user_tz":-330,"elapsed":8,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def load_word_embedding_matrix():\n"," f = open(word_embedding_path, 'rb')\n"," embedding_matrix= np.array(pickle.load(f))\n"," return embedding_matrix"],"execution_count":24,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"lrmGuYXf82E_"},"source":["# Models"]},{"cell_type":"markdown","metadata":{"id":"hx80tScO_ddi"},"source":["## RNN(LSTM/GRU) model"]},{"cell_type":"code","metadata":{"id":"K9elMDrR_laX","executionInfo":{"status":"ok","timestamp":1640608152038,"user_tz":-330,"elapsed":809,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def RNN_model(RNN_layer, maxlen, hidden_dims, l2_reg, drop_out_value_1, drop_out_value_2):\n"," main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input')\n"," embedding = Embedding(MAX_FEATURES, EMBEDDING_SIZE,\n"," weights=[EMBEDDING_MATRIX], input_length=maxlen,\n"," name='embedding' ,trainable=False)(main_input)\n"," embedding = Dropout(drop_out_value_1)(embedding)\n","\n"," x = RNN(hidden_dims)(embedding)\n","\n"," x = Dense(hidden_dims, activation='relu', init='he_normal', \n"," W_constraint = maxnorm(3), b_constraint=maxnorm(3),\n"," name='mlp')(x)\n","\n"," x = Dropout(drop_out_value_2, name='drop')(x)\n","\n"," output = Dense(2, init='he_normal',\n"," activation='softmax', name='output')(x)\n","\n"," model = Model(input=main_input, output=output ,name=\"RNN_model\")\n","\n"," model.compile(loss={'output':'categorical_crossentropy'},\n"," optimizer=Adadelta(lr=0.95, epsilon=1e-06),\n"," metrics=[\"accuracy\",\n"," tf.keras.metrics.Precision(),\n"," tf.keras.metrics.Recall(),\n"," f1])\n"," \n"," print(model.summary())\n"," return model\n","\n","def stacked_RNN_model_2(RNN_layer, maxlen, hidden_dims, l2_reg, drop_out_value_1, drop_out_value_2):\n"," main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input')\n"," embedding = Embedding(MAX_FEATURES, EMBEDDING_SIZE,\n"," weights=[EMBEDDING_MATRIX], input_length=maxlen,\n"," name='embedding' ,trainable=False)(main_input)\n","\n"," embedding = Dropout(drop_out_value_1)(embedding)\n","\n"," x = RNN_layer(hidden_dims,return_sequences=True)(embedding)\n"," x = RNN_layer(hidden_dims)(x)\n","\n"," x = Dense(hidden_dims, activation='relu', init='he_normal',\n"," W_constraint = maxnorm(3), b_constraint=maxnorm(3),\n"," name='mlp')(x)\n","\n"," x = Dropout(drop_out_value_2, name='drop')(x)\n","\n"," output = Dense(2, init='he_normal',\n"," activation='softmax', name='output')(x)\n","\n"," model = Model(input=main_input, output=output, name= \"stacked_RNN_model_2\")\n","\n"," model.compile(loss={'output':'categorical_crossentropy'},\n"," optimizer=Adadelta(lr=0.95, epsilon=1e-06),\n"," metrics=[\"accuracy\",\n"," tf.keras.metrics.Precision(),\n"," tf.keras.metrics.Recall(),\n"," f1])\n","\n"," print(model.summary())\n"," return model\n","\n","def stacked_RNN_model_3(RNN_layer, maxlen, hidden_dims, l2_reg, drop_out_value_1, drop_out_value_2):\n"," main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input')\n"," embedding = Embedding(MAX_FEATURES, EMBEDDING_SIZE,\n"," weights=[EMBEDDING_MATRIX], input_length=maxlen,\n"," name='embedding' ,trainable=False)(main_input)\n","\n"," embedding = Dropout(drop_out_value_1)(embedding)\n","\n"," x = RNN_layer(hidden_dims,return_sequences=True)(embedding)\n"," x = RNN_layer(hidden_dims,return_sequences=True)(x)\n"," x = RNN_layer(hidden_dims)(x)\n","\n"," x = Dense(hidden_dims, activation='relu', init='he_normal',\n"," W_constraint = maxnorm(3), b_constraint=maxnorm(3),\n"," name='mlp')(x)\n","\n"," x = Dropout(drop_out_value_2, name='drop')(x)\n","\n"," output = Dense(2, init='he_normal',\n"," activation='softmax', name='output')(x)\n","\n"," model = Model(input=main_input, output=output, name=\"stacked_RNN_model_3\")\n","\n"," model.compile(loss={'output':'categorical_crossentropy'},\n"," optimizer=Adadelta(lr=0.95, epsilon=1e-06),\n"," metrics=[\"accuracy\",\n"," tf.keras.metrics.Precision(),\n"," tf.keras.metrics.Recall(),\n"," f1])\n","\n"," print(model.summary())\n"," return model"],"execution_count":25,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Pt3TZsP84gRK"},"source":["## CNN+RNN(LSTM /GRU) model "]},{"cell_type":"code","metadata":{"id":"m3v2boPz4frA","executionInfo":{"status":"ok","timestamp":1640608152040,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def CNN_RNN_model(RNN_layer, maxlen, hidden_dims, l2_reg, drop_out_value_1, drop_out_value_2):\n"," main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input')\n"," embedding = Embedding(MAX_FEATURES, EMBEDDING_SIZE,\n"," weights=[EMBEDDING_MATRIX], input_length=maxlen,\n"," name='embedding' ,trainable=False)(main_input)\n","\n"," embedding = Dropout(drop_out_value_1)(embedding)\n","\n"," conv4 = Convolution1D(NB_FILTERS,\n"," 4,\n"," border_mode='valid',\n"," activation='relu',\n"," subsample_length=1,\n"," name='conv4')(embedding)\n"," maxConv4 = MaxPooling1D(pool_length=2,\n"," name='maxConv4')(conv4)\n","\n"," conv5 = Convolution1D(NB_FILTERS,\n"," 5,\n"," border_mode='valid',\n"," activation='relu',\n"," subsample_length=1,\n"," name='conv5')(embedding)\n"," maxConv5 = MaxPooling1D(pool_length=2,\n"," name='maxConv5')(conv5)\n","\n"," x = keras.layers.Concatenate(axis=1)([maxConv4, maxConv5])\n","\n"," x = Dropout(drop_out_value_2)(x)\n","\n"," x = RNN(rnn_output_size)(x)\n","\n","\n"," x = Dense(hidden_dims, activation='relu', init='he_normal',\n"," W_constraint = maxnorm(3), b_constraint=maxnorm(3),\n"," name='mlp')(x)\n","\n"," x = Dropout(drop_out_value_2, name='drop')(x)\n","\n"," output = Dense(2, init='he_normal',\n"," activation='softmax', name='output')(x)\n","\n"," model = Model(input=main_input, output=output, name= \"CNN+RNN model\")\n","\n"," model.compile(loss={'output':'categorical_crossentropy'},\n"," optimizer=Adadelta(lr=0.95, epsilon=1e-06),\n"," metrics=[\"accuracy\",\n"," tf.keras.metrics.Precision(),\n"," tf.keras.metrics.Recall(),\n"," f1])\n","\n"," return model"],"execution_count":26,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"WC0ZRhKe5a6C"},"source":["# Train and Evaluate Model"]},{"cell_type":"markdown","metadata":{"id":"jbpLZfOPGGKN"},"source":["## Custom F1 Implementation"]},{"cell_type":"code","metadata":{"id":"O0sMLlLvEGKq","executionInfo":{"status":"ok","timestamp":1640608152047,"user_tz":-330,"elapsed":24,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def f1(y_true, y_pred):\n"," def recall(y_true, y_pred):\n"," \"\"\"Recall metric.\n","\n"," Only computes a batch-wise average of recall.\n","\n"," Computes the recall, a metric for multi-label classification of\n"," how many relevant items are selected.\n"," \"\"\"\n"," true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n"," possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n"," recall = true_positives / (possible_positives + K.epsilon())\n"," return recall\n","\n"," def precision(y_true, y_pred):\n"," \"\"\"Precision metric.\n","\n"," Only computes a batch-wise average of precision.\n","\n"," Computes the precision, a metric for multi-label classification of\n"," how many selected items are relevant.\n"," \"\"\"\n"," true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n"," predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n"," precision = true_positives / (predicted_positives + K.epsilon())\n"," return precision\n"," precision = precision(y_true, y_pred)\n"," recall = recall(y_true, y_pred)\n"," return 2*((precision*recall)/(precision+recall+K.epsilon()))"],"execution_count":27,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"fqK6yZoCHIyE"},"source":["## Train and Validate model"]},{"cell_type":"code","metadata":{"id":"XTuRxzncQc5q","executionInfo":{"status":"ok","timestamp":1640608152049,"user_tz":-330,"elapsed":25,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def Train_Model(model,X_train, y_train, cross_validation = False):\n","\n"," print('Training and Testing...')\n"," print (X_train.shape, y_train.shape)\n"," \n"," es = EarlyStopping(monitor='val_f1', mode='max', verbose=1, patience=5)\n"," checkpoint = ModelCheckpoint(model_save_path, monitor='val_f1', verbose=1, save_weights_only=True, mode='max')\n"," callbacks_list = [checkpoint,es]\n","\n"," if (cross_validation):\n"," callbacks_list = [es]\n","\n"," his = model.fit(X_train, y_train, validation_split=VALIDATION_SPLIT, epochs=NB_EPOCHS, batch_size=BATCH_SIZE, callbacks=callbacks_list, verbose=1)\n"," \n","\n"," # checkpoint_path = folder_path + \"/CNN_RNN/\"+str(experiment_no)+\"_weights_best_\"+model_name+\"_\"+embedding_type+\"_\"+str(experiment_no)+\".ckpt\"\n"," # checkpoint_dir = os.path.dirname(checkpoint_path)\n","\n"," # # Create a callback that saves the model's weights\n"," # cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)\n","\n"," # # Train the model with the new callback\n"," # his = model.fit(X_train, y_train, validation_split=VALIDATION_SPLIT, epochs=NB_EPOCHS, batch_size=BATCH_SIZE, callbacks=[cp_callback])\n","\n"," return model,his"],"execution_count":28,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"u4DuyDZMkcE3"},"source":["# Cross Validation"]},{"cell_type":"code","metadata":{"id":"4TMmqz8BkTQR","executionInfo":{"status":"ok","timestamp":1640608152049,"user_tz":-330,"elapsed":25,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def Do_Cross_Validation(X,y):\n","\n"," # Define per-fold score containers\n"," loss_per_fold = []\n"," acc_per_fold = []\n"," precision_per_fold = []\n"," recall_per_fold = []\n"," f1_per_fold = []\n"," \n","\n"," kfold = KFold(n_splits=FOLDS, shuffle=True)\n","\n"," fold_no = 1\n"," inputs = X\n"," targets = y\n"," for train, test in kfold.split(inputs, targets):\n","\n"," # model = build_model()\n"," model = RNN_model(GRU, MAX_LEN, HIDDEN_DIMS, L2_REG, DROPOUT_VALUE_1, DROPOUT_VALUE_2)\n"," \n"," # RNN_layer, maxlen, hidden_dims, l2_reg, drop_out_value_1, drop_out_value_2\n","\n"," # Generate a print\n"," print('------------------------------------------------------------------------')\n"," print(f'Training for fold {fold_no} ...')\n","\n"," # Fit data to model\n"," model, his = Train_Model(model,inputs[train], targets[train], cross_validation=True)\n"," \n"," # Generate generalization metrics\n"," scores = model.evaluate(inputs[test], targets[test], verbose=0)\n","\n"," print(f\"\"\"Score for fold {fold_no}:\n"," {model.metrics_names[0]} of {scores[0]}; \n"," {model.metrics_names[1]} of {scores[1]*100}% ;\n"," {model.metrics_names[2]} of {scores[2]*100}% ;\n"," {model.metrics_names[3]} of {scores[3]*100}% ;\n"," {model.metrics_names[4]} of {scores[4]*100}% ;\n"," \"\"\")\n"," \n"," loss_per_fold.append(scores[0])\n"," acc_per_fold.append(scores[1])\n"," precision_per_fold.append(scores[2])\n"," recall_per_fold.append(scores[3])\n"," f1_per_fold.append(scores[4])\n","\n"," # Increase fold number\n"," fold_no = fold_no + 1\n","\n"," # == Provide average scores ==\n"," print('------------------------------------------------------------------------')\n"," print('Score per fold')\n"," for i in range(0, len(acc_per_fold)):\n"," print('------------------------------------------------------------------------')\n"," print(f\"\"\"> Fold {i+1} - \n"," Loss: {loss_per_fold[i]} - \n"," Accuracy: {acc_per_fold[i]}% - \n"," Precesion: {precision_per_fold[i]}% - \n"," Recall: {recall_per_fold[i]}% - \n"," F1: {f1_per_fold[i]}%\n"," \"\"\")\n"," print('------------------------------------------------------------------------')\n"," print('Average scores for all folds:')\n"," print(f'> Loss: {np.mean(loss_per_fold)}')\n"," print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')\n"," print(f'> Precision: {np.mean(precision_per_fold)}')\n"," print(f'> Recall: {np.mean(recall_per_fold)}')\n"," print(f'> F1: {np.mean(f1_per_fold)}')\n"," print('------------------------------------------------------------------------')"],"execution_count":29,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"B6CXqtIDR4jS"},"source":["# Plot Graphs"]},{"cell_type":"code","metadata":{"id":"fvl487sr8seD","executionInfo":{"status":"ok","timestamp":1640608152051,"user_tz":-330,"elapsed":27,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def Plot_graphs(metric,val_metric,metric_name):\n","\n"," epochs=range(len(metric)) # Get number of epochs\n","\n"," if metric_name == \"accuracy\":\n"," #------------------------------------------------\n"," # Plot training and validation accuracy per epoch\n"," #------------------------------------------------\n"," plt.plot(epochs, metric, 'r')\n"," plt.plot(epochs, val_metric, 'b')\n"," plt.title('Training and validation accuracy')\n"," plt.xlabel(\"Epochs\")\n"," plt.ylabel(\"Accuracy\")\n"," plt.legend([\"Accuracy\", \"Validation Accuracy\"])\n","\n"," plt.figure()\n","\n"," elif metric_name == \"loss\" :\n"," #------------------------------------------------\n"," # Plot training and validation loss per epoch\n"," #------------------------------------------------\n"," plt.plot(epochs, metric, 'r')\n"," plt.plot(epochs, val_metric, 'b')\n"," plt.title('Training and validation loss')\n"," plt.xlabel(\"Epochs\")\n"," plt.ylabel(\"Loss\")\n"," plt.legend([\"Loss\", \"Validation Loss\"])\n","\n"," plt.figure()\n","\n"," elif metric_name == \"f1\" :\n"," #------------------------------------------------\n"," # Plot training and validation loss per epoch\n"," #------------------------------------------------\n"," plt.plot(epochs, metric, 'r')\n"," plt.plot(epochs, val_metric, 'b')\n"," plt.title('Training and validation F1')\n"," plt.xlabel(\"Epochs\")\n"," plt.ylabel(\"F1\")\n"," plt.legend([\"F1\", \"Validation F1\"])\n","\n"," plt.figure()\n","\n","\n"," # Expected Output\n"," # A chart where the validation loss does not increase sharply!"],"execution_count":30,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0RS6GyZ8-mk7"},"source":["# Main Method"]},{"cell_type":"markdown","metadata":{"id":"dN_uGz3CZMCh"},"source":["## Set Hyper-Parameters"]},{"cell_type":"code","metadata":{"id":"85wsS8Mn8oeI","executionInfo":{"status":"ok","timestamp":1640610280431,"user_tz":-330,"elapsed":3297,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# EMBEDDING_MATRIX = generate_embedding_metrix()\n","EMBEDDING_MATRIX = load_word_embedding_matrix()"],"execution_count":32,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"p-x5t7AF_DGO"},"source":["## Build and Compile Model"]},{"cell_type":"code","metadata":{"id":"IuEifQn3Jc98","executionInfo":{"status":"ok","timestamp":1640610280432,"user_tz":-330,"elapsed":16,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def build_model() :\n"," model = None\n","\n"," if (model_type == \"RNN\" or model_type == \"GRU\" or model_type == \"LSTM\"):\n"," # configure layer\n"," layer = None\n"," if (model_type == \"RNN\"):\n"," layer = SimpleRNN\n"," elif (model_type == \"GRU\"):\n"," layer = GRU\n"," elif (model_type == \"LSTM\"):\n"," layer = LSTM\n"," \n"," \n"," # configure architecture\n"," if (stack_modeles == \"2\"):\n"," model = stacked_RNN_model_2(layer,MAX_LEN, HIDDEN_DIMS, L2_REG, DROPOUT_VALUE_1, DROPOUT_VALUE_2)\n"," elif (stack_modeles == \"3\"):\n"," model = stacked_RNN_model_3(layer,MAX_LEN, HIDDEN_DIMS, L2_REG, DROPOUT_VALUE_1, DROPOUT_VALUE_2)\n"," elif (apply_CNN):\n"," model = CNN_RNN_model(layer,MAX_LEN, HIDDEN_DIMS, L2_REG, DROPOUT_VALUE_1, DROPOUT_VALUE_2)\n"," else :\n"," model = RNN_model(layer,MAX_LEN, HIDDEN_DIMS, L2_REG, DROPOUT_VALUE_1, DROPOUT_VALUE_2)\n","\n"," elif (model_type == \"BiLSTM\" ):\n","\n"," # configure architecture\n"," if (stack_modeles == \"2\"):\n"," model = stacked_BiLSTM_model_2(MAX_LEN, DROPOUT_VALUE_1)\n"," elif (stack_modeles == \"3\"):\n"," model = stacked_BiLSTM_model_3(MAX_LEN, DROPOUT_VALUE_1)\n"," elif (apply_CNN):\n"," model = CNN_BiLSTM_model(MAX_LEN)\n"," else :\n"," model = BiLSTM_model(MAX_LEN, DROPOUT_VALUE_1)\n","\n"," return model"],"execution_count":33,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"d46ja9NetIpP"},"source":["## Train and Test Model (Holdout Method)"]},{"cell_type":"code","metadata":{"id":"ZB39v3N61RCY","executionInfo":{"status":"ok","timestamp":1640610280432,"user_tz":-330,"elapsed":14,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["MAX_FEATURES = EMBEDDING_MATRIX.shape[0] #vocab_size\n","VERBOSITY = 1\n","VALIDATION_SPLIT = 0.1\n","NB_EPOCHS = 3\n","FOLDS = 10\n","\n","BATCH_SIZE = 64 # 64, 128\n","NB_FILTERS = EMBEDDING_SIZE\n","FILTER_LENGTH = 4 # test with 2,3,4,5\n","HIDDEN_DIMS = NB_FILTERS * 2\n","MAX_LEN = 215 #test with other values(only this value work for now)\n","DROPOUT_VALUE_1 = 0.5 #0.8 #0.3\n","DROPOUT_VALUE_2 = 0.1\n","L2_REG= 0.01\n","# LSTM, GRU, SimpleRNN\n","RNN = GRU\n","rnn_output_size = EMBEDDING_SIZE "],"execution_count":34,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"rV8W-Nthbfh5","executionInfo":{"status":"ok","timestamp":1640610284015,"user_tz":-330,"elapsed":3596,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"1093f1e7-a163-4f2b-c84b-69516107f49f"},"source":["model = build_model()\n","# model.load_weights(model_save_path)\n","plot_model(model,to_file=\"./model.png\")"],"execution_count":35,"outputs":[{"output_type":"stream","name":"stdout","text":["WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n","\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:14: UserWarning: Update your `Conv1D` call to the Keras 2 API: `Conv1D(200, 4, activation=\"relu\", name=\"conv4\", strides=1, padding=\"valid\")`\n"," \n","/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:16: UserWarning: Update your `MaxPooling1D` call to the Keras 2 API: `MaxPooling1D(name=\"maxConv4\", pool_size=2)`\n"," app.launch_new_instance()\n","/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:23: UserWarning: Update your `Conv1D` call to the Keras 2 API: `Conv1D(200, 5, activation=\"relu\", name=\"conv5\", strides=1, padding=\"valid\")`\n","/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:25: UserWarning: Update your `MaxPooling1D` call to the Keras 2 API: `MaxPooling1D(name=\"maxConv5\", pool_size=2)`\n","/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:36: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(400, activation=\"relu\", name=\"mlp\", kernel_initializer=\"he_normal\", kernel_constraint="]},"metadata":{},"execution_count":35}]},{"cell_type":"markdown","metadata":{"id":"G61FNXmM_QSL"},"source":["### Train Model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":412},"id":"ye7IkwWamR2c","outputId":"e4542229-6793-4f96-b024-134d45542e23","executionInfo":{"status":"ok","timestamp":1640619201697,"user_tz":-330,"elapsed":8917687,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# # trained_model = Train_Model_old(model,X_train, X_test, y_train, y_test)\n","model.load_weights(model_save_path)\n","trained_model, his = Train_Model(model,X_train, y_train, cross_validation = False)\n","\n","accuracy = his.history['accuracy']\n","val_accuracy = his.history['val_accuracy']\n","loss = his.history['loss']\n","val_loss = his.history['val_loss']\n","f1 = his.history['f1']\n","val_f1 = his.history['f1']\n","\n","Plot_graphs(accuracy,val_accuracy, \"accuracy\")\n","Plot_graphs(loss,val_loss, \"loss\")\n","Plot_graphs(f1,val_f1, \"f1\")"],"execution_count":36,"outputs":[{"output_type":"stream","name":"stdout","text":["Training and Testing...\n","(120000, 215) (120000, 2)\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n","\n","Train on 108000 samples, validate on 12000 samples\n","Epoch 1/3\n","108000/108000 [==============================] - 2998s 28ms/step - loss: 0.3763 - accuracy: 0.8360 - precision: 0.8378 - recall: 0.8378 - f1: 0.8360 - val_loss: 0.3920 - val_accuracy: 0.8292 - val_precision: 0.8356 - val_recall: 0.8356 - val_f1: 0.8295\n","\n","Epoch 00001: saving model to /content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Word embedding/word embedding/word2vec200.hdf5\n","Epoch 2/3\n","108000/108000 [==============================] - 2953s 27ms/step - loss: 0.3732 - accuracy: 0.8374 - precision: 0.8359 - recall: 0.8359 - f1: 0.8374 - val_loss: 0.3963 - val_accuracy: 0.8303 - val_precision: 0.8361 - val_recall: 0.8361 - val_f1: 0.8305\n","\n","Epoch 00002: saving model to /content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Word embedding/word embedding/word2vec200.hdf5\n","Epoch 3/3\n","108000/108000 [==============================] - 2957s 27ms/step - loss: 0.3694 - accuracy: 0.8384 - precision: 0.8365 - recall: 0.8365 - f1: 0.8385 - val_loss: 0.3909 - val_accuracy: 0.8304 - val_precision: 0.8366 - val_recall: 0.8366 - val_f1: 0.8307\n","\n","Epoch 00003: saving model to /content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Word embedding/word embedding/word2vec200.hdf5\n"]},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwU1b338c+PARw2WV2QbUZl2BmBARRxBRPcQESvoGJQI1EjRmO84WYxXG/yPD655saYxRs1rjGgMVeCiVsUjcl1YxEXUBRkFBCVfUcY5vf8caqnF3qGbpieHpjv+/Xq11RXnao+XdPT3zl1qk6ZuyMiIpKpRvmugIiIHFgUHCIikhUFh4iIZEXBISIiWVFwiIhIVhQcIiKSFQWH7Dcze9rMvlbbZfPJzMrNbGQOtutmdmw0/d9m9sNMyu7D61xiZs/taz1FamK6jqNhMrMtCU+bA18Cu6Pn33D3R+q+VvWHmZUDX3f352t5uw50d/cltVXWzIqAZUATd6+ojXqK1KRxvisg+eHuLWPTNX1JmlljfRlJfaHPY/2gQ1WSxMxONbMVZvZdM/sMuN/M2prZX8xstZmtj6Y7J6zzkpl9PZqeZGb/NLPbo7LLzOzMfSxbbGYvm9lmM3vezH5tZr+vpt6Z1PE/zOx/o+09Z2YdEpZPNLOPzWytmX2/hv0z1Mw+M7OChHljzeztaHqImb1qZhvMbJWZ/crMmlazrQfM7McJz2+O1vnUzK5IKXu2mb1pZpvMbLmZTUtY/HL0c4OZbTGzE2L7NmH9YWY2x8w2Rj+HZbpvstzP7czs/ug9rDezmQnLxpjZgug9LDWzUdH8pMOCZjYt9ns2s6LokN2VZvYJMDua/8fo97Ax+oz0SVi/mZn9LPp9bow+Y83M7K9mNiXl/bxtZmPTvVepnoJD0jkSaAd0AyYTPif3R8+7AtuBX9Ww/lBgMdAB+CnwOzOzfSj7B+ANoD0wDZhYw2tmUseLgcuBw4GmwHcAzKw3cFe0/aOi1+tMGu7+OrAVOD1lu3+IpncDN0bv5wRgBHBtDfUmqsOoqD5nAN2B1P6VrcBlQBvgbOAaMzsvWnZy9LONu7d091dTtt0O+CtwZ/Te/gv4q5m1T3kPe+ybNPa2nx8mHPrsE23r51EdhgAPATdH7+FkoLy6/ZHGKUAv4KvR86cJ++lwYD6QeGj1dmAQMIzwOf5XoBJ4ELg0VsjMSoFOhH0j2XB3PRr4g/AHPDKaPhXYCRTWUP44YH3C85cIh7oAJgFLEpY1Bxw4MpuyhC+lCqB5wvLfA7/P8D2lq+MPEp5fCzwTTd8CzEhY1iLaByOr2faPgfui6VaEL/Vu1ZS9AXgi4bkDx0bTDwA/jqbvA25LKFeSWDbNdu8Afh5NF0VlGycsnwT8M5qeCLyRsv6rwKS97Zts9jPQkfAF3TZNud/G6lvT5y96Pi32e054b0fXUIc2UZnWhGDbDpSmKVcIrCf0G0EImN/U9d/bwfBQi0PSWe3uO2JPzKy5mf02avpvIhwaaZN4uCbFZ7EJd98WTbbMsuxRwLqEeQDLq6twhnX8LGF6W0KdjkrctrtvBdZW91qE1sX5ZnYIcD4w390/jupREh2++Syqx/8htD72JqkOwMcp72+omb0YHSLaCFyd4XZj2/44Zd7HhP+2Y6rbN0n2sp+7EH5n69Os2gVYmmF906naN2ZWYGa3RYe7NhFvuXSIHoXpXiv6TD8KXGpmjYAJhBaSZEnBIemknmp3E9ADGOruhxI/NFLd4afasApoZ2bNE+Z1qaH8/tRxVeK2o9dsX11hd19E+OI9k+TDVBAOeb1P+K/2UOB7+1IHQosr0R+AWUAXd28N/HfCdvd2auSnhENLiboCKzOoV6qa9vNywu+sTZr1lgPHVLPNrYTWZsyRacokvseLgTGEw3mtCa2SWB3WADtqeK0HgUsIhxC3ecphPcmMgkMy0YrQ/N8QHS//Ua5fMPoPfi4wzcyamtkJwLk5quPjwDlmNjzqyL6Vvf9t/AH4FuGL848p9dgEbDGznsA1GdbhMWCSmfWOgiu1/q0I/83viPoLLk5YtppwiOjoarb9FFBiZhebWWMzuwjoDfwlw7ql1iPtfnb3VYS+h99EnehNzCwWLL8DLjezEWbWyMw6RfsHYAEwPipfBlyQQR2+JLQKmxNadbE6VBIO+/2XmR0VtU5OiFqHREFRCfwMtTb2mYJDMnEH0Izw39xrwDN19LqXEDqY1xL6FR4lfGGks891dPeFwDcJYbCKcBx8xV5Wm07osJ3t7msS5n+H8KW+GbgnqnMmdXg6eg+zgSXRz0TXArea2WZCn8xjCetuA34C/K+Fs7mOT9n2WuAcQmthLaGz+JyUemdqb/t5IrCL0Or6gtDHg7u/Qeh8/zmwEfg78VbQDwkthPXAv5PcgkvnIUKLbyWwKKpHou8A7wBzgHXA/yP5u+4hoB+hz0z2gS4AlAOGmT0KvO/uOW/xyMHLzC4DJrv78HzX5UClFofUW2Y22MyOiQ5tjCIc1565t/VEqhMdBrwWuDvfdTmQKTikPjuScKroFsI1CNe4+5t5rZEcsMzsq4T+oM/Z++EwqYEOVYmISFbU4hARkaw0iEEOO3To4EVFRfmuhojIAWPevHlr3P2wdMsaRHAUFRUxd+7cfFdDROSAYWapow1U0aEqERHJioJDRESyouAQEZGsKDhERCQrCg4REcmKgkNERLKi4BARkaw0iOs4REQahN27Yfly+OAD+PBD2LoV/vVfa/1lFBwiIgcSd1i1KgRDLCBiP5csgZ0742U7doSbbwar3Zt1KjhEROqjtWvTh8OHH8KWLfFyhxwCxx4LPXrAOedASQl07x5+HnFErYcGKDhERPJn8+Z4GKQGxLp18XIFBVBcHMLg5JOTw6FLF2hUt93VCg4RkVzasQOWLk0fDqtWJZft0iWEwUUXxYOhe/cQGk2a5Kf+aSg4RET2V0UFlJfvGQwffACffBL6JWKOOCKEwZlnJofDMcdA8+Z5ewvZUHCIiGSishJWrkwfDh99FMIjpk2bEAjDhycfVureHQ49NH/voZbkNDii+0T/AigA7nX321KWdwUeBNpEZaa6+1NmNoT4PYENmObuT0Tr3Ah8HXDgHeByd9+Ry/chIg2EO3zxRfWd0jsSvmqaNw9B0L8/XHBBcjh06JCTTun6ImfBYWYFwK+BM4AVwBwzm+XuixKK/QB4zN3vMrPewFNAEfAuUObuFWbWEXjLzJ4EjgCuB3q7+3YzewwYDzyQq/chIgehDRv2DIfY9KZN8XJNmoRDSN27w1e+Eg+HkhI46qiDOhxqkssWxxBgibt/BGBmM4AxQGJwOBBrt7UGPgVw920JZQqjcjGNgWZmtgtoHltHRCTJ1q3huoZ0rYfVq+PlGjWCbt1CGJxwQvKhpa5dobGO6KfK5R7pBCxPeL4CGJpSZhrwnJlNAVoAI2MLzGwocB/QDZjo7hXASjO7HfgE2A485+7P5ewdiEj9tnNn6F9I13pYuTK57FFHhTA477x4q6F7dzj66HAthGQs31E6AXjA3X9mZicAD5tZX3evdPfXgT5m1gt40MyeBpoRWi3FwAbgj2Z2qbv/PnXDZjYZmAzQtWvXuno/IlLbdu8OZyal65QuLw+d1jEdOoQwGDkyuc/h2GOhZcu8vYWDTS6DYyXQJeF552heoiuBUQDu/qqZFQIdgC9iBdz9PTPbAvQlBMYyd18NYGb/AwwD9ggOd7+bqIO9rKzMU5eLSD0SG0YjXTgsXZo8jEarViEMhgyBSy5Jbj20bZu/99CA5DI45gDdzayYEBjjgYtTynwCjAAeiFoWhcDqaJ3lUed4N6AnUE448+p4M2tOOFQ1Apibw/cgIrVp7drkjujEM5a2bo2XO+SQEAQ9e8Lo0cmthxwNoyGZy1lwRF/61wHPEr7w73P3hWZ2KzDX3WcBNwH3RKfYOjDJ3d3MhgNTow7wSuBad18DrDGzx4H5QAXwJvHTdkWkPogNo5Gu9bB+fbxcQUHoX+jeHU49NTkc8jCMhmTO3A/+ozhlZWU+d64aJiK1Zvv26ofR+OyzeDmz+DAaicFQUgJFRfVqGA1JZmbz3L0s3bJ8d46LSH21a1fofE4XDumG0SgpgbPOSg6HY46BZs3y9hYkNxQcIg1ZZSWsWJH+sNKyZemH0TjppOTWw0EyjIZkTsEhcrCLDaORLhyWLEk/jMZxx8GFFya3Htq3V6e0AAoOkYPH+vXVd0pv3hwvFxtGo6QERo1Kbj004GE0JHMKDpEDSWwYjXThsGZNvFyjRqHzuXt3GDYsORw0jIbsJ316ROqb2DAa6a53SB1Go1OnEAjnn598WKm4WMNoSM4oOETywT10Si9atGc4pBtGo6QkDKORGA7HHgstWuTtLUjDpeAQybXKytCCmD8/+bF2bbxMq1YhDIYMgUsvTT60pGE0pJ5RcIjUpooKWLw4OSAWLIjf46FJE+jXD8aOhQEDwnRJCRx+uDql5YCh4BDZV19+CQsXxgPizTfhrbfCVdUQLnw77rjQghg4MDz69IGmTfNbb5H9pOAQycTWrfD228kh8e674epqCBfADRwIV18dD4kePcJ4TCIHGQWHSKqNG0MwvPlmPCjefz/eYd2hQwiGm26Kh0RxsQblkwZDwSEN2+rV8RZELCSWLo0v79QpBMOFF8ZDolMn9UdIg6bgkIbBPVwDkRgQ8+eHU2Jjjj46BMOVV4afAwaETmsRSaLgkIOPexigL/X019Wrw3KzcIOgU06JtyKOOy4M4icie6XgkAPb7t3hwrnEgHjzzdBPAWFojb594dxz4yHRv78unBPZDwoOOXDs3BmutE4Mibfegm3bwvLCQigthYsvjh9q6ttXQ2+I1DIFh9RP27cnn/46f344/XXnzrC8VasQDJMnh58DB4bDTxq8TyTn9Fcm+bdpU7i6OvFQ03vvhcNQAO3awaBBcMMN8cNNxxyj019F8kTBIXVrzZrkM5vefDMM7Bdz1FGhBTF2bDwkunTR6a8i9YiCQ3LDHVatSg6I+fPDvapjiotDSHzta/E+iSOPzF+dRSQjCg7Zf+5hKPDUC+k+/zwsNwsD+Z14IkyZEj/9tV27vFZbRPaNgkOys3t3OLSUeiHdhg1heUFBGMjvzDPjh5pKS6Fly/zWW0RqjYJDqrdrVzj9NTEkFiwIA/5BOM21f3+46KL4oaZ+/cJpsSJy0FJwSLBjB7zzTnIr4p13wtDhEC6YGzAgDMcRO/21V69wfwkRaVAUHA3R5s3hwrnEkFi0KH76a9u2IRiuvz4eEt276/RXEQEUHAe/dev27I/48MPQoQ1wxBHhGonRo+N9Et266fRXEamWguNg8tlne47ZVF4eX96tW2hBJN6RrmPHvFVXRA5MCo4DkXu4HiI1JFatipcpKYGhQ+Gaa+Id1+3b56/OInLQUHDUd5WVsGTJnhfSrVsXlhcUhE7qM85IPv310EPzW28ROWgpOOqTioowRlNiQLz5JmzZEpY3bRpOdx03Lh4S/fpBs2b5rbeINCgKjnzZsSOM9poYEm+/HeYDNG8erq6eNCkeEr16hfAQEckjBUdd2LIlnP6aeHbTwoWhhQHQunUIhm9+M94fUVISDkOJiNQzCo7atn598hDh8+fD4sXx018POyyc/nr22fFrJIqLdfqriBwwchocZjYK+AVQANzr7relLO8KPAi0icpMdfenzGwIcHesGDDN3Z+I1mkD3Av0BRy4wt1fzeX7qNbnn+95jcSyZfHlXbqEYBg/Pn646aijFBIickDLWXCYWQHwa+AMYAUwx8xmufuihGI/AB5z97vMrDfwFFAEvAuUuXuFmXUE3jKzJ929ghBEz7j7BWbWFGieq/dQxR1WrEgOiPnz4dNP42WOPRbKysId6WKHmw47LOdVExGpa7lscQwBlrj7RwBmNgMYAyQGhwOx80ZbA58CuPu2hDKFUTnMrDVwMjApKrcT2JmT2u/cCbfcEg+JtWvD/EaNwi1KTz893oo47rjQTyEi0gDkMjg6AcsTnq8AhqaUmQY8Z2ZTgBbAyNgCMxsK3Ad0AyZGrY9iYDVwv5mVAvOAb7n71tQXN7PJwGSArl27Zl/7Jk3gkUdCq+G88+Ih0b9/OONJRKSBynfn+ATgAXf/mZmdADxsZn3dvdLdXwf6mFkv4EEze5pQ34HAFHd/3cx+AUwFfpi6YXe/m6ifpKyszLOumVkYrkNnNomIJMnlcKcrgS4JzztH8xJdCTwGEHVwFwIdEgu4+3vAFkJn+ApgRRQqAI8TgiQ3FBoiInvIZXDMAbqbWXHUiT0emJVS5hNgBEDUsigEVkfrNI7mdwN6AuXu/hmw3Mx6ROuPILnPREREcixnh6qiPonrgGcJp9re5+4LzexWYK67zwJuAu4xsxsJHeCT3N3NbDgw1cx2AZXAte6+Jtr0FOCRKIw+Ai7P1XsQEZE9mXv2h/8PNGVlZT537tx8V0NE5IBhZvPcvSzdMt3STUREsqLgEBGRrCg4REQkKwoOERHJioJDRESyouAQEZGsKDhERCQrCg4REcmKgkNERLKi4BARkawoOEREJCsKDhERyYqCQ0REsqLgEBGRrCg4REQkKwoOERHJioJDRESyouAQEZGsKDhERCQrCg4REcmKgkNERLKy1+Aws3PNTAEjIiJAZi2Oi4APzeynZtYz1xUSEZH6ba/B4e6XAgOApcADZvaqmU02s1Y5r52IiNQ7GR2CcvdNwOPADKAjMBaYb2ZTclg3ERGphxrvrYCZjQYuB44FHgKGuPsXZtYcWAT8MrdVFJHasmvXLlasWMGOHTvyXRWpJwoLC+ncuTNNmjTJeJ29BgcwDvi5u7+cONPdt5nZlVnWUUTyaMWKFbRq1YqioiLMLN/VkTxzd9auXcuKFSsoLi7OeL1MDlVNA96IPTGzZmZWFL3oC9lVU0TyaceOHbRv316hIQCYGe3bt8+6BZpJcPwRqEx4vjuaJyIHIIWGJNqXz0MmwdHY3XfGnkTTTbN+JRGRyMyZMzEz3n///XxXRfZBJsGxOuogB8DMxgBrclclETnYTZ8+neHDhzN9+vScvcbu3btztu2GLpPguBr4npl9YmbLge8C38httUTkYLVlyxb++c9/8rvf/Y4ZM2YA4Uv+O9/5Dn379qV///788pfhZM05c+YwbNgwSktLGTJkCJs3b+aBBx7guuuuq9reOeecw0svvQRAy5YtuemmmygtLeXVV1/l1ltvZfDgwfTt25fJkyfj7gAsWbKEkSNHUlpaysCBA1m6dCmXXXYZM2fOrNruJZdcwp///Oc62isHlr2eVeXuS4Hjzaxl9HxLzmslIrl3ww2wYEHtbvO44+COO2os8uc//5lRo0ZRUlJC+/btmTdvHm+88Qbl5eUsWLCAxo0bs27dOnbu3MlFF13Eo48+yuDBg9m0aRPNmjWrcdtbt25l6NCh/OxnPwOgd+/e3HLLLQBMnDiRv/zlL5x77rlccsklTJ06lbFjx7Jjxw4qKyu58sor+fnPf855553Hxo0beeWVV3jwwQdrZ78cZDI5HRczOxvoAxTGOlLc/dYc1ktEDlLTp0/nW9/6FgDjx49n+vTpLFu2jKuvvprGjcNXUrt27XjnnXfo2LEjgwcPBuDQQw/d67YLCgoYN25c1fMXX3yRn/70p2zbto1169bRp08fTj31VFauXMnYsWOBcB0DwCmnnMK1117L6tWr+dOf/sS4ceOq6iPJMrkA8L+B5sBpwL3ABSScnruXdUcBvwAKgHvd/baU5V2BB4E2UZmp7v6UmQ0B7o4VA6a5+xMJ6xUAc4GV7n5OJnURkRR7aRnkwrp165g9ezbvvPMOZsbu3bsxs6pwyETjxo2prIyf6Jl4KmlhYSEFBQVV86+99lrmzp1Lly5dmDZt2l5PO73sssv4/e9/z4wZM7j//vuzfHcNRyZ9HMPc/TJgvbv/O3ACULK3laIv918DZwK9gQlm1jul2A+Ax9x9ADAe+E00/12gzN2PA0YBvzWzxJD7FvBeBnUXkXrk8ccfZ+LEiXz88ceUl5ezfPlyiouLKS0t5be//S0VFRVACJgePXqwatUq5syZA8DmzZupqKigqKiIBQsWUFlZyfLly3njjfT/x8ZCokOHDmzZsoXHH38cgFatWtG5c+eq/owvv/ySbdu2ATBp0iTuiAK1d+/UryuJySQ4YhG9zcyOAnYRxqvamyHAEnf/KDqFdwYwJqWMA7H2Z2vgUwhXpbt7RTS/MCoHgJl1Bs4mtH5E5AAyffr0qkNEMePGjWPVqlV07dqV/v37U1payh/+8AeaNm3Ko48+ypQpUygtLeWMM85gx44dnHjiiRQXF9O7d2+uv/56Bg4cmPa12rRpw1VXXUXfvn356le/mtSqefjhh7nzzjvp378/w4YN47PPPgPgiCOOoFevXlx++eW52wkHAYudZVBtAbMfEsajGkFoQThwj7vfspf1LgBGufvXo+cTgaHufl1CmY7Ac0BboAUw0t3nRcuGAvcB3YCJsUNVZvY48H+BVsB3qjtUZWaTgckAXbt2HfTxxx/X+D5FGoL33nuPXr165bsa9da2bdvo168f8+fPp3Xr1vmuTp1J97kws3nuXpaufI0tjugGTi+4+wZ3/xPhS7zn3kIjCxOAB9y9M3AW8HDsplHu/rq79wEGA/9mZoVmdg7wRSxcauLud7t7mbuXHXbYYbVUXRE5WD3//PP06tWLKVOmNKjQ2Bc1do67e6WZ/ZpwPw7c/Uvgywy3vRLokvC8czQv0ZWEPgzc/VUzKwQ6AF8k1OE9M9sC9AVOBEab2VmEQ1iHmtnvo3uGiIjss5EjR6IjE5nJpI/jBTMbZ9kPaDIH6G5mxWbWlND5PSulzCeEQ2CYWS9CGKyO1mkcze8G9ATK3f3f3L2zuxdF25ut0BARqVuZnKT8DeDbQIWZ7SCcHuvuXuNJ1e5eYWbXAc8STrW9z90XmtmtwFx3nwXcBNxjZjcS+k4mubub2XBgqpntIgyweK27a5gTEZF6IJMrx/f5FrHu/hTwVMq8WxKmFxEOP6Wu9zDw8F62/RLw0r7WTURE9k0mFwCenG5+6o2dRESkYcikj+PmhMcPgScJN3cSEcnKaaedxrPPPps074477uCaa66pdp1TTz2VuXPnAnDWWWexYcOGPcpMmzaN22+/vcbXnjlzJosWLap6fsstt/D8889nU/0a3XDDDXTq1CnpqvaD1V6Dw93PTXicQTi7aX3uqyYiB5sJEyZUjYgbM2PGDCZMmJDR+k899RRt2rTZp9dODY5bb72VkSNH7tO2UlVWVvLEE0/QpUsX/v73v9fKNtOJXVmfb5m0OFKtAHQFkYhk7YILLuCvf/0rO3eGe8OVl5fz6aefctJJJ3HNNddQVlZGnz59+NGPfpR2/aKiItasCefJ/OQnP6GkpIThw4ezePHiqjL33HMPgwcPprS0lHHjxrFt2zZeeeUVZs2axc0338xxxx3H0qVLmTRpUtUwJC+88AIDBgygX79+XHHFFXz55ZdVr/ejH/2IgQMH0q9fv2pvPPXSSy/Rp08frrnmmqR7jHz++eeMHTuW0tJSSktLeeWVVwB46KGHqq6SnzhxIkBSfSAMER/b9kknncTo0aOrhkE577zzGDRoEH369OHuu++uWueZZ55h4MCBlJaWMmLECCorK+nevTurV68GQsAde+yxVc/3VSZ9HL8kPuRHI+A4YP5+vaqI5F0+RlVv164dQ4YM4emnn2bMmDHMmDGDf/mXf8HM+MlPfkK7du3YvXs3I0aM4O2336Z///5ptzNv3jxmzJjBggULqKioYODAgQwaNAiA888/n6uuugqAH/zgB/zud79jypQpjB49mnPOOYcLLrggaVs7duxg0qRJvPDCC5SUlHDZZZdx1113ccMNNwBhrKv58+fzm9/8httvv517791ztKPp06czYcIExowZw/e+9z127dpFkyZNuP766znllFN44okn2L17N1u2bGHhwoX8+Mc/5pVXXqFDhw6sW7dur/t1/vz5vPvuuxQXFwNw33330a5dO7Zv387gwYMZN24clZWVXHXVVbz88ssUFxezbt06GjVqxKWXXsojjzzCDTfcwPPPP09paSn7e1F0Ji2OucC86PEq8F1dOyEi+yrxcFXiYarHHnuMgQMHMmDAABYuXJh0WCnVP/7xD8aOHUvz5s059NBDGT266ialvPvuu5x00kn069ePRx55hIULF9ZYn8WLF1NcXExJSRi79Wtf+xovvxw/9+f8888HYNCgQZSXl++x/s6dO3nqqac477zzOPTQQxk6dGhVP87s2bOr+m8KCgpo3bo1s2fP5sILL6RDhw5ACNO9GTJkSFVoANx5552UlpZy/PHHs3z5cj788ENee+01Tj755Kpyse1eccUVPPTQQ0AInNoYhyuT6zgeB3a4+24Io96aWXN337bfry4ieZOHUdUBGDNmDDfeeCPz589n27ZtDBo0iGXLlnH77bczZ84c2rZty6RJk/Y6BHp1Jk2axMyZMyktLeWBBx6oujvgvjrkkEOA8MWfro/h2WefZcOGDfTr1w8I4101a9aMc87J7o4PicPFV1ZWVh3OA2jRokXV9EsvvcTzzz/Pq6++SvPmzTn11FNr3FddunThiCOOYPbs2bzxxhs88sgjWdUrnYyuHAcSb7vVDKi9UxFEpEFp2bIlp512GldccUVVa2PTpk20aNGC1q1b8/nnn/P000/XuI2TTz6ZmTNnsn37djZv3syTTz5ZtWzz5s107NiRXbt2JX1JtmrVis2bN++xrR49elBeXs6SJUuAMHLuKaeckvH7mT59Ovfeey/l5eWUl5ezbNky/va3v7Ft2zZGjBjBXXfdBYTb427cuJHTTz+dP/7xj6xduxag6lBVUVER8+aFYfhmzZrFrl270r7exo0badu2Lc2bN+f999/ntddeA+D444/n5ZdfZtmyZUnbBfj617/OpZdeyoUXXlh1v5L9kUlwFCbeLjaabr7frywiDdaECRN46623qoKjtLSUAQMG0LNnTy6++GJOPHGP64KTDBw4kIsuuojS0lLOPJqMjCcAABInSURBVPPMpCHT/+M//oOhQ4dy4okn0rNnz6r548eP5z//8z8ZMGAAS5curZpfWFjI/fffz4UXXki/fv1o1KgRV199dUbvY9u2bTzzzDOcffbZVfNatGjB8OHDefLJJ/nFL37Biy++SL9+/Rg0aBCLFi2iT58+fP/73+eUU06htLSUb3/72wBcddVV/P3vf6+6X3piKyPRqFGjqKiooFevXkydOpXjjz8egMMOO4y7776b888/n9LSUi666KKqdUaPHs2WLVtqbbj4TIZV/19girvPj54PAn7l7ifUSg3qQFlZmcfOAxdpyDSsesM0d+5cbrzxRv7xj3+kXZ7tsOqZ9HHcAPzRzD4ljFN1JHBRzauIiEh9cNttt3HXXXfVSt9GTCZjVc0xs55Aj2jWYndPf/BNRETqlalTpzJ16tRa3eZe+zjM7JtAC3d/193fBVqa2bW1WgsRETlgZNI5fpW7Vw0O4+7rgatyVyURyaW99WtKw7Ivn4dMgqMg8SZOZlYANM36lUQk7woLC1m7dq3CQ4AQGmvXrqWwsDCr9TLpHH8GeNTMfhs9/wZQ80nWIlIvde7cmRUrVuz3WEVy8CgsLKRz585ZrZNJcHwXmAzETmx+m3BmlYgcYJo0aZI0dIXIvshkWPVK4HWgHBgCnA68l9tqiYhIfVVti8PMSoAJ0WMN8CiAu59WN1UTEZH6qKZDVe8D/wDOcfclAGZ2Y53USkRE6q2aDlWdD6wCXjSze8xsBOHKcRERacCqDQ53n+nu44GewIuEoUcON7O7zOwrdVVBERGpXzLpHN/q7n9w93OBzsCbhDOtRESkAcrqnuPuvt7d73b3EbmqkIiI1G9ZBYeIiIiCQ0REsqLgEBGRrCg4REQkKwoOERHJioJDRESyouAQEZGsKDhERCQrCg4REclKToPDzEaZ2WIzW2JmU9Ms72pmL5rZm2b2tpmdFc0fYmYLosdbZjY2mt8lKr/IzBaa2bdyWX8REdlTJncA3CfRvcl/DZwBrADmmNksd1+UUOwHwGPufpeZ9QaeAoqAd4Eyd68ws47AW2b2JFAB3OTu882sFTDPzP6Wsk0REcmhXLY4hgBL3P0jd98JzADGpJRx4NBoujXwKYC7b3P3imh+YVQOd1/l7vOj6c2EOxF2yuF7EBGRFLkMjk7A8oTnK9jzS34acKmZrSC0NqbEFpjZUDNbCLwDXJ0QJLHlRcAAwm1t92Bmk81srpnNXb169f69ExERqZLvzvEJwAPu3hk4C3jYzBoBuPvr7t4HGAz8m5kVxlYys5bAn4Ab3H1Tug1Ho/iWuXvZYYcdlvM3IiLSUOQyOFYCXRKed47mJboSeAzA3V8lHJbqkFjA3d8DtgB9AcysCSE0HnH3/8lJzUVEpFq5DI45QHczKzazpsB4YFZKmU+AEQBm1osQHKujdRpH87sR7kJYbmYG/A54z93/K4d1FxGRauQsOKI+ieuAZwmd2I+5+0Izu9XMRkfFbgKuMrO3gOnAJHd3YDjhTKoFwBPAte6+BjgRmAicnnC67lm5eg8iIrInC9/TB7eysjKfO3duvqshInLAMLN57l6Wblm+O8dFROQAo+AQEZGsKDhERCQrCg4REcmKgkNERLKi4BARkawoOEREJCsKDhERyYqCQ0REsqLgEBGRrCg4REQkKwoOERHJioJDRESyouAQEZGsKDhERCQrCg4REcmKgkNERLKi4BARkawoOEREJCsKDhERyYqCQ0REsqLgEBGRrCg4REQkKwoOERHJioJDRESyouAQEZGsKDhERCQrjfNdARGRhs4ddu+GnTvDY9eu+PT+PA45BL7zndqvr4JDRA5alZW18wVcFw/32n//hx2m4BCReqCysvb+I87msS+vuXt3bvZBkybQtGlmj+bN08/PZhv7+mico294BYdIPeC+b1+M+fgCr6jIzT5o3DjzL8RWrer2Czj1C98sN/vgQKHgEMmxykpYvhw++AAWL44/PvgA1qyJB0AuFBRk959xmzZ1+yWc+GXcSKfqHDAUHCK1ZNOm5GCIPT78ELZvj5dr1Qp69IDhw+GII0IHZq6+jAsK8rc/5OCl4BDJQkUFLFuWPiA+/zxerqAAiotDQIwcGX7GHkccoUMdcmDLaXCY2SjgF0ABcK+735ayvCvwINAmKjPV3Z8ysyHA3bFiwDR3fyKTbYrsL/dwCCk1GD74AJYuTT6s1KFDCIOzz04Oh6OPDv/1ixyMchYcZlYA/Bo4A1gBzDGzWe6+KKHYD4DH3P0uM+sNPAUUAe8CZe5eYWYdgbfM7EnAM9imSEZ27IAlS/YMh8WLYf36eLmmTaF7d+jdG8aODcFQUhJ+tmuXv/qL5EsuWxxDgCXu/hGAmc0AxgCJX/IOHBpNtwY+BXD3bQllCqNymW5TpIo7rFy5ZzAsXgzl5cnnznfqFMJg/Ph4MPToAd26qa9AJFEug6MTsDzh+QpgaEqZacBzZjYFaAGMjC0ws6HAfUA3YGLU+shkm7H1JwOTAbp27bpfb0Tqv82b46GQGA4ffABbt8bLtWgRwmDoULjssng4lJRAy5b5q7/IgSTfneMTgAfc/WdmdgLwsJn1dfdKd38d6GNmvYAHzezpbDbs7ncT9ZOUlZXl4JpMqWu7d4dWQmo4LF4Mn34aL9eoERQVhTA4+eTkvoejjlLHtMj+ymVwrAS6JDzvHM1LdCUwCsDdXzWzQqAD8EWsgLu/Z2ZbgL4ZblMOcGvX7hkMixeH/oidO+Pl2rYNYXDGGcnhcMwxUFiYv/qLHOxyGRxzgO5mVkz4ch8PXJxS5hNgBPBA1LIoBFZH6yyPDk91A3oC5cCGDLYpB4CdO8MZSulOa127Nl6uSZMQBD16wDnnJAdE+/ZqPYjkQ86CI/rSvw54lnDq7H3uvtDMbgXmuvss4CbgHjO7kdABPsnd3cyGA1PNbBdQCVzr7msA0m0zV+9B9o87fPZZ+nBYtixcUR1z5JEhDMaNSw6HoqLcjbcjIvvGPBdDMtYzZWVlPnfu3HxX46C1bVv64TQWLw6d1jHNmiWfrRR7dO8OrVvnr/4isiczm+fuZemW6X85yUhlJXzySfpwWJ5wnpsZdO0aAmHSpORrHjp31nhEIgcDBYck2bAh/TUPH34YLpiLad06hMGppya3Irp3Dy0LETl4KTgaoF274KOP0p+59MUX8XIFBfGO6a98Jfnw0uGHq2NapKFScByk3GH16vQd0x99lHxPhcMPD62Gc8/dc7ylJk3y9x5EpH5ScBzgtm/fc7yl2GGmDRvi5Q45JBxG6tcPLrgg+Yrptm3zV38ROfAoOA4AlZXJ4y0lPj75JHm8pc6dQyBcfHFy66FLF423JCK1Q8FRj2zalL7f4cMPwymvMS1bhjA48US44op4y6GkJIzFJCKSSwqOOlZRER9vKfXspVWr4uUaNYrfCOj005NPa+3YUR3TIpI/Co4cSb0RUCwclixJvhFQ+/YhDEaNSg6HY44J/RIiIvWNgmM/fPllvGM69RDTunXxck2bwrHHQs+eMHr0nuMtiYgcSBQce+EehuxO1/dQXp483tJRR4UwuPDC5HDo1k3jLYnIwUNfZ9WoqIATToD334ctW+LzmzcPh5MGD4ZLL00+rbVVq/zVV0Skrig4qtG4cTi0NGxYcuuhUyd1TItIw6bgqMHDD+e7BiIi9Y/GKhURkawoOEREJCsKDhERyYqCQ0REsqLgEBGRrCg4REQkKwoOERHJioJDRESyYp54F6CDlJmtBj7ex9U7AGtqsTq1RfXKjuqVHdUrOwdjvbq5+2HpFjSI4NgfZjbX3cvyXY9Uqld2VK/sqF7ZaWj10qEqERHJioJDRESyouDYu7vzXYFqqF7ZUb2yo3plp0HVS30cIiKSFbU4REQkKwoOERHJSoMNDjMbZWaLzWyJmU1Ns/wQM3s0Wv66mRUlLPu3aP5iM/tqHdfr22a2yMzeNrMXzKxbwrLdZrYgesyq43pNMrPVCa//9YRlXzOzD6PH1+q4Xj9PqNMHZrYhYVku99d9ZvaFmb1bzXIzszujer9tZgMTluVyf+2tXpdE9XnHzF4xs9KEZeXR/AVmNreO63WqmW1M+H3dkrCsxs9Ajut1c0Kd3o0+U+2iZbncX13M7MXou2ChmX0rTZncfcbcvcE9gAJgKXA00BR4C+idUuZa4L+j6fHAo9F076j8IUBxtJ2COqzXaUDzaPqaWL2i51vyuL8mAb9Ks2474KPoZ9toum1d1Sul/BTgvlzvr2jbJwMDgXerWX4W8DRgwPHA67neXxnWa1js9YAzY/WKnpcDHfK0v04F/rK/n4HarldK2XOB2XW0vzoCA6PpVsAHaf4mc/YZa6gtjiHAEnf/yN13AjOAMSllxgAPRtOPAyPMzKL5M9z9S3dfBiyJtlcn9XL3F919W/T0NaBzLb32ftWrBl8F/ubu69x9PfA3YFSe6jUBmF5Lr10jd38ZWFdDkTHAQx68BrQxs47kdn/ttV7u/kr0ulB3n69M9ld19uezWdv1qsvP1yp3nx9NbwbeAzqlFMvZZ6yhBkcnYHnC8xXsudOryrh7BbARaJ/hurmsV6IrCf9RxBSa2Vwze83MzqulOmVTr3FRk/hxM+uS5bq5rBfRIb1iYHbC7Fztr0xUV/dc7q9spX6+HHjOzOaZ2eQ81OcEM3vLzJ42sz7RvHqxv8ysOeHL908Js+tkf1k4jD4AeD1lUc4+Y42zraTUD2Z2KVAGnJIwu5u7rzSzo4HZZvaOuy+toyo9CUx39y/N7BuE1trpdfTamRgPPO7uuxPm5XN/1WtmdhohOIYnzB4e7a/Dgb+Z2fvRf+R1YT7h97XFzM4CZgLd6+i1M3Eu8L/untg6yfn+MrOWhLC6wd031ea2a9JQWxwrgS4JzztH89KWMbPGQGtgbYbr5rJemNlI4PvAaHf/Mjbf3VdGPz8CXiL8F1In9XL3tQl1uRcYlOm6uaxXgvGkHEbI4f7KRHV1z+X+yoiZ9Sf8Dse4+9rY/IT99QXwBLV3iHav3H2Tu2+Jpp8CmphZB+rB/orU9PnKyf4ysyaE0HjE3f8nTZHcfcZy0XFT3x+EltZHhEMXsQ61Pillvkly5/hj0XQfkjvHP6L2OsczqdcAQmdg95T5bYFDoukOwIfUUidhhvXqmDA9FnjN4x1xy6L6tY2m29VVvaJyPQkdlVYX+yvhNYqovrP3bJI7Lt/I9f7KsF5dCf12w1LmtwBaJUy/Aoyqw3odGfv9Eb6AP4n2XUafgVzVK1remtAP0qKu9lf03h8C7qihTM4+Y7W2cw+0B+GMgw8IX8Lfj+bdSvgvHqAQ+GP0R/QGcHTCut+P1lsMnFnH9Xoe+BxYED1mRfOHAe9EfzjvAFfWcb3+L7Awev0XgZ4J614R7cclwOV1Wa/o+TTgtpT1cr2/pgOrgF2EY8hXAlcDV0fLDfh1VO93gLI62l97q9e9wPqEz9fcaP7R0b56K/o9f7+O63VdwufrNRKCLd1noK7qFZWZRDhhJnG9XO+v4YQ+lLcTfldn1dVnTEOOiIhIVhpqH4eIiOwjBYeIiGRFwSEiIllRcIiISFYUHCIikhUFh8g+Shldd0FtjsxqZkXVjcgqkm8ackRk32139+PyXQmRuqYWh0gti+7D8NPoXgxvmNmx0fwiM5tt8XupdI3mH2FmT0QD+L1lZsOiTRWY2T3R/RaeM7NmUfnrLX5Plhl5epvSgCk4RPZds5RDVRclLNvo7v2AXwF3RPN+CTzo7v2BR4A7o/l3An9391LCvR8WRvO7A7929z7ABmBcNH8qMCDaztW5enMi1dGV4yL7yMy2uHvLNPPLgdPd/aNoILrP3L29ma0hjOm1K5q/yt07mNlqoLMnDFgZDZX9N3fvHj3/LtDE3X9sZs8AWwgjxM70aPA/kbqiFodIbng109n4MmF6N/E+ybMJYxANBOZEozeL1BkFh0huXJTw89Vo+hXCSMsAlwD/iKZfINwGGDMrMLPW1W3UzBoBXdz9ReC7hJFZ92j1iOSS/lMR2XfNzGxBwvNn3D12Sm5bM3ub0GqYEM2bAtxvZjcDq4HLo/nfAu42sysJLYtrCCOyplMA/D4KFwPudPcNtfaORDKgPg6RWhb1cZS5+5p810UkF3SoSkREsqIWh4iIZEUtDhERyYqCQ0REsqLgEBGRrCg4REQkKwoOERHJyv8Hp0j/PuyYrCgAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"needs_background":"light"}},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwU1b338c+PYd93RfZNEUQQRjBuiBpFMeIaF+KaxItRk2ji1Twx3sRoYnJ98hhziUaNRI06ceViXMNuNCoDAQRUQEQFUQFFQLZh5vf8carp6p6F6WF6epbv+/WqF92nqmtO1QzznXNO1Slzd0RERCqrUa4rICIidYuCQ0REMqLgEBGRjCg4REQkIwoOERHJiIJDREQyouCQnDOzF8zskureNpfMbLWZnZiF/bqZDYhe32NmP6vMtlX4OhPN7OWq1rOC/R5nZmuqe79SsxrnugJSN5nZ1tjblsBOoDh6/x/u/khl9+Xup2Rj2/rO3SdVx37MrA/wPtDE3XdH+34EqPT3UBoWBYdUibu3Trw2s9XAd9x9evp2ZtY48ctIROoHdVVJtUp0RZjZDWb2CTDFzDqY2d/NbL2ZfRG97hH7zGwz+070+lIz+6eZ3RFt+76ZnVLFbfua2Vwz22Jm081sspn9tZx6V6aOvzSzV6P9vWxmnWPrLzKzD8xso5n9tILzM9rMPjGzvFjZmWa2OHo9ysz+ZWabzGydmf2PmTUtZ19/MbNbY++vjz7zsZldnrbteDP7t5ltNrOPzOznsdVzo383mdlWM/ta4tzGPn+kmc0zsy+jf4+s7LmpiJkdHH1+k5ktNbPTY+tONbNl0T7XmtmPo/LO0fdnk5l9bmavmJl+l9UgnWzJhv2BjkBv4ArCz9mU6H0vYDvwPxV8fjTwLtAZ+C3wZzOzKmz7KPAm0An4OXBRBV+zMnW8ELgM6Ao0BRK/yAYDd0f7PyD6ej0og7u/AXwFHJ+230ej18XAtdHxfA04AfheBfUmqsO4qD5fBwYC6eMrXwEXA+2B8cCVZnZGtO7Y6N/27t7a3f+Vtu+OwHPAXdGx/Q54zsw6pR1DqXOzlzo3AZ4FXo4+dw3wiJkdFG3yZ0K3ZxvgEGBmVP4jYA3QBdgP+D+A5k6qQQoOyYYS4L/cfae7b3f3je7+lLtvc/ctwG3AmAo+/4G73+fuxcCDQDfCL4hKb2tmvYDDgZvdfZe7/xOYVt4XrGQdp7j7cnffDjwODI/KzwH+7u5z3X0n8LPoHJTnMeACADNrA5waleHu8939dXff7e6rgT+VUY+yfDOq3xJ3/4oQlPHjm+3ub7l7ibsvjr5eZfYLIWhWuPvDUb0eA94BvhHbprxzU5EjgNbA7dH3aCbwd6JzAxQBg82srbt/4e4LYuXdgN7uXuTur7gm3atRCg7JhvXuviPxxsxamtmfoq6czYSukfbx7po0nyReuPu26GXrDLc9APg8VgbwUXkVrmQdP4m93har0wHxfUe/uDeW97UIrYuzzKwZcBawwN0/iOpxYNQN80lUj18RWh97k1IH4IO04xttZrOirrgvgUmV3G9i3x+klX0AdI+9L+/c7LXO7h4P2fh+zyaE6gdmNsfMvhaV/zewEnjZzFaZ2Y2VOwypLgoOyYb0v/5+BBwEjHb3tiS7RsrrfqoO64COZtYyVtazgu33pY7r4vuOvman8jZ292WEX5CnkNpNBaHL6x1gYFSP/1OVOhC62+IeJbS4erp7O+Ce2H739tf6x4QuvLhewNpK1Gtv++2ZNj6xZ7/uPs/dJxC6saYSWjK4+xZ3/5G79wNOB64zsxP2sS6SAQWH1IQ2hDGDTVF/+X9l+wtGf8EXAj83s6bRX6vfqOAj+1LHJ4HTzOzoaCD7Fvb+f+tR4AeEgHoirR6bga1mNgi4spJ1eBy41MwGR8GVXv82hBbYDjMbRQishPWErrV+5ez7eeBAM7vQzBqb2XnAYEK30r54g9A6+U8za2JmxxG+RwXR92yimbVz9yLCOSkBMLPTzGxANJb1JWFcqKKuQalmCg6pCXcCLYANwOvAizX0dScSBpg3ArcCfyPcb1KWKtfR3ZcCVxHCYB3wBWHwtiKJMYaZ7r4hVv5jwi/1LcB9UZ0rU4cXomOYSejGmZm2yfeAW8xsC3Az0V/v0We3EcZ0Xo2uVDoibd8bgdMIrbKNwH8Cp6XVO2PuvosQFKcQzvsfgYvd/Z1ok4uA1VGX3STC9xPC4P90YCvwL+CP7j5rX+oimTGNKUlDYWZ/A95x96y3eETqM7U4pN4ys8PNrL+ZNYouV51A6CsXkX2gO8elPtsfeJowUL0GuNLd/53bKonUfeqqEhGRjKirSkREMtIguqo6d+7sffr0yXU1RETqlPnz529w9y7p5Q0iOPr06UNhYWGuqyEiUqeYWfqMAYC6qkREJEMKDhERyYiCQ0REMqLgEBGRjCg4REQkIwoOERHJiIJDREQy0iDu4xCpTYqKYMkSePNN2L0bjj8eBg2Ccp+qLlLLKDhEssgd3nsvhERi+fe/YceO1O26dYMTTwzLCSdA9+5l70+kNlBwiFSjTz6BefOSITFvHnzxRVjXogWMGAFXXgmjRsHhh0OjRjBjBkyfDi+8AA8/HLY9+OBkkIwZA+3a5e6YRNI1iNlx8/PzXVOOSHXbsgXmz09tTXz0UVjXqBEcckgIiMQyZAg0ruBPtZISWLw4hMiMGTBnDmzfDnl5IWQSQXLEEdCsWc0cozRsZjbf3fNLlSs4RPZu167wSz3emnj77dAVBdCvX/jlngiJww6DVq327Wvu3Amvvx6CZPr08DVLSqBlSzj22GSQDB0agkqkuik4FBxSSSUlsGJFsqspMS6xa1dY36VLsqsp8W/nztmv15dfwuzZyRbJ22+H8s6dw7hIIkg0EbRUFwWHgkPK8fHHqWMS8+aFX9IQ/rrPz08Nit69a8cVUGvXJsdHpk+HdetCef/+yUH244+HTp1yW0+puxQcCg4hBEJhYWpQrF0b1uXlwaGHJrubDj88DFJXNC5RW7iHFkiiNTJrVhiDMQvdZonWyNFHh0F6kcrISXCY2Tjg90AecL+73562fhJwFVAMbAWucPdlZtYU+BOQD5QAP3D32dFnZgPdgO3Rbk5y988qqoeCo2HauRMWLUodvH733eT6AQNSB6+HD68/v1R37w6hmGiN/Otf4f6RZs3gqKOSLZKRI0NgipSlxoPDzPKA5cDXgTXAPOACd18W26atu2+OXp8OfM/dx5nZVUC+u19mZl2BF4DD3b0kCo4fu3ulk0DBUf+VlMA776QOXi9aFH5ZAuy3X2pI5OdDx465rXNN+uoreOWVZJAsWhTK27eHsWOTLZKBA2tHN5zUDuUFRzYb4aOAle6+KqpAATAB2BMcidCItAISKTYYmBlt85mZbSK0Pt7MYn2ljnCHNWtSB68LC0PXDEDr1iEYrr02GRQ9ejTsX4itWsG4cWEB+OwzmDkzhMg//gHPPBPKe/ZMhsjxx8P+++euzlJ7ZTM4ugMfxd6vAUanbxS1Lq4DmgLHR8WLgNPN7DGgJzAy+jcRHFPMrBh4CrjVy2g2mdkVwBUAvXr1qo7jkRz54ovkoHWiNfHJJ2FdkyYwbBhcdFFy8Pqgg9T9sjddu8L554clcXd7YqB96lSYMiVsd8ghySA59lho0ya39ZbaIZtdVecA49z9O9H7i4DR7n51OdtfCJzs7peYWWPgv4GxwAdAE+Bed59qZt3dfa2ZtSEEx1/d/aGK6qKuqrpj+3ZYuDC1NbFiRXL9QQelDl4PGwbNm+euvvVRcXH4HiS6tf75zzBFSuPG4ebDxKW/o0eH4Jb6KxdjHF8Dfu7uJ0fvfwLg7r8uZ/tGwBfuXmpyBTN7DfhOfHwkKr+UMBZSZhglKDhqp+LicCVQ/AqnxYvDwC7AAQekjkuMHBn65KVm7dgBr72WDJLCwtBKad06TIeSaJEMGdKwuwPro1yMccwDBppZX2AtcD5wYVqlBrp74u/J8cCKqLwlIdS+MrOvA7ujq60aA+3dfYOZNQFOA6Zn8RikmrjDhx+mXuE0f34YtAVo2za0IK6/Ptma0ER/tUPz5mG84/jj4Ve/gs8/T96IOH06PPdc2G6//ZJXa514Yhgvkfopa8Hh7rvN7GrgJcLluA+4+1IzuwUodPdpwNVmdiJQBHwBXBJ9vCvwkpmVEELnoqi8WVTeJNrndOC+bB2DVN3GjaljEm++CevXh3VNm4Z7Cy67LNmaGDhQ02bUFR07wllnhQXCHwSJ8ZF//AMeeSSUH3hgsjVy3HHQoUPOqizVTDcAyj7btg0WLEgNilWrwjqzcBNd/M7rQw8N4SH1j3t41kiiNTJnTmhVNmoUrnRLtEiOPFJjU3WB7hxXcFSL3bth6dLUweslS8J4BYTuifjg9ciRoRtKGqZdu+CNN5ItktdfDz8rzZvDMcckWyTDh6vFWRspOBQcGXOH999PHbyePz9c+QRhoDoeEocfHh5IJFKezZth7txki2Tp0lDeqVMYQ0mMj/Trp4H22kDBoeDYq88+S7YiEv9u3BjWNW8exiXiQTFggP5zy75Zty55I+L06eHGTggz/MZvROzSJafVbLAUHAqOFFu3hnGJeGti9eqwrlGjcGll/PkShxyia/Ylu9xh+fJkiMyalZylePjw5PjIMcfs+7NOpHIUHA04OIqK4K23Ugevly0L8ztB+OsuHhIjRoRr9EVyaffu8MdNIkhefTWMmTRpEgbXEy2S/Py6MYNxXaTgaCDB4Q4rV5Z+CNGOHWF9p06lH0LUtWtu6yxSGdu2hbvYE1PH//vf4ee9bdswUWNifGTQIHWhVhcFRz0Njk8+SQ2JefPC3E4QpggfOTI1KPr21X8qqR82bAjdWYkWSeIS8AMOSLZGTjghvJeqUXDUg+DYvDlc1RQPio+iaSTz8sI4RHzwesgQNeGl4Vi1KnnZ78yZIVgABg9OtkbGjIF2pSY1kvIoOOpYcOzaFeZtig9ev/12aJpDuFwxPo/TYYeFx5yKSBi/W7w42RqZOzdcRp6XF/6/JFokRxyhm1ErouCoxcFRUhJmgI1Pz7FwYQgPCJcipj+EqHPn3NZZpC7ZuTM8BTExPvLmm+H/XcuWoRWSaJEMHaobEeMUHLUoOD7+ODUkCguTlx22ahWCIX6VU69eGpcQqU6bNoXpUBItknfeCeVduiRD5MQToXfv3NYz1xQcOQqOTZtCMMQvhf3447CuceMwb1N88Prgg/UQIpGatmZNaIkkxkjWrQvlAwYkg2Ts2HBVYkOi4KiB4NixIzzLOT54/e67yfUDB6YOXg8fHq58EpHawz2MJyZaI7Nnh8cSm4V7nBKtkaOOqv//fxUc1RwcxcUhFOKD14sWhZvtIDyrOX1cQtNKi9Q9RUXh/3eiNfKvf4WyZs3g6KOTLZIRI+pfb4GCYx+Cwz00ZeMhUVgY/gqB8Bzm/PzUoOjeXeMSIvXR1q3wyivJFsnixaG8ffswr1aiRVIf5nJTcFQhOP77v8MPyLx54UY7CNMdDB+eOnh90EG6EkOkofr003DfyIwZ4UFWH34Yynv2TL0Rcb/9clvPqlBwVCE4jjgiXO0UH7weNiw0UUVE0rnDe+8lWyMzZyZnchg6NBkkxx5bN+aDU3BUITiKijQjrIhUXXFxmFMrMT7yyivhnpLGjcMfpokgGTWqdv6uUXDUovs4RKRh2r4dXnst2SKZPz+0Ulq3Ds9lTwTJ4MG1Y3xEwaHgEJFa5vPPw0SNiRbJihWhfP/9U29E7NEjN/VTcCg4RKSW++CDZIjMmBGeygnhApxEiBx3XLiCqyYoOBQcIlKHuMOSJclurTlz4KuvwhWc+fnJIDnyyOxdsKPgUHCISB22axe88UYySN54Iwy+t2gRHqebCJJhw6rv9gAFh4JDROqRzZvDdPGJIFm6NJR36pR6I2K/flX/GgoOBYeI1GPr1iXHR6ZPh7VrQ/lnn4VZf6uivODQ8+FEROqBbt3gW98KizssXx5mvahqaFREwSEiUs+YhSuxDjooO/vXDEsiIpKRrAaHmY0zs3fNbKWZ3VjG+klm9paZLTSzf5rZ4Ki8qZlNidYtMrPjYp8ZGZWvNLO7zGrD/ZUiIg1H1oLDzPKAycApwGDggkQwxDzq7kPdfTjwW+B3Ufl3Adx9KPB14P+aWaKud0frB0bLuGwdg4iIlJbNFscoYKW7r3L3XUABMCG+gbtvjr1tBSQu8RoMzIy2+QzYBOSbWTegrbu/7uFysIeAM7J4DCIikiabwdEd+Cj2fk1UlsLMrjKz9wgtju9HxYuA082ssZn1BUYCPaPPr9nbPqP9XmFmhWZWuH79+n0+GBERCXI+OO7uk929P3ADcFNU/AAhFAqBO4HXgOIM93uvu+e7e36XbFyPJiLSQGXzcty1hFZCQo+orDwFhPEL3H03cG1ihZm9BiwHvoj2U9l9iohINctmi2MeMNDM+ppZU+B8YFp8AzMbGHs7HlgRlbc0s1bR668Du919mbuvAzab2RHR1VQXA/+bxWMQEZE0WWtxuPtuM7saeAnIAx5w96VmdgtQ6O7TgKvN7ESgiNCauCT6eFfgJTMrIbQoLort+nvAX4AWwAvRIiIiNURzVYmISJnKm6sq54PjIiJStyg4REQkIwoOERHJiIJDREQyouAQEZGMKDhERCQjCg4REcmIgkNERDKi4BARkYwoOEREJCMKDhERyYiCQ0REMqLgEBGRjCg4REQkIwoOERHJiIJDREQyouAQEZGMKDhERCQjCg4REcmIgkNERDKi4BARkYwoOEREJCMKDhERyYiCQ0REMqLgEBGRjCg4REQkIwoOERHJSFaDw8zGmdm7ZrbSzG4sY/0kM3vLzBaa2T/NbHBU3sTMHozWvW1mP4l9ZnXsM4XZrL+IiJTWOFs7NrM8YDLwdWANMM/Mprn7sthmj7r7PdH2pwO/A8YB5wLN3H2ombUElpnZY+6+OvrcWHffkK26i4hI+bLZ4hgFrHT3Ve6+CygAJsQ3cPfNsbetAE+sAlqZWWOgBbALiG8rIiI5ks3g6A58FHu/JipLYWZXmdl7wG+B70fFTwJfAeuAD4E73P3zaJ0DL5vZfDO7IluVFxGRsuV8cNzdJ7t7f+AG4KaoeBRQDBwA9AV+ZGb9onVHu/sI4BTgKjM7tqz9mtkVZlZoZoXr16/P7kGIiDQg2QyOtUDP2PseUVl5CoAzotcXAi+6e5G7fwa8CuQDuPva6N/PgGcIIVOKu9/r7vnunt+lS5d9OhAREUnKZnDMAwaaWV8zawqcD0yLb2BmA2NvxwMrotcfAsdH27QCjgDeMbNWZtYmVn4SsCSLxyAiImmydlWVu+82s6uBl4A84AF3X2pmtwCF7j4NuNrMTgSKgC+AS6KPTwammNlSwIAp7r446q56xswSdX/U3V/M1jGIiEhp5u5736qOy8/P98JC3fIhIpIJM5vv7vnp5TkfHBcRkbpFwSEiIhmpVHBEg9KNotcHmtnpZtYku1UTEZHaqLItjrlAczPrDrwMXAT8JVuVEhGR2quywWHuvg04C/iju58LDMletUREpLaqdHCY2deAicBzUVledqokIiK1WWWD44fAT4Bnonsx+gGzslctERGprSp1A6C7zwHmAESD5Bvc/fsVf0pEROqjyl5V9aiZtY2m+VhCeD7G9dmtmoiI1EaV7aoaHD074wzgBcKMtRdlrVYiIlJrVTY4mkT3bZwBTHP3IpIPXRIRkQakssHxJ2A14Sl9c82sN3oin4hIg1TZwfG7gLtiRR+Y2djsVElERGqzyg6OtzOz3yWeqGdm/5fQ+hARkQamsl1VDwBbgG9Gy2ZgSrYqJSIitVdlH+TU393Pjr3/hZktzEaFRESkdqtsi2O7mR2deGNmRwHbs1MlERGpzSrb4pgEPGRm7aL38ce8iohIA1LZq6oWAcPMrG30frOZ/RBYnM3KiYhI7ZPREwDdfXN0BznAdVmoj4iI1HL78uhYq7ZaiIhInbEvwaEpR0REGqAKxzjMbAtlB4QBLbJSIxERqdUqDA53b1NTFRERkbphX7qqRESkAVJwiIhIRhQcIiKSEQWHiIhkJKvBYWbjzOxdM1tpZjeWsX6Smb1lZgvN7J9mNjgqb2JmD0br3jazn1R2nyIikl1ZCw4zywMmA6cAg4ELEsEQ86i7D3X34cBvgd9F5ecCzdx9KDAS+A8z61PJfYqISBZls8UxCljp7qvcfRdQAEyIbxCbvgTCg6ES94w40MrMGhPuF9lFeAbIXvcpIiLZlc3g6A58FHu/JipLYWZXmdl7hBbH96PiJ4GvgHXAh8Ad7v55ZfcZ7feKxBML169fv6/HIiIikZwPjrv7ZHfvD9wA3BQVjwKKgQOAvsCPzKxfhvu9193z3T2/S5cu1VpnEZGGLJvBsRboGXvfIyorTwFwRvT6QuBFdy9y98+AV4H8KuxTRESqWTaDYx4w0Mz6mllT4HxgWnwDMxsYezseWBG9/hA4PtqmFXAE8E5l9ikiItlV2ScAZszdd5vZ1cBLQB7wgLsvNbNbgEJ3nwZcbWYnAkWkPlVwMjDFzJYSJlSc4u6LAcraZ7aOQURESjP3+j87en5+vhcWFua6GiIidYqZzXf3/PTynA+Oi4hI3aLgEBGRjCg4REQkIwoOERHJiIJDREQyouAQEZGMKDhERCQjCg4REcmIgkNERDKi4BARkYwoOEREJCMKDhERyYiCQ0REMqLgEBGRjCg4REQkIwoOERHJiIJDREQyouAQEZGMKDhERCQjCg4REcmIgqMi774L69eDe65rIiJSazTOdQVqtbPPhqVLoX17OPDAsAwcmPq6TZtc11JEpEYpOCryu9/BsmWwfDmsWAFz58Jf/5q6TbduqWGSWPr1g2bNclNvEZEsUnBU5KSTwhK3bRu8914yTJYvD8v//m/o1kpo1Ah6904Nk0TA9OoFeXk1eywiItVEwZGpli1h6NCwpNu0KTVMEstrr8GWLcntmjaFAQNKd30deCDstx+Y1dzxiIhkSMFRndq3h8MPD0ucO3z6aTJI4uHy/POwa1dy2zZtyu76Gjgw7F9EJMcUHDXBDPbfPyzHHpu6rrgYPvywdEvljTfg8cehpCS5bZcuZXd9DRgALVrU7DGJSINl3gAuNc3Pz/fCwsJcVyNzO3fCqlWpgZIImHXrUrft1avslkqfPtBYfx+ISObMbL6756eXZ/U3ipmNA34P5AH3u/vtaesnAVcBxcBW4Ap3X2ZmE4HrY5seCoxw94VmNhvoBmyP1p3k7p9l8zhyplkzOPjgsKTbsiUZIvHWymOPhbGWhMaNwxVeZbVUunfXeIqIZCxrLQ4zywOWA18H1gDzgAvcfVlsm7buvjl6fTrwPXcfl7afocBUd+8fvZ8N/NjdK92EqLMtjqpwhw0byh6kX7kStm9PbtuyZTJE0lsrnTrl7hhEpFbIRYtjFLDS3VdFFSgAJgB7giMRGpFWQFkpdgFQkMV61i9mYSykSxc48sjUdSUlsHZt6a6vhQvh6afDeEtCx45ld30NGACtW9fsMYlIrZLN4OgOfBR7vwYYnb6RmV0FXAc0BY4vYz/nEQInboqZFQNPAbd6Gc0mM7sCuAKgV69eVal//dOoEfTsGZYTTkhdV1QE779fuqUyaxY8/HDqtgccUHbXV79+4VJjEanXstlVdQ4wzt2/E72/CBjt7leXs/2FwMnufkmsbDRhbGRorKy7u681szaE4Piruz9UUV0aVFdVNmzbFrq5yhqk37AhuV2jRtC3b9ldXz17hvUiUmfkoqtqLdAz9r5HVFaeAuDutLLzgcfiBe6+Nvp3i5k9SugSqzA4ZB+1bAmHHhqWdJ9/XvYg/dy58NVXye2aNUve9JjeUunaVYP0InVINoNjHjDQzPoSAuN84ML4BmY20N1XRG/HAyti6xoB3wSOiZU1Btq7+wYzawKcBkzP4jHI3nTsCKNHhyXOPVwynN719fbb8Pe/h66xhLZty59Esl27mj0eEdmrrAWHu+82s6uBlwiX4z7g7kvN7Bag0N2nAVeb2YlAEfAFcElsF8cCHyUG1yPNgJei0MgjhMZ92ToG2QdmYSzkgANgzJjUdbt3h5se07u+XnstXE4c7z7t2rXsu+gHDIDmzWv2mEQE0A2AUtvs2FH2JJLLl4dpWxLMwk2PZbVUevfWTY8i1SAnNwCKZKx5cxgyJCzpNm8uHSYrVoSrvjbHruxu0gT69y/7cuJu3TSeIrKPFBxSd7RtCyNHhiXOPUxpX1Yr5eWXw9QtCa1alQ6UxPuOHWv2eETqKAWH1H1mYSyka1c4+ujUdSUlsGZN6bvoFyyAp55KvemxU6eyu74GDAiBIyKAxjikIdu1K9z0WNZ092vTrhzv3r3sQfq+fXXTo9RbGuMQSde0KRx0UFjSbd0abnpM7/p64olw70pCXl7ypsf0lkqPHrrpUeolBYdIWVq3huHDw5Ju48ayB+lnzw532Sc0bx6CpKxB+s6dNUgvdZaCQyRTnTqF5YgjUsvd4eOPS3d9LV0K06aF+1cS2rULATJoEAwblly6dKnZYxGpggY7xlFUVMSaNWvYsWNHjmolmWrevDk9evSgSZMmua5K5nbvhg8+KD1Iv3Rp6kO5unULU7vEw+Sgg3RfiuSExjjSrFmzhjZt2tCnTx9MXQa1nruzceNG1qxZQ9++fXNdncw1bhzuLenfH045JXXd+vWweDEsWpRcZs5MTsvSrFm4r2XYsNRQ0eXDkiMNNjh27Nih0KhDzIxOnTqxfv36XFel+nXpEqa5j091v2sXvPNOaqA89xxMmZLcpkeP1JbJoYeG8ZS8vJo/BmlQGmxwAAqNOqZBfb+aNk3OSPytbyXLP/kkGSSJUHnxxeT9KC1awCGHlA4UTRYp1ahBB4dInbP//mE5+eRk2c6dsGxZalfX00/D/fcnt+ndOzVMhg0LD97S5cJSBQqOHGrdujVbt27NdTWkrmvWDA47LCwJiSu84mGyaFGY0r6kJGzTujUMHZo6bjJ0KLRpk5vjkDpDwSFSH5mFu927d4dTT02Wb9tWunVSUAB/+lNym/79U7u5hg2DPi6WQHMAABKWSURBVH1034nsoeAA+OEPYeHC6t3n8OFw550Zf2zhwoVMmjSJbdu20b9/fx544AE6dOjAXXfdxT333EPjxo0ZPHgwBQUFzJkzhx/84AdA6P+fO3cubfTXolSkZUvIzw9Lgnt4Pkp83GTRInjmmeSzUdq2LX2Z8CGHhP1Jg6PgqGUuvvhi/vCHPzBmzBhuvvlmfvGLX3DnnXdy++238/7779OsWTM2bdoEwB133MHkyZM56qij2Lp1K831YCOpCrMwBtK7N5x+erJ861ZYsiR1MP6hh2DLluTnBg4sPXbSo4daJ/WcggOq1DLIhi+//JJNmzYxJnpi3iWXXMK5554LwKGHHsrEiRM544wzOOOMMwA46qijuO6665g4cSJnnXUWPXr0yFndpR5q3TrcHR+/Q76kBFavTu3qKiwMc3gldOhQunUyZIie2FiPKDjqiOeee465c+fy7LPPctttt/HWW29x4403Mn78eJ5//nmOOuooXnrpJQYNGpTrqkp91qhRuBqrXz8488xk+ebN8NZbqYFy//3Jubvy8sId8Ok3MerBWnWSgqMWadeuHR06dOCVV17hmGOO4eGHH2bMmDGUlJTw0UcfMXbsWI4++mgKCgrYunUrGzduZOjQoQwdOpR58+bxzjvvKDgkN9q2haOOCktCcXF4DHB83OTVV8Nz5RM6dy59z8ngwZqqvpZTcOTQtm3bUrqXrrvuOh588ME9g+P9+vVjypQpFBcX861vfYsvv/wSd+f73/8+7du352c/+xmzZs2iUaNGDBkyhFPSp7IQyaW8vORswOeckyz/4ovUMFm8GP74x/C8eQjTsxx8cOmxk65dc3McUkqDneTw7bff5uCDD85RjaSq9H2rp3bvDrMJx7u6Fi9OfaDW/vuXPQFkXZz0so7QJIciUnslWhkHHwznn58s37Ch9ASQv/99mMsLQpfWkCGlA6VTp9wcRwOh4BCR2qtzZzj++LAkFBWVngDyxRfhwQeT23TvXvomxgMP1ASQ1UTBISJ1S5MmYWqUoUNh4sRk+aeflp4A8uWXkw/Qat687Akg27fPzXHUYQoOEakf9tsPTjopLAk7d8Lbb6d2dU2dCn/+c3KbXr1KD8T3768JICug4BCR+qtZs9LPjncPT11MnwDyueeSE0C2bBlaNOmtE03pAyg4RKShMYMDDghL/BL27dtLTwD5+ONw773Jbfr1K30TY58+Da510rCOthYZO3YsL730UkrZnXfeyZVXXlnuZ4477jgSlxWfeuqpe+asivv5z3/OHXfcUeHXnjp1KsuWLdvz/uabb2b69OmZVL9Ms2fP5rTTTtvn/YjkRIsWMHIkXH55uHJr9mz4/PPwrPhp0+DWW8P6pUvhllvgrLNCl1b79nD00XDVVSFkXn8dvvoq10eTVVltcZjZOOD3QB5wv7vfnrZ+EnAVUAxsBa5w92VmNhG4PrbpocAId19oZiOBvwAtgOeBH3gdvBnlggsuoKCggJNjD+QpKCjgt7/9baU+//zzz1f5a0+dOpXTTjuNwYMHA3DLLbdUeV8i9ZpZGAPp1Qu+8Y1k+VdfpU4AuWgRPPxwuJEx8bkBA0qPnfTsWS+mWMlacJhZHjAZ+DqwBphnZtPcfVlss0fd/Z5o+9OB3wHj3P0R4JGofCgw1d0T857fDXwXeIMQHOOAF/alrrmYVf2cc87hpptuYteuXTRt2pTVq1fz8ccfc8wxx3DllVcyb948tm/fzjnnnMMvfvGLUp/v06cPhYWFdO7cmdtuu40HH3yQrl270rNnT0aOHAnAfffdx7333suuXbsYMGAADz/8MAsXLmTatGnMmTOHW2+9laeeeopf/vKXnHbaaZxzzjnMmDGDH//4x+zevZvDDz+cu+++m2bNmtGnTx8uueQSnn32WYqKinjiiScqPb3JY489xq9+9SvcnfHjx/Ob3/yG4uJivv3tb1NYWIiZcfnll3PttdeWOX28SK3TqhWMHh2WBPfSE0AuWABPPpncpn37sieAbNGixg9hX2SzxTEKWOnuqwDMrACYAOwJDnffHNu+FVBWy+ECoCDaRzegrbu/Hr1/CDiDfQyOXOjYsSOjRo3ihRdeYMKECRQUFPDNb34TM+O2226jY8eOFBcXc8IJJ7B48WIOPfTQMvczf/58CgoKWLhwIbt372bEiBF7guOss87iu9/9LgA33XQTf/7zn7nmmms4/fTT9wRF3I4dO7j00kuZMWMGBx54IBdffDF33303P/zhDwHo3LkzCxYs4I9//CN33HEH98cfTVqOjz/+mBtuuIH58+fToUMHTjrpJKZOnUrPnj1Zu3YtS5YsAdjT7VbW9PEidYIZ9O0blmgGayBMQ58+AeQDDyS7sxo1CnfApwfKAQfU2tZJNoOjO/BR7P0aYHT6RmZ2FXAd0BQ4Pn09cB4hcBL7XJO2z+5lfXEzuwK4AqBXr14VVjRXs6onuqsSwfHn6BLBxx9/nHvvvZfdu3ezbt06li1bVm5wvPLKK5x55pm0jB6oc3rseQpLlizhpptuYtOmTWzdujWlW6ws7777Ln379uXAAw8EwrTukydP3hMcZ511FgAjR47k6aefrtQxzps3j+OOO44uXboAMHHiRObOncvPfvYzVq1axTXXXMP48eM5KbqEsqzp40XqtDZt4Mgjw5JQUlJ6AsjXX4e//S25TadOpW9iHDw4XCmWYzm/qsrdJwOTzexC4CbgksQ6MxsNbHP3JVXY773AvRDmqqqm6larCRMmcO2117JgwQK2bdvGyJEjef/997njjjuYN28eHTp04NJLL2VHYvK3DF166aVMnTqVYcOG8Ze//IXZs2fvU32bRT+weXl57E7cVFVFHTp0YNGiRbz00kvcc889PP744zzwwANlTh/fuHHOf0xFqlejRuEhWAMHwtlnJ8s3bUqGSeLfu+9OnQBy0KDSlwnvv3/NVj+L+14L9Iy97xGVlaeA0O0Udz4Qm4OZtdF+KrvPWq1169aMHTuWyy+/nAsuuACAzZs306pVK9q1a8enn37KCy9U3At37LHHMnXqVLZv386WLVt49tln96zbsmUL3bp1o6ioiEceeWRPeZs2bdiSeIpbzEEHHcTq1atZuXIlwJ5p3ffFqFGjmDNnDhs2bKC4uJjHHnuMMWPGsGHDBkpKSjj77LO59dZbWbBgQcr08b/5zW/48ssv2bp16z59fZE6pX17OPZYuOYauO8+ePPN8CTGt98Oz4a//vowUD97Nvznf8LJJ4dnmiRufrz+evjrX0PXWFFR1qqZzT/l5gEDzawv4Zf7+cCF8Q3MbKC7r4jejgdWxNY1Ar4JHJMoc/d1ZrbZzI4gDI5fDPwhi8eQdRdccAFnnnnmnkHgYcOGcdhhhzFo0CB69uzJUfHnG5RhxIgRnHfeeQwbNoyuXbty+OGH71n3y1/+ktGjR9OlSxdGjx69JyzOP/98vvvd73LXXXfxZGzgrnnz5kyZMoVzzz13z+D4pEmTMjqeGTNmpEwV/8QTT3D77bczduzYPYPjEyZMYNGiRVx22WWURDdc/frXvy53+niRBi0vL7QyBg2C885Llm/cWHoCyLvuSk4A2aRJ6NqaNSs8lbEaZXVadTM7FbiTcDnuA+5+m5ndAhS6+zQz+z1wIlAEfAFc7e5Lo88eB9zu7kek7TOf5OW4LwDX7O1yXE2rXn/o+yZSgaIiWL48GSQrVsBTT1V5kL28adX1PA6pU/R9E6k55QWH7hwXEZGMNOjgaAitrfpE3y+R2qHBBkfz5s3ZuHGjfhnVEe7Oxo0bad68ea6rItLgNdgL5Hv06MGaNWtYv359rqsildS8efOUK7ZEJDcabHA0adKEvn375roaIiJ1ToPtqhIRkapRcIiISEYUHCIikpEGcQOgma0HPqjixzsDG6qxOtVF9cqM6pUZ1Ssz9bVevd29S3phgwiOfWFmhWXdOZlrqldmVK/MqF6ZaWj1UleViIhkRMEhIiIZUXDs3b25rkA5VK/MqF6ZUb0y06DqpTEOERHJiFocIiKSEQWHiIhkpEEHh5mNM7N3zWylmd1YxvpmZva3aP0bZtYntu4nUfm7ZnZyDdbpOjNbZmaLzWyGmfWOrSs2s4XRMq266pRB3S41s/WxOnwntu4SM1sRLZfUcL3+X6xOy81sU2xdVs6ZmT1gZp+Z2ZJy1puZ3RXVebGZjYity+a52lu9Jkb1ecvMXjOzYbF1q6PyhWZWWNbns1iv48zsy9j36ubYugq//1mu1/WxOi2Jfp46Ruuyeb56mtms6HfBUjP7QRnbZO9nzN0b5EJ4nO17QD+gKbAIGJy2zfeAe6LX5wN/i14PjrZvBvSN9pNXQ3UaC7SMXl+ZqFP0fmuOz9elwP+U8dmOwKro3w7R6w41Va+07a8hPMY4q+cMOBYYASwpZ/2phEcfG3AE8Ea2z1Ul63Vk4usBpyTqFb1fDXTO0fk6Dvj7vn7/q7teadt+A5hZQ+erGzAiet0GWF7G/8es/Yw15BbHKGClu69y911AATAhbZsJwIPR6yeBE8zMovICd9/p7u8DK6P9Zb1O7j7L3bdFb18Hamqe8cqcr/KcDPzD3T939y+AfwDjclSvC4DHqulrl8vd5wKfV7DJBOAhD14H2ptZN7J7rvZaL3d/Lfq6UIM/X5U4X+XZl5/L6q5XjfxsAbj7OndfEL3eArwNdE/bLGs/Yw05OLoDH8Xer6H0id+zjbvvBr4EOlXys9mqU9y3CX9RJDQ3s0Ize93MzqiG+lSlbmdHzeInzaxnhp/NZr2IuvX6AjNjxdk8ZxUpr97ZPFeZSv/5cuBlM5tvZlfkoD5fM7NFZvaCmQ2JymrF+TKzloRfvk/FimvkfFnoQj8MeCNtVdZ+xhrs8zjqOjP7FpAPjIkV93b3tWbWD5hpZm+5+3s1WK1ngcfcfaeZ/QehtXZ8DX79vTkfeNLdi2NluT5ntZKZjSUEx9Gx4qOjc9UV+IeZvRP9RV4TFhC+V1vN7FRgKjCwhr52ZXwDeNXd462TrJ8vM2tNCKsfuvvm6tx3RRpyi2Mt0DP2vkdUVuY2ZtYYaAdsrORns1UnzOxE4KfA6e6+M1Hu7mujf1cBswl/hVSXvdbN3TfG6nM/MLKyn81mvWLOJ60rIcvnrCLl1Tub56pSzOxQwvdvgrtvTJTHztVnwDNUT/dspbj7ZnffGr1+HmhiZp2pBecrUtHPVlbOl5k1IYTGI+7+dBmbZO9nLBsDN3VhIbS2VhG6LhKDakPStrmK1MHxx6PXQ0gdHF9F9QyOV6ZOhxEGAwemlXcAmkWvOwMrqN5BwsrUrVvs9ZnA654cjHs/qmOH6HXHmqpXtN0gwmCl1eA560P5g73jSR24fDPb56qS9epFGLM7Mq28FdAm9vo1YFwN1mv/xPeO8Av4w+jcVer7n616RevbEcZBWtXU+YqO/SHgzgq2ydrPWLWd3Lq4EK46WE74RfzTqOwWwl/yAM2BJ6L/SG8C/WKf/Wn0uXeBU2qwTtOBT4GF0TItKj8SeCv6j/MW8O0cnK9fA0ujOswCBsU+e3l0HlcCl9VkvaL3PwduT/tc1s4Z4a/PdUARoQ/528AkYFK03oDJUZ3fAvJr6FztrV73A1/Efr4Ko/J+0XlaFH2Pf1rD9bo69rP1OrFgK+v7X1P1ira5lHCxTPxz2T5fRxPGUBbHvlen1tTPmKYcERGRjDTkMQ4REakCBYeIiGREwSEiIhlRcIiISEYUHCIikhEFh0gVpc2su7A6Z2Y1sz7lzcgqkmuackSk6ra7+/BcV0KkpqnFIVLNoucw/DZ6FsObZjYgKu9jZjMt+SyVXlH5fmb2TDSB3yIzOzLaVZ6Z3Rc9b+FlM2sRbf99Sz6TpSBHhykNmIJDpOpapHVVnRdb96W7DwX+B7gzKvsD8KC7Hwo8AtwVld8FzHH3YYRnPyyNygcCk919CLAJODsqvxE4LNrPpGwdnEh5dOe4SBWZ2VZ3b11G+WrgeHdfFU1E94m7dzKzDYT5vIqi8nXu3tnM1gM9PDZhZTRV9j/cfWD0/gagibvfamYvAlsJM8RO9WjyP5GaohaHSHZ4Oa8zsTP2upjkmOR4whxEI4B50czNIjVGwSGSHefF/v1X9Po1wizLABOBV6LXMwiPAcbM8sysXXk7NbNGQE93nwXcQJiZtVSrRySb9JeKSNW1MLOFsfcvunviktwOZraY0Gq4ICq7BphiZtcD64HLovIfAPea2bcJLYsrCTOyliUP+GsULgbc5e6bqu2IRCpBYxwi1Swa48h39w25rotINqirSkREMqIWh4iIZEQtDhERyYiCQ0REMqLgEBGRjCg4REQkIwoOERHJyP8H532Dya97fUMAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"needs_background":"light"}},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAZQAAAEWCAYAAABBvWFzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVwV5f7A8c/XFXdFLc29wlxSUXEHxGzT283KFm21bLOysm6l2S2v1W2zfddKW0wzu5rtloloKomKChKJO66ouYuIfH9/zODvhKio5zAH+L5fL17MeeaZme8MB748z8x5HlFVjDHGmNNVyusAjDHGFA+WUIwxxviFJRRjjDF+YQnFGGOMX1hCMcYY4xeWUIwxxviFJRRTZIjIDyJyi7/reklE1ojIhQHYr4rIue7yeyLy74LUPYXj3CAi0081TlO8iH0OxQSSiOz1eVkROAgcdl/fparjCz+q4CEia4DbVfUXP+9XgTBVTfNXXRFpDKwGyqpqtj/iPM6xYoBfgf0+xTNV9Z8icj7wMtAeqKmqEshYTMGV8ToAU7ypauXc5eP98RSRMoH+I2WKnI2qWj+f8kPAJOAdYGrhhmSOx7q8jCdEJEZE0kXkMRHZDIwVkRoi8q2IZIjIX+5yfZ9tYkXkdnd5gIjMEZFRbt3VItLrFOs2EZE4EdkjIr+IyNsi8tkx4i5IjE+LyG/u/qaLSC2f9TeJyFoR2S4iw49zfTqJyGYRKe1TdqWILHWXO4rIPBHZKSKbROQtESl3jH2NE5FnfF4/4m6zUURuy1P3HyKyWER2i8h6ERnhszrO/b5TRPaKSJfca+uzfVcRWSAiu9zvXQt6bQpKVVNV9UMg+WS3NYFlCcV4qQ4QCjQC7sR5P451XzcEDgBvHWf7TkAqUAt4EfhQRI7V/XG8up8DvwM1gRHATcc5ZkFivB64FTgDKAf8C0BEWgDvuvs/yz1efv+Bo6rxwD7ggjz7/dxdPgwMcc+nC9ATuOc4cePGcKkbz0VAGJD3/s0+4GagOvAPYJCIXOGui3a/V1fVyqo6L8++Q4HvgDfcc3sF+E5EauY5h6OujSkeLKEYL+UAT6nqQVU9oKrbVfUrVd2vqnuAZ4Hux9l+raqOUdXDwMdAXeDMk6krIg2BDsCTqpqlqnOAacc6YAFjHKuqf6rqAZyumXC3/GrgW1WNU9WDwL/da3AsE4D+ACJSBejtlqGqC1V1vqpmq+oa4P184sjPtW58Saq6DyeB+p5frKouU9UcVV3qHq8g+wUnAa1Q1U/duCYAfwD/9KlzrGuTn7PcFlju17UFjMN4xO6hGC9lqGpm7gsRqQi8ClwK1HCLq4hIaTcR5LU5d0FV97sNjsr51Dte3VrADlX1vfm7HmiQ304KGONmn032+8R0lrvv3Dj2icj2Y8QLTmtkrogMAq4CFqnqWjeOpjgtgAichx3KAAuPs69cZ+WptzbP+XUCngfOx2lBlAe+LMB+c/e9Nk/ZWqCez+tjXZv8HOseiglS1kIxXsr7iOHDwHlAJ1Wtyv93sQTyKZ5NQKibKHLlm0xcpxPjJt99u8eseazKqroc5w9yL/7e3QVO19kfOE9nVQUeP5UYcLrtfH2O00JroKrVgPd89nuiR0I34nQF+moIbChAXKYYsIRigkkVnHsSO93++KcCfUD3P/4EYISIlBORLvy9i8afMU4GLhORSPcG+khO/Dv4OfAATuLybSlUAXYDe0WkGTCogDFMAgaISAs3oeWNvwpOiy1TRDriJLJcGThddGcfY9/fA01F5HoRKSMi1wEtgG8LGFuBiCMEpwWFiISISHl/HsOcGksoJpi8BlQAtgHzgR8L6bg34NzY3g48A3yB83mZ/JxyjKqaDNyLkyQ2AX8B6SfYLPcexq+qus2n/F84f+z3AGPcmAsSww/uOfwKpLnffd0DjBSRPcCTOAkod9v9OPeMfnPvaXTOs+/twGU4rbjtwKPAZXni9odGOEk99ymvAzgPXBiP2QcbjclDRL4A/lDVgLeQjClOrIViSjwR6SAi54hIKfex2j7YB+aMOWn2lJcxzudh/odzgzwdGKSqi70NyZiix7q8jDHG+IV1eRljjPGLEt3lVatWLW3cuLHXYRhjTJGycOHCbapaO295iU4ojRs3JiEhweswjDGmSBGRvCMiANblZYwxxk8CmlBE5FIRSRWRNBEZms/6hiIy0x0ue6mI9HbLO4pIovu1RESu9NlmiIgki0iSiExwPzGbO0T3ap/tjjfonDHGGD8LWEJx53F4G2ccohZAf3f4bl9PAJNUtS3QD2fCHIAkIEJVw3EG4XvfHcqhHnC/u+58oLS7Xa5HVDXc/UoM1LkZY4w5WiDvoXQE0lR1FYCITMT5wNhynzoKVHWXq+EMLkeekV9D+PugdGWACiJyCGeU1Y3+DPrQoUOkp6eTmZl54som4EJCQqhfvz5ly5b1OhRjzAkEMqHUw2eobpwPjHXKU2cEMF1EBgOV8Jnsxx1G+yOccXtucqeH3SAio4B1OOP3TFfV6T77e1ZEngRmAEPdOSdOSnp6OlWqVKFx48Yce64mUxhUle3bt5Oenk6TJk28DscYcwJe35TvD4xz5zzoDXwqIqXAmbFOVVviTH40zB1RtAZOK6cJztwLlUTkRndfw4Bmbv1Q4LH8Digid4pIgogkZGRkHLU+MzOTmjVrWjIJAiJCzZo1rbVoTBERyISygb/Pu1Cfo+dFGIg7mqk7nWgIzoRHR6hqCrAXZ8KfC4HVqpqhqodwhsvo6tbbpI6DOFO0dswvKFUdraoRqhpRu/ZRj1EDWDIJIvazMKboCGRCWQCEiUgTd+6Hfhw9teo6nLmwEZHmOAklw92mjFveCKflscat31lEKrrzgfcEUtx6dd3vAlyBc2PfGGOMjy1JGTzYdha71u3y+74DllDcex73AT/h/NGfpKrJIjJSRC53qz0M3CEiS3DmfRigzuBikcASEUkEpgD3qOo2VY3HmaRoEbDMjX+0u6/xIrLMLa+FM69FkVS6dGnCw8OPfK1Zs4bt27fTo0cPKleuzH333ed1iMaYIiZzZybPXxpLWKvyvJ3Ylbgx/p9CpkQPDhkREaF5PymfkpJC8+bNPYrIUblyZfbu3fu3sn379rF48WKSkpJISkrirbfe8ii6whcMPxNjiirNUSb/az6PvlmfNdkNuLxOPC+NO4Oml5z6gy4islBVI/KWe31T3hRQpUqViIyMJCQkxOtQjDFFRMIny4musZRrX+1ClTIH+OXFRXy9qdNpJZPjKdFjeZ3Qgw9Cop8/HxkeDq+9dtwqBw4cIDzc+aB/kyZNmDJlin9jMMYUaxsSNvH4dSv5ZFUkZ0gGo2+K47YPulG6XOmAHtcSShCqUKECif5OZMaYYm//tv2MuvZ3XpjZgWw68FinWB6f3I6q9aML5fiWUI7nBC0JY4wJBjnZOUy4fx5DRzch/XAMV9ebxwuf1ePsmJhCjcMSijHGFGHzRi/jwYeE3/d1o12FFMa/kEH04C6exGIJpQhp3Lgxu3fvJisri6lTpzJ9+nRatMg73qYxpiRY+1s6Q69fx8R1XalbajPjbp/DTe92pVQZ7561soQShPI+MpxrzZo1hRuIMSbo7Nm4h+evWcgrczsBNfl3VCyPToqgcp1Ir0Ozx4aNMaYoyMnOYexts2naYD//nRvDVY0WkTp3ByPjYqhcp7LX4QHWQjHGmKA36/VEhgwrz+IDUXSuvIwpr2bQ+fZuXod1FEsoxhgTpFb+upZHb9rE/zZ2pkHpDXx+31z6vd4FKRWcg6ZaQjHGmCCza90unum7mDcSulCWmjzdM5aHJ3WiQmg9r0M7LruHYowxQSI7M5v3ro8jrHEWLydEc/25v/Pnwr088UsMFUIreB3eCVlCMcaYIPDz8wtpW301gyZE07zqBhI+S2XsiijOalfH69AKzBJKkOnRowc//fTT38pee+01Bg0adMxtYmJiyB01uXfv3uzcufOoOiNGjGDUqFHHPfbUqVNZvnz5kddPPvkkv/zyy8mEn6/Y2FiqVat2ZDj+Cy90ZnqOi4ujXbt2lClThsmTJ5/2cYwpiv74fhWXnfk7Fw9rz77D5Zn8r3nE7mhDuxuK3gjbllCCTP/+/Zk4ceLfyiZOnEj//v0LtP33339P9erVT+nYeRPKyJEjj/zxP11RUVEkJiaSmJh4JEk1bNiQcePGcf311/vlGMYUJTtW/sUD4bNo9Y8GxG1txou9Y0nZfiZ9Xwrem+4nYgklyFx99dV89913ZGVlAc6HGTdu3EhUVBSDBg0iIiKCli1b8tRTT+W7fePGjdm2bRsAzz77LE2bNiUyMpLU1P+fTGfMmDF06NCBNm3a0LdvX/bv38/cuXOZNm0ajzzyCOHh4axcuZIBAwYcaTnMmDGDtm3b0qpVK2677TYOHjx45HhPPfUU7dq1o1WrVvzxxx8FPtfGjRvTunVrSpWyt6EpOQ7tP8QbfWdxbhi8tSSS25rNY8WygzzyXQzlq5b3OrzTYk95HYcXo9eHhobSsWNHfvjhB/r06cPEiRO59tprERGeffZZQkNDOXz4MD179mTp0qW0bt063/0sXLiQiRMnkpiYSHZ2Nu3ataN9+/YAXHXVVdxxxx0APPHEE3z44YcMHjyYyy+/nMsuu4yrr776b/vKzMxkwIABzJgxg6ZNm3LzzTfz7rvv8uCDDwJQq1YtFi1axDvvvMOoUaP44IMPjopn9uzZR4bkv+aaaxg+fPhJXztjijLNUb4fmcDDz9UkNas7F4Yu5JXRVWjVt3BGAi4M9q9hEPLt9vLt7po0aRLt2rWjbdu2JCcn/617Kq/Zs2dz5ZVXUrFiRapWrcrll19+ZF1SUhJRUVG0atWK8ePHk5ycfNx4UlNTadKkCU2bNgXglltuIS4u7sj6q666CoD27dsfc3gY3y4vSyampEmasoJLai/isv90QFX45t+/Mz2jHa36NvU6NL+yFspxeDV6fZ8+fRgyZAiLFi1i//79tG/fntWrVzNq1CgWLFhAjRo1GDBgAJmZmae0/wEDBjB16lTatGnDuHHjiI2NPa14y5d3mumlS5cmOzv7tPZlTHGSkbKNJ69ezujl3agqe3jtylkM+qQL5SoHZsZEr1kLJQhVrlyZHj16cNtttx1pnezevZtKlSpRrVo1tmzZwg8//HDcfURHRzN16lQOHDjAnj17+Oabb46s27NnD3Xr1uXQoUOMHz/+SHmVKlXYs2fPUfs677zzWLNmDWlpaQB8+umndO/e3R+nakyxdHD3QUZdFsu5LcoyZnlX7m09h7TUHB74X3fKVS7ndXgBYwklSPXv358lS5YcSSht2rShbdu2NGvWjOuvv55u3Y4/jk+7du247rrraNOmDb169aJDhw5H1j399NN06tSJbt260axZsyPl/fr146WXXqJt27asXLnySHlISAhjx47lmmuuoVWrVpQqVYq77777tM9xwYIF1K9fny+//JK77rqLli1bnvY+jfGS5ihTHptPy5qbeeS7GCJr/8myb9byxpLu1AwL9Tq8gBNV9ToGz0RERGju5zdypaSk0Lx50Xv+uzizn4kpChZP+IMhgzKZtSucFuXTeOXfO7lkeITXYQWEiCxU1aNOzu6hGGPMadiUuIXh1/7JuBXdCJW/eKdfHHeM7UqZkJL359W6vIwx5hQc2HGA/14cS1jbSny2ohMPtY8jbXUZBk2ILpHJBCyh5KskdwMGG/tZmGCjOcrE++fS7IztDP85hovqJrH8l02MSoiheqNqXofnKUsoeYSEhLB9+3b7QxYEVJXt27cTEhLidSjGAPD72GQiqy+j/5tdCS23l19fXsyUjZ05t2cjr0MLCiWzXXYc9evXJz09nYyMDK9DMTgJvn79+l6HYUq49AWbGHbdSj5bHcmZpbbywS2zGTC6K6XLlfY6tKBiCSWPsmXL0qRJ8fzQkTHm5Ozbuo8Xr1nAS3EdySGCYV1iGTa5PVXOivI6tKBkXV7GGJNHTnYOn9w5h6Z1dzMyLobLGyTyx+xt/HduDFXOquJ1eEHLWijGGONjzjtLGfJIGRL2R9KhUjKTXsqg26CuXodVJFhCMcYYYHXceh67IZ0v07tQr9QmPr37N65/swulylhHTkFZQjHGlGi703fz3DWLeHV+Z0pRk6e6x/LIpA5UOuP4wxuZowU09YrIpSKSKiJpIjI0n/UNRWSmiCwWkaUi0tst7ygiie7XEhG50mebISKSLCJJIjJBRELc8iYiEu8e6wsRKb4jsBljTtvhrMN8cMtsmjbK5Pn5MVx7dgJ//r6LEbExVDqjktfhFUkBSygiUhp4G+gFtAD6i0iLPNWeACapalugH/COW54ERKhqOHAp8L6IlBGResD97rrzgdLudgAvAK+q6rnAX8DAQJ2bMaZo+/XlxbSvlsYdn0RxTqXNxH+UzCcrI6nfoa7XoRVpgWyhdATSVHWVqmYBE4E+eeooUNVdrgZsBFDV/aqaO7FGiFsvVxmggoiUASoCG0VEgAuAyW6dj4Er/Hw+xpgibsXPa7iibjw9/9WWnYcq8cWDc5mzsxUdb7WRrv0hkAmlHrDe53W6W+ZrBHCjiKQD3wODc1eISCcRSQaWAXeraraqbgBGAeuATcAuVZ0O1AR2+iSh/I6Vu987RSRBRBLsw4vGlAw71+7i4YhYWl58FjM2t+C/F8eSsrUm177aFSklXodXbHj9+EJ/YJyq1gd6A5+KSCkAVY1X1ZZAB2CYiISISA2cVk4T4CygkojceDIHVNXRqhqhqhG1a9f268kYY4JLdmY27/SbxblNsnl1YTQ3N41nxZIDDPsphgqhFbwOr9gJZELZADTweV3fLfM1EJgEoKrzcLq3avlWUNUUYC9wPnAhsFpVM1T1EPA/oCuwHajudoMd61jGmBLkx2cSaFN9Dfd+0Z1W1daxaOIKPkiNok7rM7wOrdgKZEJZAIS5T1+Vw7l5Pi1PnXVATwARaY6TUDLcbcq45Y2AZsAat35nEano3jfpCaSoM5LjTOBqd7+3AF8H8NyMMUEq5duV9D5jAb3+HcHBw2WZMjSeX7eHE37deV6HVuwF7HMoqpotIvcBP+E8jfWRqiaLyEggQVWnAQ8DY0RkCM6N9wGqqiISCQwVkUNADnCPqm4DtonIZGARkA0sBka7h3wMmCgiz7jlHwbq3IwxwWf7ih2M6LuMd5d1ozK1GHVZLPeN70L5qjYScGGxKYDzTAFsjClasvZm8faNcxk5LZzdWoW7Wv7Gf75sQe3mtU68sTklx5oC2Oub8sYYc0o0R5k2PJ7zQzfw0NcxdAxNY8n/VvFOUrQlE4/Y0CvGmCJn6eQ/eejOvcz4qxPnlVvFdyOceyb2CLC3LKEYY4qMLUkZ/PuaFD78oxvVZRdv9J3F3Z90pWzFs70OzWBdXsaYIiBzZyYv9IolrFV5xv7RhfvbzmHFCmHw5O6UrVjW6/CMyxKKMSZoaY4y+eF5tKi9laE/xhBzZgpJ36/n1UXdCT2nhtfhmTysy8sYE5QWfpbCkHuzmL27C+eXX8HPzy7iwkc7eR2WOQ5LKMaYoLJx0WYevzaNT1Z2pZZs573r4xj4YVfKhNifq2BnXV7GmKCwf9t+nu4ZS1j7KkxY2YFHOsaxYk057hofbcmkiLCEYozxVE52DuPv+Y3z6uzkyV9j6FVvKSkzt/BCfAzVGlbzOjxzEiztG2M8M/+DJB58EOL3daNthRQ+e24r3R/o4nVY5hRZQjHGFLp18zYwtP8aJqztRp1SWxg7cA43v9eVUmWs06Qos4RijCk0ezfv5YVrEhg1pxMQyhORsTz2ZQSV60R6HZrxA/t3wBgTcDnZOYy9bTZN6+3lmTkxXNloEalzd/D07Bgq16nsdXjGT6yFYowJqLg3lzDksXIsOhBFp0pJfPVKBl3u7OZ1WCYALKEYYwJiVew6Hr1xA19t6EKD0hsYf89v9Hu9i90nKcYsoRhj/GrXul08e/ViXl/QhTLUZOQFsTz8RUcq1qrndWgmwOxfBWOMX2RnZvP+DXGENc7ipQUx9D9nASsW7uHfM2KoWKui1+GZQmAtFGPMafvlxUU89FRllmVGE1l1Cd+/mUHEzfbkVkljCcUYc8pSf1jFvwZs49utHWlSZh1fPjSPvi91tomuSihLKMaYk7Zj5V+MvHopbyd2pQK1eKFXLPd/3pmQ6g29Ds14yO6hGGMK7ND+Q7x59SzCwpQ3EyO5rdk8Viw7yKPfxxBSPcTr8IzHLKEYY05Ic5Tv/7OA1jXWcf9X3WlbfQ2Lv1zJ+ynRnHl+ba/DM0HCuryMMceV/HUaDw3cyfTtHQgru5qvH4/nn093tPsk5iiWUIwx+cpI2cZT1yzn/eRuVJU9vHrFLO75tAvlKjfxOjQTpKzLyxjzNwd3H2TUZbGEtSjL6OSu3NNqDmmpOTw4pTvlKpfzOjwTxCyhGGMA5z7J1GHxtKy5mUe+i6Fr7T9Z+vUa3lzanZphoV6HZ4oA6/IyxpD4RSpD7t5P7M5OtCifxg9PJXDpEx28DssUMZZQjCnBNi/dyhPXpPLRn90Ilb94+7pZ3Dmum83hbk6JdXkZUwId2HGA/14cS1ibCnzyZyeGtI8jbXUZ7pnY3ZKJOWWWUIwpQTRH+eKBuTQ/YzvDf46hZ53lJE/fyMsJMVRvVM3r8EwRZwnFmBJiwcfLiaq+jH5vdKV62X3MGLWYqZs6EXZRY69DM8WEtW2NKebSF2xi2HUr+Wx1JGdIBmNuns2tY7pSulxpr0MzxUxAWygicqmIpIpImogMzWd9QxGZKSKLRWSpiPR2yzuKSKL7tURErnTLz/MpTxSR3SLyoLtuhIhs8FnXO5DnZkyw27d1HyNiYmnasRpfro5gaOdYVqwrz+0fR1kyMQERsBaKiJQG3gYuAtKBBSIyTVWX+1R7Apikqu+KSAvge6AxkAREqGq2iNQFlojIN6qaCoT77H8DMMVnf6+q6qhAnZMxRUFOdg7j75vLsDHnsCEnhmvqz+OF8fVpEh3jdWimmAtkC6UjkKaqq1Q1C5gI9MlTR4Gq7nI1YCOAqu5X1Wy3PMStl1dPYKWqrvV75MYUUb+9u5TO1VO4+f1I6ob8xey3lzJpfReaRDfwOjRTAgQyodQD1vu8TnfLfI0AbhSRdJzWyeDcFSLSSUSSgWXA3T4JJlc/YEKesvvcrrOPRKRGfkGJyJ0ikiAiCRkZGSd9UsYEozVz0rmu4Vwi72nNhgOhfHzHHOJ3NSPyntZeh2ZKEK+f8uoPjFPV+kBv4FMRKQWgqvGq2hLoAAwTkSOTLYhIOeBy4Euffb0LnIPTJbYJeDm/A6rqaFWNUNWI2rVt2G1TtO3ZuIfHu8bSLKoW36wP58noWP7cVJWbR0dSqozXv96mpAnkO24D4NvOru+W+RoITAJQ1Xk43Vu1fCuoagqwFzjfp7gXsEhVt/jU26Kqh1U1BxiD0+VmTLF0OOswHw6YTViDAzw3L4ZrmiSQOn8n/5kVQ6UzKnkdnimhAplQFgBhItLEbVH0A6blqbMO514IItIcJ6FkuNuUccsbAc2ANT7b9SdPd5d78z7XlTg39o0pdma+spiI6iu4/eMozq64hfiPkvl0VSQNOp3ldWimhAvYU17uE1r3AT8BpYGPVDVZREYCCao6DXgYGCMiQ3BuvA9QVRWRSGCoiBwCcoB7VHUbgIhUwnly7K48h3xRRMLd/azJZ70xRVrajLU8ctMmpm7qTMPS6Uy8fy7XvtrFJroyQUNU83uAqmSIiIjQhIQEr8Mw5rh2rt3FM30X88bCrpQji8cvSmDIxE5UCK3gdWimhBKRhaoakbfc7toZE6SyM7N5t38cYU0O8crCaG4Ki2fF4n08Pj3GkokJSpZQjAlCPz2bQHj11dwzMZqWVdNZ+PmffPhnFHXDz/Q6NGOOycbyMiaI7Nm4h3sil/DZ6kjOLrOW/z06nyue62T3SUyRYAnFmCCx8LMU+t1WgVWHuvBkdCyPf9OF8lUbeR2WMQVmXV7GeCwnO4dX+sTS5aZzyMwpx8w3kvjPrBjKVy3vdWjGnBRroRjjoa3JGQzosYYfMmLoUyeeD+PCqBnWxuuwjDkl1kIxxiMzXlpEm9Y5/JrRireumcWUDR2pGRbqdVjGnDJLKMYUskP7DzG8WywXPRpO9TJ7if9iLfdO6m433k2RZ11exhSiNXPSub7XX8zbG8PAprN5fXY7G3vLFBvWQjGmkEx+eB7hUVVI2tuICYPn8kFqlCUTU6xYC8WYANu/bT9DohIY/Uc0HSslMeHbqpwd09XrsIzxO0soxgRQ0pQV9OsPyQejebRjLE/P6Eq5yuW8DsuYgDjlLi8RaebPQIwpTjRHef+GODpcVZ+MrOr89N+FvBAfY8nEFGun00KZDjT0VyDGFBd/rd7JHVEpfLUhmotrJvBJbCPOPL+912EZE3DHTSgi8saxVgHV/R+OMUXb3PeX0f/eGmw8HMELvWL517Rom4rXlBgnaqHcijMJ1sF81vX3fzjGFE2Hsw7z/D9m89QvkTQss5E5H6TSaWCM12EZU6hOlFAWAEmqOjfvChEZEZCIjCliNi7azI09NzFzZwz9Gs7lvdktqdbQeoNNyXOitvjVQGJ+K1S1if/DMaZo+W7EAtpElCF+Z1M+HDCbz1d3oVrDal6HZYwnTtRCqayqOwolEmOKkIO7DzKsxzxeXRRD65BUJn5ZhuaXRXkdljGeOlELZWrugoh8FeBYjCkSVvy8hq51VvHqohjubTWL+E2NaH7ZOV6HZYznTpRQfEerOzuQgRhTFHx69xzaXVyT1Zl1mDI0nreWdiekeojXYRkTFE7U5aXHWDamRNmzcQ/3Ri3h01WRRFVdwvjptWnQqZPXYRkTVE6UUNqIyG6clkoFdxn3tapq1YBGZ0wQWDQ+hX63hrDyUBee6h7LEz9GUibERi0yJq/j/laoaunCCsSYYKM5yut943h0ahfOKLWdX19bRvcHYrwOy5igZf9mGZOPjJRtDOi+mu8zuvPPM+MZOzuMmmHhXodlTFCzMSGMyWPmK4tpc342v2S05o2+s/h6o03Na0xBWEIxxpWdmZ7p06cAABfpSURBVM0TkbH0fLgNVcvsJ37iGgZPtql5jSko6/IyBlj7WzrX99rB3D0x3Bo2mzfn2NS8xpwsa6GYEu+rR+YRHlWZZXsa8/l9c/noT5ua15hTYS0UU2Id2HGAIZELeD8lmg6VkpkwrTLnXGBT8xpzqiyhmBIp+es0+l2nJB2M5pEOsTzzq03Na8zpCmiXl4hcKiKpIpImIkPzWd9QRGaKyGIRWSoivd3yjiKS6H4tEZEr3fLzfMoTRWS3iDzorgsVkZ9FZIX7vUYgz80UTZqjjL4xjg5XnMXWrOr8+EwCL/5uU/Ma4w8BSygiUhp4G+gFtAD6i0iLPNWeACapalugH/COW54ERKhqOHAp8L6IlFHVVFUNd8vbA/uBKe42Q4EZqhoGzHBfG3PEzrW7uK7RfO4aH0230BSWJCqXDI/wOixjio1AtlA6AmmqukpVs4CJQJ88dRTIHb6lGrARQFX3q2q2Wx5C/uOI9QRWqupa93Uf4GN3+WPgCr+chSkW5o1eRvg5e5iSHsHzl8by05a21Gl9htdhGVOsBDKh1APW+7xOd8t8jQBuFJF04HtgcO4KEekkIsnAMuBunwSTqx8wwef1maq6yV3eDJyZX1AicqeIJIhIQkZGxkmekilqDmcd5r8XxxJ1V3NElNljUnnshxib592YAPD6t6o/ME5V6wO9gU9FpBSAqsarakugAzBMRI6MES4i5YDLgS/z26mqKscYHVlVR6tqhKpG1K5d279nY4LKpsQtXFJnCcN/juHqBr+TuLIqnW8/3+uwjCm2AplQNgANfF7Xd8t8DQQmAajqPJzurVq+FVQ1BdgL+P4l6AUsUtUtPmVbRKQugPt9qx/OwRRRP4xcQJt2pZj7VzM+uGU2E9bY1LzGBFogE8oCIExEmrgtin7AtDx11uHcC0FEmuMklAx3mzJueSOgGbDGZ7v+/L27C3fft7jLtwBf++9UTFGRtTeLhyNi6f1UB+qU/4uErzcycFyUDZ9iTCEI2OdQVDVbRO4DfgJKAx+parKIjAQSVHUa8DAwRkSG4HRRDVBVFZFIYKiIHAJygHtUdRuAiFQCLgLuynPI54FJIjIQWAtcG6hzM8EpbcZa+l2+j4X7Y7jn/FmMmtWRCqEVvA7LmBJDnNsNJVNERIQmJCR4HYbxg88G/cag91pTVrL58JFUrnyhs9chGVNsichCVT3qmXv7pLwp0vZu3st9kYl8vDKSyKpLGP9jLRp2sWRijBe8fsrLmFO2eMIftG+YwScru/JkdCwzt7SkYZe8T6YbYwqLJRRT5GiO8vpVs+h8fRP2Hg7h11eX8p9ZMTbPuzEes99AU6RsS93OrdEr+XarMzXvR7POpdZ5NjWvMcHAWiimyIh9LZE2LbKYvrUNr1/lTM1b67yaXodljHFZQjFBLzszmyejY7lgSGsql85k/ueruf8rm5rXmGBjXV4mqK2bt4EbLt3GnN0xDDh3Nm/ObkvlOpW9DssYkw9roZigNeWx+YR3q8iS3U34bNBvjF0RZcnEmCBmLRQTdA7sOMDD0Qt4NzmaiIrLmfhNJc65oJvXYRljTsASigkqy6el0e+6HJZlRvNw+1j+G2tT8xpTVFiXlwkKmqN8cMtsIvqcxeaDNfj+PwsYlWBT8xpTlFgLxXhu59pd3BWVzKT1UVwYupBPZtSnbngHr8Myxpwka6EYT83/IIm25+7mq/Udee4SZ2reuuH5TrZpjAlyllCMJ3Kyc3j+0lgi72gGwJwxfzD0R5ua15iizLq8TKHbvHQrN/VYzy87Yrim/jxGz2lB9UYNTryhMSao2b+DplD9+EwCrcOF33Y0Z/RNcXyxtjPVG9nUvMYUB9ZCMYUia28Ww3vMZVRCDOeXX0HspF20uDza67CMMX5kCcUEXNqMtfS/fB8J+2MY1DKOl+M62NS8xhRD1uVlAurze3+j3YU1SDtwFl89Mp93kqItmRhTTFkLxQTE3s17GRy1mHFpUXSrspTxP4TSqJtNzWtMcWYtFON3iV+kEtFoKx+ndePfUbHEbm1Bo271vQ7LGBNgllCM32iO8kbfWXTq15g92RWY8fISRsbZ1LzGlBT2m278Ylvqdm7rnsY3W7pz2Rm/MzbuHGqd19brsIwxhchaKOa0zXo9kfAWWfy0JZzXrpzFtE0dbGpeY0ogSyjmlGVnZvNU91gueLAVFUtnMu+zVTzwP5ua15iSyrq8zClZH7+RGy7OYPbuGG45Zw5vxrWhyllVvA7LGOMha6GYkzZ1WDxtulRg8e6z+fTu3xiXFmnJxBhjLRRTcJk7M3k4Kp53krrTvuJyJkytSNhFNjWvMcZhCcUUSMq3K+l3TTZLM7vzUPtYnrOpeY0xeViXlzkuzVE+HDCbiH/WYdPBUL4bsYCXbWpeY0w+rIVijmnXul3cFZnMF+uj6FljEZ/+Ws+m5jXGHJO1UEy+4j9Mou05u5i8viP/vTiWnza3sal5jTHHFdCEIiKXikiqiKSJyNB81jcUkZkislhElopIb7e8o4gkul9LRORKn22qi8hkEflDRFJEpItbPkJENvhs1zuQ51Zc5WTn8EKvWCJvP48cLUXceykM+ymG0uVKex2aMSbIBazLS0RKA28DFwHpwAIRmaaqy32qPQFMUtV3RaQF8D3QGEgCIlQ1W0TqAktE5BtVzQZeB35U1atFpBxQ0Wd/r6rqqECdU3G3eelWbu6xnp93xHB1vXmM+a0F1RvZoI7GmIIJZAulI5CmqqtUNQuYCPTJU0eBqu5yNWAjgKrud5MHQIhbDxGpBkQDH7r1slR1ZwDPocSY/txC2oQLs3e04P0b4pi0zqbmNcacnEAmlHrAep/X6W6ZrxHAjSKSjtM6GZy7QkQ6iUgysAy4200wTYAMYKzbTfaBiFTy2d99btfZRyJSI7+gROROEUkQkYSMjIzTPcciL2tvFo92jOWSx9tTu9xOEqZu4M7Pom34FGPMSfP6pnx/YJyq1gd6A5+KSCkAVY1X1ZZAB2CYiITgdNG1A95V1bbAPiD33sy7wDlAOLAJeDm/A6rqaFWNUNWI2rVrB/DUgt+q2HVE1lnBSwtiuLtFHAs21qdln3O9DssYU0QFMqFsABr4vK7vlvkaCEwCUNV5ON1btXwrqGoKsBc4H6eVk66q8e7qyTgJBlXdoqqHVTUHGIPT5WaOYcLguYT3qM6K/fWY/K95vJtsU/MaY05PIBPKAiBMRJq4N8/7AdPy1FkH9AQQkeY4CSXD3aaMW94IaAasUdXNwHoROc/dview3K1X12e/V+Lc2Dd57Nu6j4FNZ3P9W11pVWUNibP30velLl6HZYwpBgL2lJf7hNZ9wE9AaeAjVU0WkZFAgqpOAx4GxojIEJwb7wNUVUUkEhgqIoeAHOAeVd3m7nowMN5NUquAW93yF0Uk3N3PGuCuQJ1bUbVkUir9bipDalY3hneLZcQvkTabojHGb0RVvY7BMxEREZqQkOB1GAGnOcrb18Xxr8mdCC21k89e3MQFD9tsisaYUyMiC1U1Im+5/XtazG1fsYOB0Sv4enN3etdewLhZTajd3JKJMcb/vH7KywRQ3JtLCG+eyfeb2/LqFbP4dnMEtZvXOvGGxhhzCiyhFEOHsw7znx6x9Lj/fCqUOsj8z1by4BSbmtcYE1jW5VXMrI/fyI2XbCVuVww3nT2Ht2fb1LzGmMJhLZRi5OvH4wnvEsLCXefyyV1z+GSlTc1rjCk81kIpBjJ3ZvJIdDxvLetOuwopTPy6AmEXRXodljGmhLGEUsT98f0q+vU9xJLM7gxpF8tzM7tQvmp5r8MyxpRA1uVVRGmOMva22bT/x5lsOFiT70Ys4JWFMZZMjDGesRZKEbRr3S4GRScxYW0UPaov5rMZdTmrnU3Na4zxlrVQipjfxybT9pxdTFrbiWcujOXnLa05q10dr8MyxhhLKEVFTnYOL/0jlm63NeWwOzXv8J9tal5jTPCwLq8iYEtSBrfErOWn7TH0rTePMbObU6OJTc1rjAku1kIJcj8/v5A2bZRZ21vy3vVxfLmuMzWaVPc6LGOMOYq1UILUof2HeKLHb7z4ewwtyqfxy4RdnH9ltNdhGWPMMVlCCUKrYtfR/7Ld/L4vhruax/FKXAQVa1X0OixjjDku6/IKMl88MJe2PaqRuq8BXz40j/eWR1syMcYUCdZCCRL7tu7jgahFfPhnFF0qL+PzH2rQONKm5jXGFB3WQgkCSyf/SUSDzXz0Zzce7xrLrC3NaBxpT3EZY4oWSyge0hzl7Wtn0fGahuzMrszPLyby7G8xlK1Y1uvQjDHmpFmXl0d2rPyLgVGpTN3kOzVvO6/DMsaYU2YtFA/MfmsJ4eft57tN7XilTyzfbGxvU/MaY4o8SyiF6HDWYUZeEEvM4PMpX+oQ8z5dyZCpMZQqYz8GY0zRZ11ehSR9wSZuvGgLs3bFcGOTObwzx6bmNcYUL/avcSGYNjyeNp3Kk7DrXD6+Yw6frrKpeY0xxY+1UAIoc2cmj3aP582l3WlbIYWJU0JoeolNzWuMKZ4soQRI6g+r6Nc3i8QD3Xmw7Syej+1ssykaY4o16/LyM81Rxt0+h/a9z2B9Zm2++ffvvLqouyUTY0yxZy0UP9qdvptBkcv4fG0kMdUX89nPdagX0dHrsIwxplBYC8VPFny8nHZN/uKLtZ14umcsv2xpTb2Iul6HZYwxhcYSymnKyc5h1GWxdB0QxiEtzax3lvPELzY1rzGm5LEur9OwNTmDW2LW8OO2GK46az4fzGlmU/MaY0qsgLZQRORSEUkVkTQRGZrP+oYiMlNEFovIUhHp7ZZ3FJFE92uJiFzps011EZksIn+ISIqIdHHLQ0XkZxFZ4X6vEchz++XFRbRpncPMba14t38ck9d3sql5jTElWsASioiUBt4GegEtgP4i0iJPtSeASaraFugHvOOWJwERqhoOXAq8LyK5ranXgR9VtRnQBkhxy4cCM1Q1DJjhvg6IZy+K5eLHwgktu4cFk9dx9+fRSCkJ1OGMMaZICGQLpSOQpqqrVDULmAj0yVNHgarucjVgI4Cq7lfVbLc8xK2HiFQDooEP3XpZqrrTrdcH+Nhd/hi4wu9n5Dq3RTlubzaHBeln0apv00AdxhhjipRA3kOpB6z3eZ0OdMpTZwQwXUQGA5WAC3NXiEgn4COgEXCTqmaLSBMgAxgrIm2AhcADqroPOFNVN7mbbwbO9P8pOa57vSvXBWrnxhhTRHn9lFd/YJyq1gd6A5+KSCkAVY1X1ZZAB2CYiITgJMB2wLtuN9k+8unaUlXFbdXkJSJ3ikiCiCRkZGQE5KSMMaYkCmRC2QA08Hld3y3zNRCYBKCq83C6t/42MYiqpgB7gfNxWjnpqhrvrp6Mk2AAtohIXQD3+9b8glLV0aoaoaoRtWvXPsVTM8YYk1cgE8oCIExEmohIOZyb7tPy1FkH9AQQkeY4CSXD3aaMW94IaAasUdXNwHoROc/dview3F2eBtziLt8CfB2Y0zLGGJOfgN1Dce953Af8BJQGPlLVZBEZCSSo6jTgYWCMiAzB6aIaoKoqIpHAUBE5BOQA96jqNnfXg4HxbpJaBdzqlj8PTBKRgcBa4NpAnZsxxpijiXO7oWSKiIjQhIQEr8MwxpgiRUQWqmpE3nKvb8obY4wpJiyhGGOM8QtLKMYYY/yiRN9DEZEMnBv4p6IWsO2EtQqfxXVyLK6TY3GdnGCNC04vtkaqetTnLkp0QjkdIpKQ300pr1lcJ8fiOjkW18kJ1rggMLFZl5cxxhi/sIRijDHGLyyhnLrRXgdwDBbXybG4To7FdXKCNS4IQGx2D8UYY4xfWAvFGGOMX1hCMcYY4xeWUPIhIpeKSKqIpInIUfOtiEh5EfnCXR8vIo191g1zy1NF5JJCjushEVkuIktFZIY7UnPuusMikuh+5R31OdBxDRCRDJ/j3+6z7hYRWeF+3ZJ32wDH9apPTH+KyE6fdQG5XiLykYhsFZGkY6wXEXnDjXmpiLTzWRfIa3WiuG5w41kmInPdCe5y161xyxNFxK+D4xUgrhgR2eXzs3rSZ91xf/4BjusRn5iS3PdTqLsukNergYjMdP8OJIvIA/nUCdx7TFXty+cLZ2TklcDZQDlgCdAiT517gPfc5X7AF+5yC7d+eaCJu5/ShRhXD6CiuzwoNy739V4Pr9cA4K18tg3FGTE6FKjhLtcorLjy1B+MMyJ2oK9XNM4cPknHWN8b+AEQoDMQH+hrVcC4uuYeD+iVG5f7eg1Qy6PrFQN8e7o/f3/HlafuP4FfC+l61QXauctVgD/z+X0M2HvMWihH6wikqeoqVc0CJuLMV+/Ld/76yUBPERG3fKKqHlTV1UCau79CiUtVZ6rqfvflfJxJzQKtINfrWC4BflbVHar6F/AzcKlHcfUHJvjp2MekqnHAjuNU6QN8oo75QHVxJowL5LU6YVyqOtc9LhTee6sg1+tYTud96e+4CuW9BaCqm1R1kbu8B0jBmY7dV8DeY5ZQjlYPWO/zOp2jfyBH6qhqNrALqFnAbQMZl6+BOP+F5AoRZ+rj+SJyhZ9iOpm4+rrN68kikjuTZ1BcL7drsAnwq09xoK7XiRwr7kBeq5OV972lwHQRWSgid3oQTxcRWSIiP4hIS7csKK6XiFTE+aP8lU9xoVwvcbri2wLxeVYF7D0WsAm2jHdE5EYgAujuU9xIVTeIyNnAryKyTFVXFlJI3wATVPWgiNyF07q7oJCOXRD9gMmqetinzMvrFbREpAdOQon0KY50r9UZwM8i8of7H3xhWITzs9orIr2BqUBYIR27IP4J/Kaqvq2ZgF8vEamMk8QeVNXd/tz38VgL5WgbgAY+r+u7ZfnWEWeq4mrA9gJuG8i4EJELgeHA5ap6MLdcVTe431cBsTj/uRRKXKq63SeWD4D2Bd02kHH56EeeLokAXq8TOVbcgbxWBSIirXF+fn1UdXtuuc+12gpMwX/dvCekqrtVda+7/D1QVkRqEQTXy3W891ZArpeIlMVJJuNV9X/5VAnceywQN4aK8hdOq20VThdI7s28lnnq3Mvfb8pPcpdb8veb8qvw3035gsTVFudGZFie8hpAeXe5FrACP92gLGBcdX2WrwTmu8uhwGo3vhrucmhhxeXWa4Zzk1QK43q5+2zMsW8y/4O/3zD9PdDXqoBxNcS5J9g1T3kloIrP8lzg0kKMq07uzw7nD/M699oV6OcfqLjc9dVw7rNUKqzr5Z77J8Brx6kTsPeY3y5ucfrCeQriT5w/zsPdspE4//UDhABfur9gvwNn+2w73N0uFehVyHH9AmwBEt2vaW55V2CZ+0u1DBhYyHE9ByS7x58JNPPZ9jb3OqYBtxZmXO7rEcDzebYL2PXC+W91E3AIp496IHA3cLe7XoC33ZiXARGFdK1OFNcHwF8+760Et/xs9zotcX/Gwws5rvt83lvz8Ul4+f38Cysut84AnId0fLcL9PWKxLlHs9TnZ9W7sN5jNvSKMcYYv7B7KMYYY/zCEooxxhi/sIRijDHGLyyhGGOM8QtLKMYYY/zCEooxAZBntOJEf452KyKNjzXKrTFesqFXjAmMA6oa7nUQxhQma6EYU4jcuTBedOfD+F1EznXLG4vIr/L/c9k0dMvPFJEp7uCHS0Skq7ur0iIyxp3zYrqIVHDr3y//PyfORI9O05RQllCMCYwKebq8rvNZt0tVWwFvAa+5ZW8CH6tqa2A88IZb/gYwS1Xb4My/keyWhwFvq2pLYCfQ1y0fCrR193N3oE7OmPzYJ+WNCQAR2auqlfMpXwNcoKqr3EH8NqtqTRHZhjPm2SG3fJOq1hKRDKC++gz06Q5L/rOqhrmvHwPKquozIvIjsBdn1N2p6g6caExhsBaKMYVPj7F8Mg76LB/m/++H/gNnnKZ2wAJ3NGxjCoUlFGMK33U+3+e5y3NxRq4GuAGY7S7PwJnOGREpLSLVjrVTESkFNFDVmcBjOKPdHtVKMiZQ7L8XYwKjgogk+rz+UVVzHx2uISJLcVoZ/d2ywcBYEXkEyABudcsfAEaLyECclsggnFFu81Ma+MxNOgK8oao7/XZGxpyA3UMxphC591AiVHWb17EY42/W5WWMMcYvrIVijDHGL6yFYowxxi8soRhjjPELSyjGGGP8whKKMcYYv7CEYowxxi/+D5TAk7PFlLEBAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"needs_background":"light"}},{"output_type":"display_data","data":{"text/plain":["
"]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"H-RSSqI7BDqE"},"source":["### Restart runtime and import Relibraries\n","There is a bug, if run time isn't restart after this point, It's going to malfunction."]},{"cell_type":"code","metadata":{"id":"AnfxE-LmAWmK","executionInfo":{"status":"ok","timestamp":1640619201703,"user_tz":-330,"elapsed":15,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# # os.kill(os.getpid(), 9)\n","\n","# exit()"],"execution_count":37,"outputs":[]},{"cell_type":"code","metadata":{"id":"Cxp9ViDsA2bb","executionInfo":{"status":"ok","timestamp":1640619201704,"user_tz":-330,"elapsed":15,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# from keras.models import Sequential,Model,load_model\n","# import pandas as pd\n","# import numpy as np\n","# from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix,precision_recall_fscore_support\n","\n","# experiment_no = 100\n","# model_name = \"RNN\"\n","# experiment_name = folder_path + \"Sentiment Analysis/CNN RNN/experiments/\" + model_name +str(experiment_no)+\"_\"+embedding_type+\"_\"+str(embedding_size)+\"_\"+str(context)\n","\n","# model_save_path = folder_path + \"Sentiment Analysis/CNN RNN/saved_models/weights_best_\"+model_name+\"_\"+embedding_type+\"_\"+str(embedding_size)+\"_\"+str(experiment_no)+\".hdf5\""],"execution_count":38,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"_VdzOT3zsB9m"},"source":["### Test Model"]},{"cell_type":"markdown","metadata":{"id":"KMtXKRpRYBx2"},"source":["#### Load Weights to Model"]},{"cell_type":"code","metadata":{"id":"Yl7JuOSe3tBl","executionInfo":{"status":"ok","timestamp":1640619450289,"user_tz":-330,"elapsed":248600,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"7618f0e9-ca70-4a9a-e977-cb3e03fb81b9"},"source":["# model_save_path = folder_path + \"Sentiment Analysis/CNN RNN/saved_models/111_weights_best_stacked_LSTM_3_fasttext_111.hdf5\"\n","model.load_weights(model_save_path)\n","# loaded_model = load_model(model_save_path,custom_objects={\"f1\": f1}, compile=True)\n","# print(\"loaded \" + model.name)\n","\n","# his2 = model.evaluate(X_train, y_train, verbose=1)\n","loss, accuracy, f1_score, precision, recall = model.evaluate(X_test, y_test, verbose=1)\n","print(\"accuracy= \",accuracy,\" precision= \",precision,\" recall= \",recall,\" f1= \",f1_score)"],"execution_count":39,"outputs":[{"output_type":"stream","name":"stdout","text":["30000/30000 [==============================] - 248s 8ms/step\n","accuracy= 0.8346999883651733 precision= 0.8364810347557068 recall= 0.8346548080444336 f1= 0.8364810347557068\n"]}]},{"cell_type":"markdown","metadata":{"id":"BtZBU5XNZaMi"},"source":["#### Get Predictions"]},{"cell_type":"code","metadata":{"id":"MmA4iDmR2czB","executionInfo":{"status":"ok","timestamp":1640619648715,"user_tz":-330,"elapsed":198465,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"69992511-d0ff-492e-bbda-68c1998957cf"},"source":["predictions = model.predict(X_test, batch_size=BATCH_SIZE, verbose=1)\n","\n","labels = np.argmax(y_test, axis=1)\n","predictions = np.argmax(predictions, axis=1)\n","\n","cm = confusion_matrix(labels,predictions )\n","\n","# classification_report\n","report = classification_report(labels, predictions, digits=4,output_dict=True)\n","report_print = classification_report(labels, predictions, digits=4)\n","print(report_print)\n","\n","# report_df = pd.DataFrame(report).transpose()\n","# report_df.to_csv(experiment_name+\".csv\")\n","\n"],"execution_count":40,"outputs":[{"output_type":"stream","name":"stdout","text":["30000/30000 [==============================] - 199s 7ms/step\n"," precision recall f1-score support\n","\n"," 0 0.7637 0.3948 0.5205 6818\n"," 1 0.8442 0.9641 0.9001 23182\n","\n"," accuracy 0.8347 30000\n"," macro avg 0.8039 0.6795 0.7103 30000\n","weighted avg 0.8259 0.8347 0.8139 30000\n","\n"]}]},{"cell_type":"code","metadata":{"id":"A5QjsfZxq4fq","executionInfo":{"status":"error","timestamp":1640619650201,"user_tz":-330,"elapsed":1503,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"colab":{"base_uri":"https://localhost:8080/","height":241},"outputId":"a44eb8f6-bb6d-40bb-92cd-8213ae0bb343"},"source":["# DataTagged_Original = pd.read_csv(all_data_path)\n","DataTagged_Original = pd.concat([lankadeepa_data,gossipLanka_data], ignore_index=True)\n","train_data_original, test_data_original = train_test_split(DataTagged_Original, test_size=0.1, random_state=0)\n","\n","predictions_series = pd.Series(predictions)\n","predictions_1 = pd.get_dummies(predictions_series).idxmax(1)"],"execution_count":41,"outputs":[{"output_type":"error","ename":"NameError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# DataTagged_Original = pd.read_csv(all_data_path)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mDataTagged_Original\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlankadeepa_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mgossipLanka_data\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0mtrain_data_original\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_data_original\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDataTagged_Original\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mpredictions_series\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'lankadeepa_data' is not defined"]}]},{"cell_type":"code","metadata":{"id":"HT-ccXWZRdak","executionInfo":{"status":"aborted","timestamp":1640619649373,"user_tz":-330,"elapsed":22,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["data_frame = pd.DataFrame({'comment': test_data_original['comment'], 'Labels': test_data_original['label'], 'Predictions': np.array(predictions_1)})\n","\n","def add_two(value):\n"," return value+2\n","data_frame[\"Predictions\"] = data_frame[\"Predictions\"].apply(add_two)\n","\n","prediction_save_name = folder_path + \"Sentiment Analysis/CNN RNN/LSTM_S3_predictions_with_labels_3.csv\"\n","data_frame.to_csv(prediction_save_name)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"qLkfB_XXPS3O"},"source":["### Print confusion matrix"]},{"cell_type":"code","metadata":{"id":"bdCcVDhPPN_m","executionInfo":{"status":"aborted","timestamp":1640619649375,"user_tz":-330,"elapsed":24,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def plot_confusion_matrix(cm,\n"," target_names,\n"," title='Confusion matrix',\n"," cmap=None,\n"," normalize=True):\n"," \"\"\"\n"," given a sklearn confusion matrix (cm), make a nice plot\n","\n"," Arguments\n"," ---------\n"," cm: confusion matrix from sklearn.metrics.confusion_matrix\n","\n"," target_names: given classification classes such as [0, 1, 2]\n"," the class names, for example: ['high', 'medium', 'low']\n","\n"," title: the text to display at the top of the matrix\n","\n"," cmap: the gradient of the values displayed from matplotlib.pyplot.cm\n"," see http://matplotlib.org/examples/color/colormaps_reference.html\n"," plt.get_cmap('jet') or plt.cm.Blues\n","\n"," normalize: If False, plot the raw numbers\n"," If True, plot the proportions\n","\n"," Usage\n"," -----\n"," plot_confusion_matrix(cm = cm, # confusion matrix created by\n"," # sklearn.metrics.confusion_matrix\n"," normalize = True, # show proportions\n"," target_names = y_labels_vals, # list of names of the classes\n"," title = best_estimator_name) # title of graph\n","\n"," Citiation\n"," ---------\n"," http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n","\n"," \"\"\"\n"," import matplotlib.pyplot as plt\n"," import numpy as np\n"," import itertools\n","\n"," accuracy = np.trace(cm) / np.sum(cm).astype('float')\n"," misclass = 1 - accuracy\n","\n"," if cmap is None:\n"," cmap = plt.get_cmap('Blues')\n","\n"," plt.figure(figsize=(8, 6))\n"," plt.imshow(cm, interpolation='nearest', cmap=cmap)\n"," plt.title(title)\n"," plt.colorbar()\n","\n"," if target_names is not None:\n"," tick_marks = np.arange(len(target_names))\n"," plt.xticks(tick_marks, target_names, rotation=45)\n"," plt.yticks(tick_marks, target_names)\n","\n"," if normalize:\n"," cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n","\n","\n"," thresh = cm.max() / 1.5 if normalize else cm.max() / 2\n"," for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n"," if normalize:\n"," plt.text(j, i, \"{:0.4f}\".format(cm[i, j]),\n"," horizontalalignment=\"center\",\n"," color=\"white\" if cm[i, j] > thresh else \"black\")\n"," else:\n"," plt.text(j, i, \"{:,}\".format(cm[i, j]),\n"," horizontalalignment=\"center\",\n"," color=\"white\" if cm[i, j] > thresh else \"black\")\n","\n","\n"," plt.tight_layout()\n"," plt.ylabel('True label')\n"," plt.xlabel('Predicted label\\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))\n"," plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aVJwG_4FPQ2f","executionInfo":{"status":"aborted","timestamp":1640619649377,"user_tz":-330,"elapsed":25,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["plot_confusion_matrix(cm, [0,1,2,3],normalize=False)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"_-x34NEZtU5C"},"source":["## Train and Test Model (Cross Validation)"]},{"cell_type":"code","metadata":{"id":"2iZDvr80VxPN","executionInfo":{"status":"aborted","timestamp":1640619649379,"user_tz":-330,"elapsed":27,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# Do_Cross_Validation(model,padded_docs,comment_labels)\n","Do_Cross_Validation(padded_docs,comment_labels)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"LR7WojLAszo7"},"source":["# Hyperparameter Tuning"]},{"cell_type":"code","metadata":{"id":"GEjuD0hzohUv","executionInfo":{"status":"aborted","timestamp":1640619649379,"user_tz":-330,"elapsed":27,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["def tune_hyperparameters(model_build_fn):\n"," # fix random seed for reproducibility\n"," seed = 7\n"," np.random.seed(seed)\n","\n"," model = KerasClassifier(build_fn=model_build_fn, maxlen=MAX_LEN, hidden_dims=HIDDEN_DIMS, l2_reg= L2_REG, verbose=VERBOSITY)\n","\n"," # optimizer = ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']\n"," maxlen_list = [210]\n"," hidden_dims_list = [300, 400, 500, 600]\n"," l2_reg_list = [0.01, 0.02]\n"," drop_out_value_list = [0.5, 0.8]\n"," # tune for kernal size 2,3,4,5\n","\n","\n"," param_grid = dict(maxlen=maxlen_list, hidden_dims = hidden_dims_list, l2_reg= l2_reg_list, drop_out_value = drop_out_value_list )\n"," grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=FOLDS, verbose = 10 )\n"," grid_result = grid.fit(padded_docs, comment_labels)\n","\n"," # summarize results\n"," print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n"," means = grid_result.cv_results_['mean_test_score']\n"," stds = grid_result.cv_results_['std_test_score']\n"," params = grid_result.cv_results_['params']\n"," for mean, stdev, param in zip(means, stds, params):\n"," print(\"%f (%f) with: %r\" % (mean, stdev, param))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"oHYpTsTDtFIr","executionInfo":{"status":"aborted","timestamp":1640619649380,"user_tz":-330,"elapsed":27,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["# tune_hyperparameters(build_BiLSTM_model)"],"execution_count":null,"outputs":[]}]}