capsule networks

Browse files

Files changed (5) hide show

sentiment-analysis-models/capsule-networks/Sinhala_text_capsules.ipynb +0 -1
sentiment-analysis-models/capsule-networks/capsule_network_with_parallel_BiLSTM.ipynb +1 -0
sentiment-analysis-models/capsule-networks/capsule_network_with_serial_BiLSTM.ipynb +1 -0
sentiment-analysis-models/capsule-networks/vanila_capsule_network.ipynb +1 -0
word-embedding-models/.DS_Store +0 -0

sentiment-analysis-models/capsule-networks/Sinhala_text_capsules.ipynb DELETED Viewed

@@ -1 +0,0 @@

- {"cells":[{"cell_type":"markdown","metadata":{"id":"k8D9EuJnBnDV"},"source":["# Installing and importing dependecies"]},{"cell_type":"code","execution_count":1,"metadata":{"id":"fp5wHxfHqnPj","executionInfo":{"status":"ok","timestamp":1684558097641,"user_tz":-330,"elapsed":9,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["i = \"1\" #@param [1,2,3,4,5]\n","i=int(i)\n","\n","embedding_type = \"fastText\" \n","sentence_type = \"GRU\" \n","Attention = True \n","EMBEDDING_SIZE = 200 \n","decoder = \"attention\""]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":21511,"status":"ok","timestamp":1684558119145,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"GgRvEbMXy9nc","outputId":"93af357a-d6b7-42ab-86b9-60cb5775d4e4"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":108262,"status":"ok","timestamp":1684558227403,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"CCi-HNCn7BCi","outputId":"c5fa6ca5-dad8-4547-fb18-84dfee28e790"},"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[33mWARNING: Skipping keras-nightly as it is not installed.\u001b[0m\u001b[33m\n","\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","\u001b[31mERROR: Could not find a version that satisfies the requirement tensorflow==1.14.0 (from versions: 2.8.0rc0, 2.8.0rc1, 2.8.0, 2.8.1, 2.8.2, 2.8.3, 2.8.4, 2.9.0rc0, 2.9.0rc1, 2.9.0rc2, 2.9.0, 2.9.1, 2.9.2, 2.9.3, 2.10.0rc0, 2.10.0rc1, 2.10.0rc2, 2.10.0rc3, 2.10.0, 2.10.1, 2.11.0rc0, 2.11.0rc1, 2.11.0rc2, 2.11.0, 2.11.1, 2.12.0rc0, 2.12.0rc1, 2.12.0, 2.13.0rc0)\u001b[0m\u001b[31m\n","\u001b[0m\u001b[31mERROR: No matching distribution found for tensorflow==1.14.0\u001b[0m\u001b[31m\n","\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting q\n"," Downloading q-2.7-py2.py3-none-any.whl (10 kB)\n","Collecting keras==2.2.2\n"," Downloading Keras-2.2.2-py2.py3-none-any.whl (299 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m299.3/299.3 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.10/dist-packages (from keras==2.2.2) (1.22.4)\n","Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.10/dist-packages (from keras==2.2.2) (1.10.1)\n","Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from keras==2.2.2) (1.16.0)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from keras==2.2.2) (6.0)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.10/dist-packages (from keras==2.2.2) (3.8.0)\n","Collecting keras-applications==1.0.4 (from keras==2.2.2)\n"," Downloading Keras_Applications-1.0.4-py2.py3-none-any.whl (43 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.8/43.8 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting keras-preprocessing==1.0.2 (from keras==2.2.2)\n"," Downloading Keras_Preprocessing-1.0.2-py2.py3-none-any.whl (26 kB)\n","Installing collected packages: q, keras-preprocessing, keras-applications, keras\n"," Attempting uninstall: keras\n"," Found existing installation: keras 2.12.0\n"," Uninstalling keras-2.12.0:\n"," Successfully uninstalled keras-2.12.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","tensorflow 2.12.0 requires keras<2.13,>=2.12.0, but you have keras 2.2.2 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed keras-2.2.2 keras-applications-1.0.4 keras-preprocessing-1.0.2 q-2.7\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting h5py<3.0.0\n"," Downloading h5py-2.10.0.tar.gz (301 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.1/301.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.10/dist-packages (from h5py<3.0.0) (1.22.4)\n","Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from h5py<3.0.0) (1.16.0)\n","Building wheels for collected packages: h5py\n"," Building wheel for h5py (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for h5py: filename=h5py-2.10.0-cp310-cp310-linux_x86_64.whl size=5619534 sha256=19772f481c694cbfa64eb4d3b19d1b7863a7a35506e6b8bda88f6282a2bb0603\n"," Stored in directory: /root/.cache/pip/wheels/21/bc/58/0d0c6056e1339f40188d136cd838c6554d9c17545196dd9110\n","Successfully built h5py\n","Installing collected packages: h5py\n"," Attempting uninstall: h5py\n"," Found existing installation: h5py 3.8.0\n"," Uninstalling h5py-3.8.0:\n"," Successfully uninstalled h5py-3.8.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","tensorflow 2.12.0 requires keras<2.13,>=2.12.0, but you have keras 2.2.2 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed h5py-2.10.0\n"]}],"source":["!pip uninstall keras-nightly\n","!pip install tensorflow==1.14.0\n","!pip install q keras==2.2.2\n","!pip install 'h5py<3.0.0'"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":550},"executionInfo":{"elapsed":5223,"status":"error","timestamp":1684558232621,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"zlXBE_-86l4q","outputId":"9c57f2da-b854-45a2-c696-ca171d1b714c"},"outputs":[{"output_type":"stream","name":"stderr","text":["Using TensorFlow backend.\n"]},{"output_type":"error","ename":"ImportError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)","\u001b[0;32m<ipython-input-4-6d24fa491cb4>\u001b[0m in \u001b[0;36m<cell line: 14>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mabsolute_import\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mactivations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mapplications\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/utils/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mnp_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_categorical\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mnp_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmulti_gpu_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmulti_gpu_model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/utils/multi_gpu_utils.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprint_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mconcatenate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLambda\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/layers/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeneric_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdeserialize_keras_object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mInput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mInputLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/engine/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mbase_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mnetwork\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_source_inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtraining_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mstandardize_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtraining_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mweighted_masked_objective\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtraining_arrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 22\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtraining_generator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtraining_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcheck_num_samples\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mcbks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeneric_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mProgbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeneric_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mslice_arrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcollections\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdeque\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcollections\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mOrderedDict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mcollections\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mIterable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeneric_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mProgbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mImportError\u001b[0m: cannot import name 'Iterable' from 'collections' (/usr/lib/python3.10/collections/__init__.py)","","\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"],"errorDetails":{"actions":[{"action":"open_url","actionText":"Open Examples","url":"/notebooks/snippets/importing_libraries.ipynb"}]}}],"source":["import pandas as pd\n","import collections\n","import gensim\n","from gensim.models.keyedvectors import KeyedVectors\n","from gensim.models.fasttext import FastText\n","from gensim.models import word2vec\n","\n","from sklearn.model_selection import train_test_split,cross_val_score, cross_val_predict, KFold, GridSearchCV\n","from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix, precision_recall_fscore_support\n","\n","import pandas as pd\n","import numpy as np\n","\n","import keras\n","from keras import backend as K\n","from keras.preprocessing.text import Tokenizer\n","from keras.preprocessing.sequence import pad_sequences\n","\n","import matplotlib.image as mpimg\n","import matplotlib.pyplot as plt\n","\n","import tensorflow as tf\n","from tensorflow.contrib.layers.python.layers import initializers"]},{"cell_type":"markdown","metadata":{"id":"nRbpY2FTDl8p"},"source":["# Load Dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"d8GzWXqBsX2V","executionInfo":{"status":"aborted","timestamp":1684558232622,"user_tz":-330,"elapsed":16,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Capsule networks/'\n","context = 5\n","posts=[]\n","labels =[]\n","\n","# facebook_data_path = (folder_path + 'embedded posts/'+sentence_type+'_'+decoder+'_'+str(i)+'.csv')\n","# dataset = pd.read_pickle(facebook_data_path)\n","# dataset=dataset.dropna()\n","# dataset = dataset[:1000]\n","# labels+=dataset[\"label\"].tolist()\n","# for j in dataset['posts']:\n","# posts.append(j)\n","\n","for i in range (1,5):\n"," facebook_data_path = (folder_path + 'embedded posts/'+sentence_type+'_'+decoder+'_'+str(i)+'.csv')\n"," dataset = pd.read_pickle(facebook_data_path)\n"," dataset=dataset.dropna()\n"," # dataset = dataset[:1000]\n"," labels+=dataset[\"label\"].tolist()\n"," for j in dataset['posts']:\n"," posts.append(j)"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":16,"status":"aborted","timestamp":1684558232622,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"ArpyPsA2iveG"},"outputs":[],"source":["len(posts)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ZYVhMTasdLEe","executionInfo":{"status":"aborted","timestamp":1684558232622,"user_tz":-330,"elapsed":15,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# import torch\n","\n","# folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Capsule networks/'\n","# context = 5\n","\n","# facebook_data_path = (folder_path + 'embedded posts/'+sentence_type+'_'+decoder+'_'+str(i)+'.csv')\n","\n","# posts=torch.tensor()\n","# labels =torch.tensor()\n","# posts = tf.constant()\n","# dataset = pd.read_pickle(facebook_data_path)\n","# dataset=dataset.dropna()\n","# labels = tf.convert_to_tensor(dataset[\"label\"],dtype='float32')\n","# posts = tf.convert_to_tensor(dataset[\"posts\"])\n","# labels+=torch.to.tolist()\n","# for j in dataset['posts']:\n","# torch.add(posts,j)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"_Okf8SRqjeFa","executionInfo":{"status":"aborted","timestamp":1684558232624,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# labels= tf.constant(dataset['label'])\n","# labels= tf.cast(labels, tf.float32)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0DDF55PhkErH","executionInfo":{"status":"aborted","timestamp":1684558232624,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# labels"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hQLyctN3akSu","executionInfo":{"status":"aborted","timestamp":1684558232624,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# posts =tf.convert_to_tensor( posts)\n","# posts = posts[...,tf.newaxis] "]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0m6FrtmcsRKu","executionInfo":{"status":"aborted","timestamp":1684558232624,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# posts=np.array(posts)\n","# labels=np.array(labels)\n","train_set, test_set , train_label, test_label = train_test_split(posts,labels, test_size=0.1, random_state=0)\n","train_set, dev_set, train_label, dev_label = train_test_split(train_set,train_label, test_size=0.1, random_state=0)\n","# else:\n","# x_train, x_test, y_train, y_test = train_test_split(posts,labels, test_size=len(labels)-1, random_state=0)"]},{"cell_type":"markdown","metadata":{"id":"XnkydX1ne969"},"source":["# Convo 2d layer"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0GVuHgh2EWw2","executionInfo":{"status":"aborted","timestamp":1684558232625,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def _get_variable_wrapper(name, shape=None, dtype=None, initializer=None,regularizer=None,trainable=True,collections=None,caching_device=None,partitioner=None,validate_shape=True,custom_getter=None):\n"," with tf.device('/cpu:0'):\n"," var = tf.get_variable(\n"," name, shape=shape, dtype=dtype, initializer=initializer,\n"," regularizer=regularizer, trainable=trainable,\n"," collections=collections, caching_device=caching_device,\n"," partitioner=partitioner, validate_shape=validate_shape,\n"," custom_getter=custom_getter\n"," )\n"," return var"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"k8F7Y8CGbsB4","executionInfo":{"status":"aborted","timestamp":1684558232625,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def _get_weights_wrapper(name, shape, dtype=tf.float32, initializer=initializers.xavier_initializer(),weights_decay_factor=None):\n"," weights = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," if weights_decay_factor is not None and weights_decay_factor > 0.0:\n"," weights_wd = tf.multiply(tf.nn.l2_loss(weights), weights_decay_factor, name=name + '/l2loss')\n"," tf.add_to_collection('losses', weights_wd)\n"," return weights"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"nkO3w5gDcC1b","executionInfo":{"status":"aborted","timestamp":1684558232625,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def _get_biases_wrapper(name, shape, dtype=tf.float32, initializer=tf.constant_initializer(0.0)):\n"," \"\"\"Wrapper over _get_variable_wrapper() to get bias.\n"," \"\"\"\n"," biases = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," return biases"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"9Z06E9a1aBVb","executionInfo":{"status":"aborted","timestamp":1684558232625,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def _conv2d_wrapper(inputs, shape, strides, padding, add_bias, activation_fn, name, stddev=0.1):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE) as scope:\n"," kernel = _get_weights_wrapper(\n"," name='weights', shape=shape, weights_decay_factor=0.0, )\n"," output = tf.nn.conv2d(inputs, filter=kernel, strides=strides, padding=padding, name='conv')\n"," if add_bias:\n"," biases = _get_biases_wrapper(name='biases', shape=[shape[-1]] )\n"," output = tf.add(output, biases, name='biasAdd')\n"," if activation_fn is not None:\n"," output = activation_fn(output, name='activation')\n"," return output"]},{"cell_type":"markdown","metadata":{"id":"7dpVU5Sb1Ddr"},"source":["# Utils"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"MBkd3MRuccQh","executionInfo":{"status":"aborted","timestamp":1684558232625,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def softmax(x, axis=-1):\n"," ex = K.exp(x - K.max(x, axis=axis, keepdims=True))\n"," return ex/K.sum(ex, axis=axis, keepdims=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hSVP0ymhc6G6","executionInfo":{"status":"aborted","timestamp":1684558232625,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def squash_v1(x, axis=-1):\n"," s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n"," scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n"," return scale * x"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"RQzm6uHwc8Rq","executionInfo":{"status":"aborted","timestamp":1684558232626,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def squash_v0(s, axis=-1, epsilon=1e-7, name=None):\n"," s_squared_norm = K.sum(K.square(s), axis, keepdims=True) + K.epsilon()\n"," safe_norm = K.sqrt(s_squared_norm)\n"," scale = 1 - tf.exp(-safe_norm)\n"," return scale * s / safe_norm"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ObeP2wKDc-Zq","executionInfo":{"status":"aborted","timestamp":1684558232626,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations):\n"," b = keras.backend.zeros_like(u_hat_vecs[:,:,:,0])\n"," if i_activations is not None:\n"," i_activations = i_activations[...,tf.newaxis]\n"," for i in range(iterations):\n"," if False:\n"," leak = tf.zeros_like(b, optimize=True)\n"," leak = tf.reduce_sum(leak, axis=1, keep_dims=True)\n"," leaky_logits = tf.concat([leak, b], axis=1)\n"," leaky_routing = tf.nn.softmax(leaky_logits, dim=1) \n"," c = tf.split(leaky_routing, [1, output_capsule_num], axis=1)[1]\n"," else:\n"," c = softmax(b, 1) \n"," outputs = squash_v1(K.batch_dot(c, u_hat_vecs, [2, 2]))\n"," if i < iterations - 1:\n"," b = b + K.batch_dot(outputs, u_hat_vecs, [2, 3]) \n"," poses = outputs \n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," return poses, activations\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"zSdVM9YCdEMp","executionInfo":{"status":"aborted","timestamp":1684558232626,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def vec_transformationByConv(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num): \n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_dim, output_capsule_dim*output_capsule_num], weights_decay_factor=0.0)\n"," u_hat_vecs = keras.backend.conv1d(poses, kernel)\n"," u_hat_vecs = keras.backend.reshape(u_hat_vecs, (-1, input_capsule_num, output_capsule_num, output_capsule_dim))\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"i-pIavredIK6","executionInfo":{"status":"aborted","timestamp":1684558232626,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def vec_transformationByMat(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num, shared=True): \n"," inputs_poses_shape = poses.get_shape().as_list()\n"," poses = poses[..., tf.newaxis, :] \n"," poses = tf.tile(poses, [1, 1, output_capsule_num, 1]) \n"," if shared:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, 1, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], input_capsule_num, 1, 1, 1])\n"," else:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_num, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], 1, 1, 1, 1])\n"," u_hat_vecs = tf.squeeze(tf.matmul(kernel, poses[...,tf.newaxis]),axis=-1)\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"mcvHPFCOdKyX","executionInfo":{"status":"aborted","timestamp":1684558232626,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def capsules_init(inputs, shape, strides, padding, pose_shape, add_bias, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses = _conv2d_wrapper(\n"," inputs,\n"," shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," strides=strides,\n"," padding=padding,\n"," add_bias=add_bias,\n"," activation_fn=None,\n"," name='pose_stacked'\n"," ) \n"," poses_shape = poses.get_shape().as_list() \n"," poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," poses = squash_v1(poses, axis=-1) \n"," activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n","\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hAJ1njh6dOpT","executionInfo":{"status":"aborted","timestamp":1684558232626,"user_tz":-330,"elapsed":16,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def capsule_fc_layer(nets, output_capsule_num, iterations, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n","\n"," u_hat_vecs = vec_transformationByConv(poses,input_pose_shape[-1], input_pose_shape[1],input_pose_shape[-1], output_capsule_num,)\n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, output_capsule_num])\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations)\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jE-SPLBwdR2a","executionInfo":{"status":"aborted","timestamp":1684558232627,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def capsule_flatten(nets):\n"," poses, activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n"," \n"," poses = tf.reshape(poses, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3], input_pose_shape[-1]]) \n"," activations = tf.reshape(activations, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3]])\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"1uMUkhf1dVJy","executionInfo":{"status":"aborted","timestamp":1684558232627,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def capsule_conv_layer(nets, shape, strides, iterations, name): \n"," # poses = _conv2d_wrapper(\n"," # inputs,\n"," # shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," # strides=strides,\n"," # padding=padding,\n"," # add_bias=add_bias,\n"," # activation_fn=None,\n"," # name='pose_stacked'\n"," # ) \n"," # poses_shape = poses.get_shape().as_list() \n"," # poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," # beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," # poses = squash_v1(poses, axis=-1) \n"," # activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," \n"," inputs_poses_shape = poses.get_shape().as_list()\n","\n"," hk_offsets = [\n"," [(h_offset + k_offset) for k_offset in range(0, shape[0])] for h_offset in\n"," range(0, inputs_poses_shape[1] + 1 - shape[0], strides[1])\n"," ]\n"," wk_offsets = [\n"," [(w_offset + k_offset) for k_offset in range(0, shape[1])] for w_offset in\n"," range(0, inputs_poses_shape[2] + 1 - shape[1], strides[2])\n"," ]\n"," \n"," inputs_poses_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," poses, hk_offsets, axis=1, name='gather_poses_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_poses_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5, 6], name='inputs_poses_patches'\n"," )\n"," inputs_poses_shape = inputs_poses_patches.get_shape().as_list()\n"," inputs_poses_patches = tf.reshape(inputs_poses_patches, [\n"," -1, shape[0]*shape[1]*shape[2], inputs_poses_shape[-1]\n"," ])\n","\n"," i_activations_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," i_activations, hk_offsets, axis=1, name='gather_activations_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_activations_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5], name='inputs_activations_patches'\n"," )\n"," i_activations_patches = tf.reshape(i_activations_patches, [\n"," -1, shape[0]*shape[1]*shape[2]]\n"," )\n"," u_hat_vecs = vec_transformationByConv(\n"," inputs_poses_patches,\n"," inputs_poses_shape[-1], shape[0]*shape[1]*shape[2],\n"," inputs_poses_shape[-1], shape[3],\n"," ) \n"," beta_a = _get_weights_wrapper(\n"," name='beta_a', shape=[1, shape[3]]\n"," )\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, shape[3], i_activations_patches)\n"," poses = tf.reshape(poses, [\n"," inputs_poses_shape[0], inputs_poses_shape[1],\n"," inputs_poses_shape[2], shape[3],\n"," inputs_poses_shape[-1]]\n"," ) \n"," activations = tf.reshape(activations, [\n"," inputs_poses_shape[0],inputs_poses_shape[1],\n"," inputs_poses_shape[2],shape[3]]\n"," ) \n"," nets = poses, activations \n"," return nets"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"HLjZkFkFtPZT","executionInfo":{"status":"aborted","timestamp":1684558232627,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# from keras.layers import Dropout, Activation, Flatten\n","# from keras.layers import Bidirectional\n","# def BiGRU_layer(nets):\n","# nets = Bidirectional (GRU (300,return_sequences=True,dropout=drop_out_value,kernel_regularizer=l2(0.01)),merge_mode='concat')(nets)\n","# return nets"]},{"cell_type":"markdown","metadata":{"id":"dqBVV8BNfGPe"},"source":["# Capsule"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jam_fKSJjYfg","executionInfo":{"status":"aborted","timestamp":1684558232627,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def capsule_model_A(X, num_classes):\n"," with tf.variable_scope('capsule_'+str(3),reuse=tf.AUTO_REUSE ): \n"," nets = _conv2d_wrapper(\n"," X, shape=[3, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," nets = capsule_flatten(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2') \n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"dbqUEzcbotXD","executionInfo":{"status":"aborted","timestamp":1684558232627,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def capsule_model_B(X, num_classes):\n"," poses_list = []\n"," for _, ngram in enumerate([3,4,5,6]):\n"," with tf.variable_scope('capsule_'+str(ngram),reuse=tf.AUTO_REUSE): \n"," nets = _conv2d_wrapper(\n"," X, shape=[ngram, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," nets = capsule_flatten(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2')\n"," poses_list.append(poses)\n"," poses = tf.reduce_mean(tf.convert_to_tensor(poses_list), axis=0) \n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," return poses, activations"]},{"cell_type":"markdown","metadata":{"id":"PARyB28Vi9VI"},"source":["# Loss functions"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"WkAyzy_3hrYf","executionInfo":{"status":"aborted","timestamp":1684558232627,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def spread_loss(labels, activations, margin):\n"," activations_shape = activations.get_shape().as_list()\n"," mask_t = tf.equal(labels, 1)\n"," mask_i = tf.equal(labels, 0) \n"," activations_t = tf.reshape(\n"," tf.boolean_mask(activations, mask_t), [activations_shape[0], 1]\n"," ) \n"," activations_i = tf.reshape(\n"," tf.boolean_mask(activations, mask_i), [activations_shape[0], activations_shape[1] - 1]\n"," ) \n"," gap_mit = tf.reduce_sum(tf.square(tf.nn.relu(margin - (activations_t - activations_i))))\n"," return gap_mit "]},{"cell_type":"code","execution_count":null,"metadata":{"id":"n2UfkOrSjFvX","executionInfo":{"status":"aborted","timestamp":1684558232627,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def cross_entropy(y, preds): \n"," y = tf.argmax(y, axis=1)\n"," loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=preds, labels=y) \n"," loss = tf.reduce_mean(loss) \n"," return loss"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qEI1tgTnjUg_","executionInfo":{"status":"aborted","timestamp":1684558232628,"user_tz":-330,"elapsed":18,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["def margin_loss(y, preds): \n"," y = tf.cast(y,tf.float32)\n"," loss = y * tf.square(tf.maximum(0., 0.9 - preds)) + \\\n"," 0.25 * (1.0 - y) * tf.square(tf.maximum(0., preds - 0.1))\n"," loss = tf.reduce_mean(tf.reduce_sum(loss, axis=1))\n"," return loss"]},{"cell_type":"markdown","metadata":{"id":"Qz6aicUlku28"},"source":["# Setup"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Un6p6OLXkPnt","executionInfo":{"status":"aborted","timestamp":1684558232628,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["class Args:\n"," embedding_type = \"static\"\n"," dataset = \"\"\n"," loss_type = \"margin_loss\"\n"," model_type = \"capsule-B\"\n"," has_test = 1\n"," has_dev = 1\n"," num_epochs = 50\n"," batch_size = 64\n"," use_orphan = False\n"," use_leaky = False\n"," learning_rate = 0.01\n"," margin = 0.2\n"," num_classes = 2\n"," # vocab_size = vocab_size\n"," vec_size = 200\n"," max_sent = 28 #max_length - may change for dataset\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"3cA5u4HrkuCb","executionInfo":{"status":"aborted","timestamp":1684558232628,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["args = Args()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"cVRu8KNo3GcR","executionInfo":{"status":"aborted","timestamp":1684558232628,"user_tz":-330,"elapsed":17,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["with tf.device('/cpu:0'):\n"," global_step = tf.train.get_or_create_global_step()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"lFGsOcj50Eov","executionInfo":{"status":"aborted","timestamp":1684558232629,"user_tz":-330,"elapsed":18,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["class BatchGenerator(object):\n"," \"\"\"Generate and hold batches.\"\"\"\n"," def __init__(self, dataset,label, batch_size,input_size, is_shuffle=True):\n"," self._dataset = dataset\n"," self._label = label\n"," self._batch_size = batch_size \n"," self._cursor = 0 \n"," self._input_size = input_size \n"," \n"," if is_shuffle:\n"," index = np.arange(len(self._dataset))\n"," np.random.shuffle(index)\n"," self._dataset = np.array(self._dataset)[index]\n"," self._label = np.array(self._label)[index]\n"," else:\n"," self._dataset = np.array(self._dataset)\n"," self._label = np.array(self._label)\n"," def next(self):\n"," if self._cursor + self._batch_size > len(self._dataset):\n"," self._cursor = 0\n"," \"\"\"Generate a single batch from the current cursor position in the data.\"\"\" \n"," batch_x = self._dataset[self._cursor : self._cursor + self._batch_size,:]\n"," batch_y = self._label[self._cursor : self._cursor + self._batch_size]\n"," self._cursor += self._batch_size\n"," return batch_x, batch_y"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":18,"status":"aborted","timestamp":1684558232629,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"wV3H1xPDUZwU"},"outputs":[],"source":["import os \n","\n","fold_no = 1\n","\n","checkpoint_path = folder_path+'/capsule/cp.h5py'\n","checkpoint_dir = os.path.dirname(checkpoint_path)\n","cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n"," save_weights_only=True,\n"," verbose=1)\n","\n","\n","# X = tf.placeholder(tf.int32, [args.batch_size, args.max_sent,args.vec_size], name=\"input_x\")\n","y = tf.placeholder(tf.int64, [args.batch_size, args.num_classes], name=\"input_y\")\n","is_training = tf.placeholder_with_default(False, shape=()) \n","learning_rate = tf.placeholder(dtype='float32') \n","margin = tf.placeholder(shape=(),dtype='float32') \n","\n","l2_loss = tf.constant(0.0)\n","\n","# w2v = np.array(w2v,dtype=np.float32)\n","# W1 = tf.Variable(w2v, trainable = False)\n","# X_embedding = np.float32(np.array([i for i in dataset['posts']]))\n","X = tf.placeholder(tf.float32, [args.batch_size, args.max_sent,args.vec_size], name=\"input_x\")\n","X = X[...,tf.newaxis]\n","poses, activations = capsule_model_B(X, args.num_classes)\n","# poses, activations = capsule_model_B(train_set, args.num_classes)\n","loss = margin_loss(y, activations) \n","y_pred = tf.argmax(activations, axis=1, name=\"y_proba\") \n","correct = tf.equal(tf.argmax(y, axis=1), y_pred, name=\"correct\")\n","accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n","\n","optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, name = 'opt'+str(fold_no)) \n","training_op = optimizer.minimize(loss, name=\"training_op\")\n","gradients, variables = zip(*optimizer.compute_gradients(loss)) \n","\n","grad_check = [tf.check_numerics(g, message='Gradient NaN Found!')\n"," for g in gradients if g is not None] + [tf.check_numerics(loss, message='Loss NaN Found')]\n","with tf.control_dependencies(grad_check):\n"," training_op = optimizer.apply_gradients(zip(gradients, variables), global_step=global_step) "]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Rhjeg8m9lKdQ","executionInfo":{"status":"aborted","timestamp":1684558232629,"user_tz":-330,"elapsed":18,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# np.transpose(dataset['posts'][3976]).shape"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"bn04tN82m-_y","executionInfo":{"status":"aborted","timestamp":1684558232629,"user_tz":-330,"elapsed":18,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# X.shape"]},{"cell_type":"markdown","metadata":{"id":"RwyuzXZBflX7"},"source":["# Train"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"1Xh_EEQEO_44","executionInfo":{"status":"aborted","timestamp":1684558232629,"user_tz":-330,"elapsed":18,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["sess = tf.InteractiveSession()\n","from keras import utils\n","import torch\n","\n","n_iterations_per_epoch = len(train_set) // args.batch_size\n","n_iterations_test = len(test_set) // args.batch_size\n","n_iterations_dev = len(dev_set) // args.batch_size \n","\n","mr_train = BatchGenerator(train_set,train_label, args.batch_size, 0) \n","mr_dev = BatchGenerator(dev_set,dev_label, args.batch_size, 0)\n","mr_test = BatchGenerator(test_set,test_label, args.batch_size, 0, is_shuffle=False)\n","\n","init = tf.global_variables_initializer()\n","sess.run(init) \n","\n","lr = args.learning_rate\n","m = args.margin\n","for epoch in range(args.num_epochs):\n"," for iteration in range(1, n_iterations_per_epoch + 1): \n"," X_batch, y_batch = mr_train.next() \n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size, 1)\n"," # y_batch =y_batch.reshape(64, 28, 200, 1)\n"," # print (X_batch.shape, y_batch.shape)\n"," y_batch = utils.to_categorical(y_batch, args.num_classes) \n"," _, loss_train, probs, capsule_pose = sess.run(\n"," [training_op, loss, activations, poses],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: True,\n"," learning_rate:lr,\n"," margin:m}) \n"," # print(\"\\rIteration: {}/{} ({:.1f}%) Loss: {:.5f}\".format(iteration, n_iterations_per_epoch,iteration * 100 / n_iterations_per_epoch,loss_train),end=\"\") \n"," loss_vals, acc_vals = [], []\n"," for iteration in range(1, n_iterations_dev + 1):\n"," X_batch, y_batch = mr_dev.next() \n"," y_batch = utils.to_categorical(y_batch, args.num_classes)\n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size, 1)\n"," loss_val, acc_val = sess.run(\n"," [loss, accuracy],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: False,\n"," margin:m})\n"," loss_vals.append(loss_val)\n"," acc_vals.append(acc_val)\n"," loss_val, acc_val = np.mean(loss_vals), np.mean(acc_vals) \n"," # print(\"\\rEpoch: {} Val accuracy: {:.1f}% Loss: {:.4f}\".format(epoch + 1, acc_val * 100, loss_val))\n"," \n"," preds_list, y_list = [], []\n"," for iteration in range(1, n_iterations_test + 1):\n"," X_batch, y_batch = mr_test.next()\n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size, 1) \n"," probs = sess.run([activations],\n"," feed_dict={X:X_batch[:,:args.max_sent],\n"," is_training: False})\n"," preds_list = preds_list + probs[0].tolist()\n"," y_list = y_list + y_batch.tolist()\n"," \n"," y_list = np.array(y_list)\n"," y_list = utils.to_categorical(y_list, args.num_classes)\n"," preds_probs = np.array(preds_list) \n"," preds_probs = (preds_probs == preds_probs.max(axis=1)[:,None]).astype(int)\n"," \n"," [precision, recall, F1, support] = precision_recall_fscore_support(y_list, preds_probs, average='samples')\n"," acc = accuracy_score(y_list, preds_probs)\n","\n"," print ('\\rER: %.3f' % acc, 'Precision: %.3f' % precision, 'Recall: %.3f' % recall, 'F1: %.3f' % F1) \n"," m = min(0.9, m + 0.1)\n"," # print (m)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"k0fVwixUfgQu","executionInfo":{"status":"aborted","timestamp":1684558232629,"user_tz":-330,"elapsed":18,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# with tf.Session() as sess:\n","\n","# init = tf.global_variables_initializer()\n","# sess.run(init)\n","\n","# for epoch in range(num_epochs): \n","\n","# for iteration in range(1, n_iterations_per_epoch + 1):\n","# X_batch, y_batch = mr_train1.next() \n","# y_batch = utils.to_categorical(y_batch, args.num_classes) \n","# _, loss_train, probs, capsule_pose = sess.run(\n","# [training_op, loss, activations, poses],\n","# feed_dict={X: X_batch[:,:args.max_sent],\n","# y: y_batch,\n","# is_training: True,\n","# learning_rate:lr,\n","# margin:m})\n"," \n","# print(\"\\rIteration: {}/{} ({:.1f}%) epoch:{} Loss: {:.5f}\".format(iteration, n_iterations_per_epoch, iteration * 100 / n_iterations_per_epoch, epoch+1, loss_train), end=\"\") \n","# loss_vals, acc_vals = [], []\n","# for iteration in range(1, n_iterations_test + 1):\n","# X_batch, y_batch = mr_test1.next() \n","# y_batch = utils.to_categorical(y_batch, args.num_classes) \n","# loss_val, acc_val = sess.run([activations],\n","# feed_dict={X:X_batch[:,:args.max_sent],\n","# is_training: False})\n","# loss_vals.append(loss_val)\n","# acc_vals.append(acc_val)\n","# loss_val, acc_val = np.mean(loss_vals), np.mean(acc_vals) \n","# print(\"\\rEpoch: {} Val accuracy: {:.1f}% Loss: {:.4f}\".format(\n","# epoch + 1, acc_val * 100, loss_val))\n"," \n","# preds_list, y_list = [], []\n","# for iteration in range(1, n_iterations_test + 1):\n","# X_batch, y_batch = mr_test.next() \n","# probs = sess.run([activations],\n","# feed_dict={X:X_batch[:,:args.max_sent],\n","# is_training: False})\n","# preds_list = preds_list + probs[0].tolist()\n","# y_list = y_list + y_batch.tolist()\n"," \n","# y_list = np.array(y_list)\n","# preds_probs = np.array(preds_list) \n","# preds_probs[np.where( preds_probs >= threshold )] = 1.0\n","# preds_probs[np.where( preds_probs < threshold )] = 0.0 \n"," \n","# [precision, recall, F1, support] = \\\n","# precision_recall_fscore_support(y_list, preds_probs, average='samples')\n","# acc = accuracy_score(y_list, preds_probs)\n","\n","# print ('\\rER: %.3f' % acc, 'Precision: %.3f' % precision, 'Recall: %.3f' % recall, 'F1: %.3f' % F1) \n","# if args.model_type == 'CNN' or args.model_type == 'KIMCNN':\n","# lr = max(1e-6, lr * 0.8)\n","# if args.loss_type == 'margin_loss': \n","# m = min(0.9, m + 0.1)\n","\n","# m = min(0.9, m + 0.1) \n","# print('m=',str(m))\n","# y_list = np.array(y_list)\n","# preds_probs = np.array(preds_list) \n","# labels = np.argmax(y_list, axis=1)\n","# predictions = np.argmax(preds_probs, axis=1)\n","\n","# accuracy_fold = accuracy_score(labels, predictions)\n","# precision_fold = precision_score(labels, predictions, average='weighted', zero_division = 0 )\n","# recall_fold = recall_score(labels, predictions, average='weighted')\n","# f1_fold = f1_score(labels, predictions, average='weighted')\n","# print(\"\\rFold: {} accuracy: {:.4f}% Precision: {:.4f} recall: {:.4f} F1: {:.4f}\".format(fold_no, accuracy_fold, precision_fold, recall_fold, f1_fold))\n"]}],"metadata":{"colab":{"collapsed_sections":["PARyB28Vi9VI"],"machine_shape":"hm","provenance":[{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/Sinhala_text_capsules.ipynb","timestamp":1621834144627}]},"environment":{"name":"tf-gpu.1-15.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf-gpu.1-15:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"nbformat":4,"nbformat_minor":0}

sentiment-analysis-models/capsule-networks/capsule_network_with_parallel_BiLSTM.ipynb ADDED Viewed

	@@ -0,0 +1 @@

+ {"cells":[{"cell_type":"markdown","metadata":{"id":"k8D9EuJnBnDV"},"source":["# Installing and importing dependecies"]},{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1650264170060,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"fp5wHxfHqnPj"},"outputs":[],"source":["embedding_type = \"MAX\" #@param [\"GRU\",\"MAX\"]\n","EMBEDDING_SIZE = 200 #@param [200,300]\n","\n","capsule_routes = \"3\" #@param [\"3\",\"4\"]\n","pool_layer = \"avg\" #@param [\"avg\",\"max\",\"min\"]\n","bilstm = \"para\" #@param [\"serial\",\"para\",\"both\"]"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":36295,"status":"ok","timestamp":1650264206349,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"GgRvEbMXy9nc","outputId":"94807269-35d6-45bc-9275-910742ee003e"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":53867,"status":"ok","timestamp":1650264260205,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"CCi-HNCn7BCi","outputId":"403bfa79-0afe-4cff-f2a2-0629d9fff84b"},"outputs":[{"name":"stdout","output_type":"stream","text":["\u001b[33mWARNING: Skipping keras-nightly as it is not installed.\u001b[0m\n","Collecting tensorflow==1.14.0\n"," Downloading tensorflow-1.14.0-cp37-cp37m-manylinux1_x86_64.whl (109.3 MB)\n","\u001b[K |████████████████████████████████| 109.3 MB 51 kB/s \n","\u001b[?25hRequirement already satisfied: gast\u003e=0.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.5.3)\n","Requirement already satisfied: keras-preprocessing\u003e=1.0.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.2)\n","Requirement already satisfied: six\u003e=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.15.0)\n","Requirement already satisfied: wrapt\u003e=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.14.0)\n","Requirement already satisfied: wheel\u003e=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.37.1)\n","Requirement already satisfied: google-pasta\u003e=0.1.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.2.0)\n","Requirement already satisfied: absl-py\u003e=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.0.0)\n","Requirement already satisfied: astor\u003e=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.8.1)\n","Requirement already satisfied: numpy\u003c2.0,\u003e=1.14.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.21.5)\n","Collecting tensorboard\u003c1.15.0,\u003e=1.14.0\n"," Downloading tensorboard-1.14.0-py3-none-any.whl (3.1 MB)\n","\u001b[K |████████████████████████████████| 3.1 MB 37.8 MB/s \n","\u001b[?25hRequirement already satisfied: grpcio\u003e=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.44.0)\n","Collecting tensorflow-estimator\u003c1.15.0rc0,\u003e=1.14.0rc0\n"," Downloading tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488 kB)\n","\u001b[K |████████████████████████████████| 488 kB 53.3 MB/s \n","\u001b[?25hRequirement already satisfied: termcolor\u003e=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.0)\n","Requirement already satisfied: protobuf\u003e=3.6.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (3.17.3)\n","Collecting keras-applications\u003e=1.0.6\n"," Downloading Keras_Applications-1.0.8-py3-none-any.whl (50 kB)\n","\u001b[K |████████████████████████████████| 50 kB 6.7 MB/s \n","\u001b[?25hRequirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras-applications\u003e=1.0.6-\u003etensorflow==1.14.0) (3.1.0)\n","Requirement already satisfied: werkzeug\u003e=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (1.0.1)\n","Requirement already satisfied: markdown\u003e=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (3.3.6)\n","Requirement already satisfied: setuptools\u003e=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (57.4.0)\n","Requirement already satisfied: importlib-metadata\u003e=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown\u003e=2.6.8-\u003etensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (4.11.3)\n","Requirement already satisfied: zipp\u003e=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata\u003e=4.4-\u003emarkdown\u003e=2.6.8-\u003etensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (3.8.0)\n","Requirement already satisfied: typing-extensions\u003e=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata\u003e=4.4-\u003emarkdown\u003e=2.6.8-\u003etensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (4.1.1)\n","Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py-\u003ekeras-applications\u003e=1.0.6-\u003etensorflow==1.14.0) (1.5.2)\n","Installing collected packages: tensorflow-estimator, tensorboard, keras-applications, tensorflow\n"," Attempting uninstall: tensorflow-estimator\n"," Found existing installation: tensorflow-estimator 2.8.0\n"," Uninstalling tensorflow-estimator-2.8.0:\n"," Successfully uninstalled tensorflow-estimator-2.8.0\n"," Attempting uninstall: tensorboard\n"," Found existing installation: tensorboard 2.8.0\n"," Uninstalling tensorboard-2.8.0:\n"," Successfully uninstalled tensorboard-2.8.0\n"," Attempting uninstall: tensorflow\n"," Found existing installation: tensorflow 2.8.0\n"," Uninstalling tensorflow-2.8.0:\n"," Successfully uninstalled tensorflow-2.8.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","kapre 0.3.7 requires tensorflow\u003e=2.0.0, but you have tensorflow 1.14.0 which is incompatible.\u001b[0m\n","Successfully installed keras-applications-1.0.8 tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0\n","Collecting q\n"," Downloading q-2.6-py2.py3-none-any.whl (6.8 kB)\n","Collecting keras==2.2.2\n"," Downloading Keras-2.2.2-py2.py3-none-any.whl (299 kB)\n","\u001b[K |████████████████████████████████| 299 kB 14.1 MB/s \n","\u001b[?25hRequirement already satisfied: six\u003e=1.9.0 in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (1.15.0)\n","Requirement already satisfied: numpy\u003e=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (1.21.5)\n","Requirement already satisfied: scipy\u003e=0.14 in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (1.4.1)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (3.1.0)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (3.13)\n","Collecting keras-applications==1.0.4\n"," Downloading Keras_Applications-1.0.4-py2.py3-none-any.whl (43 kB)\n","\u001b[K |████████████████████████████████| 43 kB 2.1 MB/s \n","\u001b[?25hCollecting keras-preprocessing==1.0.2\n"," Downloading Keras_Preprocessing-1.0.2-py2.py3-none-any.whl (26 kB)\n","Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py-\u003ekeras==2.2.2) (1.5.2)\n","Installing collected packages: keras-preprocessing, keras-applications, q, keras\n"," Attempting uninstall: keras-preprocessing\n"," Found existing installation: Keras-Preprocessing 1.1.2\n"," Uninstalling Keras-Preprocessing-1.1.2:\n"," Successfully uninstalled Keras-Preprocessing-1.1.2\n"," Attempting uninstall: keras-applications\n"," Found existing installation: Keras-Applications 1.0.8\n"," Uninstalling Keras-Applications-1.0.8:\n"," Successfully uninstalled Keras-Applications-1.0.8\n"," Attempting uninstall: keras\n"," Found existing installation: keras 2.8.0\n"," Uninstalling keras-2.8.0:\n"," Successfully uninstalled keras-2.8.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","tensorflow 1.14.0 requires keras-applications\u003e=1.0.6, but you have keras-applications 1.0.4 which is incompatible.\n","tensorflow 1.14.0 requires keras-preprocessing\u003e=1.0.5, but you have keras-preprocessing 1.0.2 which is incompatible.\n","kapre 0.3.7 requires tensorflow\u003e=2.0.0, but you have tensorflow 1.14.0 which is incompatible.\u001b[0m\n","Successfully installed keras-2.2.2 keras-applications-1.0.4 keras-preprocessing-1.0.2 q-2.6\n","Collecting h5py\u003c3.0.0\n"," Downloading h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)\n","\u001b[K |████████████████████████████████| 2.9 MB 14.3 MB/s \n","\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from h5py\u003c3.0.0) (1.15.0)\n","Requirement already satisfied: numpy\u003e=1.7 in /usr/local/lib/python3.7/dist-packages (from h5py\u003c3.0.0) (1.21.5)\n","Installing collected packages: h5py\n"," Attempting uninstall: h5py\n"," Found existing installation: h5py 3.1.0\n"," Uninstalling h5py-3.1.0:\n"," Successfully uninstalled h5py-3.1.0\n","Successfully installed h5py-2.10.0\n"]}],"source":["!pip uninstall keras-nightly\n","!pip install tensorflow==1.14.0\n","!pip install q keras==2.2.2\n","# !pip install q keras==2.3.1\n","!pip install 'h5py\u003c3.0.0'"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4679,"status":"ok","timestamp":1650264264873,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"zlXBE_-86l4q","outputId":"8e84d33a-9ac5-4d5f-e101-ab3d46cfc6ef"},"outputs":[{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n","Using TensorFlow backend.\n"]}],"source":["import pandas as pd\n","import numpy as np\n","import tensorflow as tf\n","import keras\n","\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, precision_recall_fscore_support\n","\n","from keras import backend as K\n","from keras.models import Model,load_model\n","from keras.layers import Flatten, Input, Dense,TimeDistributed, Bidirectional\n","from keras.regularizers import l2\n","from keras import callbacks\n","from keras.utils import generic_utils,plot_model\n","from keras.callbacks import ModelCheckpoint,EarlyStopping\n","from keras.layers.recurrent import LSTM\n","from keras.optimizers import Adadelta\n","\n","import matplotlib.image as mpimg\n","import matplotlib.pyplot as plt\n","\n","from tensorflow.contrib.layers.python.layers import initializers"]},{"cell_type":"markdown","metadata":{"id":"nRbpY2FTDl8p"},"source":["# Load Dataset"]},{"cell_type":"code","execution_count":5,"metadata":{"executionInfo":{"elapsed":72197,"status":"ok","timestamp":1650264337065,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"d8GzWXqBsX2V"},"outputs":[],"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Capsule networks/'\n","\n","if (embedding_type == 'GRU'):\n"," facebook_data_path = folder_path + 'embedded posts/GRU_attention_'\n","elif (embedding_type == 'MAX'):\n"," facebook_data_path = folder_path + 'embedded posts/'+'MAX_'\n","\n","context = 5\n","posts=[]\n","labels =[]\n","\n","for i in range (1,6):\n"," data_path = (facebook_data_path +str(i)+'.csv')\n"," dataset = pd.read_pickle(data_path)\n"," dataset=dataset.dropna()\n"," # dataset = dataset[:1000]\n"," labels+=dataset[\"label\"].tolist()\n"," for j in dataset['posts']:\n"," posts.append(j)\n","\n","# for i in range (1,6):\n","# facebook_data_path = (folder_path + 'embedded posts/'+sentence_type+'_'+decoder+'_'+str(i)+'.csv')\n","# dataset = pd.read_pickle(facebook_data_path)\n","# dataset=dataset.dropna()\n","# # dataset = dataset[:1000]\n","# labels+=dataset[\"label\"].tolist()\n","# for j in dataset['posts']:\n","# posts.append(j)"]},{"cell_type":"code","execution_count":6,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1650264337066,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"a2kpq4_276nB"},"outputs":[],"source":["labels = pd.get_dummies(labels).values"]},{"cell_type":"code","execution_count":7,"metadata":{"executionInfo":{"elapsed":14,"status":"ok","timestamp":1650264337067,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"0m6FrtmcsRKu"},"outputs":[],"source":["train_set, test_set , train_label, test_label = train_test_split(posts,labels, test_size=0.1, random_state=0)\n","train_set, dev_set, train_label, dev_label = train_test_split(train_set,train_label, test_size=0.1, random_state=0)"]},{"cell_type":"markdown","metadata":{"id":"XnkydX1ne969"},"source":["# Convo 2d layer"]},{"cell_type":"code","execution_count":8,"metadata":{"executionInfo":{"elapsed":12,"status":"ok","timestamp":1650264337067,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"0GVuHgh2EWw2"},"outputs":[],"source":["def _get_variable_wrapper(name, shape=None, dtype=None, initializer=None,regularizer=None,trainable=True,collections=None,caching_device=None,partitioner=None,validate_shape=True,custom_getter=None):\n"," with tf.device('/cpu:0'):\n"," var = tf.get_variable(\n"," name, shape=shape, dtype=dtype, initializer=initializer,\n"," regularizer=regularizer, trainable=trainable,\n"," collections=collections, caching_device=caching_device,\n"," partitioner=partitioner, validate_shape=validate_shape,\n"," custom_getter=custom_getter\n"," )\n"," return var"]},{"cell_type":"code","execution_count":9,"metadata":{"executionInfo":{"elapsed":11,"status":"ok","timestamp":1650264337067,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"k8F7Y8CGbsB4"},"outputs":[],"source":["def _get_weights_wrapper(name, shape, dtype=tf.float32, initializer=initializers.xavier_initializer(),weights_decay_factor=None):\n"," weights = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," if weights_decay_factor is not None and weights_decay_factor \u003e 0.0:\n"," weights_wd = tf.multiply(tf.nn.l2_loss(weights), weights_decay_factor, name=name + '/l2loss')\n"," tf.add_to_collection('losses', weights_wd)\n"," return weights"]},{"cell_type":"code","execution_count":10,"metadata":{"executionInfo":{"elapsed":12,"status":"ok","timestamp":1650264337068,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"nkO3w5gDcC1b"},"outputs":[],"source":["def _get_biases_wrapper(name, shape, dtype=tf.float32, initializer=tf.constant_initializer(0.0)):\n"," \"\"\"Wrapper over _get_variable_wrapper() to get bias.\n"," \"\"\"\n"," biases = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," return biases"]},{"cell_type":"code","execution_count":11,"metadata":{"executionInfo":{"elapsed":11,"status":"ok","timestamp":1650264337068,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"9Z06E9a1aBVb"},"outputs":[],"source":["def _conv2d_wrapper(inputs, shape, strides, padding, add_bias, activation_fn, name, stddev=0.1):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE) as scope:\n"," kernel = _get_weights_wrapper(\n"," name='weights', shape=shape, weights_decay_factor=0.0, )\n"," output = tf.nn.conv2d(inputs, filter=kernel, strides=strides, padding=padding, name='conv')\n"," if add_bias:\n"," biases = _get_biases_wrapper(name='biases', shape=[shape[-1]] )\n"," output = tf.add(output, biases, name='biasAdd')\n"," if activation_fn is not None:\n"," output = activation_fn(output, name='activation')\n"," return output"]},{"cell_type":"markdown","metadata":{"id":"7dpVU5Sb1Ddr"},"source":["# Utils"]},{"cell_type":"code","execution_count":12,"metadata":{"executionInfo":{"elapsed":11,"status":"ok","timestamp":1650264337069,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"MBkd3MRuccQh"},"outputs":[],"source":["def softmax(x, axis=-1):\n"," ex = K.exp(x - K.max(x, axis=axis, keepdims=True))\n"," return ex/K.sum(ex, axis=axis, keepdims=True)"]},{"cell_type":"code","execution_count":13,"metadata":{"executionInfo":{"elapsed":9,"status":"ok","timestamp":1650264337069,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"hSVP0ymhc6G6"},"outputs":[],"source":["def squash_v1(x, axis=-1):\n"," s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n"," scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n"," return scale * x"]},{"cell_type":"code","execution_count":14,"metadata":{"executionInfo":{"elapsed":849,"status":"ok","timestamp":1650264337909,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"RQzm6uHwc8Rq"},"outputs":[],"source":["def squash_v0(s, axis=-1, epsilon=1e-7, name=None):\n"," s_squared_norm = K.sum(K.square(s), axis, keepdims=True) + K.epsilon()\n"," safe_norm = K.sqrt(s_squared_norm)\n"," scale = 1 - tf.exp(-safe_norm)\n"," return scale * s / safe_norm"]},{"cell_type":"code","execution_count":15,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1650264337910,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"ObeP2wKDc-Zq"},"outputs":[],"source":["def routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations):\n"," b = keras.backend.zeros_like(u_hat_vecs[:,:,:,0])\n"," if i_activations is not None:\n"," i_activations = i_activations[...,tf.newaxis]\n"," for i in range(iterations):\n"," if False:\n"," leak = tf.zeros_like(b, optimize=True)\n"," leak = tf.reduce_sum(leak, axis=1, keep_dims=True)\n"," leaky_logits = tf.concat([leak, b], axis=1)\n"," leaky_routing = tf.nn.softmax(leaky_logits, dim=1) \n"," c = tf.split(leaky_routing, [1, output_capsule_num], axis=1)[1]\n"," else:\n"," c = softmax(b, 1) \n"," outputs = squash_v1(K.batch_dot(c, u_hat_vecs, [2, 2]))\n"," if i \u003c iterations - 1:\n"," b = b + K.batch_dot(outputs, u_hat_vecs, [2, 3]) \n"," poses = outputs \n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," return poses, activations\n"]},{"cell_type":"code","execution_count":16,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1650264337910,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"zSdVM9YCdEMp"},"outputs":[],"source":["def vec_transformationByConv(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num): \n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_dim, output_capsule_dim*output_capsule_num], weights_decay_factor=0.0)\n"," u_hat_vecs = keras.backend.conv1d(poses, kernel)\n"," u_hat_vecs = keras.backend.reshape(u_hat_vecs, (-1, input_capsule_num, output_capsule_num, output_capsule_dim))\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs\n"]},{"cell_type":"code","execution_count":17,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1650264337910,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"i-pIavredIK6"},"outputs":[],"source":["def vec_transformationByMat(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num, shared=True): \n"," inputs_poses_shape = poses.get_shape().as_list()\n"," poses = poses[..., tf.newaxis, :] \n"," poses = tf.tile(poses, [1, 1, output_capsule_num, 1]) \n"," if shared:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, 1, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], input_capsule_num, 1, 1, 1])\n"," else:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_num, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], 1, 1, 1, 1])\n"," u_hat_vecs = tf.squeeze(tf.matmul(kernel, poses[...,tf.newaxis]),axis=-1)\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs"]},{"cell_type":"code","execution_count":18,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1650264337911,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"mcvHPFCOdKyX"},"outputs":[],"source":["def capsules_init(inputs, shape, strides, padding, pose_shape, add_bias, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses = _conv2d_wrapper(\n"," inputs,\n"," shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," strides=strides,\n"," padding=padding,\n"," add_bias=add_bias,\n"," activation_fn=None,\n"," name='pose_stacked'\n"," ) \n"," poses_shape = poses.get_shape().as_list() \n"," poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," poses = squash_v1(poses, axis=-1) \n"," activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n","\n"," return poses, activations"]},{"cell_type":"code","execution_count":19,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1650264337911,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"hAJ1njh6dOpT"},"outputs":[],"source":["def capsule_fc_layer(nets, output_capsule_num, iterations, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n","\n"," u_hat_vecs = vec_transformationByConv(poses,input_pose_shape[-1], input_pose_shape[1],input_pose_shape[-1], output_capsule_num,)\n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, output_capsule_num])\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations)\n"," return poses, activations"]},{"cell_type":"code","execution_count":20,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337912,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"jE-SPLBwdR2a"},"outputs":[],"source":["def capsule_flatten(nets):\n"," poses, activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n"," \n"," poses = tf.reshape(poses, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3], input_pose_shape[-1]]) \n"," activations = tf.reshape(activations, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3]])\n"," return poses, activations"]},{"cell_type":"code","execution_count":21,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1650264337912,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"1uMUkhf1dVJy"},"outputs":[],"source":["def capsule_conv_layer(nets, shape, strides, iterations, name): \n"," # poses = _conv2d_wrapper(\n"," # inputs,\n"," # shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," # strides=strides,\n"," # padding=padding,\n"," # add_bias=add_bias,\n"," # activation_fn=None,\n"," # name='pose_stacked'\n"," # ) \n"," # poses_shape = poses.get_shape().as_list() \n"," # poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," # beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," # poses = squash_v1(poses, axis=-1) \n"," # activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," \n"," inputs_poses_shape = poses.get_shape().as_list()\n","\n"," hk_offsets = [\n"," [(h_offset + k_offset) for k_offset in range(0, shape[0])] for h_offset in\n"," range(0, inputs_poses_shape[1] + 1 - shape[0], strides[1])\n"," ]\n"," wk_offsets = [\n"," [(w_offset + k_offset) for k_offset in range(0, shape[1])] for w_offset in\n"," range(0, inputs_poses_shape[2] + 1 - shape[1], strides[2])\n"," ]\n"," \n"," inputs_poses_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," poses, hk_offsets, axis=1, name='gather_poses_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_poses_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5, 6], name='inputs_poses_patches'\n"," )\n"," inputs_poses_shape = inputs_poses_patches.get_shape().as_list()\n"," inputs_poses_patches = tf.reshape(inputs_poses_patches, [\n"," -1, shape[0]*shape[1]*shape[2], inputs_poses_shape[-1]\n"," ])\n","\n"," i_activations_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," i_activations, hk_offsets, axis=1, name='gather_activations_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_activations_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5], name='inputs_activations_patches'\n"," )\n"," i_activations_patches = tf.reshape(i_activations_patches, [\n"," -1, shape[0]*shape[1]*shape[2]]\n"," )\n"," u_hat_vecs = vec_transformationByConv(\n"," inputs_poses_patches,\n"," inputs_poses_shape[-1], shape[0]*shape[1]*shape[2],\n"," inputs_poses_shape[-1], shape[3],\n"," ) \n"," beta_a = _get_weights_wrapper(\n"," name='beta_a', shape=[1, shape[3]]\n"," )\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, shape[3], i_activations_patches)\n"," poses = tf.reshape(poses, [\n"," inputs_poses_shape[0], inputs_poses_shape[1],\n"," inputs_poses_shape[2], shape[3],\n"," inputs_poses_shape[-1]]\n"," ) \n"," activations = tf.reshape(activations, [\n"," inputs_poses_shape[0],inputs_poses_shape[1],\n"," inputs_poses_shape[2],shape[3]]\n"," ) \n"," nets = poses, activations \n"," return nets"]},{"cell_type":"code","execution_count":22,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1650264337912,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"Pa-4JhSSJGof"},"outputs":[],"source":["def f1(y_true, y_pred):\n"," def recall(y_true, y_pred):\n"," \"\"\"Recall metric.\n","\n"," Only computes a batch-wise average of recall.\n","\n"," Computes the recall, a metric for multi-label classification of\n"," how many relevant items are selected.\n"," \"\"\"\n"," true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n"," possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n"," recall = true_positives / (possible_positives + K.epsilon())\n"," return recall\n","\n"," def precision(y_true, y_pred):\n"," \"\"\"Precision metric.\n","\n"," Only computes a batch-wise average of precision.\n","\n"," Computes the precision, a metric for multi-label classification of\n"," how many selected items are relevant.\n"," \"\"\"\n"," true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n"," predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n"," precision = true_positives / (predicted_positives + K.epsilon())\n"," return precision\n"," precision = precision(y_true, y_pred)\n"," recall = recall(y_true, y_pred)\n"," return 2*((precision*recall)/(precision+recall+K.epsilon()))"]},{"cell_type":"markdown","metadata":{"id":"5ZVUDvuhRupy"},"source":["# BiLSTM"]},{"cell_type":"code","execution_count":23,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337913,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"KgAA0tkXRyPC"},"outputs":[],"source":["def BiLSTM_model(maxlen, drop_out_value):\n"," input= Input(shape=(args.max_sent,args.vec_size,))\n"," model = Bidirectional (LSTM (args.vec_size,return_sequences=True,dropout=drop_out_value,kernel_regularizer=l2(0.01)),merge_mode='concat')(input)\n"," model = TimeDistributed(Dense(args.vec_size,activation='relu'))(model)\n"," model = Flatten()(model)\n"," output = Dense(2,activation='softmax')(model)\n"," model = Model(input,output)\n"," model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy',\n"," tf.keras.metrics.Precision(),\n"," tf.keras.metrics.Recall(),\n"," f1])\n"," \n"," # output = Dense(1,activation='sigmoid', name='output')(model)\n"," # model = Model(input,output)\n"," # model.compile(loss={'output':'binary_crossentropy'},\n"," # optimizer=Adadelta(lr=0.95, epsilon=1e-06),\n"," # metrics=[\"accuracy\",\n"," # tf.keras.metrics.Precision(),\n"," # tf.keras.metrics.Recall(),\n"," # f1])\n"," print(model.summary())\n"," return model\n","\n","# final model\n","def stacked_BiLSTM_model_2(maxlen, drop_out_value):\n"," input= Input(shape=(args.max_sent,args.vec_size))\n"," # input = Input(shape=(256, 28, 200, 1))\n"," model = Bidirectional (LSTM (args.vec_size,return_sequences=True,dropout=drop_out_value,kernel_regularizer=l2(0.01)),merge_mode='concat')(input)\n"," model = Bidirectional (LSTM (args.vec_size,return_sequences=True),merge_mode='concat')(model)\n"," model = TimeDistributed(Dense(args.vec_size,activation='relu'))(model)\n"," model = Flatten()(model)\n"," model = Dense(args.vec_size,activation='relu')(model)\n"," output = Dense(2,activation='softmax')(model)\n"," model = Model(input,output)\n"," model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy',\n"," tf.keras.metrics.Precision(),\n"," tf.keras.metrics.Recall(),\n"," f1])\n"," \n"," # output = Dense(1,activation='sigmoid', name='output')(model)\n"," # model = Model(input,output)\n"," # model.compile(loss={'output':'binary_crossentropy'},\n"," # optimizer=Adadelta(lr=0.95, epsilon=1e-06),\n"," # metrics=[\"accuracy\",\n"," # tf.keras.metrics.Precision(),\n"," # tf.keras.metrics.Recall(),\n"," # f1])\n"," print(model.summary())\n","\n"," return model\n","\n","# final model\n","def stacked_BiLSTM_model_3(maxlen, drop_out_value):\n"," input= Input(shape=(args.max_sent,args.vec_size))\n"," # input = Input(shape=(256, 28, 200, 1))\n"," model = Bidirectional (LSTM (args.vec_size,return_sequences=True,dropout=drop_out_value,kernel_regularizer=l2(0.01)),merge_mode='concat')(input)\n"," model = Bidirectional (LSTM (args.vec_size,return_sequences=True),merge_mode='concat')(model)\n"," model = Bidirectional (LSTM (args.vec_size,return_sequences=True),merge_mode='concat')(model)\n"," model = TimeDistributed(Dense(args.vec_size,activation='relu'))(model)\n"," model = Flatten()(model)\n"," model = Dense(args.vec_size,activation='relu')(model)\n"," output = Dense(2,activation='softmax')(model)\n"," model = Model(input,output)\n"," model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy',\n"," tf.keras.metrics.Precision(),\n"," tf.keras.metrics.Recall(),\n"," f1])\n"," \n"," # output = Dense(1,activation='sigmoid', name='output')(model)\n"," # model = Model(input,output)\n"," # model.compile(loss={'output':'binary_crossentropy'},\n"," # optimizer=Adadelta(lr=0.95, epsilon=1e-06),\n"," # metrics=[\"accuracy\",\n"," # tf.keras.metrics.Precision(),\n"," # tf.keras.metrics.Recall(),\n"," # f1])\n"," print(model.summary())\n"," \n"," return model"]},{"cell_type":"code","execution_count":24,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337913,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"Xavi607m6wsj"},"outputs":[],"source":["# model = BiLSTM_model(28, 0.1)"]},{"cell_type":"code","execution_count":25,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337914,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"LVIVpWXjJPh6"},"outputs":[],"source":["def Train_Model(model,x_train, y_train, cross_validation = False):\n","\n"," print('Training and Testing...')\n"," x_train = np.array(x_train)\n"," y_train = np.array(y_train)\n"," print (x_train.shape, y_train.shape)\n"," \n"," es = EarlyStopping(monitor='val_f1', mode='max', verbose=1, patience=5)\n"," checkpoint = ModelCheckpoint(model_save_path, monitor='val_f1', verbose=1, save_weights_only=True, mode='max')\n"," callbacks_list = [checkpoint,es]\n","\n"," if (cross_validation):\n"," callbacks_list = [es]\n","\n"," his = model.fit(x_train, y_train, validation_split=0.1, epochs=args.num_epochs, batch_size=args.batch_size, callbacks=callbacks_list, verbose=1)\n"," \n"," return model,his"]},{"cell_type":"code","execution_count":26,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337914,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"WhZjBbaIJUQS"},"outputs":[],"source":["def build_model() :\n"," model = None\n","\n"," # configure architecture\n"," if (stack_modeles == \"2\"):\n"," model = stacked_BiLSTM_model_2(args.max_sent, args.dropout_value)\n"," elif (stack_modeles == \"3\"):\n"," model = stacked_BiLSTM_model_3(args.max_sent, args.dropout_value)\n"," else :\n"," model = BiLSTM_model(args.max_sent, args.dropout_value)\n","\n"," return model"]},{"cell_type":"markdown","metadata":{"id":"dqBVV8BNfGPe"},"source":["# Capsule"]},{"cell_type":"code","execution_count":27,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337914,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"jam_fKSJjYfg"},"outputs":[],"source":["def capsule_model_A(X, num_classes):\n"," with tf.variable_scope('capsule_'+str(3),reuse=tf.AUTO_REUSE ): \n"," nets = _conv2d_wrapper(\n"," X, shape=[3, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," nets = capsule_flatten(nets)\n"," # nets = BiGRU_layer(nets)\n"," # nets = Flatten()(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2') \n"," return poses, activations"]},{"cell_type":"code","execution_count":28,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1650264337915,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"_JknBR5VwEeE"},"outputs":[],"source":["# model = Bidirectional (LSTM (args.vec_size,return_sequences=True,dropout=0.1,kernel_regularizer=l2(0.01)),merge_mode='concat')(test_set)\n","# model = TimeDistributed(Dense(args.vec_size,activation='relu'))(model)\n","# model = Flatten()(model)\n","# output = Dense(2)(model)#,activation='softmax')(model)\n","# print (output)"]},{"cell_type":"code","execution_count":29,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337915,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"dbqUEzcbotXD"},"outputs":[],"source":["def capsule_model_B(X, num_classes):\n"," poses_list = []\n"," main_list = []\n"," D = tf.reshape(X, (256,28,200), name=None)\n"," print ('D:', D.shape)\n"," print ('X:', X.shape)\n","\n"," model = Bidirectional (LSTM (args.vec_size,return_sequences=True,dropout=0.1,kernel_regularizer=l2(0.01)),merge_mode='concat')(D)\n"," model = TimeDistributed(Dense(args.vec_size,activation='relu'))(model)\n"," model = Flatten()(model)\n"," print (model)\n"," # output = Dense(2, activation='softmax')(model)\n"," model = Dense(2)(model)\n"," # print (output)\n","\n"," main_list.append(model)\n","\n"," for _, ngram in enumerate([3,4,5]):\n"," with tf.variable_scope('capsule_'+str(ngram),reuse=tf.AUTO_REUSE): \n"," print(X)\n"," nets = _conv2d_wrapper(\n"," X, shape = [ngram, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," print(nets)\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," print(nets) \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," # print(nets)\n"," nets = capsule_flatten(nets)\n"," # print(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2')\n"," print(poses)\n"," poses_list.append(poses)\n","\n"," poses = tf.reduce_mean(tf.convert_to_tensor(poses_list), axis=0) \n"," # print (poses)\n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," print ('activations:', activations.shape)\n"," main_list.append(activations)\n"," print ('model:', model.shape)\n","\n"," activations = tf.reduce_mean(tf.convert_to_tensor(main_list), axis=0) \n","\n"," return poses, activations"]},{"cell_type":"markdown","metadata":{"id":"PARyB28Vi9VI"},"source":["# Loss functions"]},{"cell_type":"code","execution_count":30,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1650264337916,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"WkAyzy_3hrYf"},"outputs":[],"source":["def spread_loss(labels, activations, margin):\n"," activations_shape = activations.get_shape().as_list()\n"," mask_t = tf.equal(labels, 1)\n"," mask_i = tf.equal(labels, 0) \n"," activations_t = tf.reshape(\n"," tf.boolean_mask(activations, mask_t), [activations_shape[0], 1]\n"," ) \n"," activations_i = tf.reshape(\n"," tf.boolean_mask(activations, mask_i), [activations_shape[0], activations_shape[1] - 1]\n"," ) \n"," gap_mit = tf.reduce_sum(tf.square(tf.nn.relu(margin - (activations_t - activations_i))))\n"," return gap_mit "]},{"cell_type":"code","execution_count":31,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1650264337916,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"n2UfkOrSjFvX"},"outputs":[],"source":["def cross_entropy(y, preds): \n"," y = tf.argmax(y, axis=1)\n"," loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=preds, labels=y) \n"," loss = tf.reduce_mean(loss) \n"," return loss"]},{"cell_type":"code","execution_count":32,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1650264337916,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"qEI1tgTnjUg_"},"outputs":[],"source":["def margin_loss(y, preds): \n"," y = tf.cast(y,tf.float32)\n"," loss = y * tf.square(tf.maximum(0., 0.9 - preds)) + \\\n"," 0.25 * (1.0 - y) * tf.square(tf.maximum(0., preds - 0.1))\n"," loss = tf.reduce_mean(tf.reduce_sum(loss, axis=1))\n"," return loss"]},{"cell_type":"markdown","metadata":{"id":"Qz6aicUlku28"},"source":["# Setup"]},{"cell_type":"code","execution_count":33,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1650264337917,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"Un6p6OLXkPnt"},"outputs":[],"source":["class Args:\n"," embedding_type = \"static\"\n"," dataset = \"\"\n"," loss_type = \"margin_loss\"\n"," model_type = \"capsule-B\"\n"," has_test = 1\n"," has_dev = 1\n"," num_epochs = 50\n"," batch_size = 256\n"," use_orphan = False\n"," use_leaky = False\n"," learning_rate = 0.01\n"," margin = 0.2\n"," num_classes = 2\n"," vec_size = 200\n"," max_sent = 28 #max_length\n"," dropout_value = 0.5\n"]},{"cell_type":"code","execution_count":34,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1650264337917,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"3cA5u4HrkuCb"},"outputs":[],"source":["args = Args()"]},{"cell_type":"code","execution_count":35,"metadata":{"executionInfo":{"elapsed":19,"status":"ok","timestamp":1650264337918,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"svJIxZerJX8v"},"outputs":[],"source":["# model = build_model()\n","# plot_model(model,to_file=\"./model.png\")\n","\n","# model.load_weights(model_save_path)\n","\n","# trained_model, his = Train_Model(model,X_train, y_train, cross_validation = False)\n","# accuracy = his.history['accuracy']\n","# val_accuracy = his.history['val_accuracy']\n","# loss = his.history['loss']\n","# val_loss = his.history['val_loss']\n","# f1 = his.history['f1']\n","# val_f1 = his.history['f1']\n","\n","# Plot_graphs(accuracy,val_accuracy, \"accuracy\")\n","# Plot_graphs(loss,val_loss, \"loss\")\n","# Plot_graphs(f1,val_f1, \"f1\")\n","\n","# model.load_weights(model_save_path)\n","# loss, accuracy,precision, recall,f1 = model.evaluate(X_test, y_test, verbose=1)\n","# f1_score =2*precision*recall/(precision+recall)\n","# print(\"loss\",loss, \" accuracy\",accuracy,\" precision\",precision,\" recall\",recall,\" f1\",f1_score)"]},{"cell_type":"code","execution_count":36,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1650264337918,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"cVRu8KNo3GcR"},"outputs":[],"source":["with tf.device('/cpu:0'):\n"," global_step = tf.train.get_or_create_global_step()"]},{"cell_type":"code","execution_count":37,"metadata":{"executionInfo":{"elapsed":19,"status":"ok","timestamp":1650264337919,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"lFGsOcj50Eov"},"outputs":[],"source":["class BatchGenerator(object):\n"," \"\"\"Generate and hold batches.\"\"\"\n"," def __init__(self, dataset,label, batch_size,input_size, is_shuffle=True):\n"," self._dataset = dataset\n"," self._label = label\n"," self._batch_size = batch_size \n"," self._cursor = 0 \n"," self._input_size = input_size \n"," \n"," if is_shuffle:\n"," index = np.arange(len(self._dataset))\n"," np.random.shuffle(index)\n"," self._dataset = np.array(self._dataset)[index]\n"," self._label = np.array(self._label)[index]\n"," else:\n"," self._dataset = np.array(self._dataset)\n"," self._label = np.array(self._label)\n"," def next(self):\n"," if self._cursor + self._batch_size \u003e len(self._dataset):\n"," self._cursor = 0\n"," \"\"\"Generate a single batch from the current cursor position in the data.\"\"\" \n"," batch_x = self._dataset[self._cursor : self._cursor + self._batch_size,:]\n"," batch_y = self._label[self._cursor : self._cursor + self._batch_size]\n"," self._cursor += self._batch_size\n"," return batch_x, batch_y"]},{"cell_type":"code","execution_count":38,"metadata":{"executionInfo":{"elapsed":19,"status":"ok","timestamp":1650264337919,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"mHnF9UHNUkb7"},"outputs":[],"source":["# tf.Tensor(\n","# np.array([[1, 2, 3, 4],\n","# [5, 6, 7, 8]]), value_index=-1, dtype=tf.dtypes.int32)"]},{"cell_type":"code","execution_count":39,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":19106,"status":"ok","timestamp":1650264357006,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"wV3H1xPDUZwU","outputId":"5557707e-c3ce-46e5-c4d6-ea6b5b43d4c3"},"outputs":[{"name":"stdout","output_type":"stream","text":["D: (256, 28, 200)\n","X: (256, 28, 200, 1)\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:72: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:4048: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:131: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:3368: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n","Tensor(\"flatten_1/Reshape:0\", shape=(?, ?), dtype=float32)\n","Tensor(\"strided_slice:0\", shape=(256, 28, 200, 1), dtype=float32)\n","Tensor(\"capsule_3/conv1/activation:0\", shape=(256, 13, 1, 32), dtype=float32)\n","(\u003ctf.Tensor 'capsule_3/primary/mul:0' shape=(256, 13, 1, 16, 16) dtype=float32\u003e, \u003ctf.Tensor 'capsule_3/primary/add_2:0' shape=(256, 13, 1, 16) dtype=float32\u003e)\n","Tensor(\"capsule_3/fc2/mul_2:0\", shape=(256, 2, 16), dtype=float32)\n","Tensor(\"strided_slice:0\", shape=(256, 28, 200, 1), dtype=float32)\n","Tensor(\"capsule_4/conv1/activation:0\", shape=(256, 13, 1, 32), dtype=float32)\n","(\u003ctf.Tensor 'capsule_4/primary/mul:0' shape=(256, 13, 1, 16, 16) dtype=float32\u003e, \u003ctf.Tensor 'capsule_4/primary/add_2:0' shape=(256, 13, 1, 16) dtype=float32\u003e)\n","Tensor(\"capsule_4/fc2/mul_2:0\", shape=(256, 2, 16), dtype=float32)\n","Tensor(\"strided_slice:0\", shape=(256, 28, 200, 1), dtype=float32)\n","Tensor(\"capsule_5/conv1/activation:0\", shape=(256, 12, 1, 32), dtype=float32)\n","(\u003ctf.Tensor 'capsule_5/primary/mul:0' shape=(256, 12, 1, 16, 16) dtype=float32\u003e, \u003ctf.Tensor 'capsule_5/primary/add_2:0' shape=(256, 12, 1, 16) dtype=float32\u003e)\n","Tensor(\"capsule_5/fc2/mul_2:0\", shape=(256, 2, 16), dtype=float32)\n","activations: (256, 2)\n","model: (?, 2)\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.\u003clocals\u003e.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.where in 2.0, which has the same broadcast rule as np.where\n"]}],"source":["import os \n","import torch\n","\n","fold_no = 1\n","\n","# X = tf.placeholder(tf.int32, [args.batch_size, args.max_sent,args.vec_size], name=\"input_x\")\n","y = tf.placeholder(tf.int64, [args.batch_size, args.num_classes], name=\"input_y\")\n","is_training = tf.placeholder_with_default(False, shape=()) \n","learning_rate = tf.placeholder(dtype='float32') \n","margin = tf.placeholder(shape=(),dtype='float32') \n","\n","l2_loss = tf.constant(0.0)\n","\n","# X= Input(shape=(args.max_sent,args.vec_size))\n","# # print(input)\n","# model = Bidirectional (LSTM (args.vec_size,return_sequences=True,dropout=0.1,kernel_regularizer=l2(0.01)),merge_mode='concat')(X)\n","# # print(model)\n","# model = TimeDistributed(Dense(args.vec_size,activation='relu'))(model)\n","# # print(model)\n","# model = tf.ensure_shape(model, (256,28,200), name=None)\n","# # print(model)\n","# model = model[...,tf.newaxis]\n","# poses, activations = capsule_model_B(model, args.num_classes)\n","\n","X = tf.placeholder(tf.float32, [args.batch_size, args.max_sent,args.vec_size], name=\"input_x\")\n","X = X[...,tf.newaxis]\n","poses, activations = capsule_model_B(X, args.num_classes)\n","loss = margin_loss(y, activations) \n","y_pred = tf.argmax(activations, axis=1, name=\"y_proba\") \n","correct = tf.equal(tf.argmax(y, axis=1), y_pred, name=\"correct\")\n","accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n","\n","optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, name = 'opt'+str(fold_no)) \n","training_op = optimizer.minimize(loss, name=\"training_op\")\n","gradients, variables = zip(*optimizer.compute_gradients(loss)) \n","\n","grad_check = [tf.check_numerics(g, message='Gradient NaN Found!')\n"," for g in gradients if g is not None] + [tf.check_numerics(loss, message='Loss NaN Found')]\n","with tf.control_dependencies(grad_check):\n"," training_op = optimizer.apply_gradients(zip(gradients, variables), global_step=global_step) "]},{"cell_type":"code","execution_count":40,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1650264357007,"user":{"displayName":"Gihan Ayeshmantha","userId":"16004878281960957449"},"user_tz":-330},"id":"Rhjeg8m9lKdQ"},"outputs":[],"source":["# np.transpose(dataset['posts'][3976]).shape"]},{"cell_type":"markdown","metadata":{"id":"RwyuzXZBflX7"},"source":["# Train"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"1Xh_EEQEO_44"},"outputs":[{"name":"stdout","output_type":"stream","text":["0.30000000000000004\n","accuracy: 0.7968% Precision: 0.7867 recall: 0.7968 F1: 0.7662\n","0.4\n","accuracy: 0.8029% Precision: 0.7896 recall: 0.8029 F1: 0.7890\n","0.5\n","accuracy: 0.8027% Precision: 0.7901 recall: 0.8027 F1: 0.7911\n","0.6\n","accuracy: 0.8049% Precision: 0.7918 recall: 0.8049 F1: 0.7888\n","0.7\n","accuracy: 0.8042% Precision: 0.7972 recall: 0.8042 F1: 0.7757\n","0.7999999999999999\n","accuracy: 0.7997% Precision: 0.7987 recall: 0.7997 F1: 0.7634\n","0.8999999999999999\n","accuracy: 0.7932% Precision: 0.7946 recall: 0.7932 F1: 0.7507\n","0.9\n","accuracy: 0.8040% Precision: 0.7908 recall: 0.8040 F1: 0.7864\n","0.9\n","accuracy: 0.8057% Precision: 0.7955 recall: 0.8057 F1: 0.7821\n","0.9\n","accuracy: 0.8035% Precision: 0.7963 recall: 0.8035 F1: 0.7749\n","0.9\n","accuracy: 0.8044% Precision: 0.7912 recall: 0.8044 F1: 0.7879\n","0.9\n","accuracy: 0.8033% Precision: 0.7899 recall: 0.8033 F1: 0.7880\n","0.9\n","accuracy: 0.8052% Precision: 0.7925 recall: 0.8052 F1: 0.7867\n","0.9\n","accuracy: 0.8047% Precision: 0.7961 recall: 0.8047 F1: 0.7784\n"]}],"source":["saver = tf.train.Saver()\n","save_path = folder_path + \"/Tests/\"+embedding_type+\"_\"+capsule_routes+\"_\"+pool_layer+\"/model.ckpt\"\n","\n","sess = tf.InteractiveSession()\n","from keras import utils\n","\n","old_acc,old_pre,old_rec,old_f1 = 0,0,0,0\n","\n","n_iterations_per_epoch = len(train_set) // args.batch_size\n","n_iterations_test = len(test_set) // args.batch_size\n","n_iterations_dev = len(dev_set) // args.batch_size \n","\n","mr_train = BatchGenerator(train_set,train_label, args.batch_size, 0) \n","mr_dev = BatchGenerator(dev_set,dev_label, args.batch_size, 0)\n","mr_test = BatchGenerator(test_set,test_label, args.batch_size, 0, is_shuffle=False)\n","\n","init = tf.global_variables_initializer()\n","sess.run(init) \n","\n","lr = args.learning_rate\n","m = args.margin\n","for epoch in range(args.num_epochs):\n"," for iteration in range(1, n_iterations_per_epoch + 1): \n"," X_batch, y_batch = mr_train.next() \n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size,1)\n"," # X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size) \n"," _, loss_train, probs, capsule_pose = sess.run(\n"," [training_op, loss, activations, poses],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: True,\n"," learning_rate:lr,\n"," margin:m}) \n"," # print(\"\\rIteration: {}/{} ({:.1f}%) Loss: {:.5f}\".format(iteration, n_iterations_per_epoch,iteration * 100 / n_iterations_per_epoch,loss_train),end=\"\") \n"," loss_vals, acc_vals = [], []\n"," for iteration in range(1, n_iterations_dev + 1):\n"," X_batch, y_batch = mr_dev.next() \n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size,1)\n"," # X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size)\n"," loss_val, acc_val = sess.run(\n"," [loss, accuracy],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: False,\n"," margin:m})\n"," loss_vals.append(loss_val)\n"," acc_vals.append(acc_val)\n"," loss_val, acc_val = np.mean(loss_vals), np.mean(acc_vals) \n"," # print(\"\\rEpoch: {} Val accuracy: {:.1f}% Loss: {:.4f}\".format(epoch + 1, acc_val * 100, loss_val))\n"," \n"," preds_list, y_list = [], []\n"," for iteration in range(1, n_iterations_test + 1):\n"," X_batch, y_batch = mr_test.next()\n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size,1) \n"," # X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size) \n"," probs = sess.run([activations],\n"," feed_dict={X:X_batch[:,:args.max_sent],\n"," is_training: False})\n"," preds_list = preds_list + probs[0].tolist()\n"," y_list = y_list + y_batch.tolist()\n"," \n"," y_list = np.array(y_list)\n"," preds_probs = np.array(preds_list) \n"," # preds_probs = (preds_probs == preds_probs.max(axis=1)[:,None]).astype(int)\n"," \n"," # [precision, recall, F1, support] = precision_recall_fscore_support(y_list, preds_probs, average='samples')\n"," # acc = accuracy_score(y_list, preds_probs)\n"," # print ('\\rER: %.3f' % acc, 'Precision: %.3f' % precision, 'Recall: %.3f' % recall, 'F1: %.3f' % F1) \n"," m = min(0.9, m + 0.1)\n"," print (m)\n","\n"," labels = np.argmax(y_list, axis=1)\n"," predictions = np.argmax(preds_probs, axis=1)\n","\n"," accuracy_fold = accuracy_score(labels, predictions)\n"," precision_fold = precision_score(labels, predictions, average='weighted', zero_division = 0 )\n"," recall_fold = recall_score(labels, predictions, average='weighted')\n"," f1_fold = f1_score(labels, predictions, average='weighted')\n"," print(\"\\raccuracy: {:.4f}% Precision: {:.4f} recall: {:.4f} F1: {:.4f}\".format(accuracy_fold, precision_fold, recall_fold, f1_fold))\n","\n"," if not (accuracy_fold\u003cold_acc and precision_fold\u003cold_pre and recall_fold\u003cold_rec and f1_fold\u003cold_f1):\n"," saver.save(sess, save_path)\n","\n"," old_acc = accuracy_fold\n"," old_pre = precision_fold\n"," old_rec = recall_fold\n"," old_f1 = f1_fold\n"]}],"metadata":{"colab":{"collapsed_sections":["XnkydX1ne969","7dpVU5Sb1Ddr","5ZVUDvuhRupy","PARyB28Vi9VI"],"machine_shape":"hm","name":"Copy of under construction Sinhala_text_capsules.ipynb","provenance":[{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/Sinhala_text_capsules.ipynb","timestamp":1621834144627}],"version":""},"environment":{"name":"tf-gpu.1-15.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf-gpu.1-15:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"nbformat":4,"nbformat_minor":0}

sentiment-analysis-models/capsule-networks/capsule_network_with_serial_BiLSTM.ipynb ADDED Viewed

	@@ -0,0 +1 @@

+ {"cells":[{"cell_type":"markdown","metadata":{"id":"k8D9EuJnBnDV"},"source":["# Installing and importing dependecies"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"fp5wHxfHqnPj"},"outputs":[],"source":["embedding_type = \"MAX\" #@param [\"GRU\",\"MAX\"]\n","EMBEDDING_SIZE = 200 #@param [200,300]\n","\n","capsule_routes = \"3\" #@param [\"3\",\"4\"]\n","pool_layer = \"max\" #@param [\"avg\",\"max\",\"min\"]\n","bilstm = \"serial\" #@param [\"serial\",\"para\",\"both\"]"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":32726,"status":"ok","timestamp":1651116122236,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"GgRvEbMXy9nc","outputId":"1d25d72c-2cfc-49e4-a3c3-134072bd9663"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":39823,"status":"ok","timestamp":1651116162039,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"CCi-HNCn7BCi","outputId":"bedeb176-57a1-41e2-e4df-04c311b6f3a0"},"outputs":[{"name":"stdout","output_type":"stream","text":["\u001b[33mWARNING: Skipping keras-nightly as it is not installed.\u001b[0m\n","Collecting tensorflow==1.14.0\n"," Downloading tensorflow-1.14.0-cp37-cp37m-manylinux1_x86_64.whl (109.3 MB)\n","\u001b[K |████████████████████████████████| 109.3 MB 86 kB/s \n","\u001b[?25hRequirement already satisfied: six\u003e=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.15.0)\n","Collecting keras-applications\u003e=1.0.6\n"," Downloading Keras_Applications-1.0.8-py3-none-any.whl (50 kB)\n","\u001b[K |████████████████████████████████| 50 kB 5.9 MB/s \n","\u001b[?25hRequirement already satisfied: wrapt\u003e=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.14.0)\n","Requirement already satisfied: google-pasta\u003e=0.1.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.2.0)\n","Requirement already satisfied: protobuf\u003e=3.6.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (3.17.3)\n","Requirement already satisfied: termcolor\u003e=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.0)\n","Requirement already satisfied: absl-py\u003e=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.0.0)\n","Collecting tensorboard\u003c1.15.0,\u003e=1.14.0\n"," Downloading tensorboard-1.14.0-py3-none-any.whl (3.1 MB)\n","\u001b[K |████████████████████████████████| 3.1 MB 74.5 MB/s \n","\u001b[?25hRequirement already satisfied: astor\u003e=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.8.1)\n","Requirement already satisfied: wheel\u003e=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.37.1)\n","Collecting tensorflow-estimator\u003c1.15.0rc0,\u003e=1.14.0rc0\n"," Downloading tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488 kB)\n","\u001b[K |████████████████████████████████| 488 kB 55.6 MB/s \n","\u001b[?25hRequirement already satisfied: grpcio\u003e=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.44.0)\n","Requirement already satisfied: numpy\u003c2.0,\u003e=1.14.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.21.6)\n","Requirement already satisfied: gast\u003e=0.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.5.3)\n","Requirement already satisfied: keras-preprocessing\u003e=1.0.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.2)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras-applications\u003e=1.0.6-\u003etensorflow==1.14.0) (3.1.0)\n","Requirement already satisfied: werkzeug\u003e=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (1.0.1)\n","Requirement already satisfied: markdown\u003e=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (3.3.6)\n","Requirement already satisfied: setuptools\u003e=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (57.4.0)\n","Requirement already satisfied: importlib-metadata\u003e=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown\u003e=2.6.8-\u003etensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (4.11.3)\n","Requirement already satisfied: zipp\u003e=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata\u003e=4.4-\u003emarkdown\u003e=2.6.8-\u003etensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (3.8.0)\n","Requirement already satisfied: typing-extensions\u003e=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata\u003e=4.4-\u003emarkdown\u003e=2.6.8-\u003etensorboard\u003c1.15.0,\u003e=1.14.0-\u003etensorflow==1.14.0) (4.2.0)\n","Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py-\u003ekeras-applications\u003e=1.0.6-\u003etensorflow==1.14.0) (1.5.2)\n","Installing collected packages: tensorflow-estimator, tensorboard, keras-applications, tensorflow\n"," Attempting uninstall: tensorflow-estimator\n"," Found existing installation: tensorflow-estimator 2.8.0\n"," Uninstalling tensorflow-estimator-2.8.0:\n"," Successfully uninstalled tensorflow-estimator-2.8.0\n"," Attempting uninstall: tensorboard\n"," Found existing installation: tensorboard 2.8.0\n"," Uninstalling tensorboard-2.8.0:\n"," Successfully uninstalled tensorboard-2.8.0\n"," Attempting uninstall: tensorflow\n"," Found existing installation: tensorflow 2.8.0\n"," Uninstalling tensorflow-2.8.0:\n"," Successfully uninstalled tensorflow-2.8.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","kapre 0.3.7 requires tensorflow\u003e=2.0.0, but you have tensorflow 1.14.0 which is incompatible.\u001b[0m\n","Successfully installed keras-applications-1.0.8 tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0\n","Collecting q\n"," Downloading q-2.6-py2.py3-none-any.whl (6.8 kB)\n","Collecting keras==2.2.2\n"," Downloading Keras-2.2.2-py2.py3-none-any.whl (299 kB)\n","\u001b[K |████████████████████████████████| 299 kB 13.8 MB/s \n","\u001b[?25hRequirement already satisfied: scipy\u003e=0.14 in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (1.4.1)\n","Collecting keras-applications==1.0.4\n"," Downloading Keras_Applications-1.0.4-py2.py3-none-any.whl (43 kB)\n","\u001b[K |████████████████████████████████| 43 kB 1.9 MB/s \n","\u001b[?25hRequirement already satisfied: numpy\u003e=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (1.21.6)\n","Requirement already satisfied: six\u003e=1.9.0 in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (1.15.0)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (3.1.0)\n","Collecting keras-preprocessing==1.0.2\n"," Downloading Keras_Preprocessing-1.0.2-py2.py3-none-any.whl (26 kB)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras==2.2.2) (3.13)\n","Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py-\u003ekeras==2.2.2) (1.5.2)\n","Installing collected packages: keras-preprocessing, keras-applications, q, keras\n"," Attempting uninstall: keras-preprocessing\n"," Found existing installation: Keras-Preprocessing 1.1.2\n"," Uninstalling Keras-Preprocessing-1.1.2:\n"," Successfully uninstalled Keras-Preprocessing-1.1.2\n"," Attempting uninstall: keras-applications\n"," Found existing installation: Keras-Applications 1.0.8\n"," Uninstalling Keras-Applications-1.0.8:\n"," Successfully uninstalled Keras-Applications-1.0.8\n"," Attempting uninstall: keras\n"," Found existing installation: keras 2.8.0\n"," Uninstalling keras-2.8.0:\n"," Successfully uninstalled keras-2.8.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","tensorflow 1.14.0 requires keras-applications\u003e=1.0.6, but you have keras-applications 1.0.4 which is incompatible.\n","tensorflow 1.14.0 requires keras-preprocessing\u003e=1.0.5, but you have keras-preprocessing 1.0.2 which is incompatible.\n","kapre 0.3.7 requires tensorflow\u003e=2.0.0, but you have tensorflow 1.14.0 which is incompatible.\u001b[0m\n","Successfully installed keras-2.2.2 keras-applications-1.0.4 keras-preprocessing-1.0.2 q-2.6\n","Collecting h5py\u003c3.0.0\n"," Downloading h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)\n","\u001b[K |████████████████████████████████| 2.9 MB 14.3 MB/s \n","\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from h5py\u003c3.0.0) (1.15.0)\n","Requirement already satisfied: numpy\u003e=1.7 in /usr/local/lib/python3.7/dist-packages (from h5py\u003c3.0.0) (1.21.6)\n","Installing collected packages: h5py\n"," Attempting uninstall: h5py\n"," Found existing installation: h5py 3.1.0\n"," Uninstalling h5py-3.1.0:\n"," Successfully uninstalled h5py-3.1.0\n","Successfully installed h5py-2.10.0\n"]}],"source":["!pip uninstall keras-nightly\n","!pip install tensorflow==1.14.0\n","!pip install q keras==2.2.2\n","!pip install 'h5py\u003c3.0.0'"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2884,"status":"ok","timestamp":1651116164912,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"zlXBE_-86l4q","outputId":"c3205ff0-33c3-4a88-93a2-ef3c6cc41095"},"outputs":[{"name":"stderr","output_type":"stream","text":["Using TensorFlow backend.\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n"," np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"]}],"source":["import pandas as pd\n","import collections\n","import gensim\n","from gensim.models.keyedvectors import KeyedVectors\n","from gensim.models.fasttext import FastText\n","from gensim.models import word2vec\n","\n","from sklearn.model_selection import train_test_split,cross_val_score, cross_val_predict, KFold, GridSearchCV\n","from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix, precision_recall_fscore_support\n","\n","import pandas as pd\n","import numpy as np\n","\n","import keras\n","from keras import backend as K\n","from keras.models import Model,load_model\n","from keras.layers import Flatten, Input, Dense,TimeDistributed, Bidirectional\n","from keras.regularizers import l2\n","from keras import callbacks\n","from keras.utils import generic_utils,plot_model\n","from keras.callbacks import ModelCheckpoint,EarlyStopping\n","from keras.layers.recurrent import LSTM\n","from keras.optimizers import Adadelta\n","\n","import tensorflow as tf\n","from tensorflow.contrib.layers.python.layers import initializers"]},{"cell_type":"markdown","metadata":{"id":"nRbpY2FTDl8p"},"source":["# Load Dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"d8GzWXqBsX2V"},"outputs":[],"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Capsule networks/'\n","\n","if (embedding_type == 'GRU'):\n"," facebook_data_path = folder_path + 'embedded posts/GRU_attention_'\n","elif (embedding_type == 'MAX'):\n"," facebook_data_path = folder_path + 'embedded posts/'+'MAX_'\n","\n","context = 5\n","posts=[]\n","labels =[]\n","\n","# facebook_data_path = (facebook_data_path +str(i)+'.csv')\n","# dataset = pd.read_pickle(facebook_data_path)\n","# dataset=dataset.dropna()\n","# dataset = dataset[:1000]\n","# labels+=dataset[\"label\"].tolist()\n","# for j in dataset['posts']:\n","# posts.append(j)\n","\n","for i in range (1,6):\n"," data_path = (facebook_data_path +str(i)+'.csv')\n"," dataset = pd.read_pickle(data_path)\n"," dataset=dataset.dropna()\n"," # dataset = dataset[:1000]\n"," labels+=dataset[\"label\"].tolist()\n"," for j in dataset['posts']:\n"," posts.append(j)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0m6FrtmcsRKu"},"outputs":[],"source":["# posts=np.array(posts)\n","# labels=np.array(labels)\n","train_set, test_set , train_label, test_label = train_test_split(posts,labels, test_size=0.1, random_state=0)\n","train_set, dev_set, train_label, dev_label = train_test_split(train_set,train_label, test_size=0.1, random_state=0)"]},{"cell_type":"markdown","metadata":{"id":"XnkydX1ne969"},"source":["# Convo 2d layer"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0GVuHgh2EWw2"},"outputs":[],"source":["def _get_variable_wrapper(name, shape=None, dtype=None, initializer=None,regularizer=None,trainable=True,collections=None,caching_device=None,partitioner=None,validate_shape=True,custom_getter=None):\n"," with tf.device('/cpu:0'):\n"," var = tf.get_variable(\n"," name, shape=shape, dtype=dtype, initializer=initializer,\n"," regularizer=regularizer, trainable=trainable,\n"," collections=collections, caching_device=caching_device,\n"," partitioner=partitioner, validate_shape=validate_shape,\n"," custom_getter=custom_getter\n"," )\n"," return var"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"k8F7Y8CGbsB4"},"outputs":[],"source":["def _get_weights_wrapper(name, shape, dtype=tf.float32, initializer=initializers.xavier_initializer(),weights_decay_factor=None):\n"," weights = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," if weights_decay_factor is not None and weights_decay_factor \u003e 0.0:\n"," weights_wd = tf.multiply(tf.nn.l2_loss(weights), weights_decay_factor, name=name + '/l2loss')\n"," tf.add_to_collection('losses', weights_wd)\n"," return weights"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"nkO3w5gDcC1b"},"outputs":[],"source":["def _get_biases_wrapper(name, shape, dtype=tf.float32, initializer=tf.constant_initializer(0.0)):\n"," \"\"\"Wrapper over _get_variable_wrapper() to get bias.\n"," \"\"\"\n"," biases = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," return biases"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"9Z06E9a1aBVb"},"outputs":[],"source":["def _conv2d_wrapper(inputs, shape, strides, padding, add_bias, activation_fn, name, stddev=0.1):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE) as scope:\n"," kernel = _get_weights_wrapper(\n"," name='weights', shape=shape, weights_decay_factor=0.0, )\n"," output = tf.nn.conv2d(inputs, filter=kernel, strides=strides, padding=padding, name='conv')\n"," if add_bias:\n"," biases = _get_biases_wrapper(name='biases', shape=[shape[-1]] )\n"," output = tf.add(output, biases, name='biasAdd')\n"," if activation_fn is not None:\n"," output = activation_fn(output, name='activation')\n"," return output"]},{"cell_type":"markdown","metadata":{"id":"7dpVU5Sb1Ddr"},"source":["# Utils"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"MBkd3MRuccQh"},"outputs":[],"source":["def softmax(x, axis=-1):\n"," ex = K.exp(x - K.max(x, axis=axis, keepdims=True))\n"," return ex/K.sum(ex, axis=axis, keepdims=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hSVP0ymhc6G6"},"outputs":[],"source":["def squash_v1(x, axis=-1):\n"," s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n"," scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n"," return scale * x"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"RQzm6uHwc8Rq"},"outputs":[],"source":["def squash_v0(s, axis=-1, epsilon=1e-7, name=None):\n"," s_squared_norm = K.sum(K.square(s), axis, keepdims=True) + K.epsilon()\n"," safe_norm = K.sqrt(s_squared_norm)\n"," scale = 1 - tf.exp(-safe_norm)\n"," return scale * s / safe_norm"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ObeP2wKDc-Zq"},"outputs":[],"source":["def routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations):\n"," b = keras.backend.zeros_like(u_hat_vecs[:,:,:,0])\n"," if i_activations is not None:\n"," i_activations = i_activations[...,tf.newaxis]\n"," for i in range(iterations):\n"," if False:\n"," leak = tf.zeros_like(b, optimize=True)\n"," leak = tf.reduce_sum(leak, axis=1, keep_dims=True)\n"," leaky_logits = tf.concat([leak, b], axis=1)\n"," leaky_routing = tf.nn.softmax(leaky_logits, dim=1) \n"," c = tf.split(leaky_routing, [1, output_capsule_num], axis=1)[1]\n"," else:\n"," c = softmax(b, 1) \n"," outputs = squash_v1(K.batch_dot(c, u_hat_vecs, [2, 2]))\n"," if i \u003c iterations - 1:\n"," b = b + K.batch_dot(outputs, u_hat_vecs, [2, 3]) \n"," poses = outputs \n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," return poses, activations\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"zSdVM9YCdEMp"},"outputs":[],"source":["def vec_transformationByConv(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num): \n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_dim, output_capsule_dim*output_capsule_num], weights_decay_factor=0.0)\n"," u_hat_vecs = keras.backend.conv1d(poses, kernel)\n"," u_hat_vecs = keras.backend.reshape(u_hat_vecs, (-1, input_capsule_num, output_capsule_num, output_capsule_dim))\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"i-pIavredIK6"},"outputs":[],"source":["def vec_transformationByMat(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num, shared=True): \n"," inputs_poses_shape = poses.get_shape().as_list()\n"," poses = poses[..., tf.newaxis, :] \n"," poses = tf.tile(poses, [1, 1, output_capsule_num, 1]) \n"," if shared:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, 1, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], input_capsule_num, 1, 1, 1])\n"," else:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_num, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], 1, 1, 1, 1])\n"," u_hat_vecs = tf.squeeze(tf.matmul(kernel, poses[...,tf.newaxis]),axis=-1)\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"mcvHPFCOdKyX"},"outputs":[],"source":["def capsules_init(inputs, shape, strides, padding, pose_shape, add_bias, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses = _conv2d_wrapper(\n"," inputs,\n"," shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," strides=strides,\n"," padding=padding,\n"," add_bias=add_bias,\n"," activation_fn=None,\n"," name='pose_stacked'\n"," ) \n"," poses_shape = poses.get_shape().as_list() \n"," poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," poses = squash_v1(poses, axis=-1) \n"," activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n","\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hAJ1njh6dOpT"},"outputs":[],"source":["def capsule_fc_layer(nets, output_capsule_num, iterations, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n","\n"," u_hat_vecs = vec_transformationByConv(poses,input_pose_shape[-1], input_pose_shape[1],input_pose_shape[-1], output_capsule_num,)\n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, output_capsule_num])\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations)\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jE-SPLBwdR2a"},"outputs":[],"source":["def capsule_flatten(nets):\n"," poses, activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n"," \n"," poses = tf.reshape(poses, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3], input_pose_shape[-1]]) \n"," activations = tf.reshape(activations, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3]])\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"1uMUkhf1dVJy"},"outputs":[],"source":["def capsule_conv_layer(nets, shape, strides, iterations, name): \n"," # poses = _conv2d_wrapper(\n"," # inputs,\n"," # shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," # strides=strides,\n"," # padding=padding,\n"," # add_bias=add_bias,\n"," # activation_fn=None,\n"," # name='pose_stacked'\n"," # ) \n"," # poses_shape = poses.get_shape().as_list() \n"," # poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," # beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," # poses = squash_v1(poses, axis=-1) \n"," # activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," \n"," inputs_poses_shape = poses.get_shape().as_list()\n","\n"," hk_offsets = [\n"," [(h_offset + k_offset) for k_offset in range(0, shape[0])] for h_offset in\n"," range(0, inputs_poses_shape[1] + 1 - shape[0], strides[1])\n"," ]\n"," wk_offsets = [\n"," [(w_offset + k_offset) for k_offset in range(0, shape[1])] for w_offset in\n"," range(0, inputs_poses_shape[2] + 1 - shape[1], strides[2])\n"," ]\n"," \n"," inputs_poses_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," poses, hk_offsets, axis=1, name='gather_poses_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_poses_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5, 6], name='inputs_poses_patches'\n"," )\n"," inputs_poses_shape = inputs_poses_patches.get_shape().as_list()\n"," inputs_poses_patches = tf.reshape(inputs_poses_patches, [\n"," -1, shape[0]*shape[1]*shape[2], inputs_poses_shape[-1]\n"," ])\n","\n"," i_activations_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," i_activations, hk_offsets, axis=1, name='gather_activations_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_activations_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5], name='inputs_activations_patches'\n"," )\n"," i_activations_patches = tf.reshape(i_activations_patches, [\n"," -1, shape[0]*shape[1]*shape[2]]\n"," )\n"," u_hat_vecs = vec_transformationByConv(\n"," inputs_poses_patches,\n"," inputs_poses_shape[-1], shape[0]*shape[1]*shape[2],\n"," inputs_poses_shape[-1], shape[3],\n"," ) \n"," beta_a = _get_weights_wrapper(\n"," name='beta_a', shape=[1, shape[3]]\n"," )\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, shape[3], i_activations_patches)\n"," poses = tf.reshape(poses, [\n"," inputs_poses_shape[0], inputs_poses_shape[1],\n"," inputs_poses_shape[2], shape[3],\n"," inputs_poses_shape[-1]]\n"," ) \n"," activations = tf.reshape(activations, [\n"," inputs_poses_shape[0],inputs_poses_shape[1],\n"," inputs_poses_shape[2],shape[3]]\n"," ) \n"," nets = poses, activations \n"," return nets"]},{"cell_type":"markdown","metadata":{"id":"dqBVV8BNfGPe"},"source":["# Capsule"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jam_fKSJjYfg"},"outputs":[],"source":["def capsule_model_A(X, num_classes):\n"," with tf.variable_scope('capsule_'+str(3),reuse=tf.AUTO_REUSE ): \n"," nets = _conv2d_wrapper(\n"," X, shape=[3, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," nets = capsule_flatten(nets)\n"," # nets = BiGRU_layer(nets)\n"," # nets = Flatten()(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2') \n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"dbqUEzcbotXD"},"outputs":[],"source":["def capsule_model_B(X, num_classes):\n"," poses_list = []\n"," #bilstm\n"," for _, ngram in enumerate([3,4,5]):\n"," with tf.variable_scope('capsule_'+str(ngram),reuse=tf.AUTO_REUSE): \n"," nets = _conv2d_wrapper(\n"," X, shape=[ngram, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," nets = capsule_flatten(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2')\n"," poses_list.append(poses)\n"," #reduce_max,min\n"," # poses = tf.reduce_max(tf.convert_to_tensor(poses_list), axis=0) \n"," poses = tf.reduce_max(tf.convert_to_tensor(poses_list), axis=0) \n"," # poses = tf.reduce_mean(tf.convert_to_tensor(poses_list), axis=0) \n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," return poses, activations"]},{"cell_type":"markdown","metadata":{"id":"PARyB28Vi9VI"},"source":["# Loss functions"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"WkAyzy_3hrYf"},"outputs":[],"source":["def spread_loss(labels, activations, margin):\n"," activations_shape = activations.get_shape().as_list()\n"," mask_t = tf.equal(labels, 1)\n"," mask_i = tf.equal(labels, 0) \n"," activations_t = tf.reshape(\n"," tf.boolean_mask(activations, mask_t), [activations_shape[0], 1]\n"," ) \n"," activations_i = tf.reshape(\n"," tf.boolean_mask(activations, mask_i), [activations_shape[0], activations_shape[1] - 1]\n"," ) \n"," gap_mit = tf.reduce_sum(tf.square(tf.nn.relu(margin - (activations_t - activations_i))))\n"," return gap_mit "]},{"cell_type":"code","execution_count":null,"metadata":{"id":"n2UfkOrSjFvX"},"outputs":[],"source":["def cross_entropy(y, preds): \n"," y = tf.argmax(y, axis=1)\n"," loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=preds, labels=y) \n"," loss = tf.reduce_mean(loss) \n"," return loss"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qEI1tgTnjUg_"},"outputs":[],"source":["def margin_loss(y, preds): \n"," y = tf.cast(y,tf.float32)\n"," loss = y * tf.square(tf.maximum(0., 0.9 - preds)) + \\\n"," 0.25 * (1.0 - y) * tf.square(tf.maximum(0., preds - 0.1))\n"," loss = tf.reduce_mean(tf.reduce_sum(loss, axis=1))\n"," return loss"]},{"cell_type":"markdown","metadata":{"id":"Qz6aicUlku28"},"source":["# Setup"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Un6p6OLXkPnt"},"outputs":[],"source":["class Args:\n"," embedding_type = \"static\"\n"," dataset = \"\"\n"," loss_type = \"margin_loss\"\n"," model_type = \"capsule-B\"\n"," has_test = 1\n"," has_dev = 1\n"," num_epochs = 50\n"," batch_size = 256\n"," use_orphan = False\n"," use_leaky = False\n"," learning_rate = 0.01\n"," margin = 0.2\n"," num_classes = 2\n"," # vocab_size = vocab_size\n"," vec_size = 200\n"," max_sent = 28 #max_length\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"3cA5u4HrkuCb"},"outputs":[],"source":["args = Args()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"cVRu8KNo3GcR"},"outputs":[],"source":["with tf.device('/cpu:0'):\n"," global_step = tf.train.get_or_create_global_step()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"lFGsOcj50Eov"},"outputs":[],"source":["class BatchGenerator(object):\n"," \"\"\"Generate and hold batches.\"\"\"\n"," def __init__(self, dataset,label, batch_size,input_size, is_shuffle=True):\n"," self._dataset = dataset\n"," self._label = label\n"," self._batch_size = batch_size \n"," self._cursor = 0 \n"," self._input_size = input_size \n"," \n"," if is_shuffle:\n"," index = np.arange(len(self._dataset))\n"," np.random.shuffle(index)\n"," self._dataset = np.array(self._dataset)[index]\n"," self._label = np.array(self._label)[index]\n"," else:\n"," self._dataset = np.array(self._dataset)\n"," self._label = np.array(self._label)\n"," def next(self):\n"," if self._cursor + self._batch_size \u003e len(self._dataset):\n"," self._cursor = 0\n"," \"\"\"Generate a single batch from the current cursor position in the data.\"\"\" \n"," batch_x = self._dataset[self._cursor : self._cursor + self._batch_size,:]\n"," batch_y = self._label[self._cursor : self._cursor + self._batch_size]\n"," self._cursor += self._batch_size\n"," return batch_x, batch_y"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":13347,"status":"ok","timestamp":1651116250113,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"wV3H1xPDUZwU","outputId":"9377d57e-ab56-4ad1-c7fe-835edcf2cc2d"},"outputs":[{"name":"stdout","output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:72: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:515: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:4048: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:131: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:3368: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n","WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.\u003clocals\u003e.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.where in 2.0, which has the same broadcast rule as np.where\n"]}],"source":["import os \n","import torch\n","\n","fold_no = 1\n","\n","y = tf.placeholder(tf.int64, [args.batch_size, args.num_classes], name=\"input_y\")\n","is_training = tf.placeholder_with_default(False, shape=()) \n","learning_rate = tf.placeholder(dtype='float32') \n","margin = tf.placeholder(shape=(),dtype='float32') \n","\n","l2_loss = tf.constant(0.0)\n","\n","X= Input(shape=(args.max_sent,args.vec_size))\n","model = Bidirectional (LSTM (args.vec_size,return_sequences=True,dropout=0.1,kernel_regularizer=l2(0.01)),merge_mode='concat')(X)\n","model = TimeDistributed(Dense(args.vec_size,activation='relu'))(model)\n","model = tf.ensure_shape(model, (256,28,200), name=None)\n","\n","model = model[...,tf.newaxis]\n","poses, activations = capsule_model_B(model, args.num_classes)\n","\n","# X = tf.placeholder(tf.float32, [args.batch_size, args.max_sent,args.vec_size], name=\"input_x\")\n","# X = X[...,tf.newaxis]\n","# poses, activations = capsule_model_B(X, args.num_classes)\n","# poses, activations = capsule_model_B(train_set, args.num_classes)\n","loss = margin_loss(y, activations) \n","y_pred = tf.argmax(activations, axis=1, name=\"y_proba\") \n","correct = tf.equal(tf.argmax(y, axis=1), y_pred, name=\"correct\")\n","accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n","\n","optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, name = 'opt'+str(fold_no)) \n","training_op = optimizer.minimize(loss, name=\"training_op\")\n","gradients, variables = zip(*optimizer.compute_gradients(loss)) \n","\n","grad_check = [tf.check_numerics(g, message='Gradient NaN Found!')\n"," for g in gradients if g is not None] + [tf.check_numerics(loss, message='Loss NaN Found')]\n","with tf.control_dependencies(grad_check):\n"," training_op = optimizer.apply_gradients(zip(gradients, variables), global_step=global_step) "]},{"cell_type":"markdown","metadata":{"id":"RwyuzXZBflX7"},"source":["# Train"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"1Xh_EEQEO_44"},"outputs":[{"name":"stdout","output_type":"stream","text":["0.30000000000000004\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.4\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.5\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.6\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.7\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.7999999999999999\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.8999999999999999\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.9\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.9\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n","0.9\n","accuracy: 0.7439% Precision: 0.5534 recall: 0.7439 F1: 0.6346\n"]}],"source":["saver = tf.train.Saver()\n","save_path = folder_path + \"/Tests/\"+embedding_type+\"_\"+capsule_routes+\"_\"+pool_layer+\"/model.ckpt\"\n","\n","sess = tf.InteractiveSession()\n","from keras import utils\n","\n","# saver.restore(sess, save_path)\n","\n","old_acc,old_pre,old_rec,old_f1 = 0,0,0,0\n","\n","n_iterations_per_epoch = len(train_set) // args.batch_size\n","n_iterations_test = len(test_set) // args.batch_size\n","n_iterations_dev = len(dev_set) // args.batch_size \n","\n","mr_train = BatchGenerator(train_set,train_label, args.batch_size, 0) \n","mr_dev = BatchGenerator(dev_set,dev_label, args.batch_size, 0)\n","mr_test = BatchGenerator(test_set,test_label, args.batch_size, 0, is_shuffle=False)\n","\n","init = tf.global_variables_initializer()\n","sess.run(init) \n","\n","lr = args.learning_rate\n","m = args.margin\n","for epoch in range(args.num_epochs):\n"," for iteration in range(1, n_iterations_per_epoch + 1): \n"," X_batch, y_batch = mr_train.next() \n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size)\n"," y_batch = utils.to_categorical(y_batch, args.num_classes) \n"," _, loss_train, probs, capsule_pose = sess.run(\n"," [training_op, loss, activations, poses],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: True,\n"," learning_rate:lr,\n"," margin:m}) \n"," # print(\"\\rIteration: {}/{} ({:.1f}%) Loss: {:.5f}\".format(iteration, n_iterations_per_epoch,iteration * 100 / n_iterations_per_epoch,loss_train),end=\"\") \n"," loss_vals, acc_vals = [], []\n"," for iteration in range(1, n_iterations_dev + 1):\n"," X_batch, y_batch = mr_dev.next() \n"," y_batch = utils.to_categorical(y_batch, args.num_classes)\n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size)\n"," loss_val, acc_val = sess.run(\n"," [loss, accuracy],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: False,\n"," margin:m})\n"," loss_vals.append(loss_val)\n"," acc_vals.append(acc_val)\n"," loss_val, acc_val = np.mean(loss_vals), np.mean(acc_vals) \n"," # print(\"\\rEpoch: {} Val accuracy: {:.1f}% Loss: {:.4f}\".format(epoch + 1, acc_val * 100, loss_val))\n"," \n"," preds_list, y_list = [], []\n"," for iteration in range(1, n_iterations_test + 1):\n"," X_batch, y_batch = mr_test.next()\n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size) \n"," probs = sess.run([activations],\n"," feed_dict={X:X_batch[:,:args.max_sent],\n"," is_training: False})\n"," preds_list = preds_list + probs[0].tolist()\n"," y_list = y_list + y_batch.tolist()\n"," \n"," y_list = np.array(y_list)\n"," y_list = utils.to_categorical(y_list, args.num_classes)\n"," preds_probs = np.array(preds_list) \n"," # preds_probs = (preds_probs == preds_probs.max(axis=1)[:,None]).astype(int)\n"," \n"," # [precision, recall, F1, support] = precision_recall_fscore_support(y_list, preds_probs, average='samples')\n"," # acc = accuracy_score(y_list, preds_probs)\n","\n"," # print ('\\rER: %.3f' % acc, 'Precision: %.3f' % precision, 'Recall: %.3f' % recall, 'F1: %.3f' % F1) \n"," m = min(0.9, m + 0.1)\n"," print (m)\n"," \n"," labels = np.argmax(y_list, axis=1)\n"," predictions = np.argmax(preds_probs, axis=1)\n","\n"," accuracy_fold = accuracy_score(labels, predictions)\n"," precision_fold = precision_score(labels, predictions, average='weighted', zero_division = 0 )\n"," recall_fold = recall_score(labels, predictions, average='weighted')\n"," f1_fold = f1_score(labels, predictions, average='weighted')\n"," print(\"\\raccuracy: {:.4f}% Precision: {:.4f} recall: {:.4f} F1: {:.4f}\".format(accuracy_fold, precision_fold, recall_fold, f1_fold))\n"," \n"," if not (accuracy_fold\u003cold_acc and precision_fold\u003cold_pre and recall_fold\u003cold_rec and f1_fold\u003cold_f1):\n"," saver.save(sess, save_path)\n","\n"," old_acc = accuracy_fold\n"," old_pre = precision_fold\n"," old_rec = recall_fold\n"," old_f1 = f1_fold\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"k0fVwixUfgQu"},"outputs":[],"source":["# with tf.Session() as sess:\n","\n","# init = tf.global_variables_initializer()\n","# sess.run(init)\n","\n","# for epoch in range(num_epochs): \n","\n","# for iteration in range(1, n_iterations_per_epoch + 1):\n","# X_batch, y_batch = mr_train1.next() \n","# y_batch = utils.to_categorical(y_batch, args.num_classes) \n","# _, loss_train, probs, capsule_pose = sess.run(\n","# [training_op, loss, activations, poses],\n","# feed_dict={X: X_batch[:,:args.max_sent],\n","# y: y_batch,\n","# is_training: True,\n","# learning_rate:lr,\n","# margin:m})\n"," \n","# print(\"\\rIteration: {}/{} ({:.1f}%) epoch:{} Loss: {:.5f}\".format(iteration, n_iterations_per_epoch, iteration * 100 / n_iterations_per_epoch, epoch+1, loss_train), end=\"\") \n","# loss_vals, acc_vals = [], []\n","# for iteration in range(1, n_iterations_test + 1):\n","# X_batch, y_batch = mr_test1.next() \n","# y_batch = utils.to_categorical(y_batch, args.num_classes) \n","# loss_val, acc_val = sess.run([activations],\n","# feed_dict={X:X_batch[:,:args.max_sent],\n","# is_training: False})\n","# loss_vals.append(loss_val)\n","# acc_vals.append(acc_val)\n","# loss_val, acc_val = np.mean(loss_vals), np.mean(acc_vals) \n","# print(\"\\rEpoch: {} Val accuracy: {:.1f}% Loss: {:.4f}\".format(\n","# epoch + 1, acc_val * 100, loss_val))\n"," \n","# preds_list, y_list = [], []\n","# for iteration in range(1, n_iterations_test + 1):\n","# X_batch, y_batch = mr_test.next() \n","# probs = sess.run([activations],\n","# feed_dict={X:X_batch[:,:args.max_sent],\n","# is_training: False})\n","# preds_list = preds_list + probs[0].tolist()\n","# y_list = y_list + y_batch.tolist()\n"," \n","# y_list = np.array(y_list)\n","# preds_probs = np.array(preds_list) \n","# preds_probs[np.where( preds_probs \u003e= threshold )] = 1.0\n","# preds_probs[np.where( preds_probs \u003c threshold )] = 0.0 \n"," \n","# [precision, recall, F1, support] = \\\n","# precision_recall_fscore_support(y_list, preds_probs, average='samples')\n","# acc = accuracy_score(y_list, preds_probs)\n","\n","# print ('\\rER: %.3f' % acc, 'Precision: %.3f' % precision, 'Recall: %.3f' % recall, 'F1: %.3f' % F1) \n","# if args.model_type == 'CNN' or args.model_type == 'KIMCNN':\n","# lr = max(1e-6, lr * 0.8)\n","# if args.loss_type == 'margin_loss': \n","# m = min(0.9, m + 0.1)\n","\n","# m = min(0.9, m + 0.1) \n","# print('m=',str(m))\n","# y_list = np.array(y_list)\n","# preds_probs = np.array(preds_list) \n","# labels = np.argmax(y_list, axis=1)\n","# predictions = np.argmax(preds_probs, axis=1)\n","\n","# accuracy_fold = accuracy_score(labels, predictions)\n","# precision_fold = precision_score(labels, predictions, average='weighted', zero_division = 0 )\n","# recall_fold = recall_score(labels, predictions, average='weighted')\n","# f1_fold = f1_score(labels, predictions, average='weighted')\n","# print(\"\\rFold: {} accuracy: {:.4f}% Precision: {:.4f} recall: {:.4f} F1: {:.4f}\".format(fold_no, accuracy_fold, precision_fold, recall_fold, f1_fold))\n"]}],"metadata":{"colab":{"collapsed_sections":["7dpVU5Sb1Ddr","PARyB28Vi9VI"],"machine_shape":"hm","name":"serial Sinhala_text_capsules.ipynb","provenance":[{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/Sinhala_text_capsules.ipynb","timestamp":1621834144627}],"version":""},"environment":{"name":"tf-gpu.1-15.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf-gpu.1-15:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"nbformat":4,"nbformat_minor":0}

sentiment-analysis-models/capsule-networks/vanila_capsule_network.ipynb ADDED Viewed

	@@ -0,0 +1 @@

+ {"cells":[{"cell_type":"markdown","metadata":{"id":"k8D9EuJnBnDV"},"source":["# Installing and importing dependecies"]},{"cell_type":"code","execution_count":2,"metadata":{"executionInfo":{"elapsed":626,"status":"ok","timestamp":1684563372538,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"fp5wHxfHqnPj"},"outputs":[],"source":["embedding_type = \"GRU\" #@param [\"GRU\",\"MAX\"]\n","EMBEDDING_SIZE = \"300\" #@param [200,300]\n","\n","capsule_routes = \"3\" #@param [\"3\",\"4\"]\n","pool_layer = \"min\" #@param [\"avg\",\"max\",\"min\"]"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":24720,"status":"ok","timestamp":1684563397732,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"GgRvEbMXy9nc","outputId":"77030479-dcef-4ecc-aad0-e5d384e1d091"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":86594,"status":"ok","timestamp":1684563484320,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"CCi-HNCn7BCi","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"9b8abe99-8501-4d44-ced2-209dc922327e"},"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[33mWARNING: Skipping keras-nightly as it is not installed.\u001b[0m\u001b[33m\n","\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: tensorflow==2.12.0 in /usr/local/lib/python3.10/dist-packages (2.12.0)\n","Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (1.4.0)\n","Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (1.6.3)\n","Requirement already satisfied: flatbuffers>=2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (23.3.3)\n","Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (0.4.0)\n","Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (0.2.0)\n","Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (1.54.0)\n","Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (3.8.0)\n","Requirement already satisfied: jax>=0.3.15 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (0.4.8)\n","Requirement already satisfied: keras<2.13,>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (2.12.0)\n","Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (16.0.0)\n","Requirement already satisfied: numpy<1.24,>=1.22 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (1.22.4)\n","Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (3.3.0)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (23.1)\n","Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (3.20.3)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (67.7.2)\n","Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (1.16.0)\n","Requirement already satisfied: tensorboard<2.13,>=2.12 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (2.12.2)\n","Requirement already satisfied: tensorflow-estimator<2.13,>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (2.12.0)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (2.3.0)\n","Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (4.5.0)\n","Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (1.14.1)\n","Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12.0) (0.32.0)\n","Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow==2.12.0) (0.40.0)\n","Requirement already satisfied: ml-dtypes>=0.0.3 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow==2.12.0) (0.1.0)\n","Requirement already satisfied: scipy>=1.7 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow==2.12.0) (1.10.1)\n","Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12.0) (2.17.3)\n","Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12.0) (1.0.0)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12.0) (3.4.3)\n","Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12.0) (2.27.1)\n","Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12.0) (0.7.0)\n","Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12.0) (1.8.1)\n","Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12.0) (2.3.0)\n","Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (5.3.0)\n","Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (0.3.0)\n","Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (4.9)\n","Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (1.3.1)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (1.26.15)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (2022.12.7)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (2.0.12)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (3.4)\n","Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (2.1.2)\n","Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (0.5.0)\n","Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow==2.12.0) (3.2.2)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting q\n"," Downloading q-2.7-py2.py3-none-any.whl (10 kB)\n","Requirement already satisfied: keras==2.12.0 in /usr/local/lib/python3.10/dist-packages (2.12.0)\n","Installing collected packages: q\n","Successfully installed q-2.7\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting h5py<3.0.0\n"," Downloading h5py-2.10.0.tar.gz (301 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.1/301.1 kB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.10/dist-packages (from h5py<3.0.0) (1.22.4)\n","Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from h5py<3.0.0) (1.16.0)\n","Building wheels for collected packages: h5py\n"," Building wheel for h5py (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for h5py: filename=h5py-2.10.0-cp310-cp310-linux_x86_64.whl size=5620017 sha256=6d135394d9fa71e4a4cceb099474bf54920929a918b1f7c214b5746c223c4d71\n"," Stored in directory: /root/.cache/pip/wheels/21/bc/58/0d0c6056e1339f40188d136cd838c6554d9c17545196dd9110\n","Successfully built h5py\n","Installing collected packages: h5py\n"," Attempting uninstall: h5py\n"," Found existing installation: h5py 3.8.0\n"," Uninstalling h5py-3.8.0:\n"," Successfully uninstalled h5py-3.8.0\n","Successfully installed h5py-2.10.0\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["h5py"]}}},"metadata":{}}],"source":["!pip uninstall keras-nightly\n","!pip install tensorflow==2.12.0\n","!pip install q keras==2.12.0\n","!pip install 'h5py<3.0.0'\n","\n","# !pip install tensorflow==1.14.0\n","# !pip install q keras==2.2.2"]},{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":3420,"status":"error","timestamp":1684564736591,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"zlXBE_-86l4q","colab":{"base_uri":"https://localhost:8080/","height":383},"outputId":"6f02caa0-b32c-40d4-965f-d1844b0a3ea4"},"outputs":[{"output_type":"error","ename":"ModuleNotFoundError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)","\u001b[0;32m<ipython-input-1-7c994c2ac32f>\u001b[0m in \u001b[0;36m<cell line: 17>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras_preprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msequence\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpad_sequences\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimage\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmpimg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'keras_preprocessing'","","\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"],"errorDetails":{"actions":[{"action":"open_url","actionText":"Open Examples","url":"/notebooks/snippets/importing_libraries.ipynb"}]}}],"source":["import pandas as pd\n","import collections\n","import gensim\n","from gensim.models.keyedvectors import KeyedVectors\n","from gensim.models.fasttext import FastText\n","from gensim.models import word2vec\n","\n","from sklearn.model_selection import train_test_split,cross_val_score, cross_val_predict, KFold, GridSearchCV\n","from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix, precision_recall_fscore_support\n","\n","import pandas as pd\n","import numpy as np\n","\n","import keras\n","from keras import backend as K\n","from keras.preprocessing.text import Tokenizer\n","from keras_preprocessing.sequence import pad_sequences\n","\n","import matplotlib.image as mpimg\n","import matplotlib.pyplot as plt\n","\n","import tensorflow as tf\n","from tensorflow.keras import initializers"]},{"cell_type":"markdown","metadata":{"id":"nRbpY2FTDl8p"},"source":["# Load Dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485839,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"d8GzWXqBsX2V"},"outputs":[],"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Capsule networks/'\n","\n","if (embedding_type == 'GRU'):\n"," facebook_data_path = folder_path + 'embedded posts/GRU_attention_'\n","elif (embedding_type == 'MAX'):\n"," facebook_data_path = folder_path + 'embedded posts/'+'MAX_'\n","\n","context = 5\n","posts=[]\n","labels =[]\n","\n","# facebook_data_path = (facebook_data_path +str(i)+'.csv')\n","# dataset = pd.read_pickle(facebook_data_path)\n","# dataset=dataset.dropna()\n","# dataset = dataset[:1000]\n","# labels+=dataset[\"label\"].tolist()\n","# for j in dataset['posts']:\n","# posts.append(j)\n","\n","for i in range (1,6):\n"," data_path = (facebook_data_path +str(i)+'.csv')\n"," dataset = pd.read_pickle(data_path)\n"," dataset=dataset.dropna()\n"," # dataset = dataset[:1000]\n"," labels+=dataset[\"label\"].tolist()\n"," for j in dataset['posts']:\n"," posts.append(j)"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485839,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"0m6FrtmcsRKu"},"outputs":[],"source":["# posts=np.array(posts)\n","# labels=np.array(labels)\n","train_set, test_set , train_label, test_label = train_test_split(posts,labels, test_size=0.1, random_state=0)\n","train_set, dev_set, train_label, dev_label = train_test_split(train_set,train_label, test_size=0.1, random_state=0)"]},{"cell_type":"markdown","metadata":{"id":"XnkydX1ne969"},"source":["# Convo 2d layer"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485839,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"0GVuHgh2EWw2"},"outputs":[],"source":["def _get_variable_wrapper(name, shape=None, dtype=None, initializer=None,regularizer=None,trainable=True,collections=None,caching_device=None,partitioner=None,validate_shape=True,custom_getter=None):\n"," with tf.device('/cpu:0'):\n"," var = tf.get_variable(\n"," name, shape=shape, dtype=dtype, initializer=initializer,\n"," regularizer=regularizer, trainable=trainable,\n"," collections=collections, caching_device=caching_device,\n"," partitioner=partitioner, validate_shape=validate_shape,\n"," custom_getter=custom_getter\n"," )\n"," return var"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485839,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"k8F7Y8CGbsB4"},"outputs":[],"source":["def _get_weights_wrapper(name, shape, dtype=tf.float32, initializer=tf.keras.initializers.GlorotUniform(),weights_decay_factor=None):\n"," weights = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," if weights_decay_factor is not None and weights_decay_factor > 0.0:\n"," weights_wd = tf.multiply(tf.nn.l2_loss(weights), weights_decay_factor, name=name + '/l2loss')\n"," tf.add_to_collection('losses', weights_wd)\n"," return weights"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485839,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"nkO3w5gDcC1b"},"outputs":[],"source":["def _get_biases_wrapper(name, shape, dtype=tf.float32, initializer=tf.constant_initializer(0.0)):\n"," \"\"\"Wrapper over _get_variable_wrapper() to get bias.\n"," \"\"\"\n"," biases = _get_variable_wrapper(name=name, shape=shape, dtype=dtype, initializer=initializer)\n"," return biases"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":14,"status":"aborted","timestamp":1684563485840,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"9Z06E9a1aBVb"},"outputs":[],"source":["def _conv2d_wrapper(inputs, shape, strides, padding, add_bias, activation_fn, name, stddev=0.1):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE) as scope:\n"," kernel = _get_weights_wrapper(\n"," name='weights', shape=shape, weights_decay_factor=0.0, )\n"," output = tf.nn.conv2d(inputs, filter=kernel, strides=strides, padding=padding, name='conv')\n"," if add_bias:\n"," biases = _get_biases_wrapper(name='biases', shape=[shape[-1]] )\n"," output = tf.add(output, biases, name='biasAdd')\n"," if activation_fn is not None:\n"," output = activation_fn(output, name='activation')\n"," return output"]},{"cell_type":"markdown","metadata":{"id":"7dpVU5Sb1Ddr"},"source":["# Utils"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":14,"status":"aborted","timestamp":1684563485840,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"MBkd3MRuccQh"},"outputs":[],"source":["def softmax(x, axis=-1):\n"," ex = K.exp(x - K.max(x, axis=axis, keepdims=True))\n"," return ex/K.sum(ex, axis=axis, keepdims=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":14,"status":"aborted","timestamp":1684563485840,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"hSVP0ymhc6G6"},"outputs":[],"source":["def squash_v1(x, axis=-1):\n"," s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n"," scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n"," return scale * x"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485840,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"RQzm6uHwc8Rq"},"outputs":[],"source":["def squash_v0(s, axis=-1, epsilon=1e-7, name=None):\n"," s_squared_norm = K.sum(K.square(s), axis, keepdims=True) + K.epsilon()\n"," safe_norm = K.sqrt(s_squared_norm)\n"," scale = 1 - tf.exp(-safe_norm)\n"," return scale * s / safe_norm"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485840,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"ObeP2wKDc-Zq"},"outputs":[],"source":["def routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations):\n"," b = keras.backend.zeros_like(u_hat_vecs[:,:,:,0])\n"," if i_activations is not None:\n"," i_activations = i_activations[...,tf.newaxis]\n"," for i in range(iterations):\n"," if False:\n"," leak = tf.zeros_like(b, optimize=True)\n"," leak = tf.reduce_sum(leak, axis=1, keep_dims=True)\n"," leaky_logits = tf.concat([leak, b], axis=1)\n"," leaky_routing = tf.nn.softmax(leaky_logits, dim=1) \n"," c = tf.split(leaky_routing, [1, output_capsule_num], axis=1)[1]\n"," else:\n"," c = softmax(b, 1) \n"," outputs = squash_v1(K.batch_dot(c, u_hat_vecs, [2, 2]))\n"," if i < iterations - 1:\n"," b = b + K.batch_dot(outputs, u_hat_vecs, [2, 3]) \n"," poses = outputs \n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," return poses, activations\n"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485840,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"zSdVM9YCdEMp"},"outputs":[],"source":["def vec_transformationByConv(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num): \n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_dim, output_capsule_dim*output_capsule_num], weights_decay_factor=0.0)\n"," u_hat_vecs = keras.backend.conv1d(poses, kernel)\n"," u_hat_vecs = keras.backend.reshape(u_hat_vecs, (-1, input_capsule_num, output_capsule_num, output_capsule_dim))\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs\n"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485840,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"i-pIavredIK6"},"outputs":[],"source":["def vec_transformationByMat(poses, input_capsule_dim, input_capsule_num, output_capsule_dim, output_capsule_num, shared=True): \n"," inputs_poses_shape = poses.get_shape().as_list()\n"," poses = poses[..., tf.newaxis, :] \n"," poses = tf.tile(poses, [1, 1, output_capsule_num, 1]) \n"," if shared:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, 1, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], input_capsule_num, 1, 1, 1])\n"," else:\n"," kernel = _get_weights_wrapper(name='weights', shape=[1, input_capsule_num, output_capsule_num, output_capsule_dim, input_capsule_dim], weights_decay_factor=0.0)\n"," kernel = tf.tile(kernel, [inputs_poses_shape[0], 1, 1, 1, 1])\n"," u_hat_vecs = tf.squeeze(tf.matmul(kernel, poses[...,tf.newaxis]),axis=-1)\n"," u_hat_vecs = keras.backend.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n"," return u_hat_vecs"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":14,"status":"aborted","timestamp":1684563485841,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"mcvHPFCOdKyX"},"outputs":[],"source":["def capsules_init(inputs, shape, strides, padding, pose_shape, add_bias, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses = _conv2d_wrapper(\n"," inputs,\n"," shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," strides=strides,\n"," padding=padding,\n"," add_bias=add_bias,\n"," activation_fn=None,\n"," name='pose_stacked'\n"," ) \n"," poses_shape = poses.get_shape().as_list() \n"," poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," poses = squash_v1(poses, axis=-1) \n"," activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n","\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":14,"status":"aborted","timestamp":1684563485841,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"hAJ1njh6dOpT"},"outputs":[],"source":["def capsule_fc_layer(nets, output_capsule_num, iterations, name):\n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n","\n"," u_hat_vecs = vec_transformationByConv(poses,input_pose_shape[-1], input_pose_shape[1],input_pose_shape[-1], output_capsule_num,)\n"," beta_a = _get_weights_wrapper(name='beta_a', shape=[1, output_capsule_num])\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, output_capsule_num, i_activations)\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485841,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"jE-SPLBwdR2a"},"outputs":[],"source":["def capsule_flatten(nets):\n"," poses, activations = nets\n"," input_pose_shape = poses.get_shape().as_list()\n"," \n"," poses = tf.reshape(poses, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3], input_pose_shape[-1]]) \n"," activations = tf.reshape(activations, [\n"," -1, input_pose_shape[1]*input_pose_shape[2]*input_pose_shape[3]])\n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485841,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"1uMUkhf1dVJy"},"outputs":[],"source":["def capsule_conv_layer(nets, shape, strides, iterations, name): \n"," # poses = _conv2d_wrapper(\n"," # inputs,\n"," # shape=shape[0:-1] + [shape[-1] * pose_shape],\n"," # strides=strides,\n"," # padding=padding,\n"," # add_bias=add_bias,\n"," # activation_fn=None,\n"," # name='pose_stacked'\n"," # ) \n"," # poses_shape = poses.get_shape().as_list() \n"," # poses = tf.reshape(poses, [-1, poses_shape[1], poses_shape[2], shape[-1], pose_shape]) \n"," # beta_a = _get_weights_wrapper(name='beta_a', shape=[1, shape[-1]]) \n"," # poses = squash_v1(poses, axis=-1) \n"," # activations = K.sqrt(K.sum(K.square(poses), axis=-1)) + beta_a \n"," with tf.variable_scope(name,reuse=tf.AUTO_REUSE): \n"," poses, i_activations = nets\n"," \n"," inputs_poses_shape = poses.get_shape().as_list()\n","\n"," hk_offsets = [\n"," [(h_offset + k_offset) for k_offset in range(0, shape[0])] for h_offset in\n"," range(0, inputs_poses_shape[1] + 1 - shape[0], strides[1])\n"," ]\n"," wk_offsets = [\n"," [(w_offset + k_offset) for k_offset in range(0, shape[1])] for w_offset in\n"," range(0, inputs_poses_shape[2] + 1 - shape[1], strides[2])\n"," ]\n"," \n"," inputs_poses_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," poses, hk_offsets, axis=1, name='gather_poses_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_poses_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5, 6], name='inputs_poses_patches'\n"," )\n"," inputs_poses_shape = inputs_poses_patches.get_shape().as_list()\n"," inputs_poses_patches = tf.reshape(inputs_poses_patches, [\n"," -1, shape[0]*shape[1]*shape[2], inputs_poses_shape[-1]\n"," ])\n","\n"," i_activations_patches = tf.transpose(\n"," tf.gather(\n"," tf.gather(\n"," i_activations, hk_offsets, axis=1, name='gather_activations_height_kernel'\n"," ), wk_offsets, axis=3, name='gather_activations_width_kernel'\n"," ), perm=[0, 1, 3, 2, 4, 5], name='inputs_activations_patches'\n"," )\n"," i_activations_patches = tf.reshape(i_activations_patches, [\n"," -1, shape[0]*shape[1]*shape[2]]\n"," )\n"," u_hat_vecs = vec_transformationByConv(\n"," inputs_poses_patches,\n"," inputs_poses_shape[-1], shape[0]*shape[1]*shape[2],\n"," inputs_poses_shape[-1], shape[3],\n"," ) \n"," beta_a = _get_weights_wrapper(\n"," name='beta_a', shape=[1, shape[3]]\n"," )\n"," poses, activations = routing(u_hat_vecs, beta_a, iterations, shape[3], i_activations_patches)\n"," poses = tf.reshape(poses, [\n"," inputs_poses_shape[0], inputs_poses_shape[1],\n"," inputs_poses_shape[2], shape[3],\n"," inputs_poses_shape[-1]]\n"," ) \n"," activations = tf.reshape(activations, [\n"," inputs_poses_shape[0],inputs_poses_shape[1],\n"," inputs_poses_shape[2],shape[3]]\n"," ) \n"," nets = poses, activations \n"," return nets"]},{"cell_type":"markdown","metadata":{"id":"dqBVV8BNfGPe"},"source":["# Capsule"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":13,"status":"aborted","timestamp":1684563485841,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"jam_fKSJjYfg"},"outputs":[],"source":["def capsule_model_A(X, num_classes):\n"," with tf.variable_scope('capsule_'+str(3),reuse=tf.AUTO_REUSE ): \n"," nets = _conv2d_wrapper(\n"," X, shape=[3, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," nets = capsule_flatten(nets)\n"," # nets = BiGRU_layer(nets)\n"," # nets = Flatten()(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2') \n"," return poses, activations"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485841,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"dbqUEzcbotXD"},"outputs":[],"source":["def capsule_model_B(X, num_classes):\n"," poses_list = []\n"," #bilstm\n"," for _, ngram in enumerate([3,4,5,6]):\n"," with tf.variable_scope('capsule_'+str(ngram),reuse=tf.AUTO_REUSE): \n"," nets = _conv2d_wrapper(\n"," X, shape=[ngram, 200, 1, 32], strides=[1, 2, 1, 1], padding='VALID', \n"," add_bias=True, activation_fn=tf.nn.relu, name='conv1'\n"," )\n"," nets = capsules_init(nets, shape=[1, 1, 32, 16], strides=[1, 1, 1, 1], \n"," padding='VALID', pose_shape=16, add_bias=True, name='primary') \n"," nets = capsule_conv_layer(nets, shape=[3, 1, 16, 16], strides=[1, 1, 1, 1], iterations=3, name='conv2')\n"," nets = capsule_flatten(nets)\n"," poses, activations = capsule_fc_layer(nets, num_classes, 3, 'fc2')\n"," poses_list.append(poses)\n"," #reduce_max,min\n"," # poses = tf.reduce_max(tf.convert_to_tensor(poses_list), axis=0) \n"," poses = tf.reduce_min(tf.convert_to_tensor(poses_list), axis=0) \n"," # poses = tf.reduce_mean(tf.convert_to_tensor(poses_list), axis=0) \n"," activations = K.sqrt(K.sum(K.square(poses), 2))\n"," return poses, activations"]},{"cell_type":"markdown","metadata":{"id":"PARyB28Vi9VI"},"source":["# Loss functions"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":11,"status":"aborted","timestamp":1684563485841,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"WkAyzy_3hrYf"},"outputs":[],"source":["def spread_loss(labels, activations, margin):\n"," activations_shape = activations.get_shape().as_list()\n"," mask_t = tf.equal(labels, 1)\n"," mask_i = tf.equal(labels, 0) \n"," activations_t = tf.reshape(\n"," tf.boolean_mask(activations, mask_t), [activations_shape[0], 1]\n"," ) \n"," activations_i = tf.reshape(\n"," tf.boolean_mask(activations, mask_i), [activations_shape[0], activations_shape[1] - 1]\n"," ) \n"," gap_mit = tf.reduce_sum(tf.square(tf.nn.relu(margin - (activations_t - activations_i))))\n"," return gap_mit "]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485842,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"n2UfkOrSjFvX"},"outputs":[],"source":["def cross_entropy(y, preds): \n"," y = tf.argmax(y, axis=1)\n"," loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=preds, labels=y) \n"," loss = tf.reduce_mean(loss) \n"," return loss"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485842,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"qEI1tgTnjUg_"},"outputs":[],"source":["def margin_loss(y, preds): \n"," y = tf.cast(y,tf.float32)\n"," loss = y * tf.square(tf.maximum(0., 0.9 - preds)) + \\\n"," 0.25 * (1.0 - y) * tf.square(tf.maximum(0., preds - 0.1))\n"," loss = tf.reduce_mean(tf.reduce_sum(loss, axis=1))\n"," return loss"]},{"cell_type":"markdown","metadata":{"id":"Qz6aicUlku28"},"source":["# Setup"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485842,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"Un6p6OLXkPnt"},"outputs":[],"source":["class Args:\n"," embedding_type = \"static\"\n"," dataset = \"\"\n"," loss_type = \"margin_loss\"\n"," model_type = \"capsule-B\"\n"," has_test = 1\n"," has_dev = 1\n"," num_epochs = 50\n"," batch_size = 256\n"," use_orphan = False\n"," use_leaky = False\n"," learning_rate = 0.01\n"," margin = 0.2\n"," num_classes = 2\n"," # vocab_size = vocab_size\n"," vec_size = 200\n"," max_sent = 28 #max_length\n"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485842,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"3cA5u4HrkuCb"},"outputs":[],"source":["args = Args()"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485842,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"cVRu8KNo3GcR"},"outputs":[],"source":["with tf.device('/cpu:0'):\n"," global_step = tf.compat.v1.train.get_or_create_global_step()"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485842,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"lFGsOcj50Eov"},"outputs":[],"source":["class BatchGenerator(object):\n"," \"\"\"Generate and hold batches.\"\"\"\n"," def __init__(self, dataset,label, batch_size,input_size, is_shuffle=True):\n"," self._dataset = dataset\n"," self._label = label\n"," self._batch_size = batch_size \n"," self._cursor = 0 \n"," self._input_size = input_size \n"," \n"," if is_shuffle:\n"," index = np.arange(len(self._dataset))\n"," np.random.shuffle(index)\n"," self._dataset = np.array(self._dataset)[index]\n"," self._label = np.array(self._label)[index]\n"," else:\n"," self._dataset = np.array(self._dataset)\n"," self._label = np.array(self._label)\n"," def next(self):\n"," if self._cursor + self._batch_size > len(self._dataset):\n"," self._cursor = 0\n"," \"\"\"Generate a single batch from the current cursor position in the data.\"\"\" \n"," batch_x = self._dataset[self._cursor : self._cursor + self._batch_size,:]\n"," batch_y = self._label[self._cursor : self._cursor + self._batch_size]\n"," self._cursor += self._batch_size\n"," return batch_x, batch_y"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":12,"status":"aborted","timestamp":1684563485842,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"},"user_tz":-330},"id":"wV3H1xPDUZwU"},"outputs":[],"source":["import os \n","\n","import tensorflow.compat.v1 as tf\n","tf.disable_v2_behavior()\n","\n","# X = tf.placeholder(tf.int32, [args.batch_size, args.max_sent,args.vec_size], name=\"input_x\")\n","y = tf.placeholder(tf.int64, [args.batch_size, args.num_classes], name=\"input_y\")\n","is_training = tf.placeholder_with_default(False, shape=()) \n","learning_rate = tf.placeholder(dtype='float32') \n","margin = tf.placeholder(shape=(),dtype='float32') \n","\n","l2_loss = tf.constant(0.0)\n","\n","X = tf.placeholder(tf.float32, [args.batch_size, args.max_sent,args.vec_size], name=\"input_x\")\n","X = X[...,tf.newaxis]\n","poses, activations = capsule_model_B(X, args.num_classes)\n","loss = margin_loss(y, activations) \n","y_pred = tf.argmax(activations, axis=1, name=\"y_proba\") \n","correct = tf.equal(tf.argmax(y, axis=1), y_pred, name=\"correct\")\n","accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n","\n","optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, name = 'opt1') \n","training_op = optimizer.minimize(loss, name=\"training_op\")\n","gradients, variables = zip(*optimizer.compute_gradients(loss)) \n","\n","grad_check = [tf.check_numerics(g, message='Gradient NaN Found!')\n"," for g in gradients if g is not None] + [tf.check_numerics(loss, message='Loss NaN Found')]\n","with tf.control_dependencies(grad_check):\n"," training_op = optimizer.apply_gradients(zip(gradients, variables), global_step=global_step) "]},{"cell_type":"markdown","metadata":{"id":"RwyuzXZBflX7"},"source":["# Train"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"1Xh_EEQEO_44","executionInfo":{"status":"aborted","timestamp":1684563485843,"user_tz":-330,"elapsed":13,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["saver = tf.train.Saver()\n","save_path = folder_path + \"/Tests/\"+embedding_type+\"_\"+capsule_routes+\"_\"+pool_layer+\"/model.ckpt\"\n","\n","\n","sess = tf.InteractiveSession()\n","from keras import utils\n","\n","# saver.restore(sess, save_path)\n","\n","old_acc,old_pre,old_rec,old_f1 = 0,0,0,0\n","\n","n_iterations_per_epoch = len(train_set) // args.batch_size\n","n_iterations_test = len(test_set) // args.batch_size\n","n_iterations_dev = len(dev_set) // args.batch_size \n","\n","mr_train = BatchGenerator(train_set,train_label, args.batch_size, 0) \n","mr_dev = BatchGenerator(dev_set,dev_label, args.batch_size, 0)\n","mr_test = BatchGenerator(test_set,test_label, args.batch_size, 0, is_shuffle=False)\n","\n","init = tf.global_variables_initializer()\n","sess.run(init) \n","\n","lr = args.learning_rate\n","m = args.margin\n","for epoch in range(args.num_epochs):\n"," for iteration in range(1, n_iterations_per_epoch + 1): \n"," X_batch, y_batch = mr_train.next() \n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size, 1)\n"," y_batch = utils.to_categorical(y_batch, args.num_classes) \n"," _, loss_train, probs, capsule_pose = sess.run(\n"," [training_op, loss, activations, poses],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: True,\n"," learning_rate:lr,\n","\n"," margin:m}) \n"," # print(\"\\rIteration: {}/{} ({:.1f}%) Loss: {:.5f}\".format(iteration, n_iterations_per_epoch,iteration * 100 / n_iterations_per_epoch,loss_train),end=\"\") \n"," loss_vals, acc_vals = [], []\n"," for iteration in range(1, n_iterations_dev + 1):\n"," X_batch, y_batch = mr_dev.next() \n"," y_batch = utils.to_categorical(y_batch, args.num_classes)\n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size, 1)\n"," loss_val, acc_val = sess.run(\n"," [loss, accuracy],\n"," feed_dict={X: X_batch[:,:args.max_sent],\n"," y: y_batch,\n"," is_training: False,\n"," margin:m})\n"," loss_vals.append(loss_val)\n"," acc_vals.append(acc_val)\n"," loss_val, acc_val = np.mean(loss_vals), np.mean(acc_vals) \n"," # print(\"\\rEpoch: {} Val accuracy: {:.1f}% Loss: {:.4f}\".format(epoch + 1, acc_val * 100, loss_val))\n"," \n"," preds_list, y_list = [], []\n"," for iteration in range(1, n_iterations_test + 1):\n"," X_batch, y_batch = mr_test.next()\n"," X_batch =X_batch.reshape(args.batch_size, args.max_sent, args.vec_size, 1) \n"," probs = sess.run([activations],\n"," feed_dict={X:X_batch[:,:args.max_sent],\n"," is_training: False})\n"," preds_list = preds_list + probs[0].tolist()\n"," y_list = y_list + y_batch.tolist()\n"," \n"," y_list = np.array(y_list)\n"," y_list = utils.to_categorical(y_list, args.num_classes)\n"," preds_probs = np.array(preds_list) \n"," # preds_probs = (preds_probs == preds_probs.max(axis=1)[:,None]).astype(int)\n"," \n"," # [precision, recall, F1, support] = precision_recall_fscore_support(y_list, preds_probs, average='samples')\n"," # acc = accuracy_score(y_list, preds_probs)\n","\n"," # print ('\\rER: %.3f' % acc, 'Precision: %.3f' % precision, 'Recall: %.3f' % recall, 'F1: %.3f' % F1) \n"," m = min(0.9, m + 0.1)\n"," print (m)\n"," \n"," labels = np.argmax(y_list, axis=1)\n"," predictions = np.argmax(preds_probs, axis=1)\n","\n"," accuracy_fold = accuracy_score(labels, predictions)\n"," precision_fold = precision_score(labels, predictions, average='weighted', zero_division = 0 )\n"," recall_fold = recall_score(labels, predictions, average='weighted')\n"," f1_fold = f1_score(labels, predictions, average='weighted')\n"," print(\"\\raccuracy: {:.4f}% Precision: {:.4f} recall: {:.4f} F1: {:.4f}\".format(accuracy_fold, precision_fold, recall_fold, f1_fold))\n"," \n"," if not (accuracy_fold<old_acc and precision_fold<old_pre and recall_fold<old_rec and f1_fold<old_f1):\n"," saver.save(sess, save_path)\n","\n"," old_acc = accuracy_fold\n"," old_pre = precision_fold\n"," old_rec = recall_fold\n"," old_f1 = f1_fold\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"k0fVwixUfgQu","executionInfo":{"status":"aborted","timestamp":1684563485843,"user_tz":-330,"elapsed":13,"user":{"displayName":"Gihan Weeraprameshwara","userId":"07090364678417433377"}}},"outputs":[],"source":["# with tf.Session() as sess:\n","\n","# init = tf.global_variables_initializer()\n","# sess.run(init)\n","\n","# for epoch in range(num_epochs): \n","\n","# for iteration in range(1, n_iterations_per_epoch + 1):\n","# X_batch, y_batch = mr_train1.next() \n","# y_batch = utils.to_categorical(y_batch, args.num_classes) \n","# _, loss_train, probs, capsule_pose = sess.run(\n","# [training_op, loss, activations, poses],\n","# feed_dict={X: X_batch[:,:args.max_sent],\n","# y: y_batch,\n","# is_training: True,\n","# learning_rate:lr,\n","# margin:m})\n"," \n","# print(\"\\rIteration: {}/{} ({:.1f}%) epoch:{} Loss: {:.5f}\".format(iteration, n_iterations_per_epoch, iteration * 100 / n_iterations_per_epoch, epoch+1, loss_train), end=\"\") \n","# loss_vals, acc_vals = [], []\n","# for iteration in range(1, n_iterations_test + 1):\n","# X_batch, y_batch = mr_test1.next() \n","# y_batch = utils.to_categorical(y_batch, args.num_classes) \n","# loss_val, acc_val = sess.run([activations],\n","# feed_dict={X:X_batch[:,:args.max_sent],\n","# is_training: False})\n","# loss_vals.append(loss_val)\n","# acc_vals.append(acc_val)\n","# loss_val, acc_val = np.mean(loss_vals), np.mean(acc_vals) \n","# print(\"\\rEpoch: {} Val accuracy: {:.1f}% Loss: {:.4f}\".format(\n","# epoch + 1, acc_val * 100, loss_val))\n"," \n","# preds_list, y_list = [], []\n","# for iteration in range(1, n_iterations_test + 1):\n","# X_batch, y_batch = mr_test.next() \n","# probs = sess.run([activations],\n","# feed_dict={X:X_batch[:,:args.max_sent],\n","# is_training: False})\n","# preds_list = preds_list + probs[0].tolist()\n","# y_list = y_list + y_batch.tolist()\n"," \n","# y_list = np.array(y_list)\n","# preds_probs = np.array(preds_list) \n","# preds_probs[np.where( preds_probs >= threshold )] = 1.0\n","# preds_probs[np.where( preds_probs < threshold )] = 0.0 \n"," \n","# [precision, recall, F1, support] = \\\n","# precision_recall_fscore_support(y_list, preds_probs, average='samples')\n","# acc = accuracy_score(y_list, preds_probs)\n","\n","# print ('\\rER: %.3f' % acc, 'Precision: %.3f' % precision, 'Recall: %.3f' % recall, 'F1: %.3f' % F1) \n","# if args.model_type == 'CNN' or args.model_type == 'KIMCNN':\n","# lr = max(1e-6, lr * 0.8)\n","# if args.loss_type == 'margin_loss': \n","# m = min(0.9, m + 0.1)\n","\n","# m = min(0.9, m + 0.1) \n","# print('m=',str(m))\n","# y_list = np.array(y_list)\n","# preds_probs = np.array(preds_list) \n","# labels = np.argmax(y_list, axis=1)\n","# predictions = np.argmax(preds_probs, axis=1)\n","\n","# accuracy_fold = accuracy_score(labels, predictions)\n","# precision_fold = precision_score(labels, predictions, average='weighted', zero_division = 0 )\n","# recall_fold = recall_score(labels, predictions, average='weighted')\n","# f1_fold = f1_score(labels, predictions, average='weighted')\n","# print(\"\\rFold: {} accuracy: {:.4f}% Precision: {:.4f} recall: {:.4f} F1: {:.4f}\".format(fold_no, accuracy_fold, precision_fold, recall_fold, f1_fold))\n"]}],"metadata":{"colab":{"collapsed_sections":["PARyB28Vi9VI"],"machine_shape":"hm","provenance":[{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/Sinhala_text_capsules.ipynb","timestamp":1621834144627}]},"environment":{"name":"tf-gpu.1-15.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf-gpu.1-15:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"nbformat":4,"nbformat_minor":0}

word-embedding-models/.DS_Store CHANGED Viewed

Binary files a/word-embedding-models/.DS_Store and b/word-embedding-models/.DS_Store differ