File size: 37,021 Bytes
21861c8 | 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"preprocess.ipynb","provenance":[{"file_id":"1Wd79wkxXF6CyKVFhYCid1jpIsv0HWRH-","timestamp":1625292490447},{"file_id":"1liafFKE2IZeDAgr_AibZfKykYLvHVzD3","timestamp":1624675505095},{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/cnn_rnn_multiclass.ipynb","timestamp":1621843389117},{"file_id":"153374pBL1OZuAJMWKK_ae0D9_DUq4P0P","timestamp":1590420423516}],"collapsed_sections":["0BYvE-D5QAc0","xXiAlHRZ7X_p","nGRZ8HrfREsw","jbpLZfOPGGKN","B6CXqtIDR4jS","d46ja9NetIpP","_VdzOT3zsB9m","qLkfB_XXPS3O"]},"environment":{"name":"tf2-2-2-gpu.2-2.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf2-2-2-gpu.2-2:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"cells":[{"cell_type":"markdown","metadata":{"id":"p7y2rE1vNZWc"},"source":["# Folder Paths"]},{"cell_type":"code","metadata":{"id":"IDq--mTz9k24","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535703014,"user_tz":-330,"elapsed":3385,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"98086339-ca77-445d-c28f-18fa4898e03b"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"code","metadata":{"id":"gWPD3dgEqB5d","executionInfo":{"status":"ok","timestamp":1641535703015,"user_tz":-330,"elapsed":9,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/'"],"execution_count":3,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0BYvE-D5QAc0"},"source":["# Dependencies"]},{"cell_type":"code","metadata":{"id":"Cf2gZocJff8x","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535712544,"user_tz":-330,"elapsed":9536,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"53ee4184-ca35-4090-a6b5-7f73beccc637"},"source":["!pip uninstall keras-nightly\n","!pip install tensorflow==1.14.0\n","!pip install q keras==2.3.1\n","!pip install 'h5py<3.0.0'"],"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[33mWARNING: Skipping keras-nightly as it is not installed.\u001b[0m\n","Requirement already satisfied: tensorflow==1.14.0 in /usr/local/lib/python3.7/dist-packages (1.14.0)\n","Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.12.0)\n","Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.15.0)\n","Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.2)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.0)\n","Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.2.0)\n","Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.14.0)\n","Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.14.0)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.37.0)\n","Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.19.5)\n","Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.13.3)\n","Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.42.0)\n","Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.4.0)\n","Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.8.1)\n","Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.0.8)\n","Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (3.17.3)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras-applications>=1.0.6->tensorflow==1.14.0) (2.10.0)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (1.0.1)\n","Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (57.4.0)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.3.6)\n","Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (4.8.2)\n","Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.10.0.2)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.6.0)\n","Requirement already satisfied: q in /usr/local/lib/python3.7/dist-packages (2.6)\n","Requirement already satisfied: keras==2.3.1 in /usr/local/lib/python3.7/dist-packages (2.3.1)\n","Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.4.1)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (2.10.0)\n","Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.15.0)\n","Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.19.5)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (3.13)\n","Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.0.8)\n","Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.1.2)\n","Requirement already satisfied: h5py<3.0.0 in /usr/local/lib/python3.7/dist-packages (2.10.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from h5py<3.0.0) (1.15.0)\n","Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from h5py<3.0.0) (1.19.5)\n"]}]},{"cell_type":"code","metadata":{"id":"gZ9YDtD6qn4t","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535714481,"user_tz":-330,"elapsed":1950,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"c5739ca6-e627-4ae8-aac2-c3a0fb7c3b21"},"source":["from __future__ import print_function\n","\n","import collections\n","import pickle\n","import re\n","import random\n","import sys\n","import os \n","import time\n","\n","import pandas as pd\n","import numpy as np\n","from numpy import array\n","from numpy import asarray\n","from numpy import zeros\n","from numpy import cumsum\n","\n","import matplotlib.image as mpimg\n","import matplotlib.pyplot as plt\n","\n","import torch\n","import torch.nn as nn\n","from torch import optim\n","import torch.nn.functional as F\n","torch.set_printoptions(precision=8)\n","\n","import nltk\n","nltk.download('punkt')\n","import nltk.data\n"],"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data] Package punkt is already up-to-date!\n"]}]},{"cell_type":"markdown","metadata":{"id":"xXiAlHRZ7X_p"},"source":["# Load Data"]},{"cell_type":"code","metadata":{"id":"GqqHP9rwpbdt","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535723488,"user_tz":-330,"elapsed":9012,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"878eaccb-57b9-4b37-95eb-20ec96d0cee3"},"source":["facebook_data_path = folder_path + 'Word embedding/labeled_corpus.csv'\n","all_data = pd.read_csv(facebook_data_path)"],"execution_count":6,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (31,32) have mixed types.Specify dtype option on import or set low_memory=False.\n"," interactivity=interactivity, compiler=compiler, result=result)\n"]}]},{"cell_type":"code","source":["all_data.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":617},"id":"Et0w2UcXKViX","executionInfo":{"status":"ok","timestamp":1641535863777,"user_tz":-330,"elapsed":497,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"909f44ab-42b7-4066-a47e-ef1c30d22ad0"},"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n"," <div id=\"df-f4968c51-4cf8-4da1-b146-b4dcdf34c077\">\n"," <div class=\"colab-df-container\">\n"," <div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>Unnamed: 0</th>\n"," <th>Unnamed: 0.1</th>\n"," <th>Unnamed: 0.1.1</th>\n"," <th>Unnamed: 0.1.1.1</th>\n"," <th>Page Name</th>\n"," <th>User Name</th>\n"," <th>Facebook Id</th>\n"," <th>Likes at Posting</th>\n"," <th>Created</th>\n"," <th>Type</th>\n"," <th>Likes</th>\n"," <th>Comments</th>\n"," <th>Shares</th>\n"," <th>Love</th>\n"," <th>Wow</th>\n"," <th>Haha</th>\n"," <th>Sad</th>\n"," <th>Angry</th>\n"," <th>Thankful</th>\n"," <th>Video Share Status</th>\n"," <th>Post Views</th>\n"," <th>Total Views</th>\n"," <th>Total Views for all Crossposts</th>\n"," <th>Video Length</th>\n"," <th>URL</th>\n"," <th>Message</th>\n"," <th>Link</th>\n"," <th>Final Link</th>\n"," <th>Link Text</th>\n"," <th>Description</th>\n"," <th>Sponsor Id</th>\n"," <th>Sponsor Name</th>\n"," <th>Overperforming Score</th>\n"," <th>FilteredMessage</th>\n"," <th>TotalReacts</th>\n"," <th>Percentage Love</th>\n"," <th>Percentage Wow</th>\n"," <th>Percentage Sad</th>\n"," <th>Percentage Angry</th>\n"," <th>Positive_Vector</th>\n"," <th>Negative_Vector</th>\n"," <th>Resultant_Vector</th>\n"," <th>label</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>0</td>\n"," <td>53</td>\n"," <td>88</td>\n"," <td>88</td>\n"," <td>Dinakshie Priyasad</td>\n"," <td>Dinakshie.live</td>\n"," <td>119840598026067</td>\n"," <td>NaN</td>\n"," <td>2011-01-01 00:35:06 IST</td>\n"," <td>Status</td>\n"," <td>31</td>\n"," <td>9</td>\n"," <td>0</td>\n"," <td>1</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>https://www.facebook.com/Dinakshie.live/posts/...</td>\n"," <td>HHHHapppppyyyyyyyyyyy.....newww yearrrrr....to...</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>-9.41</td>\n"," <td>hhhhapppppyyyyyyyyyyy.....newww yearrrrr....to...</td>\n"," <td>1</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>1</td>\n"," <td>380</td>\n"," <td>609</td>\n"," <td>609</td>\n"," <td>Dinakshie Priyasad</td>\n"," <td>Dinakshie.live</td>\n"," <td>119840598026067</td>\n"," <td>NaN</td>\n"," <td>2010-12-25 02:26:02 IST</td>\n"," <td>Status</td>\n"," <td>21</td>\n"," <td>10</td>\n"," <td>0</td>\n"," <td>1</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>https://www.facebook.com/Dinakshie.live/posts/...</td>\n"," <td>merry Christmas everyone....</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>-12.06</td>\n"," <td>merry christmas everyone....</td>\n"," <td>1</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>2</td>\n"," <td>394</td>\n"," <td>639</td>\n"," <td>639</td>\n"," <td>Y FM</td>\n"," <td>yfm927</td>\n"," <td>21693380989</td>\n"," <td>NaN</td>\n"," <td>2010-12-24 14:44:09 IST</td>\n"," <td>Status</td>\n"," <td>130</td>\n"," <td>27</td>\n"," <td>0</td>\n"," <td>1</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>https://www.facebook.com/yfm927/posts/18304848...</td>\n"," <td>Somehow, not only for Christmas, But all the l...</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>-1.47</td>\n"," <td>somehow, not only for christmas, but all the l...</td>\n"," <td>1</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>3</td>\n"," <td>532</td>\n"," <td>868</td>\n"," <td>868</td>\n"," <td>Yureni Noshika</td>\n"," <td>yureninoshika</td>\n"," <td>17873179905</td>\n"," <td>NaN</td>\n"," <td>2010-12-21 11:31:07 IST</td>\n"," <td>Status</td>\n"," <td>151</td>\n"," <td>21</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>1</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>https://www.facebook.com/yureninoshika/posts/1...</td>\n"," <td>I feel over the mooonnnn...such a dream cme tr...</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>-7.97</td>\n"," <td>i feel over the mooonnnn...such a dream cme tr...</td>\n"," <td>1</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>-1.0</td>\n"," <td>0</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>4</td>\n"," <td>768</td>\n"," <td>1293</td>\n"," <td>1293</td>\n"," <td>Sri Lanka Cricket</td>\n"," <td>srilankacricket</td>\n"," <td>142325002465382</td>\n"," <td>NaN</td>\n"," <td>2010-12-14 17:20:01 IST</td>\n"," <td>Link</td>\n"," <td>18</td>\n"," <td>5</td>\n"," <td>1</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>1</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>0</td>\n"," <td>NaN</td>\n"," <td>https://www.facebook.com/srilankacricket/posts...</td>\n"," <td>Match officials announced for ICC Cricket Worl...</td>\n"," <td>http://www.srilankacricket.lk/news/de-silva-dh...</td>\n"," <td>NaN</td>\n"," <td>De Silva & Dharmasena to officiate in the CWC ...</td>\n"," <td>Match officials announced for ICC Cricket Worl...</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>-27.04</td>\n"," <td>match officials announced for icc cricket worl...</td>\n"," <td>1</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>0.0</td>\n"," <td>1.0</td>\n"," <td>-1.0</td>\n"," <td>0</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>\n"," <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f4968c51-4cf8-4da1-b146-b4dcdf34c077')\"\n"," title=\"Convert this dataframe to an interactive table.\"\n"," style=\"display:none;\">\n"," \n"," <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n"," width=\"24px\">\n"," <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n"," <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n"," </svg>\n"," </button>\n"," \n"," <style>\n"," .colab-df-container {\n"," display:flex;\n"," flex-wrap:wrap;\n"," gap: 12px;\n"," }\n","\n"," .colab-df-convert {\n"," background-color: #E8F0FE;\n"," border: none;\n"," border-radius: 50%;\n"," cursor: pointer;\n"," display: none;\n"," fill: #1967D2;\n"," height: 32px;\n"," padding: 0 0 0 0;\n"," width: 32px;\n"," }\n","\n"," .colab-df-convert:hover {\n"," background-color: #E2EBFA;\n"," box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n"," fill: #174EA6;\n"," }\n","\n"," [theme=dark] .colab-df-convert {\n"," background-color: #3B4455;\n"," fill: #D2E3FC;\n"," }\n","\n"," [theme=dark] .colab-df-convert:hover {\n"," background-color: #434B5C;\n"," box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n"," filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n"," fill: #FFFFFF;\n"," }\n"," </style>\n","\n"," <script>\n"," const buttonEl =\n"," document.querySelector('#df-f4968c51-4cf8-4da1-b146-b4dcdf34c077 button.colab-df-convert');\n"," buttonEl.style.display =\n"," google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n"," async function convertToInteractive(key) {\n"," const element = document.querySelector('#df-f4968c51-4cf8-4da1-b146-b4dcdf34c077');\n"," const dataTable =\n"," await google.colab.kernel.invokeFunction('convertToInteractive',\n"," [key], {});\n"," if (!dataTable) return;\n","\n"," const docLinkHtml = 'Like what you see? Visit the ' +\n"," '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n"," + ' to learn more about interactive tables.';\n"," element.innerHTML = '';\n"," dataTable['output_type'] = 'display_data';\n"," await google.colab.output.renderOutput(dataTable, element);\n"," const docLink = document.createElement('div');\n"," docLink.innerHTML = docLinkHtml;\n"," element.appendChild(docLink);\n"," }\n"," </script>\n"," </div>\n"," </div>\n"," "],"text/plain":[" Unnamed: 0 Unnamed: 0.1 ... Resultant_Vector label\n","0 0 53 ... 1.0 1\n","1 1 380 ... 1.0 1\n","2 2 394 ... 1.0 1\n","3 3 532 ... -1.0 0\n","4 4 768 ... -1.0 0\n","\n","[5 rows x 43 columns]"]},"metadata":{},"execution_count":9}]},{"cell_type":"code","source":["all_data['Thankful'].sum()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zmJkeehdITzX","executionInfo":{"status":"ok","timestamp":1641535988190,"user_tz":-330,"elapsed":3,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"e454f29a-b201-479e-a938-3f724abb3337"},"execution_count":15,"outputs":[{"output_type":"execute_result","data":{"text/plain":["12933"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","source":["all_data"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":170},"id":"Jr76lr_uIX01","executionInfo":{"status":"error","timestamp":1641535448140,"user_tz":-330,"elapsed":6,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"dfe53222-453a-42a1-8a4c-d5348d33f539"},"execution_count":9,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)","\u001b[0;32m<ipython-input-9-7e769d2371b6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mall_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLikes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mTypeError\u001b[0m: 'Series' object is not callable"]}]},{"cell_type":"code","metadata":{"id":"E7_YqDlVtwQ2"},"source":["all_data =all_data.rename({'FilteredMessage': 'posts'}, axis=1)\n","all_data.drop(all_data.iloc[:,0 :33], inplace = True, axis = 1)\n","all_data.drop(all_data.iloc[:,1 :9], inplace = True, axis = 1)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"nGRZ8HrfREsw"},"source":["## Preprocessing functions"]},{"cell_type":"code","metadata":{"id":"Pdzl-_Xx-Qw7"},"source":["# Function to strip punctuation and numbers\n","def strip_punctuation(text):\n"," for char in text:\n"," if not ((3456<=ord(char)<=3583) or (97<=ord(char)<=122) or (ord(char)==32)):\n"," text = text.replace(char, ' ')\n"," \n"," text = \" \".join(text.split())\n"," return (text)\n","\n","def divide_sentences(text):\n"," tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n"," sentences = tokenizer.tokenize(text)\n"," return sentences\n","\n","def text_preprocessing(post):\n"," sentences = divide_sentences(post) \n"," messages = [strip_punctuation(sentence) for sentence in sentences]\n"," messages_splitted = []\n","\n"," for message in messages:\n"," # print (message)\n"," lines = []\n"," try:\n"," words = message.split()\n"," if words == []:\n"," continue\n"," lines += words\n"," except:\n"," continue\n"," messages_splitted.append(lines)\n","\n"," return messages_splitted"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8NaxnAbRMC3F"},"source":["sample=all_data[:400000]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"ySQusrZCl0El","executionInfo":{"status":"ok","timestamp":1633238352999,"user_tz":-330,"elapsed":419,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"7b8f2945-6e2a-4ca6-f29c-a4ffb2f809d7"},"source":["sample.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>posts</th>\n"," <th>label</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>hhhhapppppyyyyyyyyyyy.....newww yearrrrr....to...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>merry christmas everyone....</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>somehow, not only for christmas, but all the l...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>i feel over the mooonnnn...such a dream cme tr...</td>\n"," <td>0</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>match officials announced for icc cricket worl...</td>\n"," <td>0</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" posts label\n","0 hhhhapppppyyyyyyyyyyy.....newww yearrrrr....to... 1\n","1 merry christmas everyone.... 1\n","2 somehow, not only for christmas, but all the l... 1\n","3 i feel over the mooonnnn...such a dream cme tr... 0\n","4 match officials announced for icc cricket worl... 0"]},"metadata":{},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"8Sl2si23rmk4"},"source":["# sample=all_data.sample(200000)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lfMMGf9brr1l","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1630309181077,"user_tz":-330,"elapsed":36,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"c657f851-41ae-4e47-d83b-19f8b12788dc"},"source":["sample.info()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 200000 entries, 389371 to 165627\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 posts 200000 non-null object\n"," 1 label 200000 non-null int64 \n","dtypes: int64(1), object(1)\n","memory usage: 4.6+ MB\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"cg6naaRwnSEU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1633238406060,"user_tz":-330,"elapsed":38009,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"d9766c27-5a83-4add-fcdf-0e51db374808"},"source":["sample['posts'] = [text_preprocessing(post) for post in sample['posts']]"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"," \"\"\"Entry point for launching an IPython kernel.\n"]}]},{"cell_type":"code","metadata":{"id":"ybp9RIR2sAxq","colab":{"base_uri":"https://localhost:8080/","height":423},"executionInfo":{"status":"ok","timestamp":1633238959136,"user_tz":-330,"elapsed":399,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"3a2a11c8-4896-49cc-983e-ab33d55443be"},"source":["sample.head(100)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>posts</th>\n"," <th>label</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>[]</td>\n"," <td>0</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>[]</td>\n"," <td>0</td>\n"," </tr>\n"," <tr>\n"," <th>...</th>\n"," <td>...</td>\n"," <td>...</td>\n"," </tr>\n"," <tr>\n"," <th>95</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>96</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>97</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>98</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>99</th>\n"," <td>[]</td>\n"," <td>1</td>\n"," </tr>\n"," </tbody>\n","</table>\n","<p>100 rows × 2 columns</p>\n","</div>"],"text/plain":[" posts label\n","0 [] 1\n","1 [] 1\n","2 [] 1\n","3 [] 0\n","4 [] 0\n",".. ... ...\n","95 [] 1\n","96 [] 1\n","97 [] 1\n","98 [] 1\n","99 [] 1\n","\n","[100 rows x 2 columns]"]},"metadata":{},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"ibIMQO9qyZsI"},"source":["len(sample['posts'][0])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"QKnM-VPKCDHp"},"source":["sample=sample.reset_index()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"3F_5YB01CdZw","executionInfo":{"status":"ok","timestamp":1630310009315,"user_tz":-330,"elapsed":488,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"90c49edf-ddff-4ad5-f55d-f85a3c40e4ce"},"source":["d.drop('level_0', inplace=False, axis=1)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>posts</th>\n"," <th>label</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>[[ලංකාව, සමඟ, යුද, සම්බන්ධතා, ප්රවර්ධනය, අමෙරි...</td>\n"," <td>0</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>[[forever, in, debt, to, my, wonderful, family...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>[[මට, බලයක්, තියෙනව, නම්, මම, කැබිනට්, මණ්ඩලයේ...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>[[hi, guys, the, icc, women, s, world, cup, ki...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>[[y, after, school, ආනන්ද, බාලිකා, විද්යාලය]]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>...</th>\n"," <td>...</td>\n"," <td>...</td>\n"," </tr>\n"," <tr>\n"," <th>199995</th>\n"," <td>[[ඒ, ගෑනි, හොඳ, කෙනෙක්, නෙවෙයි], [හැමෝම, එක්ක,...</td>\n"," <td>0</td>\n"," </tr>\n"," <tr>\n"," <th>199996</th>\n"," <td>[[live, everyday, like, its, your, last, captu...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>199997</th>\n"," <td>[[කලින්, වැරදි, හදා, ගන්න, විල්පත්තුවේ, කපපු, ...</td>\n"," <td>0</td>\n"," </tr>\n"," <tr>\n"," <th>199998</th>\n"," <td>[[nights, hearts, are, you, also, among, them]]</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>199999</th>\n"," <td>[[රටට, ඔච්චර, හොදයි, නම්, ඇයි, හංගලා, ගිවිසුම්...</td>\n"," <td>1</td>\n"," </tr>\n"," </tbody>\n","</table>\n","<p>200000 rows × 2 columns</p>\n","</div>"],"text/plain":[" posts label\n","0 [[ලංකාව, සමඟ, යුද, සම්බන්ධතා, ප්රවර්ධනය, අමෙරි... 0\n","1 [[forever, in, debt, to, my, wonderful, family... 1\n","2 [[මට, බලයක්, තියෙනව, නම්, මම, කැබිනට්, මණ්ඩලයේ... 1\n","3 [[hi, guys, the, icc, women, s, world, cup, ki... 1\n","4 [[y, after, school, ආනන්ද, බාලිකා, විද්යාලය]] 1\n","... ... ...\n","199995 [[ඒ, ගෑනි, හොඳ, කෙනෙක්, නෙවෙයි], [හැමෝම, එක්ක,... 0\n","199996 [[live, everyday, like, its, your, last, captu... 1\n","199997 [[කලින්, වැරදි, හදා, ගන්න, විල්පත්තුවේ, කපපු, ... 0\n","199998 [[nights, hearts, are, you, also, among, them]] 1\n","199999 [[රටට, ඔච්චර, හොදයි, නම්, ඇයි, හංගලා, ගිවිසුම්... 1\n","\n","[200000 rows x 2 columns]"]},"metadata":{},"execution_count":61}]},{"cell_type":"code","metadata":{"id":"mhfLcW92r4R1"},"source":["sample.to_pickle(folder_path+ 'Word embedding/preprocessed_set.csv')"],"execution_count":null,"outputs":[]}]} |