{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"preprocess.ipynb","provenance":[{"file_id":"1Wd79wkxXF6CyKVFhYCid1jpIsv0HWRH-","timestamp":1625292490447},{"file_id":"1liafFKE2IZeDAgr_AibZfKykYLvHVzD3","timestamp":1624675505095},{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/cnn_rnn_multiclass.ipynb","timestamp":1621843389117},{"file_id":"153374pBL1OZuAJMWKK_ae0D9_DUq4P0P","timestamp":1590420423516}],"collapsed_sections":["0BYvE-D5QAc0","xXiAlHRZ7X_p","nGRZ8HrfREsw","jbpLZfOPGGKN","B6CXqtIDR4jS","d46ja9NetIpP","_VdzOT3zsB9m","qLkfB_XXPS3O"]},"environment":{"name":"tf2-2-2-gpu.2-2.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf2-2-2-gpu.2-2:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"cells":[{"cell_type":"markdown","metadata":{"id":"p7y2rE1vNZWc"},"source":["# Folder Paths"]},{"cell_type":"code","metadata":{"id":"IDq--mTz9k24","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535703014,"user_tz":-330,"elapsed":3385,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"98086339-ca77-445d-c28f-18fa4898e03b"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"code","metadata":{"id":"gWPD3dgEqB5d","executionInfo":{"status":"ok","timestamp":1641535703015,"user_tz":-330,"elapsed":9,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}}},"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/'"],"execution_count":3,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0BYvE-D5QAc0"},"source":["# Dependencies"]},{"cell_type":"code","metadata":{"id":"Cf2gZocJff8x","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535712544,"user_tz":-330,"elapsed":9536,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"53ee4184-ca35-4090-a6b5-7f73beccc637"},"source":["!pip uninstall keras-nightly\n","!pip install tensorflow==1.14.0\n","!pip install q keras==2.3.1\n","!pip install 'h5py<3.0.0'"],"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[33mWARNING: Skipping keras-nightly as it is not installed.\u001b[0m\n","Requirement already satisfied: tensorflow==1.14.0 in /usr/local/lib/python3.7/dist-packages (1.14.0)\n","Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.12.0)\n","Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.15.0)\n","Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.2)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.1.0)\n","Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.2.0)\n","Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.14.0)\n","Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.14.0)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.37.0)\n","Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.19.5)\n","Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.13.3)\n","Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.42.0)\n","Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.4.0)\n","Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (0.8.1)\n","Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (1.0.8)\n","Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.14.0) (3.17.3)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras-applications>=1.0.6->tensorflow==1.14.0) (2.10.0)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (1.0.1)\n","Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (57.4.0)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.3.6)\n","Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (4.8.2)\n","Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.10.0.2)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.6.0)\n","Requirement already satisfied: q in /usr/local/lib/python3.7/dist-packages (2.6)\n","Requirement already satisfied: keras==2.3.1 in /usr/local/lib/python3.7/dist-packages (2.3.1)\n","Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.4.1)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (2.10.0)\n","Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.15.0)\n","Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.19.5)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (3.13)\n","Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.0.8)\n","Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from keras==2.3.1) (1.1.2)\n","Requirement already satisfied: h5py<3.0.0 in /usr/local/lib/python3.7/dist-packages (2.10.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from h5py<3.0.0) (1.15.0)\n","Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from h5py<3.0.0) (1.19.5)\n"]}]},{"cell_type":"code","metadata":{"id":"gZ9YDtD6qn4t","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535714481,"user_tz":-330,"elapsed":1950,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"c5739ca6-e627-4ae8-aac2-c3a0fb7c3b21"},"source":["from __future__ import print_function\n","\n","import collections\n","import pickle\n","import re\n","import random\n","import sys\n","import os \n","import time\n","\n","import pandas as pd\n","import numpy as np\n","from numpy import array\n","from numpy import asarray\n","from numpy import zeros\n","from numpy import cumsum\n","\n","import matplotlib.image as mpimg\n","import matplotlib.pyplot as plt\n","\n","import torch\n","import torch.nn as nn\n","from torch import optim\n","import torch.nn.functional as F\n","torch.set_printoptions(precision=8)\n","\n","import nltk\n","nltk.download('punkt')\n","import nltk.data\n"],"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data] Package punkt is already up-to-date!\n"]}]},{"cell_type":"markdown","metadata":{"id":"xXiAlHRZ7X_p"},"source":["# Load Data"]},{"cell_type":"code","metadata":{"id":"GqqHP9rwpbdt","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1641535723488,"user_tz":-330,"elapsed":9012,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"878eaccb-57b9-4b37-95eb-20ec96d0cee3"},"source":["facebook_data_path = folder_path + 'Word embedding/labeled_corpus.csv'\n","all_data = pd.read_csv(facebook_data_path)"],"execution_count":6,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (31,32) have mixed types.Specify dtype option on import or set low_memory=False.\n"," interactivity=interactivity, compiler=compiler, result=result)\n"]}]},{"cell_type":"code","source":["all_data.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":617},"id":"Et0w2UcXKViX","executionInfo":{"status":"ok","timestamp":1641535863777,"user_tz":-330,"elapsed":497,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"909f44ab-42b7-4066-a47e-ef1c30d22ad0"},"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Unnamed: 0Unnamed: 0.1Unnamed: 0.1.1Unnamed: 0.1.1.1Page NameUser NameFacebook IdLikes at PostingCreatedTypeLikesCommentsSharesLoveWowHahaSadAngryThankfulVideo Share StatusPost ViewsTotal ViewsTotal Views for all CrosspostsVideo LengthURLMessageLinkFinal LinkLink TextDescriptionSponsor IdSponsor NameOverperforming ScoreFilteredMessageTotalReactsPercentage LovePercentage WowPercentage SadPercentage AngryPositive_VectorNegative_VectorResultant_Vectorlabel
00538888Dinakshie PriyasadDinakshie.live119840598026067NaN2011-01-01 00:35:06 ISTStatus3190100000NaN000NaNhttps://www.facebook.com/Dinakshie.live/posts/...HHHHapppppyyyyyyyyyyy.....newww yearrrrr....to...NaNNaNNaNNaNNaNNaN-9.41hhhhapppppyyyyyyyyyyy.....newww yearrrrr....to...11.00.00.00.01.00.01.01
11380609609Dinakshie PriyasadDinakshie.live119840598026067NaN2010-12-25 02:26:02 ISTStatus21100100000NaN000NaNhttps://www.facebook.com/Dinakshie.live/posts/...merry Christmas everyone....NaNNaNNaNNaNNaNNaN-12.06merry christmas everyone....11.00.00.00.01.00.01.01
22394639639Y FMyfm92721693380989NaN2010-12-24 14:44:09 ISTStatus130270100000NaN000NaNhttps://www.facebook.com/yfm927/posts/18304848...Somehow, not only for Christmas, But all the l...NaNNaNNaNNaNNaNNaN-1.47somehow, not only for christmas, but all the l...11.00.00.00.01.00.01.01
33532868868Yureni Noshikayureninoshika17873179905NaN2010-12-21 11:31:07 ISTStatus151210000010NaN000NaNhttps://www.facebook.com/yureninoshika/posts/1...I feel over the mooonnnn...such a dream cme tr...NaNNaNNaNNaNNaNNaN-7.97i feel over the mooonnnn...such a dream cme tr...10.00.00.01.00.01.0-1.00
4476812931293Sri Lanka Cricketsrilankacricket142325002465382NaN2010-12-14 17:20:01 ISTLink1851000010NaN000NaNhttps://www.facebook.com/srilankacricket/posts...Match officials announced for ICC Cricket Worl...http://www.srilankacricket.lk/news/de-silva-dh...NaNDe Silva & Dharmasena to officiate in the CWC ...Match officials announced for ICC Cricket Worl...NaNNaN-27.04match officials announced for icc cricket worl...10.00.00.01.00.01.0-1.00
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "],"text/plain":[" Unnamed: 0 Unnamed: 0.1 ... Resultant_Vector label\n","0 0 53 ... 1.0 1\n","1 1 380 ... 1.0 1\n","2 2 394 ... 1.0 1\n","3 3 532 ... -1.0 0\n","4 4 768 ... -1.0 0\n","\n","[5 rows x 43 columns]"]},"metadata":{},"execution_count":9}]},{"cell_type":"code","source":["all_data['Thankful'].sum()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zmJkeehdITzX","executionInfo":{"status":"ok","timestamp":1641535988190,"user_tz":-330,"elapsed":3,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"e454f29a-b201-479e-a938-3f724abb3337"},"execution_count":15,"outputs":[{"output_type":"execute_result","data":{"text/plain":["12933"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","source":["all_data"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":170},"id":"Jr76lr_uIX01","executionInfo":{"status":"error","timestamp":1641535448140,"user_tz":-330,"elapsed":6,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"dfe53222-453a-42a1-8a4c-d5348d33f539"},"execution_count":9,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mall_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLikes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mTypeError\u001b[0m: 'Series' object is not callable"]}]},{"cell_type":"code","metadata":{"id":"E7_YqDlVtwQ2"},"source":["all_data =all_data.rename({'FilteredMessage': 'posts'}, axis=1)\n","all_data.drop(all_data.iloc[:,0 :33], inplace = True, axis = 1)\n","all_data.drop(all_data.iloc[:,1 :9], inplace = True, axis = 1)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"nGRZ8HrfREsw"},"source":["## Preprocessing functions"]},{"cell_type":"code","metadata":{"id":"Pdzl-_Xx-Qw7"},"source":["# Function to strip punctuation and numbers\n","def strip_punctuation(text):\n"," for char in text:\n"," if not ((3456<=ord(char)<=3583) or (97<=ord(char)<=122) or (ord(char)==32)):\n"," text = text.replace(char, ' ')\n"," \n"," text = \" \".join(text.split())\n"," return (text)\n","\n","def divide_sentences(text):\n"," tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n"," sentences = tokenizer.tokenize(text)\n"," return sentences\n","\n","def text_preprocessing(post):\n"," sentences = divide_sentences(post) \n"," messages = [strip_punctuation(sentence) for sentence in sentences]\n"," messages_splitted = []\n","\n"," for message in messages:\n"," # print (message)\n"," lines = []\n"," try:\n"," words = message.split()\n"," if words == []:\n"," continue\n"," lines += words\n"," except:\n"," continue\n"," messages_splitted.append(lines)\n","\n"," return messages_splitted"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8NaxnAbRMC3F"},"source":["sample=all_data[:400000]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"ySQusrZCl0El","executionInfo":{"status":"ok","timestamp":1633238352999,"user_tz":-330,"elapsed":419,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"7b8f2945-6e2a-4ca6-f29c-a4ffb2f809d7"},"source":["sample.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
postslabel
0hhhhapppppyyyyyyyyyyy.....newww yearrrrr....to...1
1merry christmas everyone....1
2somehow, not only for christmas, but all the l...1
3i feel over the mooonnnn...such a dream cme tr...0
4match officials announced for icc cricket worl...0
\n","
"],"text/plain":[" posts label\n","0 hhhhapppppyyyyyyyyyyy.....newww yearrrrr....to... 1\n","1 merry christmas everyone.... 1\n","2 somehow, not only for christmas, but all the l... 1\n","3 i feel over the mooonnnn...such a dream cme tr... 0\n","4 match officials announced for icc cricket worl... 0"]},"metadata":{},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"8Sl2si23rmk4"},"source":["# sample=all_data.sample(200000)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lfMMGf9brr1l","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1630309181077,"user_tz":-330,"elapsed":36,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"c657f851-41ae-4e47-d83b-19f8b12788dc"},"source":["sample.info()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["\n","Int64Index: 200000 entries, 389371 to 165627\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 posts 200000 non-null object\n"," 1 label 200000 non-null int64 \n","dtypes: int64(1), object(1)\n","memory usage: 4.6+ MB\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"cg6naaRwnSEU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1633238406060,"user_tz":-330,"elapsed":38009,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"d9766c27-5a83-4add-fcdf-0e51db374808"},"source":["sample['posts'] = [text_preprocessing(post) for post in sample['posts']]"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"," \"\"\"Entry point for launching an IPython kernel.\n"]}]},{"cell_type":"code","metadata":{"id":"ybp9RIR2sAxq","colab":{"base_uri":"https://localhost:8080/","height":423},"executionInfo":{"status":"ok","timestamp":1633238959136,"user_tz":-330,"elapsed":399,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"3a2a11c8-4896-49cc-983e-ab33d55443be"},"source":["sample.head(100)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
postslabel
0[]1
1[]1
2[]1
3[]0
4[]0
.........
95[]1
96[]1
97[]1
98[]1
99[]1
\n","

100 rows × 2 columns

\n","
"],"text/plain":[" posts label\n","0 [] 1\n","1 [] 1\n","2 [] 1\n","3 [] 0\n","4 [] 0\n",".. ... ...\n","95 [] 1\n","96 [] 1\n","97 [] 1\n","98 [] 1\n","99 [] 1\n","\n","[100 rows x 2 columns]"]},"metadata":{},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"ibIMQO9qyZsI"},"source":["len(sample['posts'][0])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"QKnM-VPKCDHp"},"source":["sample=sample.reset_index()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"3F_5YB01CdZw","executionInfo":{"status":"ok","timestamp":1630310009315,"user_tz":-330,"elapsed":488,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"90c49edf-ddff-4ad5-f55d-f85a3c40e4ce"},"source":["d.drop('level_0', inplace=False, axis=1)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
postslabel
0[[ලංකාව, සමඟ, යුද, සම්බන්ධතා, ප්රවර්ධනය, අමෙරි...0
1[[forever, in, debt, to, my, wonderful, family...1
2[[මට, බලයක්, තියෙනව, නම්, මම, කැබිනට්, මණ්ඩලයේ...1
3[[hi, guys, the, icc, women, s, world, cup, ki...1
4[[y, after, school, ආනන්ද, බාලිකා, විද්යාලය]]1
.........
199995[[ඒ, ගෑනි, හොඳ, කෙනෙක්, නෙවෙයි], [හැමෝම, එක්ක,...0
199996[[live, everyday, like, its, your, last, captu...1
199997[[කලින්, වැරදි, හදා, ගන්න, විල්පත්තුවේ, කපපු, ...0
199998[[nights, hearts, are, you, also, among, them]]1
199999[[රටට, ඔච්චර, හොදයි, නම්, ඇයි, හංගලා, ගිවිසුම්...1
\n","

200000 rows × 2 columns

\n","
"],"text/plain":[" posts label\n","0 [[ලංකාව, සමඟ, යුද, සම්බන්ධතා, ප්රවර්ධනය, අමෙරි... 0\n","1 [[forever, in, debt, to, my, wonderful, family... 1\n","2 [[මට, බලයක්, තියෙනව, නම්, මම, කැබිනට්, මණ්ඩලයේ... 1\n","3 [[hi, guys, the, icc, women, s, world, cup, ki... 1\n","4 [[y, after, school, ආනන්ද, බාලිකා, විද්යාලය]] 1\n","... ... ...\n","199995 [[ඒ, ගෑනි, හොඳ, කෙනෙක්, නෙවෙයි], [හැමෝම, එක්ක,... 0\n","199996 [[live, everyday, like, its, your, last, captu... 1\n","199997 [[කලින්, වැරදි, හදා, ගන්න, විල්පත්තුවේ, කපපු, ... 0\n","199998 [[nights, hearts, are, you, also, among, them]] 1\n","199999 [[රටට, ඔච්චර, හොදයි, නම්, ඇයි, හංගලා, ගිවිසුම්... 1\n","\n","[200000 rows x 2 columns]"]},"metadata":{},"execution_count":61}]},{"cell_type":"code","metadata":{"id":"mhfLcW92r4R1"},"source":["sample.to_pickle(folder_path+ 'Word embedding/preprocessed_set.csv')"],"execution_count":null,"outputs":[]}]}