File size: 21,288 Bytes
04dad43 | 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"preprocess.ipynb","provenance":[{"file_id":"1FmITstmFuNHHEYpdDmVbszNYZG_dzArH","timestamp":1623981135335}],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"8bxoDwcdM2IR"},"source":["# **Initialization**"]},{"cell_type":"code","metadata":{"id":"I31GH_7zy8YM"},"source":["import pandas as pd\n","import re\n","from openpyxl import load_workbook, worksheet\n","import string\n","import unicodedata\n","import numpy as np"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"pRn4xy4E_Tn6","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1633261927639,"user_tz":-330,"elapsed":2488,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"}},"outputId":"00a16c9d-3bb1-4d12-bd91-96015b7c34c1"},"source":["import nltk\n","nltk.download('punkt')\n","import nltk.data"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data] Unzipping tokenizers/punkt.zip.\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UM4arc1rzMLj","executionInfo":{"status":"ok","timestamp":1633261604261,"user_tz":-330,"elapsed":62815,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"}},"outputId":"7e6838c9-1d98-49b4-8e26-12381f800fff"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Nvs7pMoyzPLt","executionInfo":{"status":"ok","timestamp":1633261622702,"user_tz":-330,"elapsed":18454,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"}},"outputId":"46b08c60-c4cd-4ce7-d7bb-d2e83a030167"},"source":["source_path = 'drive/MyDrive/Sencat/data/normalized_cleaned_corpus.csv'\n","dest_path = 'drive/MyDrive/Sencat/data/labeled_corpus.csv'\n","\n","source_df = pd.read_csv(source_path, encoding = \"utf-8\")\n","source_df.info()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (30,31) have mixed types.Specify dtype option on import or set low_memory=False.\n"," interactivity=interactivity, compiler=compiler, result=result)\n"]},{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 542871 entries, 0 to 542870\n","Data columns (total 39 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 Unnamed: 0 542871 non-null int64 \n"," 1 Unnamed: 0.1 542871 non-null int64 \n"," 2 Unnamed: 0.1.1 542871 non-null int64 \n"," 3 Page Name 542871 non-null object \n"," 4 User Name 534191 non-null object \n"," 5 Facebook Id 542871 non-null int64 \n"," 6 Likes at Posting 491120 non-null float64\n"," 7 Created 542871 non-null object \n"," 8 Type 542871 non-null object \n"," 9 Likes 542871 non-null int64 \n"," 10 Comments 542871 non-null int64 \n"," 11 Shares 542871 non-null int64 \n"," 12 Love 542871 non-null int64 \n"," 13 Wow 542871 non-null int64 \n"," 14 Haha 542871 non-null int64 \n"," 15 Sad 542871 non-null int64 \n"," 16 Angry 542871 non-null int64 \n"," 17 Thankful 542871 non-null int64 \n"," 18 Video Share Status 141290 non-null object \n"," 19 Post Views 542871 non-null int64 \n"," 20 Total Views 542871 non-null int64 \n"," 21 Total Views for all Crossposts 542871 non-null int64 \n"," 22 Video Length 141290 non-null object \n"," 23 URL 536086 non-null object \n"," 24 Message 542871 non-null object \n"," 25 Link 528618 non-null object \n"," 26 Final Link 41282 non-null object \n"," 27 Link Text 416410 non-null object \n"," 28 Description 220599 non-null object \n"," 29 Sponsor Id 194 non-null float64\n"," 30 Sponsor Name 183 non-null object \n"," 31 Overperforming Score 542871 non-null object \n"," 32 FilteredMessage 542871 non-null object \n"," 33 TotalReacts 542871 non-null int64 \n"," 34 Percentage Love 542871 non-null float64\n"," 35 Percentage Wow 542871 non-null float64\n"," 36 Percentage Haha 542871 non-null float64\n"," 37 Percentage Sad 542871 non-null float64\n"," 38 Percentage Angry 542871 non-null float64\n","dtypes: float64(7), int64(17), object(15)\n","memory usage: 161.5+ MB\n"]}]},{"cell_type":"code","metadata":{"id":"rOaOOownOigx"},"source":["k=source_df[:150000]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rqmcyKRGPecU","executionInfo":{"status":"ok","timestamp":1633261623808,"user_tz":-330,"elapsed":1145,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"}},"outputId":"ac65e209-1da7-4a56-b205-1ddffc7d5ce0"},"source":["count=0\n","for i in range(150000):\n"," count+=k['Thankful'][i]\n","count\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["7"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"mKoLvqg0On-T","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1633261683040,"user_tz":-330,"elapsed":416,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"}},"outputId":"eed52949-3fd4-44dc-a76a-edfeee33eb8a"},"source":["k.info()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 150000 entries, 0 to 149999\n","Data columns (total 39 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 Unnamed: 0 150000 non-null int64 \n"," 1 Unnamed: 0.1 150000 non-null int64 \n"," 2 Unnamed: 0.1.1 150000 non-null int64 \n"," 3 Page Name 150000 non-null object \n"," 4 User Name 147141 non-null object \n"," 5 Facebook Id 150000 non-null int64 \n"," 6 Likes at Posting 133749 non-null float64\n"," 7 Created 150000 non-null object \n"," 8 Type 150000 non-null object \n"," 9 Likes 150000 non-null int64 \n"," 10 Comments 150000 non-null int64 \n"," 11 Shares 150000 non-null int64 \n"," 12 Love 150000 non-null int64 \n"," 13 Wow 150000 non-null int64 \n"," 14 Haha 150000 non-null int64 \n"," 15 Sad 150000 non-null int64 \n"," 16 Angry 150000 non-null int64 \n"," 17 Thankful 150000 non-null int64 \n"," 18 Video Share Status 48161 non-null object \n"," 19 Post Views 150000 non-null int64 \n"," 20 Total Views 150000 non-null int64 \n"," 21 Total Views for all Crossposts 150000 non-null int64 \n"," 22 Video Length 48161 non-null object \n"," 23 URL 148091 non-null object \n"," 24 Message 150000 non-null object \n"," 25 Link 145292 non-null object \n"," 26 Final Link 4275 non-null object \n"," 27 Link Text 112452 non-null object \n"," 28 Description 56456 non-null object \n"," 29 Sponsor Id 28 non-null float64\n"," 30 Sponsor Name 26 non-null object \n"," 31 Overperforming Score 150000 non-null object \n"," 32 FilteredMessage 150000 non-null object \n"," 33 TotalReacts 150000 non-null int64 \n"," 34 Percentage Love 150000 non-null float64\n"," 35 Percentage Wow 150000 non-null float64\n"," 36 Percentage Haha 150000 non-null float64\n"," 37 Percentage Sad 150000 non-null float64\n"," 38 Percentage Angry 150000 non-null float64\n","dtypes: float64(7), int64(17), object(15)\n","memory usage: 44.6+ MB\n"]}]},{"cell_type":"markdown","metadata":{"id":"XWfqDF6DdPfy"},"source":["# **Generating Sentiment Vector Values**"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ow9VM8kh-_6y","executionInfo":{"status":"ok","timestamp":1633261971306,"user_tz":-330,"elapsed":12875,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"}},"outputId":"d807e804-f26f-4605-f940-f71c6daec4d1"},"source":["tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n","SEN_COUNT = 0\n","for post in k['Message']:\n"," SEN_COUNT += len(tokenizer.tokenize(post))\n","print (SEN_COUNT)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["235660\n"]}]},{"cell_type":"code","metadata":{"id":"xbWDx5mJRQPN"},"source":["# Function to strip unwanted characters\n","def strip_text(text):\n"," for char in text:\n"," if not ((3456<=ord(char)<=3583) or (ord(char)==32)):\n"," text = text.replace(char, ' ')\n"," \n"," text = \" \".join(text.split())\n"," return (text)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"sLrhQIW05jYE","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1631760407816,"user_tz":-330,"elapsed":10,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"4f494ad2-ae89-4b7b-a45a-643fac9343f7"},"source":["import os, sys\n","from google.colab import drive\n","drive.mount('/content/drive')\n","nb_path = '/content/notebookss'\n","#os.symlink('/content/drive/My Drive/Colab Notebooks', nb_path)\n","#sys.path.insert(0,nb_path)\n","\n","#!pip install --target=$nb_path sinling"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"code","metadata":{"id":"N_WsFxNzzYHI"},"source":["def create_vectors(source_df, text_column):\n"," source_df[text_column] = [strip_text(text) for text in source_df[text_column]]\n"," #source_df['Words'] = [frozenset(text.strip().split()) for text in source_df[text_column]]\n"," source_df['Positive_Vector'] = source_df['Percentage Love'] + source_df['Percentage Wow'] #+ source_df['Percentage Haha'] \n"," source_df['Negative_Vector'] = source_df['Percentage Sad'] + source_df['Percentage Angry']\n"," source_df['Resultant_Vector'] = source_df['Positive_Vector'] - source_df['Negative_Vector']\n","\n"," # source_df['Positive'] = source_df['Love'] + source_df['Haha'] #+ source_df['Percentage Wow'] #\n"," # source_df['Negative'] = source_df['Sad'] + source_df['Angry']\n","\n"," # min = source_df['Resultant_Vector'].min()\n"," # max = source_df['Resultant_Vector'].max()\n"," # source_df['Resultant_Vector'] = (source_df['Resultant_Vector'] - source_df['Resultant_Vector'].min()) / (source_df['Resultant_Vector'].max() - source_df['Resultant_Vector'].min())\n"," # source_df['Star_Rating'] = (source_df['Resultant_Vector'])*4 + 1\n"," \n"," source_df = source_df.dropna(axis=0, subset=[text_column])\n"," source_df = source_df[source_df.FilteredMessage != '']\n","\n"," #, row['Resultant_Vector'], row['Star_Rating']\n"," #source_df['Dict'] = [dict.fromkeys(row['Words'], [row['Resultant_Vector']]) for index, row in source_df.iterrows()]\n"," \n"," return source_df"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Md2ub4-D9UUZ"},"source":["source_df = create_vectors(source_df, 'FilteredMessage')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"rwYE-NevNqrq"},"source":["# **Creating Labels**"]},{"cell_type":"code","metadata":{"id":"-C8E2VQjLdPD"},"source":["def create_labels(source_df):\n"," # If resultant vector < 0, label is 1. Else label is 2\n"," source_df['label'] = (np.where(source_df['Resultant_Vector'] < 0, 1, 2))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"VG1QvmeTReAN"},"source":["create_labels(source_df)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"aiaD3xrJfBho"},"source":["# **Saving Dataframe to File**"]},{"cell_type":"code","metadata":{"id":"XqFxy7s51l53"},"source":["source_df =source_df.rename({'FilteredMessage': 'comment'}, axis=1)\n","source_df.drop(source_df.iloc[:,0 :32], inplace = True, axis = 1)\n","source_df.drop(source_df.iloc[:,1 :10], inplace = True, axis = 1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"HtF5bJnf1n3F","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1631760425383,"user_tz":-330,"elapsed":22,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"391b55a8-dd82-4e00-9c2e-5c7b4d993da1"},"source":["source_df.info()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 381891 entries, 27 to 542870\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 comment 381891 non-null object\n"," 1 label 381891 non-null int64 \n","dtypes: int64(1), object(1)\n","memory usage: 8.7+ MB\n"]}]},{"cell_type":"code","metadata":{"id":"O6nQ5-a1RWV-","colab":{"base_uri":"https://localhost:8080/","height":202},"executionInfo":{"status":"ok","timestamp":1631760426504,"user_tz":-330,"elapsed":1139,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"f5468e55-343b-4c76-dc41-40a5cce42e1a"},"source":["source_df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>comment</th>\n"," <th>label</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>27</th>\n"," <td>විශ්වාසනීය වෙනසක්</td>\n"," <td>2</td>\n"," </tr>\n"," <tr>\n"," <th>33</th>\n"," <td>අපේ යාලුවො හැමෝටම උදාවුන මේ නත්තල සුභ නත්තලක් ...</td>\n"," <td>2</td>\n"," </tr>\n"," <tr>\n"," <th>34</th>\n"," <td>දෙව්රේ වෙලීලා සේනක පෙරේරා</td>\n"," <td>2</td>\n"," </tr>\n"," <tr>\n"," <th>37</th>\n"," <td>මා මුව අගින් නාමල් උඩුගම</td>\n"," <td>2</td>\n"," </tr>\n"," <tr>\n"," <th>38</th>\n"," <td>මේ ඇස් දිහා බලන් ශිහාන් මිහිරංග</td>\n"," <td>2</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" comment label\n","27 විශ්වාසනීය වෙනසක් 2\n","33 අපේ යාලුවො හැමෝටම උදාවුන මේ නත්තල සුභ නත්තලක් ... 2\n","34 දෙව්රේ වෙලීලා සේනක පෙරේරා 2\n","37 මා මුව අගින් නාමල් උඩුගම 2\n","38 මේ ඇස් දිහා බලන් ශිහාන් මිහිරංග 2"]},"metadata":{},"execution_count":27}]},{"cell_type":"code","metadata":{"id":"SSf6lahTRqIw","colab":{"base_uri":"https://localhost:8080/","height":202},"executionInfo":{"status":"ok","timestamp":1631760426505,"user_tz":-330,"elapsed":11,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"8a5df760-c914-4dc2-e51c-434f8374f8c1"},"source":["source_df.tail()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>comment</th>\n"," <th>label</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>542865</th>\n"," <td>මාධ්ය අධ්යක්ෂ වරයා ජ්යෙෂ්ඨ මාධ්යවේදී ධර්මශ්රි ...</td>\n"," <td>2</td>\n"," </tr>\n"," <tr>\n"," <th>542867</th>\n"," <td>එක්දහස් නවසිය හතළිහ සහ එක්දහස් නවසිය හතළිස් පහ...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>542868</th>\n"," <td>මාත් දේශපාලඥයෙක් උනත් ඇත්ත කියන්න එපැයි තක්කඩි...</td>\n"," <td>2</td>\n"," </tr>\n"," <tr>\n"," <th>542869</th>\n"," <td>වැල්ලවතේ නිවාස සංකීර්ණයකින් බිමට වැටී දැරියක ම...</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>542870</th>\n"," <td>රාජපක්ෂට සහය දීමට ගත් තීරණය අද තමන් කනගාටු වන ...</td>\n"," <td>2</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" comment label\n","542865 මාධ්ය අධ්යක්ෂ වරයා ජ්යෙෂ්ඨ මාධ්යවේදී ධර්මශ්රි ... 2\n","542867 එක්දහස් නවසිය හතළිහ සහ එක්දහස් නවසිය හතළිස් පහ... 1\n","542868 මාත් දේශපාලඥයෙක් උනත් ඇත්ත කියන්න එපැයි තක්කඩි... 2\n","542869 වැල්ලවතේ නිවාස සංකීර්ණයකින් බිමට වැටී දැරියක ම... 1\n","542870 රාජපක්ෂට සහය දීමට ගත් තීරණය අද තමන් කනගාටු වන ... 2"]},"metadata":{},"execution_count":28}]},{"cell_type":"code","metadata":{"id":"wDm6OMFRxrvt"},"source":["source_df.to_csv(dest_path)"],"execution_count":null,"outputs":[]}]} |