File size: 14,274 Bytes
508de5b | 1 | {"cells":[{"cell_type":"markdown","metadata":{"id":"3zR8-aK009p9"},"source":["# User Parameters"]},{"cell_type":"code","execution_count":267,"metadata":{"executionInfo":{"elapsed":558,"status":"ok","timestamp":1640535427889,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"dbSiWEihH0zu"},"outputs":[],"source":["i = \"5\" #@param [1,2,3,4,5]\n","embedding_type = \"word2vec\" #@param [\"fastText\",\"word2vec\"]\n","model_type = \"AVG\" #@param [\"MAX\",\"MIN\", \"AVG\" ]\n","EMBEDDING_SIZE = \"200\" #@param [\"200\", \"300\" ]\n","\n","\n","i=int(i)\n","EMBEDDING_SIZE = int(EMBEDDING_SIZE)\n"]},{"cell_type":"markdown","metadata":{"id":"p7y2rE1vNZWc"},"source":["# Folder Paths"]},{"cell_type":"code","execution_count":268,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3200,"status":"ok","timestamp":1640535431836,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"IDq--mTz9k24","outputId":"6c34ec46-23ed-46d5-951e-d414697b4077"},"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":269,"metadata":{"executionInfo":{"elapsed":13,"status":"ok","timestamp":1640535431837,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"gWPD3dgEqB5d"},"outputs":[],"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/'\n","\n","embedding_path= (folder_path + 'Word embedding/embedded posts/'+embedding_type+'/'+model_type+'_'+str(i)+'.csv')\n","word_embedding_path= (folder_path + 'Word embedding/'+embedding_type+'/'+embedding_type+str(EMBEDDING_SIZE))"]},{"cell_type":"code","execution_count":270,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"elapsed":13,"status":"ok","timestamp":1640535431839,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"WbxGy6fAHT-Q","outputId":"897430ae-c20f-4f54-df91-8287fb905aec"},"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/Word embedding/embedded posts/word2vec/AVG_5.csv'"]},"metadata":{},"execution_count":270}],"source":["embedding_path"]},{"cell_type":"markdown","metadata":{"id":"0BYvE-D5QAc0"},"source":["# Dependencies"]},{"cell_type":"code","execution_count":271,"metadata":{"executionInfo":{"elapsed":11,"status":"ok","timestamp":1640535431840,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"gZ9YDtD6qn4t"},"outputs":[],"source":["from __future__ import print_function\n","\n","import collections\n","import pickle\n","\n","import gensim\n","from gensim.models.fasttext import FastText\n","from gensim.models import word2vec\n","\n","import pandas as pd\n","import numpy as np\n","from numpy import array\n","from numpy import asarray\n","from numpy import zeros\n","from numpy import cumsum\n","\n","import tensorflow as tf\n","import keras"]},{"cell_type":"markdown","metadata":{"id":"xXiAlHRZ7X_p"},"source":["# Load Data"]},{"cell_type":"code","execution_count":272,"metadata":{"executionInfo":{"elapsed":2736,"status":"ok","timestamp":1640535434566,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"GqqHP9rwpbdt"},"outputs":[],"source":["facebook_data_path = folder_path + 'Word embedding/preprocessed_set.csv'\n","all_data = pd.read_pickle(facebook_data_path)"]},{"cell_type":"code","execution_count":273,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":9,"status":"ok","timestamp":1640535434567,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"2z3xR2c2aAaC","outputId":"5eb47c40-6578-480c-9a99-84fbcd9d7b82"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["4"]},"metadata":{},"execution_count":273}],"source":["len(all_data['posts'][3])"]},{"cell_type":"code","execution_count":274,"metadata":{"executionInfo":{"elapsed":606,"status":"ok","timestamp":1640535435168,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"UkilaQMGa9jW"},"outputs":[],"source":["all_data=all_data.drop('level_0',axis=1)"]},{"cell_type":"markdown","metadata":{"id":"Shq4w6GcqbIq"},"source":["# Create Input"]},{"cell_type":"markdown","metadata":{"id":"tvUoARYaRM2c"},"source":["## Embedding functions"]},{"cell_type":"code","execution_count":275,"metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1640535435169,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"2W3owALoR96d"},"outputs":[],"source":["window_size = 40\n","min_word = 5\n","down_sampling = 1e-2"]},{"cell_type":"code","execution_count":276,"metadata":{"executionInfo":{"elapsed":556,"status":"ok","timestamp":1640535435721,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"Di94yXN5SALw"},"outputs":[],"source":["from gensim.models import FastText,Word2Vec\n","if (embedding_type=='fastText'):\n"," embedding_model = FastText(size=EMBEDDING_SIZE,\n"," window=window_size,\n"," min_count=min_word,\n"," sample=down_sampling,\n"," sg=1)\n"," embedding_model = FastText.load(word_embedding_path)\n","else:\n"," embedding_model = Word2Vec(size=EMBEDDING_SIZE,\n"," window=window_size,\n"," min_count=min_word,\n"," sample=down_sampling,\n"," sg=1)\n"," embedding_model = Word2Vec.load(word_embedding_path)"]},{"cell_type":"code","execution_count":277,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1640535435721,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"a4fGMBAbSN1x"},"outputs":[],"source":["def max_pool(sentence):\n"," sentence_embedding = []\n"," for i in range (EMBEDDING_SIZE):\n"," sentence_embedding.append(max(word[i] for word in sentence))\n"," return sentence_embedding"]},{"cell_type":"code","execution_count":278,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1640535435722,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"BOO2pt5U8qeZ"},"outputs":[],"source":["def min_pool(sentence):\n"," sentence_embedding = []\n"," for i in range (EMBEDDING_SIZE):\n"," sentence_embedding.append(min(word[i] for word in sentence))\n"," return sentence_embedding"]},{"cell_type":"code","execution_count":279,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1640535435722,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"hxdiRyjv8qQN"},"outputs":[],"source":["def avg(lst, len_lst):\n"," return sum(lst) / len_lst\n","\n","def avg_pool(sentence):\n"," sentence_embedding = []\n"," for i in range (EMBEDDING_SIZE):\n"," sentence_embedding.append(avg((word[i] for word in sentence), len(sentence)))\n"," return sentence_embedding"]},{"cell_type":"code","execution_count":280,"metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1640535435723,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"eLYOoiKqA3aI"},"outputs":[],"source":["def sentence_embedding(sentence, model_type):\n"," line = []\n"," for i in range(len(sentence)):\n"," try:\n"," line.append(embedding_model.wv[sentence[i]])\n"," except KeyError:\n"," continue\n"," sentence = line\n"," # print (model_type == 'MAX', len(sentence))\n"," if (len(sentence)==0):\n"," return None\n"," if model_type == 'MAX':\n"," # print ('x')\n"," sentence_embedding = max_pool(sentence)\n"," elif model_type == 'MIN':\n"," sentence_embedding = min_pool(sentence)\n"," else:\n"," sentence_embedding = avg_pool(sentence)\n"," # print (sentence_embedding)\n","\n"," return sentence_embedding"]},{"cell_type":"code","execution_count":281,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1640535435723,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"6FW0tXi2t90O"},"outputs":[],"source":["ZEROS = np.zeros(EMBEDDING_SIZE)\n","SoP = ZEROS - 1\n","EoP = SoP + 2"]},{"cell_type":"code","execution_count":282,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1640535435723,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"WthWhf3-Aa1a"},"outputs":[],"source":["def get_embeddings(sentences, model_type):\n"," embeddings=[SoP]\n"," for i in range (len(sentences)):\n"," sen_emb=sentence_embedding(sentences[i], model_type)\n"," if (sen_emb!=None):\n"," embeddings.append(np.array(sen_emb))\n"," if (len(embeddings)==1):\n"," return None\n"," else:\n"," embeddings.append(EoP)\n"," if len(embeddings) < 28:\n"," for x in range(28 - len(embeddings)):\n"," embeddings.append(ZEROS)\n"," return np.array(embeddings)"]},{"cell_type":"markdown","metadata":{"id":"0cvorEy_tc5G"},"source":["### Input embedding"]},{"cell_type":"code","execution_count":283,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":90310,"status":"ok","timestamp":1640535526026,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"tX3FTQZnTOeH","outputId":"e1fe99eb-4167-4450-dcec-ea6935874658"},"outputs":[{"output_type":"stream","name":"stdout","text":["16\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"," after removing the cwd from sys.path.\n","/usr/local/lib/python3.7/dist-packages/pandas/core/internals/blocks.py:849: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n"," arr_value = np.array(value)\n"]},{"output_type":"stream","name":"stdout","text":["17\n","18\n","19\n"]}],"source":["for j in range (4*(i-1),4*i):\n"," print(j)\n"," m = [get_embeddings(post, model_type) for post in all_data['posts'][(j)*10000:(j+1)*10000]]\n"," all_data['posts'][j*10000:(j+1)*10000] = m"]},{"cell_type":"code","execution_count":284,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":16,"status":"ok","timestamp":1640535526026,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"HtGlmMiqbk5e","outputId":"baf71a2d-d5d2-4eba-a9a6-300918752ef8"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["['ලංකාව',\n"," 'සමඟ',\n"," 'යුද',\n"," 'සම්බන්ධතා',\n"," 'ප්රවර්ධනය',\n"," 'අමෙරිකාව',\n"," 'දෙපාරක්',\n"," 'හිතයි']"]},"metadata":{},"execution_count":284}],"source":["all_data['posts'][0][0]"]},{"cell_type":"code","execution_count":285,"metadata":{"executionInfo":{"elapsed":11766,"status":"ok","timestamp":1640535537788,"user":{"displayName":"Vihanga Jayawickrama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11172562172528032273"},"user_tz":-330},"id":"8hqI-LX--Eia"},"outputs":[],"source":["all_data[(i-1)*40000:(i)*40000].to_pickle(embedding_path)"]}],"metadata":{"colab":{"collapsed_sections":[],"name":"post pool embedding.ipynb","provenance":[{"file_id":"1Wd79wkxXF6CyKVFhYCid1jpIsv0HWRH-","timestamp":1625292490447},{"file_id":"1liafFKE2IZeDAgr_AibZfKykYLvHVzD3","timestamp":1624675505095},{"file_id":"https://github.com/GihanAyesh/sinhala_sentiment_anlaysis_tallip/blob/master/cnn_rnn_multiclass.ipynb","timestamp":1621843389117},{"file_id":"153374pBL1OZuAJMWKK_ae0D9_DUq4P0P","timestamp":1590420423516}]},"environment":{"name":"tf2-2-2-gpu.2-2.m50","type":"gcloud","uri":"gcr.io/deeplearning-platform-release/tf2-2-2-gpu.2-2:m50"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"nbformat":4,"nbformat_minor":0} |