Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+doc2vec_model_opinion_corpus[[:space:]](1).d2v filter=lfs diff=lfs merge=lfs -text
+weights.best.from_scratch1[[:space:]](1).hdf5 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,6 @@
 ---
-title: Deceptive Rev
-emoji: 👁
-colorFrom: gray
-colorTo: pink
-sdk: gradio
-sdk_version: 4.16.0
+title: deceptive-rev
 app_file: app.py
-pinned: false
+sdk: gradio
+sdk_version: 3.44.4
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,263 @@
+from flask import Flask, redirect, render_template, request, jsonify
+import requests
+from datetime import datetime
+import pandas as pd
+import numpy as np
+from gensim.models import Doc2Vec
+import snowballstemmer, re
+from bs4 import BeautifulSoup
+import re, sys
+from tensorflow.keras.models import load_model
+import joblib
+import gradio as gr
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36, Opera/9.80 (Windows NT 6.1; WOW64) Presto/2.12.388 Version/12.18'
+}
+
+app = Flask(__name__)
+
+def getsoup(url):
+    response = requests.get(url, headers=headers)
+    Status_Code = response.status_code
+    print(url)
+    print(Status_Code)
+    
+    if Status_Code == 200:
+      soup = BeautifulSoup(response.content, features="lxml")
+    else:
+      soup = getsoup(url)
+    return soup 
+
+def getLastPageNumber(soup, site):
+    pageNumber = []
+    if site == 'flipkart':
+        review_number = int(soup.find("span", "_2_R_DZ").text.strip().replace(',', '').split()[-2])
+        if review_number <=10:
+            lastPage = 1
+        else:
+            link = soup.find(attrs={"class": "_2MImiq _1Qnn1K"})
+            pageNumber = link.find('span').text.strip().replace(',', '').split()
+            lastPage1 = pageNumber[len(pageNumber)-1]
+            lastPage = int(lastPage1)
+    elif site == 'amazon':
+        review_number = int(soup.find("div", {"data-hook": "cr-filter-info-review-rating-count"}).text.strip().replace(',', '').split()[-3])
+        if review_number <=10:
+            lastPage = 1
+        else:
+            lastPage = review_number // 10
+    if lastPage > 500:
+        lastPage = 2
+    return lastPage
+
+
+def geturllist(url, lastPage):
+    urllistPages = []
+    url = url[:-1]
+    for i in range(1,lastPage+1):
+      urllistPages.append (url + str(i))
+    return urllistPages
+
+
+def getReviews(soup, site, url):
+    if site == 'flipkart':
+        #Extracting the Titles
+        title_sec = soup.find_all("p",'_2-N8zT')
+        title = []
+        for s in title_sec:
+            title.append(s.text)
+
+        author_sec = soup.find_all("p","_2sc7ZR _2V5EHH")
+        author = []
+        for r in author_sec:
+            author.append(r.text)
+
+        Review_text_sec = soup.find_all("div",'t-ZTKy')
+        text = []
+        for t in Review_text_sec:
+            text.append(t.text)
+            
+        Rating = soup.find_all("div", {"class": ["_3LWZlK _1BLPMq", "_3LWZlK _32lA32 _1BLPMq", "_3LWZlK _1rdVr6 _1BLPMq"]})    
+        rate = []
+        for d in Rating:
+            rate.append(d.text)
+
+        Date_sec = soup.find_all(lambda tag: tag.name == 'p' and tag.get('class') == ['_2sc7ZR'])    
+        date = []
+        for d in Date_sec:
+          date.append(d.text)
+
+        help_sec = soup.find_all(lambda tag: tag.name == 'div' and tag.get('class') == ['_1LmwT9'])    
+        help1 = []
+        for d in help_sec:
+          help1.append(d.text)
+    
+    elif site == 'amazon':
+        n_ = 0
+        title_sec = soup.find_all(attrs={"data-hook": "review-title", "class": "a-size-base a-link-normal review-title a-color-base review-title-content a-text-bold"})
+        title = []
+        for s in title_sec:
+            title.append(s.text.replace('\n', ''))
+        n_ = len(title)
+
+        author_sec = soup.find_all(attrs = {"class": "a-profile-name"})
+        author = []
+        for r in author_sec:
+            author.append(r.text)
+        while(1):
+            if len(author) > n_:
+                author.pop(0)
+            else:
+                break
+
+        Review_text_sec = soup.find_all(attrs={"data-hook": "review-body", "class": "a-size-base review-text review-text-content"})
+        text = []
+        for t in Review_text_sec:
+            text.append(t.text.replace('\n', ''))
+
+        Rating = soup.find_all(attrs={"data-hook": "review-star-rating"})    
+        rate = []
+        for d in Rating:
+            rate.append(d.text)
+
+        Date_sec = soup.find_all(attrs={"data-hook": "review-date"})    
+        date = []
+        for d in Date_sec:
+          date.append(d.text) 
+
+        help_sec = soup.find_all(attrs={"data-hook": "helpful-vote-statement"})    
+        help1 = []
+        for d in help_sec:
+             help1.append(d.text.replace('\n          ', '')) 
+        while(1):
+            if len(help1) < n_:
+                help1.append(0)
+            else:
+                break
+        
+    url1 = []
+    url1 = [url] * len(date)
+
+    collate = {'Date': date, 'URL': url1, 'Review_Title': title, 'Author': author, 'Rating': rate, 'Review_text': text, 'Review_helpful': help1}          
+    collate_df = pd.DataFrame.from_dict(collate)
+    return collate_df
+
+
+def preprocess_text(text):
+    stemmer = snowballstemmer.EnglishStemmer()
+    text = " ".join(stemmer.stemWords(re.sub('[!"#%\'()*+,-./:;<=>?@[\\]^_`{|}~1234567890’”“′‘\\\\]', ' ', text).split(' ')))
+    stop_words = set(["may", "also", "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "across","among", "beside", "however", "yet", "within"] + list('abcdefghijklmnopqrstuvwxyz'))
+    stop_list = stemmer.stemWords(stop_words)
+    stop_words.update(stop_list)
+    text = " ".join(filter(None, filter(lambda word: word not in stop_words, text.lower().split(' '))))
+    return text.split(' ')
+
+def vectorize_comments_(df, d2v_model):
+    y = []
+    comments = []
+    for i in range(0, len(df)): 
+        
+        print(i)
+        label = 'SENT_%s' %i
+        comments.append(d2v_model.docvecs[label])
+    return comments
+
+def scraper(url):
+    df2 = []      
+    soup = getsoup(url)
+    site = url.split('.')[1]
+    if site == 'flipkart':
+        url = url + '&page=1'
+    elif site == 'amazon':
+        url = url + '&pageNumber=1'
+    product = url.split('/')[3]
+    lastPage = 1
+    urllistPages = geturllist(url, lastPage)
+    x = 1
+    for url in urllistPages:
+        soup = getsoup(url)
+        df1 = getReviews(soup, site, url)         
+        if x == 1:
+            df3 = []
+            df3 = df1
+        else:                        
+            df2 = df3
+            result = df2.append(df1, ignore_index=True)
+            df3 = result
+        x += 1
+    
+    loaded_model = load_model('weights.best.from_scratch1 (1).hdf5')
+
+    preprocessed_arr = [preprocess_text(x) for x in list(df3['Review_text'])]
+
+    doc2vec_model = Doc2Vec.load("doc2vec_model_opinion_corpus (1).d2v")
+
+    textData = vectorize_comments_(preprocessed_arr, doc2vec_model)
+
+    textData_array = np.array(textData)
+
+    num_vectors = textData_array.shape[0]
+    textData_3d = textData_array.reshape((num_vectors, 1, -1))
+
+    new_shape = (textData_array.shape[0], 380, 512)
+
+    X_test3_reshaped = np.zeros(new_shape, dtype=textData_3d.dtype)
+    X_test3_reshaped[:, :textData_3d.shape[1], :textData_3d.shape[2]] = textData_3d
+
+    predictions = np.rint(loaded_model.predict(X_test3_reshaped))
+
+    argMax = []
+
+    for i in predictions:
+        argMax.append(np.argmax(i))
+
+    print(argMax)
+    print(list(df3['Review_text'])[3])
+
+    arr = []
+    for i, j in enumerate(argMax):
+        if j == 2 or j == 1:
+            arr.append(list(df3['Review_text'])[i])
+    return arr
+
+
+# @app.route('/', methods=['GET'])
+# def index():
+#     results = []
+#     if request.args.get('url'):
+#         results = scraper(request.args.get('url'))
+#     return results
+    
+
+# if __name__ == "__main__":
+#     app.run(debug=True)
+
+
+
+def index():
+    results = []
+    if request.args.get('url'):
+        results = scraper(request.args.get('url'))
+    return results
+
+
+inputs_image_url = [
+    gr.Textbox(type="text", label="Image URL"),
+]
+
+outputs_result_dict = [
+    gr.Textbox(type="text", label="Result Dictionary"),
+]
+
+interface_image_url = gr.Interface(
+    fn=index,
+    inputs=inputs_image_url,
+    outputs=outputs_result_dict,
+    title="Dark review detection",
+    cache_examples=False,
+)
+
+gr.TabbedInterface(
+    [interface_image_url],
+    tab_names=['Reviews inference']
+).queue().launch()

doc2vec_model_opinion_corpus (1).d2v

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deabad8d2bf4677f8f6f7069da1f928f6ce1fe45722f0839c99a2615f37f28ea
+size 29196813

init.py

@@ -0,0 +1,239 @@
+import pandas as pd
+import numpy as np
+from string import ascii_lowercase
+from gensim.models import Doc2Vec
+from gensim.models import doc2vec
+from gensim.models import KeyedVectors
+import snowballstemmer, re
+import requests
+from bs4 import BeautifulSoup
+import re, sys
+from tensorflow.keras.models import load_model
+import joblib
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36, Opera/9.80 (Windows NT 6.1; WOW64) Presto/2.12.388 Version/12.18'
+}
+
+def getsoup(url):
+    response = requests.get(url, headers=headers)
+    Status_Code = response.status_code
+    print(url)
+    print(Status_Code)
+    
+    if Status_Code == 200:
+      soup = BeautifulSoup(response.content, features="lxml")
+    else:
+      soup = getsoup(url)
+    return soup 
+
+#Get Last Page number
+def getLastPageNumber(soup, site):
+    pageNumber = []
+    if site == 'flipkart':
+        review_number = int(soup.find("span", "_2_R_DZ").text.strip().replace(',', '').split()[-2])
+        if review_number <=10:
+            lastPage = 1
+        else:
+            link = soup.find(attrs={"class": "_2MImiq _1Qnn1K"})
+            pageNumber = link.find('span').text.strip().replace(',', '').split()
+            lastPage1 = pageNumber[len(pageNumber)-1]
+            lastPage = int(lastPage1)
+    elif site == 'amazon':
+        review_number = int(soup.find("div", {"data-hook": "cr-filter-info-review-rating-count"}).text.strip().replace(',', '').split()[-3])
+        if review_number <=10:
+            lastPage = 1
+        else:
+            lastPage = review_number // 10
+    if lastPage > 500:
+        lastPage = 2
+    return lastPage
+
+
+def geturllist(url, lastPage):
+    urllistPages = []
+    url = url[:-1]
+    for i in range(1,lastPage+1):
+      urllistPages.append (url + str(i))
+    return urllistPages
+
+
+def getReviews(soup, site, url):
+    if site == 'flipkart':
+        #Extracting the Titles
+        title_sec = soup.find_all("p",'_2-N8zT')
+        title = []
+        for s in title_sec:
+            title.append(s.text)
+
+        #Extracting the Author names
+        author_sec = soup.find_all("p","_2sc7ZR _2V5EHH")
+        author = []
+        for r in author_sec:
+            author.append(r.text)
+
+        #Extracting the Text
+        Review_text_sec = soup.find_all("div",'t-ZTKy')
+        text = []
+        for t in Review_text_sec:
+            text.append(t.text)
+            
+        #Extracting the Star rating  
+        Rating = soup.find_all("div", {"class": ["_3LWZlK _1BLPMq", "_3LWZlK _32lA32 _1BLPMq", "_3LWZlK _1rdVr6 _1BLPMq"]})    
+        rate = []
+        for d in Rating:
+            rate.append(d.text)
+
+        #Extracting the Date
+        Date_sec = soup.find_all(lambda tag: tag.name == 'p' and tag.get('class') == ['_2sc7ZR'])    
+        date = []
+        for d in Date_sec:
+          date.append(d.text)
+
+        #Extracting the Helpful rating
+        help_sec = soup.find_all(lambda tag: tag.name == 'div' and tag.get('class') == ['_1LmwT9'])    
+        help1 = []
+        for d in help_sec:
+          help1.append(d.text)
+    
+    elif site == 'amazon':
+        n_ = 0
+        title_sec = soup.find_all(attrs={"data-hook": "review-title", "class": "a-size-base a-link-normal review-title a-color-base review-title-content a-text-bold"})
+        title = []
+        for s in title_sec:
+            title.append(s.text.replace('\n', ''))
+        n_ = len(title)
+
+        author_sec = soup.find_all(attrs = {"class": "a-profile-name"})
+        author = []
+        for r in author_sec:
+            author.append(r.text)
+        while(1):
+            if len(author) > n_:
+                author.pop(0)
+            else:
+                break
+
+        Review_text_sec = soup.find_all(attrs={"data-hook": "review-body", "class": "a-size-base review-text review-text-content"})
+        text = []
+        for t in Review_text_sec:
+            text.append(t.text.replace('\n', ''))
+
+        Rating = soup.find_all(attrs={"data-hook": "review-star-rating"})    
+        rate = []
+        for d in Rating:
+            rate.append(d.text)
+
+        Date_sec = soup.find_all(attrs={"data-hook": "review-date"})    
+        date = []
+        for d in Date_sec:
+          date.append(d.text) 
+
+        help_sec = soup.find_all(attrs={"data-hook": "helpful-vote-statement"})    
+        help1 = []
+        for d in help_sec:
+             help1.append(d.text.replace('\n          ', '')) 
+        while(1):
+            if len(help1) < n_:
+                help1.append(0)
+            else:
+                break
+        
+    url1 = []
+    url1 = [url] * len(date)
+
+    collate = {'Date': date, 'URL': url1, 'Review_Title': title, 'Author': author, 'Rating': rate, 'Review_text': text, 'Review_helpful': help1}          
+    collate_df = pd.DataFrame.from_dict(collate)
+    return collate_df
+
+def scraper(url):
+    df2 = []      
+    soup = getsoup(url)
+    site = url.split('.')[1]
+    if site == 'flipkart':
+        url = url + '&page=1'
+    elif site == 'amazon':
+        url = url + '&pageNumber=1'
+    product = url.split('/')[3]
+    lastPage = 1
+    urllistPages = geturllist(url, lastPage)
+    x = 1
+    for url in urllistPages:
+        soup = getsoup(url)
+        df1 = getReviews(soup, site, url)         
+        if x == 1:
+            df3 = []
+            df3 = df1
+        else:                        
+            df2 = df3
+            result = df2.append(df1, ignore_index=True)
+            df3 = result
+        x += 1
+    print(list(df3['Review_text']))
+    return list(df3['Review_text'])
+
+arr = scraper('https://www.amazon.in/Redmi-inches-Ready-Smart-L32R8-FVIN/product-review/B0BVMLNGXR/ref=lp_90117314031_1_1?pf_rd_p=9e034799-55e2-4ab2-b0d0-eb42f95b2d05&pf_rd_r=V81TJ2VTRM0BYHQ6XX8S&sbo=RZvfv%2F%2FHxDF%2BO5021pAnSA%3D%3D&th=1')
+
+TaggedDocument = doc2vec.TaggedDocument
+
+loaded_model = load_model('weights.best.from_scratch1 (1).hdf5')
+
+def preprocess_text(text):
+    stemmer = snowballstemmer.EnglishStemmer()
+    text = " ".join(stemmer.stemWords(re.sub('[!"#%\'()*+,-./:;<=>?@[\\]^_`{|}~1234567890’”“′‘\\\\]', ' ', text).split(' ')))
+
+    stop_words = set(["may", "also", "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "across", "among", "beside", "however", "yet", "within"] + list(ascii_lowercase))
+    stop_list = stemmer.stemWords(stop_words)
+    stop_words.update(stop_list)
+    text = " ".join(filter(None, filter(lambda word: word not in stop_words, text.lower().split(' '))))
+
+    return text.split(' ')
+
+
+# arr = ['This is not a nice product', 'This is brilliant product', "I have had this tv for a year and a half. The tv worked smoothly and recently, it's display stopped working. I have contacted the company and they assured a repair within 8-10 days replacing the display panel. Then came the worst after sale experience I had ever. They didn't respond to my calls and whenever, I raised a grievance, I was just given different dates and extensions.", ''' In monotheistic belief systems, God is usually viewed as the supreme being, creator, and principal object of faith. In polytheistic belief systems, a god is "a spirit or being believed to have created, or for controlling some part of the universe or life, for which such a deity is often worshipped ''']
+
+preprocessed_arr = [preprocess_text(x) for x in arr]
+
+doc2vec_model = Doc2Vec.load("doc2vec_model_opinion_corpus (1).d2v")
+
+def vectorize_comments_(df, d2v_model):
+    y = []
+    comments = []
+    for i in range(0, len(df)):
+        print(i)
+        label = 'SENT_%s' %i
+        comments.append(d2v_model.docvecs[label])
+
+    return comments
+
+textData = vectorize_comments_(preprocessed_arr, doc2vec_model)
+
+import numpy as np
+
+textData_array = np.array(textData)
+
+num_vectors = textData_array.shape[0]
+textData_3d = textData_array.reshape((num_vectors, 1, -1))
+
+new_shape = (textData_array.shape[0], 380, 512)
+
+X_test3_reshaped = np.zeros(new_shape, dtype=textData_3d.dtype)
+X_test3_reshaped[:, :textData_3d.shape[1], :textData_3d.shape[2]] = textData_3d
+
+predictions = np.rint(loaded_model.predict(X_test3_reshaped))
+
+argMax = []
+
+for i in predictions:
+    argMax.append(np.argmax(i))
+
+def returnRequirements(comments, preds):
+    arr = []
+    for i, j in enumerate(preds):
+        if j == 3 or j == 0:
+            arr.append(comments[i])
+    return arr
+
+# 3 & 0 is deceptive rest aren't
+print(argMax)
+print(returnRequirements(arr, argMax))

review_detection.ipynb

@@ -0,0 +1,1383 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "aqGqpcYIpf_q",
+        "outputId": "a6d87acc-4df9-4abf-973c-c791c5461af9"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Downloading deceptive-opinion-spam-corpus.zip to /content\n",
+            "\r  0% 0.00/456k [00:00<?, ?B/s]\n",
+            "\r100% 456k/456k [00:00<00:00, 111MB/s]\n"
+          ]
+        }
+      ],
+      "source": [
+        "!kaggle datasets download -d rtatman/deceptive-opinion-spam-corpus"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zu4NTMHapmms"
+      },
+      "outputs": [],
+      "source": [
+        "import zipfile\n",
+        "zip_ref = zipfile.ZipFile('/content/deceptive-opinion-spam-corpus.zip', 'r')\n",
+        "zip_ref.extractall('/content')\n",
+        "zip_ref.close()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3hIl0gHep9q6"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "import numpy as np\n",
+        "from keras.preprocessing import sequence\n",
+        "from keras.layers import TimeDistributed, GlobalAveragePooling1D, GlobalAveragePooling2D, BatchNormalization\n",
+        "from keras.layers import LSTM\n",
+        "from keras.layers import Conv1D, MaxPooling1D, Conv2D, MaxPooling2D, AveragePooling1D\n",
+        "from keras.layers import Embedding\n",
+        "from keras.layers import Dropout, Flatten, Bidirectional, Dense, Activation, TimeDistributed\n",
+        "from keras.models import Model, Sequential\n",
+        "from tensorflow.keras.utils import to_categorical\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.preprocessing import LabelEncoder\n",
+        "from nltk.corpus import stopwords\n",
+        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
+        "from nltk.stem.wordnet import WordNetLemmatizer\n",
+        "from string import ascii_lowercase\n",
+        "from collections import Counter\n",
+        "from gensim.models import Word2Vec\n",
+        "from gensim.models import Doc2Vec\n",
+        "from gensim.models import doc2vec\n",
+        "from gensim.models import KeyedVectors\n",
+        "import itertools, nltk, snowballstemmer, re\n",
+        "import random\n",
+        "\n",
+        "TaggedDocument = doc2vec.TaggedDocument"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fzwpJb7EqCjc"
+      },
+      "outputs": [],
+      "source": [
+        "class LabeledLineSentence(object):\n",
+        "    def __init__(self, sources):\n",
+        "        self.sources = sources\n",
+        "\n",
+        "        flipped = {}\n",
+        "\n",
+        "        for key, value in sources.items():\n",
+        "            if value not in flipped:\n",
+        "                flipped[value] = [key]\n",
+        "            else:\n",
+        "                raise Exception('Non-unique prefix encountered')\n",
+        "\n",
+        "    def __iter__(self):\n",
+        "        for source, prefix in self.sources.items():\n",
+        "            with utils.smart_open(source) as fin:\n",
+        "                for item_no, line in enumerate(fin):\n",
+        "                    yield TaggedDocument(utils.to_unicode(line).split(), [prefix + '_%s' % item_no])\n",
+        "\n",
+        "    def to_array(self):\n",
+        "        self.sentences = []\n",
+        "        for source, prefix in self.sources.items():\n",
+        "            with utils.smart_open(source) as fin:\n",
+        "                for item_no, line in enumerate(fin):\n",
+        "                    self.sentences.append(TaggedDocument(utils.to_unicode(line).split(), [prefix + '_%s' % item_no]))\n",
+        "        return self.sentences\n",
+        "\n",
+        "    def sentences_perm(self):\n",
+        "        shuffled = list(self.sentences)\n",
+        "        random.shuffle(shuffled)\n",
+        "        return shuffled"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e_auMclPqmrI"
+      },
+      "outputs": [],
+      "source": [
+        "data = pd.read_csv(\"/content/deceptive-opinion.csv\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "G-EIqcKVrbEl"
+      },
+      "outputs": [],
+      "source": [
+        "data['polarity'] = np.where(data['polarity']=='positive', 1, 0)\n",
+        "data['deceptive'] = np.where(data['deceptive']=='truthful', 1, 0)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 300
+        },
+        "id": "2REEjGj9rck1",
+        "outputId": "a5679d0f-f0b9-4c21-8005-42058e2cc4fc"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "  <div id=\"df-6614128d-b81e-41f1-9a92-25467585644c\" class=\"colab-df-container\">\n",
+              "    <div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th></th>\n",
+              "      <th>deceptive</th>\n",
+              "      <th>polarity</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>count</th>\n",
+              "      <td>1600.000000</td>\n",
+              "      <td>1600.000000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>mean</th>\n",
+              "      <td>0.500000</td>\n",
+              "      <td>0.500000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>std</th>\n",
+              "      <td>0.500156</td>\n",
+              "      <td>0.500156</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>min</th>\n",
+              "      <td>0.000000</td>\n",
+              "      <td>0.000000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>25%</th>\n",
+              "      <td>0.000000</td>\n",
+              "      <td>0.000000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>50%</th>\n",
+              "      <td>0.500000</td>\n",
+              "      <td>0.500000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>75%</th>\n",
+              "      <td>1.000000</td>\n",
+              "      <td>1.000000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>max</th>\n",
+              "      <td>1.000000</td>\n",
+              "      <td>1.000000</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>\n",
+              "    <div class=\"colab-df-buttons\">\n",
+              "\n",
+              "  <div class=\"colab-df-container\">\n",
+              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6614128d-b81e-41f1-9a92-25467585644c')\"\n",
+              "            title=\"Convert this dataframe to an interactive table.\"\n",
+              "            style=\"display:none;\">\n",
+              "\n",
+              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
+              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
+              "  </svg>\n",
+              "    </button>\n",
+              "\n",
+              "  <style>\n",
+              "    .colab-df-container {\n",
+              "      display:flex;\n",
+              "      gap: 12px;\n",
+              "    }\n",
+              "\n",
+              "    .colab-df-convert {\n",
+              "      background-color: #E8F0FE;\n",
+              "      border: none;\n",
+              "      border-radius: 50%;\n",
+              "      cursor: pointer;\n",
+              "      display: none;\n",
+              "      fill: #1967D2;\n",
+              "      height: 32px;\n",
+              "      padding: 0 0 0 0;\n",
+              "      width: 32px;\n",
+              "    }\n",
+              "\n",
+              "    .colab-df-convert:hover {\n",
+              "      background-color: #E2EBFA;\n",
+              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
+              "      fill: #174EA6;\n",
+              "    }\n",
+              "\n",
+              "    .colab-df-buttons div {\n",
+              "      margin-bottom: 4px;\n",
+              "    }\n",
+              "\n",
+              "    [theme=dark] .colab-df-convert {\n",
+              "      background-color: #3B4455;\n",
+              "      fill: #D2E3FC;\n",
+              "    }\n",
+              "\n",
+              "    [theme=dark] .colab-df-convert:hover {\n",
+              "      background-color: #434B5C;\n",
+              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
+              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
+              "      fill: #FFFFFF;\n",
+              "    }\n",
+              "  </style>\n",
+              "\n",
+              "    <script>\n",
+              "      const buttonEl =\n",
+              "        document.querySelector('#df-6614128d-b81e-41f1-9a92-25467585644c button.colab-df-convert');\n",
+              "      buttonEl.style.display =\n",
+              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
+              "\n",
+              "      async function convertToInteractive(key) {\n",
+              "        const element = document.querySelector('#df-6614128d-b81e-41f1-9a92-25467585644c');\n",
+              "        const dataTable =\n",
+              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
+              "                                                    [key], {});\n",
+              "        if (!dataTable) return;\n",
+              "\n",
+              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
+              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
+              "          + ' to learn more about interactive tables.';\n",
+              "        element.innerHTML = '';\n",
+              "        dataTable['output_type'] = 'display_data';\n",
+              "        await google.colab.output.renderOutput(dataTable, element);\n",
+              "        const docLink = document.createElement('div');\n",
+              "        docLink.innerHTML = docLinkHtml;\n",
+              "        element.appendChild(docLink);\n",
+              "      }\n",
+              "    </script>\n",
+              "  </div>\n",
+              "\n",
+              "\n",
+              "<div id=\"df-cc66d435-72a8-4146-ba22-ea8ef5610445\">\n",
+              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cc66d435-72a8-4146-ba22-ea8ef5610445')\"\n",
+              "            title=\"Suggest charts\"\n",
+              "            style=\"display:none;\">\n",
+              "\n",
+              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
+              "     width=\"24px\">\n",
+              "    <g>\n",
+              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
+              "    </g>\n",
+              "</svg>\n",
+              "  </button>\n",
+              "\n",
+              "<style>\n",
+              "  .colab-df-quickchart {\n",
+              "      --bg-color: #E8F0FE;\n",
+              "      --fill-color: #1967D2;\n",
+              "      --hover-bg-color: #E2EBFA;\n",
+              "      --hover-fill-color: #174EA6;\n",
+              "      --disabled-fill-color: #AAA;\n",
+              "      --disabled-bg-color: #DDD;\n",
+              "  }\n",
+              "\n",
+              "  [theme=dark] .colab-df-quickchart {\n",
+              "      --bg-color: #3B4455;\n",
+              "      --fill-color: #D2E3FC;\n",
+              "      --hover-bg-color: #434B5C;\n",
+              "      --hover-fill-color: #FFFFFF;\n",
+              "      --disabled-bg-color: #3B4455;\n",
+              "      --disabled-fill-color: #666;\n",
+              "  }\n",
+              "\n",
+              "  .colab-df-quickchart {\n",
+              "    background-color: var(--bg-color);\n",
+              "    border: none;\n",
+              "    border-radius: 50%;\n",
+              "    cursor: pointer;\n",
+              "    display: none;\n",
+              "    fill: var(--fill-color);\n",
+              "    height: 32px;\n",
+              "    padding: 0;\n",
+              "    width: 32px;\n",
+              "  }\n",
+              "\n",
+              "  .colab-df-quickchart:hover {\n",
+              "    background-color: var(--hover-bg-color);\n",
+              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
+              "    fill: var(--button-hover-fill-color);\n",
+              "  }\n",
+              "\n",
+              "  .colab-df-quickchart-complete:disabled,\n",
+              "  .colab-df-quickchart-complete:disabled:hover {\n",
+              "    background-color: var(--disabled-bg-color);\n",
+              "    fill: var(--disabled-fill-color);\n",
+              "    box-shadow: none;\n",
+              "  }\n",
+              "\n",
+              "  .colab-df-spinner {\n",
+              "    border: 2px solid var(--fill-color);\n",
+              "    border-color: transparent;\n",
+              "    border-bottom-color: var(--fill-color);\n",
+              "    animation:\n",
+              "      spin 1s steps(1) infinite;\n",
+              "  }\n",
+              "\n",
+              "  @keyframes spin {\n",
+              "    0% {\n",
+              "      border-color: transparent;\n",
+              "      border-bottom-color: var(--fill-color);\n",
+              "      border-left-color: var(--fill-color);\n",
+              "    }\n",
+              "    20% {\n",
+              "      border-color: transparent;\n",
+              "      border-left-color: var(--fill-color);\n",
+              "      border-top-color: var(--fill-color);\n",
+              "    }\n",
+              "    30% {\n",
+              "      border-color: transparent;\n",
+              "      border-left-color: var(--fill-color);\n",
+              "      border-top-color: var(--fill-color);\n",
+              "      border-right-color: var(--fill-color);\n",
+              "    }\n",
+              "    40% {\n",
+              "      border-color: transparent;\n",
+              "      border-right-color: var(--fill-color);\n",
+              "      border-top-color: var(--fill-color);\n",
+              "    }\n",
+              "    60% {\n",
+              "      border-color: transparent;\n",
+              "      border-right-color: var(--fill-color);\n",
+              "    }\n",
+              "    80% {\n",
+              "      border-color: transparent;\n",
+              "      border-right-color: var(--fill-color);\n",
+              "      border-bottom-color: var(--fill-color);\n",
+              "    }\n",
+              "    90% {\n",
+              "      border-color: transparent;\n",
+              "      border-bottom-color: var(--fill-color);\n",
+              "    }\n",
+              "  }\n",
+              "</style>\n",
+              "\n",
+              "  <script>\n",
+              "    async function quickchart(key) {\n",
+              "      const quickchartButtonEl =\n",
+              "        document.querySelector('#' + key + ' button');\n",
+              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
+              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
+              "      try {\n",
+              "        const charts = await google.colab.kernel.invokeFunction(\n",
+              "            'suggestCharts', [key], {});\n",
+              "      } catch (error) {\n",
+              "        console.error('Error during call to suggestCharts:', error);\n",
+              "      }\n",
+              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
+              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
+              "    }\n",
+              "    (() => {\n",
+              "      let quickchartButtonEl =\n",
+              "        document.querySelector('#df-cc66d435-72a8-4146-ba22-ea8ef5610445 button');\n",
+              "      quickchartButtonEl.style.display =\n",
+              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
+              "    })();\n",
+              "  </script>\n",
+              "</div>\n",
+              "\n",
+              "    </div>\n",
+              "  </div>\n"
+            ],
+            "text/plain": [
+              "         deceptive     polarity\n",
+              "count  1600.000000  1600.000000\n",
+              "mean      0.500000     0.500000\n",
+              "std       0.500156     0.500156\n",
+              "min       0.000000     0.000000\n",
+              "25%       0.000000     0.000000\n",
+              "50%       0.500000     0.500000\n",
+              "75%       1.000000     1.000000\n",
+              "max       1.000000     1.000000"
+            ]
+          },
+          "execution_count": 15,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "df = data.sample(frac=1)\n",
+        "df.describe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gqCqI9xSriAb"
+      },
+      "outputs": [],
+      "source": [
+        "def create_class(c):\n",
+        "    if c['polarity'] == 1 and c['deceptive'] == 1:\n",
+        "        return [1,1]\n",
+        "    elif c['polarity'] == 1 and c['deceptive'] == 0:\n",
+        "        return [1,0]\n",
+        "    elif c['polarity'] == 0 and c['deceptive'] == 1:\n",
+        "        return [0,1]\n",
+        "    else:\n",
+        "        return [0,0]\n",
+        "\n",
+        "def specific_class(c):\n",
+        "    if c['polarity'] == 1 and c['deceptive'] == 1: # Actually Deceptive ---> 0\n",
+        "        return \"TRUE_POSITIVE\"\n",
+        "    elif c['polarity'] == 1 and c['deceptive'] == 0: # Actually Not Deceptive ---> 1\n",
+        "        return \"FALSE_POSITIVE\"\n",
+        "    elif c['polarity'] == 0 and c['deceptive'] == 1: # Actually Not Deceptive ---> 2\n",
+        "        return \"TRUE_NEGATIVE\"\n",
+        "    else:  # Actually Deceptive ---> 3\n",
+        "        return \"FALSE_NEGATIVE\"\n",
+        "\n",
+        "data['final_class'] = data.apply(create_class, axis=1)\n",
+        "data['given_class'] = data.apply(specific_class, axis=1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0KtN7332rkOJ"
+      },
+      "outputs": [],
+      "source": [
+        "from sklearn import preprocessing\n",
+        "\n",
+        "label_encoder = preprocessing.LabelEncoder()\n",
+        "\n",
+        "data['given_class'] = label_encoder.fit_transform(data['given_class'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 251,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "MW0O6_v3EM2G",
+        "outputId": "ab5238a9-8afb-4af5-8c49-69bab79c8caa"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "0       [1, 1]\n",
+              "1       [1, 1]\n",
+              "2       [1, 1]\n",
+              "3       [1, 1]\n",
+              "4       [1, 1]\n",
+              "         ...  \n",
+              "1595    [0, 0]\n",
+              "1596    [0, 0]\n",
+              "1597    [0, 0]\n",
+              "1598    [0, 0]\n",
+              "1599    [0, 0]\n",
+              "Name: final_class, Length: 1600, dtype: object"
+            ]
+          },
+          "execution_count": 251,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "data['final_class']"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "F24YJHdermPy"
+      },
+      "outputs": [],
+      "source": [
+        "Y = data['given_class']\n",
+        "encoder = LabelEncoder()\n",
+        "encoder.fit(Y)\n",
+        "encoded_Y = encoder.transform(Y)\n",
+        "dummy_y = to_categorical(encoded_Y)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 247,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "wmCJ8i83D1x4",
+        "outputId": "89d49003-3a75-421d-884d-e0d032cab6ca"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "array([[0., 0., 0., 1.],\n",
+              "       [0., 0., 0., 1.],\n",
+              "       [0., 0., 0., 1.],\n",
+              "       ...,\n",
+              "       [1., 0., 0., 0.],\n",
+              "       [1., 0., 0., 0.],\n",
+              "       [1., 0., 0., 0.]], dtype=float32)"
+            ]
+          },
+          "execution_count": 247,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "dummy_y"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "U0OJ5Qhbrocj"
+      },
+      "outputs": [],
+      "source": [
+        "textData = pd.DataFrame(list(data['text']))\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c_WdJiovrwpN"
+      },
+      "outputs": [],
+      "source": [
+        "stemmer = snowballstemmer.EnglishStemmer()\n",
+        "\n",
+        "stop = stopwords.words('english')\n",
+        "stop.extend(['may','also','zero','one','two','three','four','five','six','seven','eight','nine','ten','across','among','beside','however','yet','within']+list(ascii_lowercase))\n",
+        "stoplist = stemmer.stemWords(stop)\n",
+        "stoplist = set(stoplist)\n",
+        "stop = set(sorted(stop + list(stoplist)))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "pUJtMqjZryE9",
+        "outputId": "295b19f7-94fa-447b-b964-4017b0593401"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
+            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
+          ]
+        },
+        {
+          "data": {
+            "text/plain": [
+              "True"
+            ]
+          },
+          "execution_count": 23,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "nltk.download('stopwords')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_FG8kbvgr9HS"
+      },
+      "outputs": [],
+      "source": [
+        "textData[0].replace('[!\"#%\\'()*+,-./:;<=>?@\\[\\]^_`{|}~1234567890’”“′‘\\\\\\]',' ',inplace=True,regex=True)\n",
+        "wordlist = filter(None, \" \".join(list(set(list(itertools.chain(*textData[0].str.split(' ')))))).split(\" \"))\n",
+        "data['stemmed_text_data'] = [' '.join(filter(None,filter(lambda word: word not in stop, line))) for line in textData[0].str.lower().str.split(' ')]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fjUfTsjEsDh2"
+      },
+      "outputs": [],
+      "source": [
+        "minimum_count = 1\n",
+        "str_frequencies = pd.DataFrame(list(Counter(filter(None,list(itertools.chain(*data['stemmed_text_data'].str.split(' '))))).items()),columns=['word','count'])\n",
+        "low_frequency_words = set(str_frequencies[str_frequencies['count'] < minimum_count]['word'])\n",
+        "data['stemmed_text_data'] = [' '.join(filter(None,filter(lambda word: word not in low_frequency_words, line))) for line in data['stemmed_text_data'].str.split(' ')]\n",
+        "data['stemmed_text_data'] = [\" \".join(stemmer.stemWords(re.sub('[!\"#%\\'()*+,-./:;<=>?@\\[\\]^_`{|}~1234567890’”“′‘\\\\\\]',' ', next_text).split(' '))) for next_text in data['stemmed_text_data']]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GX-Gd8M6sEpp"
+      },
+      "outputs": [],
+      "source": [
+        "lmtzr = WordNetLemmatizer()\n",
+        "w = re.compile(\"\\w+\",re.I)\n",
+        "\n",
+        "def label_sentences(df, input_point):\n",
+        "    labeled_sentences = []\n",
+        "    list_sen = []\n",
+        "    for index, datapoint in df.iterrows():\n",
+        "        tokenized_words = re.findall(w,datapoint[input_point].lower())\n",
+        "        labeled_sentences.append(TaggedDocument(words=tokenized_words, tags=['SENT_%s' %index]))\n",
+        "        list_sen.append(tokenized_words)\n",
+        "    return labeled_sentences, list_sen\n",
+        "\n",
+        "def train_doc2vec_model(labeled_sentences):\n",
+        "    model = Doc2Vec(min_count=1, window=9, vector_size=512, sample=1e-4, negative=5, workers=7)\n",
+        "    model.build_vocab(labeled_sentences)\n",
+        "    pretrained_weights = model.wv.vectors\n",
+        "    vocab_size, embedding_size = pretrained_weights.shape\n",
+        "    model.train(labeled_sentences, total_examples=vocab_size, epochs=400)\n",
+        "\n",
+        "    return model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2C_5s3UOsGfU"
+      },
+      "outputs": [],
+      "source": [
+        "textData = data['stemmed_text_data'].to_frame().reset_index()\n",
+        "sen, corpus = label_sentences(textData, 'stemmed_text_data')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qpb3aMvW_jdj"
+      },
+      "outputs": [],
+      "source": [
+        "sen"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JeK3t6_HsNv9"
+      },
+      "outputs": [],
+      "source": [
+        "doc2vec_model = train_doc2vec_model(sen)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DyX1XG1usQMm"
+      },
+      "outputs": [],
+      "source": [
+        "doc2vec_model.save(\"doc2vec_model_opinion_corpus.d2v\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "l0OAFencszum"
+      },
+      "outputs": [],
+      "source": [
+        "doc2vec_model = Doc2Vec.load(\"doc2vec_model_opinion_corpus.d2v\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "ZJTp1POIs1sQ",
+        "outputId": "5788b8b8-3f66-4d4a-e22a-6bff020adf1e"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/sklearn/feature_extraction/text.py:528: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
+            "  warnings.warn(\n"
+          ]
+        }
+      ],
+      "source": [
+        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+        "from sklearn.decomposition import TruncatedSVD\n",
+        "\n",
+        "tfidf1 = TfidfVectorizer(tokenizer=lambda i:i, lowercase=False, ngram_range=(1,1))\n",
+        "result_train1 = tfidf1.fit_transform(corpus)\n",
+        "\n",
+        "tfidf2 = TfidfVectorizer(tokenizer=lambda i:i, lowercase=False, ngram_range=(1,2))\n",
+        "result_train2 = tfidf2.fit_transform(corpus)\n",
+        "\n",
+        "tfidf3 = TfidfVectorizer(tokenizer=lambda i:i, lowercase=False, ngram_range=(1,3))\n",
+        "result_train3 = tfidf3.fit_transform(corpus)\n",
+        "\n",
+        "svd = TruncatedSVD(n_components=512, n_iter=40, random_state=34)\n",
+        "tfidf_data1 = svd.fit_transform(result_train1)\n",
+        "tfidf_data2 = svd.fit_transform(result_train2)\n",
+        "tfidf_data3 = svd.fit_transform(result_train3)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "io0D71F00Wv8"
+      },
+      "outputs": [],
+      "source": [
+        "nlp = spacy.load(\"en_core_web_sm\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QR6PqZREs3EA"
+      },
+      "outputs": [],
+      "source": [
+        "from sklearn.feature_extraction.text import CountVectorizer\n",
+        "import spacy\n",
+        "\n",
+        "nlp = spacy.load(\"en_core_web_sm\")\n",
+        "temp_textData = pd.DataFrame(list(data['text']))\n",
+        "\n",
+        "overall_pos_tags_tokens = []\n",
+        "overall_pos = []\n",
+        "overall_tokens = []\n",
+        "overall_dep = []\n",
+        "\n",
+        "for i in range(1600):\n",
+        "    doc = nlp(temp_textData[0][i])\n",
+        "    given_pos_tags_tokens = []\n",
+        "    given_pos = []\n",
+        "    given_tokens = []\n",
+        "    given_dep = []\n",
+        "    for token in doc:\n",
+        "        output = \"%s_%s\" % (token.pos_, token.tag_)\n",
+        "        given_pos_tags_tokens.append(output)\n",
+        "        given_pos.append(token.pos_)\n",
+        "        given_tokens.append(token.tag_)\n",
+        "        given_dep.append(token.dep_)\n",
+        "\n",
+        "    overall_pos_tags_tokens.append(given_pos_tags_tokens)\n",
+        "    overall_pos.append(given_pos)\n",
+        "    overall_tokens.append(given_tokens)\n",
+        "    overall_dep.append(given_dep)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "O6C2OJ8KEzHk"
+      },
+      "outputs": [],
+      "source": [
+        "overall_tokens"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "4PxkBoQgs4wV",
+        "outputId": "07f59ab7-dbc0-4b41-e712-31a165e6ddf2"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/sklearn/feature_extraction/text.py:528: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
+            "  warnings.warn(\n"
+          ]
+        }
+      ],
+      "source": [
+        "import numpy as np\n",
+        "from sklearn.feature_extraction.text import CountVectorizer\n",
+        "from sklearn.preprocessing import MinMaxScaler\n",
+        "\n",
+        "count = CountVectorizer(tokenizer=lambda i: i, lowercase=False)\n",
+        "pos_tags_data = count.fit_transform(overall_pos_tags_tokens).todense()\n",
+        "pos_data = count.fit_transform(overall_pos).todense()\n",
+        "tokens_data = count.fit_transform(overall_tokens).todense()\n",
+        "dep_data = count.fit_transform(overall_dep).todense()\n",
+        "\n",
+        "min_max_scaler = MinMaxScaler()\n",
+        "\n",
+        "normalized_pos_tags_data = min_max_scaler.fit_transform(np.asarray(pos_tags_data))\n",
+        "normalized_pos_data = min_max_scaler.fit_transform(np.asarray(pos_data))\n",
+        "normalized_tokens_data = min_max_scaler.fit_transform(np.asarray(tokens_data))\n",
+        "normalized_dep_data = min_max_scaler.fit_transform(np.asarray(dep_data))\n",
+        "\n",
+        "# Convert the scaled data to numpy arrays\n",
+        "normalized_pos_tags_data = np.asarray(normalized_pos_tags_data)\n",
+        "normalized_pos_data = np.asarray(normalized_pos_data)\n",
+        "normalized_tokens_data = np.asarray(normalized_tokens_data)\n",
+        "normalized_dep_data = np.asarray(normalized_dep_data)\n",
+        "\n",
+        "final_pos_tags_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
+        "final_pos_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
+        "final_tokens_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
+        "final_dep_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
+        "\n",
+        "# Assign the converted arrays to the final arrays\n",
+        "final_pos_tags_data[:normalized_pos_tags_data.shape[0], :normalized_pos_tags_data.shape[1]] = normalized_pos_tags_data\n",
+        "final_pos_data[:normalized_pos_data.shape[0], :normalized_pos_data.shape[1]] = normalized_pos_data\n",
+        "final_tokens_data[:normalized_tokens_data.shape[0], :normalized_tokens_data.shape[1]] = normalized_tokens_data\n",
+        "final_dep_data[:normalized_dep_data.shape[0], :normalized_dep_data.shape[1]] = normalized_dep_data\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "jQdeLCgas6HD",
+        "outputId": "b1079e7c-c4fa-413d-bf89-9f5dc8fe36d8"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "370\n"
+          ]
+        }
+      ],
+      "source": [
+        "maxlength = []\n",
+        "for i in range(0,len(sen)):\n",
+        "    maxlength.append(len(sen[i][0]))\n",
+        "\n",
+        "print(max(maxlength))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "X5y1kjW-s7bJ"
+      },
+      "outputs": [],
+      "source": [
+        "doc2vec_model = Doc2Vec.load(\"doc2vec_model_opinion_corpus.d2v\")\n",
+        "\n",
+        "def vectorize_comments(df,d2v_model):\n",
+        "    y = []\n",
+        "    comments = []\n",
+        "    for i in range(0,df.shape[0]):\n",
+        "        label = 'SENT_%s' %i\n",
+        "        comments.append(d2v_model.docvecs[label])\n",
+        "    df['vectorized_comments'] = comments\n",
+        "\n",
+        "    return df\n",
+        "\n",
+        "textData = vectorize_comments(textData,doc2vec_model)\n",
+        "print (textData.head(2))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SUfiSDENs8cg"
+      },
+      "outputs": [],
+      "source": [
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.model_selection import cross_validate,GridSearchCV\n",
+        "\n",
+        "X_train, X_test, y_train, y_test = train_test_split(textData[\"vectorized_comments\"].T.tolist(),\n",
+        "                                                                     dummy_y,\n",
+        "                                                                     test_size=0.1,\n",
+        "                                                                     random_state=56)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1ID0T5d0s-iS"
+      },
+      "outputs": [],
+      "source": [
+        "X = np.array(textData[\"vectorized_comments\"].T.tolist()).reshape((1,1600,512))\n",
+        "y = np.array(dummy_y).reshape((1600,4))\n",
+        "X_train2 = np.array(X_train).reshape((1,1440,512))\n",
+        "y_train2 = np.array(y_train).reshape((1,1440,4))\n",
+        "X_test2 = np.array(X_test).reshape((1,160,512))\n",
+        "y_test2 = np.array(y_test).reshape((1,160,4))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GlYOfPhGs_lR"
+      },
+      "outputs": [],
+      "source": [
+        "from sklearn.model_selection import StratifiedKFold\n",
+        "Xtemp = df[\"vectorized_comments\"].T.tolist()\n",
+        "ytemp = data['given_class']\n",
+        "training_indices = []\n",
+        "testing_indices = []\n",
+        "\n",
+        "skf = StratifiedKFold(n_splits=10)\n",
+        "skf.get_n_splits(Xtemp, ytemp)\n",
+        "\n",
+        "for train_index, test_index in skf.split(Xtemp, ytemp):\n",
+        "    training_indices.append(train_index)\n",
+        "    testing_indices.append(test_index)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 238,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "-avX-WdT_Z2P",
+        "outputId": "a2a227a5-9ff7-4ee0-8cd0-d6b8fb157363"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "160"
+            ]
+          },
+          "execution_count": 238,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "len(testing_indices[2])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EIVEsyLiRjn7"
+      },
+      "outputs": [],
+      "source": [
+        "training_indices"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 211,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "iPc15Lwv7-L2",
+        "outputId": "5b5578e0-27b7-4bad-92e7-ef42dda066ed"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "<12x12 sparse matrix of type '<class 'numpy.float64'>'\n",
+              "\twith 20 stored elements in Compressed Sparse Row format>"
+            ]
+          },
+          "execution_count": 211,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "result_train1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 212,
+      "metadata": {
+        "id": "8UwiqmnhtAz6"
+      },
+      "outputs": [],
+      "source": [
+        "def extractTrainingAndTestingData(givenIndex):\n",
+        "    X_train3 = np.zeros(shape=(1440, max(maxlength)+10, 512)).astype(np.float32)\n",
+        "    Y_train3 = np.zeros(shape=(1440, 4)).astype(np.float32)\n",
+        "    X_test3 = np.zeros(shape=(160, max(maxlength)+10, 512)).astype(np.float32)\n",
+        "    Y_test3 = np.zeros(shape=(160, 4)).astype(np.float32)\n",
+        "\n",
+        "    empty_word = np.zeros(512).astype(np.float32)\n",
+        "\n",
+        "    count_i = 0\n",
+        "    for i in training_indices[givenIndex]:\n",
+        "        len1 = len(sen[i][0])\n",
+        "        average_vector1 = np.zeros(512).astype(np.float32)\n",
+        "        average_vector2 = np.zeros(512).astype(np.float32)\n",
+        "        average_vector3 = np.zeros(512).astype(np.float32)\n",
+        "        for j in range(max(maxlength)+10):\n",
+        "            if j < len1:\n",
+        "                X_train3[count_i,j,:] = doc2vec_model[sen[i][0][j]]\n",
+        "                average_vector1 += result_train1[i, tfidf1.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
+        "                average_vector2 += result_train2[i, tfidf2.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
+        "                average_vector3 += result_train3[i, tfidf3.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
+        "            #elif j >= len1 and j < len1 + 379:\n",
+        "            #    X_train3[count_i,j,:] = glove_data[i, j-len1, :]\n",
+        "            elif j == len1:\n",
+        "                X_train3[count_i,j,:] = tfidf_data1[i]\n",
+        "            elif j == len1 + 1:\n",
+        "                X_train3[count_i,j,:] = tfidf_data2[i]\n",
+        "            elif j == len1+2:\n",
+        "                X_train3[count_i,j,:] = tfidf_data3[i]\n",
+        "            elif j == len1+3:\n",
+        "                X_train3[count_i,j,:] = average_vector1\n",
+        "            elif j == len1+4:\n",
+        "                X_train3[count_i,j,:] = average_vector2\n",
+        "            elif j == len1+5:\n",
+        "                X_train3[count_i,j,:] = average_vector3\n",
+        "            elif j == len1+6:\n",
+        "                X_train3[count_i,j,:] = final_pos_tags_data[i]\n",
+        "            elif j == len1+7:\n",
+        "                X_train3[count_i,j,:] = final_pos_data[i]\n",
+        "            elif j == len1+8:\n",
+        "                X_train3[count_i,j,:] = final_tokens_data[i]\n",
+        "            elif j == len1+9:\n",
+        "                X_train3[count_i,j,:] = final_dep_data[i]\n",
+        "            else:\n",
+        "                X_train3[count_i,j,:] = empty_word\n",
+        "\n",
+        "        Y_train3[count_i,:] = dummy_y[i]\n",
+        "        count_i += 1\n",
+        "\n",
+        "\n",
+        "    count_i = 0\n",
+        "    for i in testing_indices[givenIndex]:\n",
+        "        len1 = len(sen[i][0])\n",
+        "        average_vector1 = np.zeros(512).astype(np.float32)\n",
+        "        average_vector2 = np.zeros(512).astype(np.float32)\n",
+        "        average_vector3 = np.zeros(512).astype(np.float32)\n",
+        "        for j in range(max(maxlength)+10):\n",
+        "            if j < len1:\n",
+        "                X_test3[count_i,j,:] = doc2vec_model[sen[i][0][j]]\n",
+        "                average_vector1 += result_train1[i, tfidf1.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
+        "                average_vector2 += result_train2[i, tfidf2.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
+        "                average_vector3 += result_train3[i, tfidf3.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
+        "            #elif j >= len1 and j < len1 + 379:\n",
+        "            #    X_test3[count_i,j,:] = glove_data[i, j-len1, :]\n",
+        "            elif j == len1:\n",
+        "                X_test3[count_i,j,:] = tfidf_data1[i]\n",
+        "            elif j == len1 + 1:\n",
+        "                X_test3[count_i,j,:] = tfidf_data2[i]\n",
+        "            elif j == len1+2:\n",
+        "                X_test3[count_i,j,:] = tfidf_data3[i]\n",
+        "            elif j == len1+3:\n",
+        "                X_test3[count_i,j,:] = average_vector1\n",
+        "            elif j == len1+4:\n",
+        "                X_test3[count_i,j,:] = average_vector2\n",
+        "            elif j == len1+5:\n",
+        "                X_test3[count_i,j,:] = average_vector3\n",
+        "            elif j == len1+6:\n",
+        "                X_test3[count_i,j,:] = final_pos_tags_data[i]\n",
+        "            elif j == len1+7:\n",
+        "                X_test3[count_i,j,:] = final_pos_data[i]\n",
+        "            elif j == len1+8:\n",
+        "                X_test3[count_i,j,:] = final_tokens_data[i]\n",
+        "            elif j == len1+9:\n",
+        "                X_test3[count_i,j,:] = final_dep_data[i]\n",
+        "            else:\n",
+        "                X_test3[count_i,j,:] = empty_word\n",
+        "\n",
+        "        Y_test3[count_i,:] = dummy_y[i]\n",
+        "        count_i += 1\n",
+        "\n",
+        "    return X_train3, X_test3, Y_train3, Y_test3"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_ZQ6S5IhtB_8"
+      },
+      "outputs": [],
+      "source": [
+        "model = Sequential()\n",
+        "model.add(Conv1D(filters=128, kernel_size=9, padding='same', activation='relu', input_shape=(max(maxlength)+10,512)))\n",
+        "model.add(Dropout(0.25))\n",
+        "model.add(MaxPooling1D(pool_size=2))\n",
+        "model.add(Dropout(0.25))\n",
+        "model.add(Conv1D(filters=128, kernel_size=7, padding='same', activation='relu'))\n",
+        "model.add(Dropout(0.25))\n",
+        "model.add(MaxPooling1D(pool_size=2))\n",
+        "model.add(Dropout(0.25))\n",
+        "model.add(Conv1D(filters=128, kernel_size=5, padding='same', activation='relu'))\n",
+        "model.add(Dropout(0.25))\n",
+        "model.add(Bidirectional(LSTM(50, dropout=0.25, recurrent_dropout=0.2)))\n",
+        "model.add(Dense(4, activation='softmax'))\n",
+        "model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['accuracy'])\n",
+        "print(model.summary())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "G2XuZvBOtDMs"
+      },
+      "outputs": [],
+      "source": [
+        "from sklearn.metrics import accuracy_score\n",
+        "from keras.callbacks import ModelCheckpoint\n",
+        "\n",
+        "final_accuracies = []\n",
+        "\n",
+        "filename = 'weights.best.from_scratch%s.hdf5' % 9\n",
+        "checkpointer = ModelCheckpoint(filepath=filename, verbose=1, save_best_only=True)\n",
+        "X_train3, X_test3, Y_train3, Y_test3 = extractTrainingAndTestingData(9)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kGGf09dktEmS"
+      },
+      "outputs": [],
+      "source": [
+        "history = model.fit(X_train3, Y_train3, epochs=15, batch_size=512, callbacks=[checkpointer], validation_data=(X_test3, Y_test3), verbose=1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ug5x2h7xtGNb"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(X_test3, Y_test3)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 207,
+      "metadata": {
+        "id": "UVKUsrk0tHil"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib.pyplot as plt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BPcCy47XtKcL"
+      },
+      "outputs": [],
+      "source": [
+        "model.load_weights(filename)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "csp-z21UtLUT"
+      },
+      "outputs": [],
+      "source": [
+        "for i in range(10):\n",
+        "    filename = 'weights.best.from_scratch%s.hdf5' % i\n",
+        "    checkpointer = ModelCheckpoint(filepath=filename, verbose=1, save_best_only=True)\n",
+        "    X_train3, X_test3, Y_train3, Y_test3 = extractTrainingAndTestingData(i)\n",
+        "    model.fit(X_train3, Y_train3, epochs=10, batch_size=512, callbacks=[checkpointer], validation_data=(X_test3, Y_test3))\n",
+        "    model.load_weights(filename)\n",
+        "    predicted = np.rint(model.predict(X_test3))\n",
+        "    final_accuracies.append(accuracy_score(Y_test3, predicted))\n",
+        "    print(accuracy_score(Y_test3, predicted))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "aKhWVJjFLE_9",
+        "outputId": "0a8053a8-3bae-46b1-b154-31fd2030465b"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "380"
+            ]
+          },
+          "execution_count": 162,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "len(X_test3[0])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "73d3h_lhL5r8",
+        "outputId": "8da114d8-2901-49b2-fbcd-90b821b392ad"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "160"
+            ]
+          },
+          "execution_count": 161,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "len(Y_test3)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "EGeLg-HXtMlu",
+        "outputId": "c41250cd-b6e6-4b92-a775-d010fbdc803a"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "0.8875\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(sum(final_accuracies) / len(final_accuracies))"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

vercel.json

@@ -0,0 +1,5 @@
+{
+    "rewrites": [
+      { "source": "/(.*)", "destination": "/api/app" }
+    ]
+  }

weights.best.from_scratch1 (1).hdf5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f84bea0167801a32295fa5321735e25b05f9b69b4c440dcdfe5f48da0db08b61
+size 10380808