init.py

import pandas as pd
import numpy as np
from string import ascii_lowercase
from gensim.models import Doc2Vec
from gensim.models import doc2vec
from gensim.models import KeyedVectors
import snowballstemmer, re
import requests
from bs4 import BeautifulSoup
import re, sys
from tensorflow.keras.models import load_model
import joblib

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36, Opera/9.80 (Windows NT 6.1; WOW64) Presto/2.12.388 Version/12.18'
}

def getsoup(url):
    response = requests.get(url, headers=headers)
    Status_Code = response.status_code
    print(url)
    print(Status_Code)
    
    if Status_Code == 200:
      soup = BeautifulSoup(response.content, features="lxml")
    else:
      soup = getsoup(url)
    return soup 

#Get Last Page number
def getLastPageNumber(soup, site):
    pageNumber = []
    if site == 'flipkart':
        review_number = int(soup.find("span", "_2_R_DZ").text.strip().replace(',', '').split()[-2])
        if review_number <=10:
            lastPage = 1
        else:
            link = soup.find(attrs={"class": "_2MImiq _1Qnn1K"})
            pageNumber = link.find('span').text.strip().replace(',', '').split()
            lastPage1 = pageNumber[len(pageNumber)-1]
            lastPage = int(lastPage1)
    elif site == 'amazon':
        review_number = int(soup.find("div", {"data-hook": "cr-filter-info-review-rating-count"}).text.strip().replace(',', '').split()[-3])
        if review_number <=10:
            lastPage = 1
        else:
            lastPage = review_number // 10
    if lastPage > 500:
        lastPage = 2
    return lastPage


def geturllist(url, lastPage):
    urllistPages = []
    url = url[:-1]
    for i in range(1,lastPage+1):
      urllistPages.append (url + str(i))
    return urllistPages


def getReviews(soup, site, url):
    if site == 'flipkart':
        #Extracting the Titles
        title_sec = soup.find_all("p",'_2-N8zT')
        title = []
        for s in title_sec:
            title.append(s.text)

        #Extracting the Author names
        author_sec = soup.find_all("p","_2sc7ZR _2V5EHH")
        author = []
        for r in author_sec:
            author.append(r.text)

        #Extracting the Text
        Review_text_sec = soup.find_all("div",'t-ZTKy')
        text = []
        for t in Review_text_sec:
            text.append(t.text)
            
        #Extracting the Star rating  
        Rating = soup.find_all("div", {"class": ["_3LWZlK _1BLPMq", "_3LWZlK _32lA32 _1BLPMq", "_3LWZlK _1rdVr6 _1BLPMq"]})    
        rate = []
        for d in Rating:
            rate.append(d.text)

        #Extracting the Date
        Date_sec = soup.find_all(lambda tag: tag.name == 'p' and tag.get('class') == ['_2sc7ZR'])    
        date = []
        for d in Date_sec:
          date.append(d.text)

        #Extracting the Helpful rating
        help_sec = soup.find_all(lambda tag: tag.name == 'div' and tag.get('class') == ['_1LmwT9'])    
        help1 = []
        for d in help_sec:
          help1.append(d.text)
    
    elif site == 'amazon':
        n_ = 0
        title_sec = soup.find_all(attrs={"data-hook": "review-title", "class": "a-size-base a-link-normal review-title a-color-base review-title-content a-text-bold"})
        title = []
        for s in title_sec:
            title.append(s.text.replace('\n', ''))
        n_ = len(title)

        author_sec = soup.find_all(attrs = {"class": "a-profile-name"})
        author = []
        for r in author_sec:
            author.append(r.text)
        while(1):
            if len(author) > n_:
                author.pop(0)
            else:
                break

        Review_text_sec = soup.find_all(attrs={"data-hook": "review-body", "class": "a-size-base review-text review-text-content"})
        text = []
        for t in Review_text_sec:
            text.append(t.text.replace('\n', ''))

        Rating = soup.find_all(attrs={"data-hook": "review-star-rating"})    
        rate = []
        for d in Rating:
            rate.append(d.text)

        Date_sec = soup.find_all(attrs={"data-hook": "review-date"})    
        date = []
        for d in Date_sec:
          date.append(d.text) 

        help_sec = soup.find_all(attrs={"data-hook": "helpful-vote-statement"})    
        help1 = []
        for d in help_sec:
             help1.append(d.text.replace('\n          ', '')) 
        while(1):
            if len(help1) < n_:
                help1.append(0)
            else:
                break
        
    url1 = []
    url1 = [url] * len(date)

    collate = {'Date': date, 'URL': url1, 'Review_Title': title, 'Author': author, 'Rating': rate, 'Review_text': text, 'Review_helpful': help1}          
    collate_df = pd.DataFrame.from_dict(collate)
    return collate_df

def scraper(url):
    df2 = []      
    soup = getsoup(url)
    site = url.split('.')[1]
    if site == 'flipkart':
        url = url + '&page=1'
    elif site == 'amazon':
        url = url + '&pageNumber=1'
    product = url.split('/')[3]
    lastPage = 1
    urllistPages = geturllist(url, lastPage)
    x = 1
    for url in urllistPages:
        soup = getsoup(url)
        df1 = getReviews(soup, site, url)         
        if x == 1:
            df3 = []
            df3 = df1
        else:                        
            df2 = df3
            result = df2.append(df1, ignore_index=True)
            df3 = result
        x += 1
    print(list(df3['Review_text']))
    return list(df3['Review_text'])

arr = scraper('https://www.amazon.in/Redmi-inches-Ready-Smart-L32R8-FVIN/product-review/B0BVMLNGXR/ref=lp_90117314031_1_1?pf_rd_p=9e034799-55e2-4ab2-b0d0-eb42f95b2d05&pf_rd_r=V81TJ2VTRM0BYHQ6XX8S&sbo=RZvfv%2F%2FHxDF%2BO5021pAnSA%3D%3D&th=1')

TaggedDocument = doc2vec.TaggedDocument

loaded_model = load_model('weights.best.from_scratch1 (1).hdf5')

def preprocess_text(text):
    stemmer = snowballstemmer.EnglishStemmer()
    text = " ".join(stemmer.stemWords(re.sub('[!"#%\'()*+,-./:;<=>?@[\\]^_`{|}~1234567890’”“′‘\\\\]', ' ', text).split(' ')))

    stop_words = set(["may", "also", "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "across", "among", "beside", "however", "yet", "within"] + list(ascii_lowercase))
    stop_list = stemmer.stemWords(stop_words)
    stop_words.update(stop_list)
    text = " ".join(filter(None, filter(lambda word: word not in stop_words, text.lower().split(' '))))

    return text.split(' ')


# arr = ['This is not a nice product', 'This is brilliant product', "I have had this tv for a year and a half. The tv worked smoothly and recently, it's display stopped working. I have contacted the company and they assured a repair within 8-10 days replacing the display panel. Then came the worst after sale experience I had ever. They didn't respond to my calls and whenever, I raised a grievance, I was just given different dates and extensions.", ''' In monotheistic belief systems, God is usually viewed as the supreme being, creator, and principal object of faith. In polytheistic belief systems, a god is "a spirit or being believed to have created, or for controlling some part of the universe or life, for which such a deity is often worshipped ''']

preprocessed_arr = [preprocess_text(x) for x in arr]

doc2vec_model = Doc2Vec.load("doc2vec_model_opinion_corpus (1).d2v")

def vectorize_comments_(df, d2v_model):
    y = []
    comments = []
    for i in range(0, len(df)):
        print(i)
        label = 'SENT_%s' %i
        comments.append(d2v_model.docvecs[label])

    return comments

textData = vectorize_comments_(preprocessed_arr, doc2vec_model)

import numpy as np

textData_array = np.array(textData)

num_vectors = textData_array.shape[0]
textData_3d = textData_array.reshape((num_vectors, 1, -1))

new_shape = (textData_array.shape[0], 380, 512)

X_test3_reshaped = np.zeros(new_shape, dtype=textData_3d.dtype)
X_test3_reshaped[:, :textData_3d.shape[1], :textData_3d.shape[2]] = textData_3d

predictions = np.rint(loaded_model.predict(X_test3_reshaped))

argMax = []

for i in predictions:
    argMax.append(np.argmax(i))

def returnRequirements(comments, preds):
    arr = []
    for i, j in enumerate(preds):
        if j == 3 or j == 0:
            arr.append(comments[i])
    return arr

# 3 & 0 is deceptive rest aren't
print(argMax)
print(returnRequirements(arr, argMax))