File size: 2,032 Bytes
696a732 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
# downloading the stopwords from nltk
nltk.download('stopwords')
# donwloading the resource word_tokenize
nltk.download('punkt_tab')
# donwloading the resource wordnet
nltk.download('wordnet')
# setting the stopwords dictionary to english language
stop_words_en = set(stopwords.words('english'))
# Define additional stopwords
additional_exclusions = {'persona', 'team', 'minecraft', 'arma', 'bean', 'boss','ps4','wild','moon','2d','wasteland','ultra','military','editor','allows','solid','journey','expert','bungie','season','terraria','minor','hunting','kojima','deck','destiny','remaster','complain','fall','slay','gta','atlus','spire','terrarium','rockstar','biome','mafia','castle',
'access','instead','idea','sorry','ca','dlc','dayz','until','valve','tutorial','concept','swing','horde','regionlockchina','unless','sims','ark','was','company','customer','state','software','advertised','region','survivor','2016','fighter','headshot','zombie','alpha','trailer','planet','pubg','microtransactions','payday','too','camping','ubisoft','galaxy','rome','skyrim','vr','h1z1','vegas','warband','flight','creation','forum','netcode','sky','elite','china','chinese','hero','policy','pile','untill','zombies','xcom','rust','capcom'}
stop_words_en = stop_words_en.union(additional_exclusions)
def filter_review(reviews):
#Tokenize the text
tokens = word_tokenize(reviews.lower())
#Remove stop words
filtered_tokens = [token for token in tokens if token not in stop_words_en]
#Lemmatize the tokens (to mantain the context)
lemmatizer = WordNetLemmatizer()
lemmatized_tokens = [lemmatizer.lemmatize(token) for token in filtered_tokens]
# Remove extra words after lemmatizer
final_tokens = [token for token in lemmatized_tokens if token not in stop_words_en]
#Join the tokens back into a string
processed_text = ' '.join(final_tokens)
return processed_text |