feat:added main files

app.py

@@ -0,0 +1,30 @@
+import streamlit as st
+import pandas as pd
+from preprocess_data import preprocess_text,get_stopwords
+from datasets import load_dataset
+
+dataset = load_dataset('danielcd99/imdb')
+
+dataframes = {}
+for split in dataset.keys():
+    # Convert the dataset split to a pandas DataFrame
+    df = dataset[split].to_pandas()
+    dataframes[split] = df
+
+TITLE_TEXT = f"IMDB reviews"
+DESCRIPTION_TEXT = f"Esta é uma aplicação para o trabalho de NLP. Utilizamos a base de dados de reviews do IMDb com 50.000 comentários entre positivos e negativos (a base está balanceada). Por meio desta interface é possível visualizar como os exemplos da nossa base de teste foram classificados com um BERT treinado para esta task."
+
+st.title(TITLE_TEXT)
+st.write(DESCRIPTION_TEXT)
+
+if st.button('Encontre exemplos!'):
+    df = df.sample(5)
+    get_stopwords()
+    df['preprocessed_review'] = df['review'].copy()
+    df['preprocessed_review'] = df['preprocessed_review'].apply(preprocess_text)
+    cols = ['review','preprocessed_review','sentiment']    
+
+
+
+    st.table(df[cols])
+

preprocess_data.py

@@ -0,0 +1,81 @@
+import re
+import nltk
+from nltk.corpus import stopwords
+from nltk.stem import PorterStemmer
+
+
+def lowercase_text(text):
+    return text.lower()
+
+def remove_html(text):
+    return re.sub(r'<[^<]+?>', '', text)
+
+def remove_url(text):
+    return re.sub(r'http[s]?://\S+|www\.\S+', '', text)
+
+def remove_punctuations(text):
+    tokens_list = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
+    for char in text:
+        if char in tokens_list:
+            text = text.replace(char, ' ')
+
+    return text
+
+def remove_emojis(text):
+    emojis = re.compile("["
+                        u"\U0001F600-\U0001F64F"
+                        u"\U0001F300-\U0001F5FF"
+                        u"\U0001F680-\U0001F6FF"
+                        u"\U0001F1E0-\U0001F1FF"
+                        u"\U00002500-\U00002BEF"
+                        u"\U00002702-\U000027B0"
+                        u"\U00002702-\U000027B0"
+                        u"\U000024C2-\U0001F251"
+                        u"\U0001f926-\U0001f937"
+                        u"\U00010000-\U0010ffff"
+                        u"\u2640-\u2642"
+                        u"\u2600-\u2B55"
+                        u"\u200d"
+                        u"\u23cf"
+                        u"\u23e9"
+                        u"\u231a"
+                        u"\ufe0f"
+                        u"\u3030"
+                        "]+", re.UNICODE)
+
+    text = re.sub(emojis, '', text)
+    return text
+
+def remove_stop_words(text):
+    stop_words = stopwords.words('english')
+    new_text = ''
+    for word in text.split():
+        if word not in stop_words:
+            new_text += ''.join(f'{word} ')
+
+    return new_text.strip()
+
+def stem_words(text):
+    stemmer = PorterStemmer()
+    new_text = ''
+    for word in text.split():
+        new_text += ''.join(f'{stemmer.stem(word)} ')
+
+    return new_text
+
+def get_stopwords():
+    nltk.download('stopwords')
+
+def preprocess_text(text):
+    text = lowercase_text(text)
+    text = remove_html(text)
+    text = remove_url(text)
+    text = remove_punctuations(text)
+    text = remove_emojis(text)
+    text = remove_stop_words(text)
+    text = stem_words(text)
+
+    return text
+
+if __name__ == "__main__":
+    pass