File size: 2,207 Bytes
230defe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | import joblib
import re
import os
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer, WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch
nltk.download('punkt_tab')
nltk.download('stopwords')
nltk.download('wordnet')
# Get the directory of this file
current_dir = os.path.dirname(os.path.abspath(__file__))
# Load the trained model and vectorizer
model_path = os.path.join(current_dir, "model (2).pkl")
vectorizer_path = os.path.join(current_dir, "tokenizer (2).pkl")
model = joblib.load(model_path)
vectorizer = joblib.load(vectorizer_path)
# Preprocessing function
def preprocess_text(text, use_stemming=False, use_lemmatization=True):
text = text.lower()
text = re.sub(r'\W', ' ', text)
words = word_tokenize(text)
stop_words = set(stopwords.words('english'))
stop_words.discard('not') # Keep 'not' for sentiment analysis
words = [word for word in words if word not in stop_words]
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()
if use_stemming:
words = [stemmer.stem(word) for word in words]
elif use_lemmatization:
words = [lemmatizer.lemmatize(word) for word in words]
return " ".join(words)
# Prediction function
def predict_sentiment(analyser):
"""Predicts sentiment using the trained BERT model."""
processed_text = preprocess_text(analyser.sentence) # ✅ Preprocess the text
# ✅ Tokenize input text (Replacing vectorizer.transform)
inputs = vectorizer(processed_text, truncation=True, padding="max_length", max_length=256, return_tensors="pt")
# ✅ Move inputs to the correct device
#inputs = {key: val.to(device) for key, val in inputs.items()}
# ✅ Get model prediction
with torch.no_grad():
outputs = model(**inputs)
prediction = torch.argmax(outputs.logits, dim=1).item()
# ✅ Convert prediction to sentiment label
sentiment_labels = ["Negative", "Neutral", "Positive"]
return sentiment_labels[prediction] |