ctr-ll4 / src /utils /nlp.py
sanjin7's picture
Upload src/ with huggingface_hub
cea4a4b
raw
history blame contribute delete
522 Bytes
from langdetect import detect
from loguru import logger
from nltk.sentiment import SentimentIntensityAnalyzer
from textblob import TextBlob
def detect_language(text: str) -> None | str:
try:
return detect(text)
except Exception:
logger.debug(f"No language features: {text}")
return None
sia = SentimentIntensityAnalyzer()
def get_sentiment(text: str) -> dict:
sentiment = sia.polarity_scores(text)
sentiment["subjectivity"] = TextBlob(text).subjectivity
return sentiment