import re
from autocorrect import Speller
import nltk
from nltk.corpus import stopwords
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk import word_tokenize
import string
import nltk
import os
import nltk
nltk_data_dir = os.path.expanduser("~/app/nltk_data")
os.makedirs(nltk_data_dir, exist_ok=True)
nltk.data.path.append(nltk_data_dir)
nltk.download("stopwords", download_dir=nltk_data_dir)
nltk.download("punkt_tab", download_dir=nltk_data_dir)
nltk.download("punkt", download_dir=nltk_data_dir)
nltk.download("wordnet", download_dir=nltk_data_dir)


def remove_html_tags(text):
    html_pattern = r'<.*?>'
    without_html = re.sub(pattern=html_pattern, repl=' ', string=text)
    return without_html

def convert_to_lower(text):
    return text.lower()

def remove_urls(text):
    url_pattern = r'https?://\S+|www\.\S+'
    without_urls = re.sub(pattern=url_pattern, repl=' ', string=text)
    return without_urls

def spell_checker(text):
    spellChecker = Speller(lang="en")
    correct_words = []
    for word in nltk.word_tokenize(text):
        correct_word = spellChecker(word)
        correct_words.append(correct_word)
    correct_spell_text = " ".join(correct_words)
    return correct_spell_text


def remove_punctuation(text):
    return text.translate(str.maketrans('', '', string.punctuation))


def remove_stopwords(text):
    removed = []
    stop_words = list(stopwords.words("english"))
    tokens = word_tokenize(text)
    for i in range(len(tokens)):
        if tokens[i] not in stop_words:
            removed.append(tokens[i])
    return " ".join(removed)


def lemmatizing(text):
    lemmatizer = WordNetLemmatizer()
    tokens = word_tokenize(text)
    for i in range(len(tokens)):
        lemma_word = lemmatizer.lemmatize(tokens[i])
        tokens[i] = lemma_word
    return " ".join(tokens)


def clean(text):
    cleaned_text=convert_to_lower(text)
    cleaned_text=remove_html_tags(cleaned_text)
    cleaned_text=remove_urls(cleaned_text)
    cleaned_text=remove_punctuation(cleaned_text)
    cleaned_text=remove_stopwords(cleaned_text)
    cleaned_text=lemmatizing(cleaned_text)
    cleaned_text= spell_checker(cleaned_text)
    return  cleaned_text