Update app.py

app.py

@@ -3,10 +3,8 @@ import numpy as np
 import re
 import json
 
-
 import nltk
 from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
 from nltk.stem import WordNetLemmatizer
 
 from datasets import load_dataset
@@ -36,11 +34,6 @@ def download_nltk_data():
         stopwords.words('english')
     except LookupError:
         nltk.download('stopwords')
-    # try:
-    #     word_tokenize("test")
-    # except LookupError:
-    #     nltk.download('punkt', force=True)
-
     try:
         WordNetLemmatizer().lemmatize("test")
     except LookupError:
@@ -53,6 +46,10 @@ def clean_text(text):
     text = re.sub(r'[^\w\s]', '', text)
     return text
 
+def simple_tokenize(text):
+    """Tokenizes text using regex (splits on word boundaries, avoids NLTK punkt)."""
+    return re.findall(r'\b\w+\b', text)
+
 def process_tokens(tokens, stop_words, lemmatizer):
     """Removes stopwords and performs lemmatization on a list of tokens."""
     tokens = [word for word in tokens if word not in stop_words]
@@ -113,7 +110,7 @@ def engineer_features(df):
     df['title'] = df['title'].fillna('No Title')
     df['text'] = df['title'] + ' ' + df['Description']
     df['text'] = df['text'].apply(clean_text)
-    df['tokens'] = df['text'].apply(word_tokenize)
+    df['tokens'] = df['text'].apply(simple_tokenize)
     df['tokens'] = df['tokens'].apply(lambda x: process_tokens(x, stop_words, lemmatizer))
     df['processed_text'] = df['tokens'].apply(lambda x: ' '.join(x))