Update app.py

app.py

@@ -3,30 +3,44 @@ import pandas as pd
 import numpy as np
 import re
 import nltk
+import os
 from nltk.corpus import stopwords
 from nltk.stem import WordNetLemmatizer
-from nltk.tokenize import word_tokenize, sent_tokenize
+from nltk.tokenize import word_tokenize
 from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
 from wordcloud import WordCloud
 import matplotlib.pyplot as plt
 import io
 
-
-# Download all required NLTK resources with error handling
-try:
-    nltk.data.find('tokenizers/punkt')
-except LookupError:
-    nltk.download('punkt')
-    
-try:
-    nltk.data.find('corpora/stopwords')
-except LookupError:
-    nltk.download('stopwords')
+@st.cache_resource
+def setup_nltk():
+    # Set NLTK data path
+    nltk_data_path = os.path.join(os.getcwd(), 'nltk_data')
+    os.makedirs(nltk_data_path, exist_ok=True)
+    nltk.data.path.append(nltk_data_path)
     
-try:
-    nltk.data.find('corpora/wordnet')
-except LookupError:
-    nltk.download('wordnet')
+    # Download required resources with retries
+    required = ['punkt', 'stopwords', 'wordnet', 'omw-1.4']
+    for resource in required:
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                nltk.data.find(f'tokenizers/punkt/PY3/english.pickle')
+                break
+            except LookupError:
+                try:
+                    nltk.download(resource, download_dir=nltk_data_path)
+                    # Special handling for punkt
+                    if resource == 'punkt':
+                        nltk.download('punkt_tab', download_dir=nltk_data_path)
+                except Exception as e:
+                    if attempt == max_retries - 1:
+                        st.error(f"Failed to download NLTK resource {resource} after {max_retries} attempts")
+                        raise
+                    continue
+
+# Run initialization before anything else
+setup_nltk()
 
 # Initialize lemmatizer
 lemmatizer = WordNetLemmatizer()