Fix NLTK data download and path issues

Dockerfile

@@ -41,6 +41,13 @@ COPY --chown=user:user requirements.txt .
 # Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
+# Download NLTK data during build
+RUN python -c "import nltk; nltk.download('punkt', download_dir='/home/user/app/nltk_data')"
+RUN python -c "import nltk; nltk.download('stopwords', download_dir='/home/user/app/nltk_data')"
+
+# Set NLTK_DATA environment variable
+ENV NLTK_DATA=/home/user/app/nltk_data
+
 # Copy application files
 COPY --chown=user:user . .
 

app.py

@@ -78,16 +78,52 @@ try:
         logger.info("NLTK data verified successfully")
     except LookupError as e:
         logger.warning(f"NLTK data missing: {e}")
-        # Attempt to download missing data
         try:
             nltk.download('punkt', download_dir=str(NLTK_DATA_DIR))
+            nltk.data.path.append(str(NLTK_DATA_DIR))
+            logger.info("Successfully downloaded NLTK punkt data")
+        except Exception as download_error:
+            logger.error(f"Failed to download NLTK data: {download_error}")
+            # Continue without NLTK data if download fails
+
+    try:
+        nltk.data.find('corpora/stopwords')
+        logger.info("NLTK stopwords is available")
+    except LookupError as e:
+        logger.warning(f"NLTK data missing: {e}")
+        try:
             nltk.download('stopwords', download_dir=str(NLTK_DATA_DIR))
+            nltk.data.path.append(str(NLTK_DATA_DIR))
+            logger.info("Successfully downloaded NLTK stopwords data")
+        except Exception as download_error:
+            logger.error(f"Failed to download NLTK data: {download_error}")
+            # Continue without NLTK data if download fails
+
+    try:
+        nltk.data.find('corpora/wordnet')
+        logger.info("NLTK wordnet is available")
+    except LookupError as e:
+        logger.warning(f"NLTK data missing: {e}")
+        try:
             nltk.download('wordnet', download_dir=str(NLTK_DATA_DIR))
+            nltk.data.path.append(str(NLTK_DATA_DIR))
+            logger.info("Successfully downloaded NLTK wordnet data")
+        except Exception as download_error:
+            logger.error(f"Failed to download NLTK data: {download_error}")
+            # Continue without NLTK data if download fails
+
+    try:
+        nltk.data.find('taggers/averaged_perceptron_tagger')
+        logger.info("NLTK averaged_perceptron_tagger is available")
+    except LookupError as e:
+        logger.warning(f"NLTK data missing: {e}")
+        try:
             nltk.download('averaged_perceptron_tagger', download_dir=str(NLTK_DATA_DIR))
-            logger.info("Successfully downloaded NLTK data")
+            nltk.data.path.append(str(NLTK_DATA_DIR))
+            logger.info("Successfully downloaded NLTK averaged_perceptron_tagger data")
         except Exception as download_error:
             logger.error(f"Failed to download NLTK data: {download_error}")
-            raise
+            # Continue without NLTK data if download fails
 
 except ImportError as e:
     logger.error(f"Failed to import required modules: {e}")