Fix NLTK data handling and Dockerfile configuration

Dockerfile

@@ -38,15 +38,26 @@ RUN useradd -m -u 1000 user && \
 # Copy requirements first to leverage Docker cache
 COPY --chown=user:user requirements.txt .
 
-# Install Python dependencies
-RUN pip install --no-cache-dir -r requirements.txt
+# Install Python dependencies and NLTK data as root
+USER root
 
-# Download NLTK data during build
-RUN python -c "import nltk; nltk.download('punkt', download_dir='/home/user/app/nltk_data')"
-RUN python -c "import nltk; nltk.download('stopwords', download_dir='/home/user/app/nltk_data')"
+# Create NLTK data directory with proper permissions
+RUN mkdir -p /usr/share/nltk_data/tokenizers \
+    && chmod -R 777 /usr/share/nltk_data
+
+# Install NLTK and download data
+RUN pip install --no-cache-dir -r requirements.txt \
+    && python -c "import nltk; nltk.download('punkt', download_dir='/usr/share/nltk_data')" \
+    && python -c "import nltk; nltk.download('stopwords', download_dir='/usr/share/nltk_data')" \
+    && python -c "import nltk; nltk.download('wordnet', download_dir='/usr/share/nltk_data')" \
+    && python -c "import nltk; nltk.download('averaged_perceptron_tagger', download_dir='/usr/share/nltk_data')" \
+    && chmod -R 755 /usr/share/nltk_data
 
 # Set NLTK_DATA environment variable
-ENV NLTK_DATA=/home/user/app/nltk_data
+ENV NLTK_DATA=/usr/share/nltk_data
+
+# Switch back to non-root user
+USER user
 
 # Copy application files
 COPY --chown=user:user . .

app.py

@@ -42,7 +42,8 @@ app.add_middleware(
 BASE_DIR = Path("/home/user/app/data")
 UPLOAD_DIR = BASE_DIR / "uploads"
 PROCESSED_DIR = BASE_DIR / "processed"
-NLTK_DATA_DIR = Path(os.getenv("NLTK_DATA", "/app/nltk_data"))
+# Use system NLTK data directory that we'll populate in the Dockerfile
+NLTK_DATA_DIR = Path("/usr/share/nltk_data")
 
 # Ensure directories exist with proper permissions
 for directory in [BASE_DIR, UPLOAD_DIR, PROCESSED_DIR]:
@@ -67,63 +68,25 @@ try:
     
     # Initialize NLTK data
     import nltk
-    nltk.data.path.append(str(NLTK_DATA_DIR))
+    
+    # Set NLTK data path - system path first, then user path
+    nltk.data.path = [str(NLTK_DATA_DIR)] + nltk.data.path
     
     # Verify NLTK data is available
-    try:
-        nltk.data.find('tokenizers/punkt')
-        nltk.data.find('corpora/stopwords')
-        nltk.data.find('corpora/wordnet')
-        nltk.data.find('taggers/averaged_perceptron_tagger')
-        logger.info("NLTK data verified successfully")
-    except LookupError as e:
-        logger.warning(f"NLTK data missing: {e}")
-        try:
-            nltk.download('punkt', download_dir=str(NLTK_DATA_DIR))
-            nltk.data.path.append(str(NLTK_DATA_DIR))
-            logger.info("Successfully downloaded NLTK punkt data")
-        except Exception as download_error:
-            logger.error(f"Failed to download NLTK data: {download_error}")
-            # Continue without NLTK data if download fails
-
-    try:
-        nltk.data.find('corpora/stopwords')
-        logger.info("NLTK stopwords is available")
-    except LookupError as e:
-        logger.warning(f"NLTK data missing: {e}")
-        try:
-            nltk.download('stopwords', download_dir=str(NLTK_DATA_DIR))
-            nltk.data.path.append(str(NLTK_DATA_DIR))
-            logger.info("Successfully downloaded NLTK stopwords data")
-        except Exception as download_error:
-            logger.error(f"Failed to download NLTK data: {download_error}")
-            # Continue without NLTK data if download fails
-
-    try:
-        nltk.data.find('corpora/wordnet')
-        logger.info("NLTK wordnet is available")
-    except LookupError as e:
-        logger.warning(f"NLTK data missing: {e}")
-        try:
-            nltk.download('wordnet', download_dir=str(NLTK_DATA_DIR))
-            nltk.data.path.append(str(NLTK_DATA_DIR))
-            logger.info("Successfully downloaded NLTK wordnet data")
-        except Exception as download_error:
-            logger.error(f"Failed to download NLTK data: {download_error}")
-            # Continue without NLTK data if download fails
+    required_nltk_data = [
+        'tokenizers/punkt',
+        'corpora/stopwords',
+        'corpora/wordnet',
+        'taggers/averaged_perceptron_tagger'
+    ]
 
-    try:
-        nltk.data.find('taggers/averaged_perceptron_tagger')
-        logger.info("NLTK averaged_perceptron_tagger is available")
-    except LookupError as e:
-        logger.warning(f"NLTK data missing: {e}")
+    for resource in required_nltk_data:
         try:
-            nltk.download('averaged_perceptron_tagger', download_dir=str(NLTK_DATA_DIR))
-            nltk.data.path.append(str(NLTK_DATA_DIR))
-            logger.info("Successfully downloaded NLTK averaged_perceptron_tagger data")
-        except Exception as download_error:
-            logger.error(f"Failed to download NLTK data: {download_error}")
-            # Continue without NLTK data if download fails
+            nltk.data.find(resource)
+            logger.info(f"NLTK resource found: {resource}")
+        except LookupError as e:
+            logger.warning(f"NLTK resource not found: {resource}")
+            # Don't try to download at runtime - should be handled in Dockerfile
 
 except ImportError as e:
     logger.error(f"Failed to import required modules: {e}")