Fix NLTK data installation and path handling

Dockerfile

@@ -48,16 +48,25 @@ RUN mkdir -p /usr/local/share/nltk_data \
 # Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Download NLTK data as root
+# Install system dependencies for NLTK
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Download and install NLTK data as root
 RUN python -c "import nltk; nltk.download('punkt', download_dir='/usr/local/share/nltk_data')" \
     && python -c "import nltk; nltk.download('stopwords', download_dir='/usr/local/share/nltk_data')" \
     && python -c "import nltk; nltk.download('wordnet', download_dir='/usr/local/share/nltk_data')" \
     && python -c "import nltk; nltk.download('averaged_perceptron_tagger', download_dir='/usr/local/share/nltk_data')" \
+    && python -c "import nltk; nltk.download('punkt_tab', download_dir='/usr/local/share/nltk_data')" \
     && chmod -R 755 /usr/local/share/nltk_data
 
 # Set NLTK_DATA environment variable
 ENV NLTK_DATA=/usr/local/share/nltk_data
 
+# Verify NLTK data is accessible
+RUN python -c "import nltk; nltk.data.path.append('/usr/local/share/nltk_data'); nltk.data.find('tokenizers/punkt')"
+
 # Switch to non-root user
 USER user
 

app.py

@@ -66,18 +66,35 @@ try:
     from summarizer import document_summarizer
     from vector_store import vector_store
     
-    # Initialize NLTK data
+        # Initialize NLTK data
     import nltk
     
     # Set NLTK data path - system path first, then user path
-    nltk.data.path = [str(NLTK_DATA_DIR)] + nltk.data.path
+    nltk_data_paths = [
+        str(NLTK_DATA_DIR),
+        '/usr/local/share/nltk_data',
+        '/usr/share/nltk_data',
+        '/usr/local/nltk_data',
+        '/usr/local/share/nltk_data',
+        '/usr/local/lib/nltk_data',
+        '/usr/share/nltk_data',
+        '/usr/local/share/nltk_data',
+        '/usr/lib/nltk_data',
+        '/usr/local/lib/nltk_data',
+        '/root/nltk_data',
+        '/home/user/nltk_data'
+    ]
+    
+    # Add all possible NLTK data paths
+    nltk.data.path = list(dict.fromkeys(nltk_data_paths + nltk.data.path))
     
     # Verify NLTK data is available
     required_nltk_data = [
         'tokenizers/punkt',
         'corpora/stopwords',
         'corpora/wordnet',
-        'taggers/averaged_perceptron_tagger'
+        'taggers/averaged_perceptron_tagger',
+        'tokenizers/punkt_tab/english'
     ]
 
     for resource in required_nltk_data:
@@ -86,7 +103,16 @@ try:
             logger.info(f"NLTK resource found: {resource}")
         except LookupError as e:
             logger.warning(f"NLTK resource not found: {resource}")
-            # Don't try to download at runtime - should be handled in Dockerfile
+            # Try to download the resource if not found
+            try:
+                resource_name = resource.split('/')[-1].split('.')[0]
+                logger.info(f"Attempting to download NLTK resource: {resource_name}")
+                nltk.download(resource_name, download_dir=str(NLTK_DATA_DIR))
+                nltk.data.path.append(str(NLTK_DATA_DIR))
+                nltk.data.find(resource)  # Try to find it again after download
+                logger.info(f"Successfully downloaded NLTK resource: {resource}")
+            except Exception as download_error:
+                logger.error(f"Failed to download NLTK resource {resource}: {str(download_error)}")
 
 except ImportError as e:
     logger.error(f"Failed to import required modules: {e}")