Add NLTK package

app/search/bm25_search.py

@@ -6,16 +6,21 @@ import string
 from typing import List, Set, Optional
 from nltk.corpus import stopwords
 from nltk.stem import WordNetLemmatizer
-
+import os
 
 def download_nltk_resources():
     """
     Downloads required NLTK resources synchronously.
     """
     resources = ['punkt', 'stopwords', 'wordnet', 'omw-1.4']
+
+    nltk_data_path = "/tmp/nltk_data"  # Temporary directory for Hugging Face Spaces
+    os.makedirs(nltk_data_path, exist_ok=True)
+    nltk.data.path.append(nltk_data_path)
+          
     for resource in resources:
         try:
-            nltk.download(resource, quiet=True)
+            nltk.download(resource, download_dir=nltk_data_path, quiet=True)
         except Exception as e:
             print(f"Error downloading {resource}: {str(e)}")
 
@@ -33,6 +38,7 @@ class BM25_search:
         """
         # Ensure NLTK resources are downloaded only once
         if not BM25_search.nltk_resources_downloaded:
+            
             download_nltk_resources()
             BM25_search.nltk_resources_downloaded = True  # Mark as downloaded