using pre-tokenized chunks

Dockerfile

@@ -6,9 +6,6 @@ COPY requirements.txt .
 RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 
-# Download the spaCy English model
-RUN python -m spacy download en_core_web_sm
-
 COPY . .
 
 # Let Streamlit accept connections from everywhere, on port 7860 (HF Spaces default)

rag_ui.py

@@ -21,8 +21,6 @@ from huggingface_hub import hf_hub_download
 from rank_bm25 import BM25Okapi
 import io
 from docx import Document
-import spacy
-from functools import lru_cache
 import hashlib
 
 # Caching for search results function
@@ -31,16 +29,9 @@ def cached_search(query, chunk_idx_pool_tuple, n_final):
     return hybrid_search(query, chunk_idx_pool=list(chunk_idx_pool_tuple) if chunk_idx_pool_tuple else None, n_final=n_final)
 
 ############### TOKENIZER AND NORM FUNCTION ##############
-nlp = spacy.load("en_core_web_sm", disable=['ner', 'parser'])  # fast, enough for tokenization
 
-@lru_cache(maxsize=2048)
-def _spacy_tokenize(text):
-    # spaCy removes punctuation, does lemmatization, drops stopwords
-    doc = nlp(text.lower())
-    return [tok.lemma_ for tok in doc if tok.is_alpha and not tok.is_stop]
-
-def tokenize(text):
-    return _spacy_tokenize(text)
+def query_tokenize(text):
+    return re.findall(r"\w+", text.lower())
 
 def l2_normalize(vecs, axis=1, epsilon=1e-10):
     norms = np.linalg.norm(vecs, ord=2, axis=axis, keepdims=True)
@@ -86,9 +77,8 @@ def load_search_data():
             fout.write(fidx.read())
     faiss_index = faiss.read_index("/tmp/zotero_chunks.index")
 
-    # Tokenize all texts for BM25
-    texts = [c["text"] for c in chunks][:100]
-    tokenized_texts = [tokenize(text) for text in texts]
+    # get tokens for BM25
+    tokenized_texts = [c["tokens"] for c in chunks]
     bm25 = BM25Okapi(tokenized_texts)
 
     return chunks, faiss_index, bm25
@@ -124,7 +114,7 @@ source_key_map = dict(zip(source_labels, sources_sorted)) # Map label to (author
 
 ########### BM25-BASED SPARSE SEARCH ###########
 def sparse_search(query, chunk_idx_pool=None, k=TOPK_SPARSE):
-    query_tokens = tokenize(query)
+    query_tokens = query_tokenize(query)
     if chunk_idx_pool is None:
         scores = bm25.get_scores(query_tokens)
         idxs = np.argsort(scores)[::-1][:k]

requirements.txt

@@ -7,5 +7,4 @@ tqdm
 huggingface_hub
 rank_bm25
 python-docx
-fpdf2
-spacy
+fpdf2