Update app.py

app.py

@@ -78,6 +78,7 @@ text_splitter = RecursiveCharacterTextSplitter(
     chunk_size=1600,  
     chunk_overlap=150,  
     add_start_index=True
+    separators=["\n\n", "\n", ". ", " ", ""] 
 )
 def split_document_with_metadata(document):
     # Split the document text into chunks.
@@ -110,10 +111,10 @@ vectorstore = Chroma.from_documents(
 print("Created new vector store and persisted embeddings.")
 
 # Create a BM25 retriever from the document splits.
-bm25_retriever = BM25Retriever.from_documents(all_splits)
+bm25_retriever = BM25Retriever.from_documents(all_splits,search_kwargs={"score_threshold": 0.7})
 ensemble_retriever = EnsembleRetriever(
     retrievers=[
-        vectorstore.as_retriever(),
+        vectorstore.as_retriever(search_kwargs={"score_threshold": 0.5},),
         bm25_retriever
     ],
     weights=[0.9, 0.1]