Checkin version 2

README.md

@@ -12,16 +12,19 @@ short_description: Create an intelligent Bible study assistant that utilizes LL
 
 ## <h1 align="center" id="heading">An Agentic Bible Study Tool Built with LangChain and LangGraph</h1>
 
-Create an intelligent Bible study assistant that utilizes LLMs to enhance contextual understanding of scripture. This tool will enable users to pose questions, and the AI will provide answers grounded in the Bible, by accurately identifying and synthesizing information from relevant verses, chapters, and cross-references, promoting deeper comprehension and reducing misinterpretations.
+Welcome to the Bible Study Tool, an interactive platform designed to deepen your understanding of the Bible, with a special focus on the book of Genesis (Phase I). Powered by advanced AI technology, this tool offers a variety of features to enrich your study experience:
+
+Ask Questions: Receive detailed answers about Genesis through an AI-driven retrieval system that pulls from a comprehensive database of Bible verses.
+Internet Search: Broaden your perspective by exploring additional context and related topics from the web.
+Quiz Mode: Challenge yourself with personalized quizzes on specific verse ranges—just type "start quiz on <verse range>" (e.g., "start quiz on Genesis 1:1-5") to get started.
+Built with a user-friendly chat interface, this tool makes Bible study engaging and accessible for everyone, whether you’re a beginner or a seasoned scholar. Dive in and let the Bible Study Tool guide you on your journey!
 
 
-### Phase I
-    - Book of Genesis
-    - Examples of questions:
-        - How did GOD create the whole universe based on Genesis?
-        - Why LORD God make man leave garden?
-        - How did the Israelites, led by Jacob, end up in Egypt, and what role did Joseph play in their settlement there?
 
 
 ## Ship 🚢
-Check out the prototype at https://huggingface.co/spaces/kcheng0816/BibleStudy
+Check out the prototype at https://huggingface.co/spaces/kcheng0816/BibleStudy
+
+
+
+

app.py

@@ -1,83 +1,174 @@
 import os
+import re
+import random
+import uuid
 from dotenv import load_dotenv
 import chainlit as cl
-
-import pandas as pd
-from langchain_community.vectorstores import FAISS
-from langchain_openai.embeddings import OpenAIEmbeddings
-from langchain_core.documents import Document
-from langchain_community.document_loaders import DirectoryLoader
-from langchain_community.document_loaders import BSHTMLLoader
-from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain.docstore.document import Document
+from bs4 import BeautifulSoup
 from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_qdrant import QdrantVectorStore
 from qdrant_client import QdrantClient
-from qdrant_client.http.models import Distance, VectorParams
-from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
-from langchain_cohere import CohereRerank
+from qdrant_client.http.models import VectorParams, Distance
+from qdrant_client.http.models import PointStruct
+from langchain.storage import LocalFileStore
+from langchain.embeddings import CacheBackedEmbeddings
+from qdrant_client.http.models import Filter, FieldCondition, MatchValue, MatchAny
 from langchain.prompts import ChatPromptTemplate
-from langchain_openai import ChatOpenAI
-from langchain.chat_models import init_chat_model
-from langchain_core.rate_limiters import InMemoryRateLimiter
-from langgraph.graph import START, StateGraph, END
-from typing_extensions import List, TypedDict
-from langchain_core.documents import Document
-from langchain_core.messages import HumanMessage
+from langchain_core.runnables import RunnablePassthrough
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnableLambda
+from langchain_core.messages import HumanMessage, AIMessage, SystemMessage, ToolMessage
 from langchain_core.tools import tool
-from langgraph.prebuilt import ToolNode
+from langchain_community.tools.tavily_search import TavilySearchResults
+from functools import partial
+from typing import Any, Callable, List, Optional, TypedDict, Union
 from langchain_core.messages import AnyMessage
 from langgraph.graph.message import add_messages
 from typing import TypedDict, Annotated
-from langchain_core.documents import Document
+from langgraph.prebuilt import ToolNode
+from langgraph.graph import StateGraph, END
+import json
 
-#Load API Keys
+# Load API Keys
 load_dotenv()
+os.environ["LANGCHAIN_PROJECT"] = f"AIE5- Bible Study Tool - {uuid.uuid4().hex[0:8]}"
+os.environ["LANGCHAIN_TRACING_V2"] = "true"
 
-#Load downloaded html pages of the book Genesis in Bible
 path = "data/"
-loader = DirectoryLoader(path, glob="*.html")
-docs = loader.load()
-
-#Text Splitter
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size = 750,
-    chunk_overlap = 100
-)
-
-split_documents = text_splitter.split_documents(docs)
-len(split_documents)
-
-#fine tuned embedding model
+book = "Genesis"
+collection_name = "genesis_study"
+
+# Load Genesis documents (unchanged from original)
+def load_genesis_documents(path, book_name):
+    documents = []
+    for file in os.listdir(path):
+        if file.endswith(".html"):
+            file_path = os.path.join(path, file)
+            with open(file_path, "r", encoding="utf-8") as f:
+                soup = BeautifulSoup(f, "html.parser")
+                p_tags = soup.find_all("p", align="left")
+                for p_tag in p_tags:
+                    verse_texts = [content.strip() for content in p_tag.contents 
+                                   if isinstance(content, str) and content.strip()]
+                    for verse in verse_texts:
+                        match = re.match(r"\[(\d+):(\d+)\]\s*(.*)", verse)
+                        if match:
+                            chapter = int(match.group(1))
+                            verse_num = int(match.group(2))
+                            text = match.group(3)
+                            doc = Document(
+                                page_content=text,
+                                metadata={"book": book_name, "chapter": chapter, "verse": verse_num}
+                            )
+                            documents.append(doc)
+    return documents
+
+documents = load_genesis_documents(path, book)
+
+# Initialize embeddings
 huggingface_embeddings = HuggingFaceEmbeddings(model_name="kcheng0816/finetuned_arctic_genesis")
+dimension = len(huggingface_embeddings.embed_query("test"))
 
-#vector datastore
+# Set up Qdrant client and collection
 client = QdrantClient(":memory:")
 client.create_collection(
-    collection_name="genesis_bible",
-    vectors_config=VectorParams(size=1024, distance=Distance.COSINE),
+    collection_name=collection_name,
+    vectors_config=VectorParams(size=dimension, distance=Distance.COSINE)
 )
 
-vector_store = QdrantVectorStore(
-    client=client,
-    collection_name="genesis_bible",
-    embedding=huggingface_embeddings,
-)
-
-_ = vector_store.add_documents(documents=split_documents)
-
-#Retrieve
-retriever = vector_store.as_retriever(search_kwargs={"k": 5})
-
+# Generate and upload embeddings
+embeddings = huggingface_embeddings.embed_documents([doc.page_content for doc in documents])
+points = [
+    PointStruct(
+        id=str(uuid.uuid5(uuid.NAMESPACE_DNS, f"{doc.metadata['chapter']}_{doc.metadata['verse']}")),
+        vector=embedding,
+        payload={
+            "text": doc.page_content,
+            "book": doc.metadata["book"],
+            "chapter": doc.metadata["chapter"],
+            "verse": doc.metadata["verse"]
+        }
+    )
+    for embedding, doc in zip(embeddings, documents)
+]
+client.upsert(collection_name=collection_name, points=points)
 
-def retrieve_adjusted(state):
-  compressor = CohereRerank(model="rerank-v3.5")
-  compression_retriever = ContextualCompressionRetriever(
-    base_compressor=compressor, base_retriever=retriever, search_kwargs={"k": 5}
-  )
-  retrieved_docs = compression_retriever.invoke(state["question"])
-  return {"context" : retrieved_docs}
+# Cached embedder
+safe_namespace = "AIE5_BibleStudyTool"
+store = LocalFileStore("./cache/")
+cached_embedder = CacheBackedEmbeddings.from_bytes_store(
+    huggingface_embeddings, store, namespace=safe_namespace, batch_size=32
+)
 
-#RAG prompt
+# Retrieval functions (unchanged from original)
+def parse_verse_reference(ref: str):
+    match = re.match(r"(\w+(?:\s\w+)?)\s(\d+):([\d,-]+)", ref)
+    if not match:
+        return None
+    book, chapter, verse_part = match.groups()
+    chapter = int(chapter)
+    verses = []
+    for part in verse_part.split(','):
+        if '-' in part:
+            start, end = map(int, part.split('-'))
+            verses.extend(range(start, end + 1))
+        else:
+            verses.append(int(part))
+    return book, chapter, verses
+
+def retrieve_verse_content(verse_range: str, client: QdrantClient):
+    parsed = parse_verse_reference(verse_range)
+    if not parsed:
+        return "Invalid verse range format."
+    book, chapter, verses = parsed
+    filter = Filter(
+        must=[
+            FieldCondition(key="book", match=MatchValue(value=book)),
+            FieldCondition(key="chapter", match=MatchValue(value=chapter)),
+            FieldCondition(key="verse", match=MatchAny(any=verses))
+        ]
+    )
+    search_result = client.scroll(
+        collection_name=collection_name,
+        scroll_filter=filter,
+        limit=len(verses)
+    )
+    if not search_result[0]:
+        return "No verses found for the specified range."
+    sorted_points = sorted(search_result[0], key=lambda p: p.payload["verse"])
+    docs = [
+        Document(
+            page_content=p.payload["text"],
+            metadata=p.payload
+        )
+        for p in sorted_points
+    ]
+    return docs
+
+def retrieve_documents(question: str, collection_name: str, client: QdrantClient):
+    reference_match = re.search(r"(\w+)\s?(\d+):\s?([\d,-]+)", question)
+    if reference_match:
+        verse_range = reference_match.group(1) + ' ' + reference_match.group(2) + ':' + reference_match.group(3)
+        return retrieve_verse_content(verse_range, client)
+    else:
+        query_vector = cached_embedder.embed_query(question)
+        search_result = client.query_points(
+            collection_name=collection_name,
+            query=query_vector,
+            limit=5,
+            with_payload=True
+        ).points
+        if search_result:
+            return [
+                Document(
+                    page_content=point.payload["text"],
+                    metadata=point.payload
+                )
+                for point in search_result
+            ]
+        return "No relevant documents found."
+
+# RAG setup (unchanged from original)
 RAG_PROMPT = """\
 You are a helpful assistant who answers questions based on provided context. You must only use the provided context, and cannot use your own knowledge.
 
@@ -89,105 +180,307 @@ You are a helpful assistant who answers questions based on provided context. You
 """
 rag_prompt = ChatPromptTemplate.from_template(RAG_PROMPT)
 
+from langchain_openai import ChatOpenAI
+from langchain.chat_models import init_chat_model
+from langchain_core.rate_limiters import InMemoryRateLimiter
 
-#llm for RAG
 rate_limiter = InMemoryRateLimiter(
-    requests_per_second=1,  # <-- make a request once every 1 seconds!!
-    check_every_n_seconds=0.1,  # Wake up every 100 ms to check whether allowed to make a request,
-    max_bucket_size=10,  # Controls the maximum burst size.
+    requests_per_second=1,
+    check_every_n_seconds=0.1,
+    max_bucket_size=10,
 )
-llm = init_chat_model("gpt-4o-mini", rate_limiter=rate_limiter)
 
+chat_model = init_chat_model("gpt-4o-mini", rate_limiter=rate_limiter)
+
+def create_retriever_runnable(collection_name: str, client: QdrantClient) -> RunnableLambda:
+    return RunnableLambda(lambda question: retrieve_documents(question, collection_name, client))
 
-def generate(state):
-  docs_content = "\n\n".join(doc.page_content for doc in state["context"])
-  messages = rag_prompt.format_messages(question=state["question"], context=docs_content)
-  response = llm.invoke(messages)
-  return {"response" : response.content}
+retrieval_runnable = create_retriever_runnable(collection_name, client)
 
-#Build RAG graph
-class State(TypedDict):
-  question: str
-  context: List[Document]
-  response: str
+def format_docs(docs):
+    if isinstance(docs, str):
+        return docs
+    return "\n\n".join(f"Genesis {doc.metadata['chapter']}:{doc.metadata['verse']} - {doc.page_content}" for doc in docs)
 
-graph_builder = StateGraph(State).add_sequence([retrieve_adjusted, generate])
-graph_builder.add_edge(START, "retrieve_adjusted")
-graph = graph_builder.compile()
+rag_chain = (
+    {"context": retrieval_runnable | RunnableLambda(format_docs), "question": RunnablePassthrough()}
+    | RunnablePassthrough.assign(response=rag_prompt | chat_model | StrOutputParser())
+)
 
+# Tools
+def format_contexts(docs):
+    return "\n\n".join(docs) if isinstance(docs, list) else docs
 
 @tool
-def ai_rag_tool(question: str) -> str:
-    """Useful for when you need to answer questions about Bible """
-    response = graph.invoke({"question": question})
+def ai_rag_tool(question: str):
+    """Useful for when you need to answer questions about Bible"""
+    response = rag_chain.invoke(question)
     return {
         "message": [HumanMessage(content=response["response"])],
-        "context": response["context"]
+        "context": format_contexts(response["context"])
     }
 
-tool_belt = [
-    ai_rag_tool
-]
+tavily_tool = TavilySearchResults(max_results=5)
+
+def _generate_quiz_question(verse_range: str, client: QdrantClient):
+    docs = retrieve_verse_content(verse_range, client)
+    if isinstance(docs, str):
+        return {"error": docs}
+    verse_content = "\n".join(
+        f"{doc.metadata['book']} {doc.metadata['chapter']}:{doc.metadata['verse']} - {doc.page_content}"
+        for doc in docs
+    )
+    quiz_prompt = ChatPromptTemplate.from_template(
+        "Based on the following Bible verse(s), generate a multiple-choice quiz question with 4 options (A, B, C, D) "
+        "and indicate the correct answer:\n\n"
+        "{verse_content}\n\n"
+        "Format your response as follows:\n"
+        "Question: [Your question here]\n"
+        "A: [Option A]\n"
+        "B: [Option B]\n"
+        "C: [Option C]\n"
+        "D: [Option D]\n"
+        "Correct Answer: [Letter of correct answer]\n"
+        "Explanation: [Brief explanation of why the answer is correct]\n"
+    )
+    response = (quiz_prompt | chat_model).invoke({"verse_content": verse_content})
+    response_text = response.content.strip()
+    lines = response_text.split("\n")
+    question = ""
+    options = {}
+    correct_answer = ""
+    explanation = ""
+    for line in lines:
+        line = line.strip()
+        if line.startswith("Question:"):
+            question = line[len("Question:"):].strip()
+        elif line.startswith(("A:", "B:", "C:", "D:")):
+            key, value = line.split(":", 1)
+            options[key.strip()] = value.strip()
+        elif line.startswith("Correct Answer:"):
+            correct_answer = line[len("Correct Answer:"):].strip()
+        elif line.startswith("Explanation:"):
+            explanation = line[len("Explanation:"):].strip()
+    return {
+        "quiz_question": question,
+        "options": options,
+        "correct_answer": correct_answer,
+        "explanation": explanation,
+        "verse_range": verse_range,
+        "verse_content": verse_content
+    }
 
-#llm for agent reasoning
-llm = init_chat_model("gpt-4o", temperature=0, rate_limiter=rate_limiter)
-llm_with_tools = llm.bind_tools(tool_belt)
+generate_quiz_question_tool = partial(_generate_quiz_question, client=client)
+
+@tool
+def generate_quiz_question(verse_range: str):
+    """Generate a quiz question based on the content of the specified verse range."""
+    quiz_data = generate_quiz_question_tool(verse_range)
+    return json.dumps(quiz_data)
 
+tool_belt = [ai_rag_tool, tavily_tool, generate_quiz_question]
 
+# LLM for agent reasoning
+llm = init_chat_model("gpt-4o", temperature=0, rate_limiter=rate_limiter)
+llm_with_tools = llm.bind_tools(tool_belt)
 
-#Build an agent graph
+# Define the state
 class AgentState(TypedDict):
     messages: Annotated[list[AnyMessage], add_messages]
-    context:List[Document]
+    in_quiz: bool
+    quiz_question: Optional[dict]
+    verse_range: Optional[str]
+    quiz_score: int
+    quiz_total: int
+    waiting_for_answer: bool
+
+# System message
+system_message = SystemMessage(content="""You are a Bible study assistant. You can answer questions about the Bible, search the internet for related information, or generate quiz questions based on specific verse ranges.
+
+- Use the 'ai_rag_tool' to answer questions about the Bible.
+- Use the 'tavily_tool' to search the internet for additional information.
+- Use the 'generate_quiz_question' tool when the user requests to start a quiz on a specific verse range, such as 'start quiz on Genesis 1:1-10'.
+
+When the user requests a quiz, extract the verse range from their message and pass it to the 'generate_quiz_question' tool.""")
+
+
+from typing import Optional
+from typing_extensions import TypedDict
+from langgraph.graph.message import AnyMessage, add_messages
+from typing import Annotated
 
 
 def call_mode(state):
-    messages = state["messages"]
+    last_message = state["messages"][-1]
+    
+    if state.get("in_quiz", False):
+        if state.get("waiting_for_answer", False):
+            # Process the user's answer
+            quiz_data = state["quiz_question"]
+            user_answer = last_message.content.strip().upper()
+            correct_answer = quiz_data["correct_answer"]
+            new_quiz_total = state["quiz_total"] + 1
+            if user_answer == correct_answer:
+                new_quiz_score = state["quiz_score"] + 1
+                feedback = f"Correct! {quiz_data['explanation']}"
+            else:
+                new_quiz_score = state["quiz_score"]
+                feedback = f"Incorrect. The correct answer is {correct_answer}. {quiz_data['explanation']}"
+            return {
+                "messages": [
+                    AIMessage(content=feedback),
+                    AIMessage(content="Would you like another question? Type 'Yes' to continue or 'No' to end the quiz.")
+                ],
+                "quiz_total": new_quiz_total,
+                "quiz_score": new_quiz_score,
+                "waiting_for_answer": False,
+                "quiz_question": state["quiz_question"],
+                "in_quiz": True,
+                "verse_range": state["verse_range"]
+            }
+        else:
+            # Handle the user's decision to continue or stop the quiz
+            user_input = last_message.content.strip().lower()
+            if user_input == "yes":
+                # Generate a new quiz question
+                verse_range = state["verse_range"]
+                quiz_data_str = generate_quiz_question(verse_range)
+                quiz_data = json.loads(quiz_data_str)
+                question = quiz_data["quiz_question"]
+                options = "\n".join([f"{k}: {v}" for k, v in quiz_data["options"].items()])
+                verse_content = quiz_data["verse_content"]
+                message_to_user = (
+                    f"Based on the following verse(s):\n\n{verse_content}\n\n"
+                    f"Here's your quiz question:\n\n{question}\n\n{options}\n\n"
+                    "Please select your answer (A, B, C, or D)."
+                )
+                return {
+                    "messages": [AIMessage(content=message_to_user)],
+                    "quiz_question": quiz_data,
+                    "waiting_for_answer": True,
+                    "quiz_total": state["quiz_total"],
+                    "quiz_score": state["quiz_score"],
+                    "in_quiz": True,
+                    "verse_range": state["verse_range"]
+                }
+            elif user_input == "no":
+                # End the quiz and provide a summary
+                score = state["quiz_score"]
+                total = state["quiz_total"]
+                continue_message = "Ask me anything about Genesis or type 'start quiz on <verse range>' (e.g., 'start quiz on Genesis 1:1-5') for a trivia challenge."
+                if total > 0:
+                    percentage = (score / total) * 100
+                    if percentage == 100:
+                        feedback = "Excellent! You got all questions correct. Please continue your Bible study!"
+                    elif percentage >= 80:
+                        feedback = "Great job! You have a strong understanding. Please continue your Bible study!"
+                    elif percentage >= 50:
+                        feedback = "Good effort! Keep practicing to improve. Please continue your Bible study!"
+                    else:
+                        feedback = "Don’t worry, keep your Bible studying and you’ll get better!"
+                    summary = f"You got {score} out of {total} questions correct. {feedback} \n\n {continue_message}"
+                else:
+                    summary = "No questions were attempted."
+                return {
+                    "messages": [AIMessage(content=summary)],
+                    "in_quiz": False,
+                    "quiz_question": None,
+                    "verse_range": None,
+                    "quiz_score": 0,
+                    "quiz_total": 0,
+                    "waiting_for_answer": False
+                }
+            else:
+                # Handle invalid input
+                return {
+                    "messages": [AIMessage(content="Please type 'Yes' to continue or 'No' to end the quiz.")],
+                    "quiz_total": state["quiz_total"],
+                    "quiz_score": state["quiz_score"],
+                    "waiting_for_answer": False,
+                    "quiz_question": state["quiz_question"],
+                    "in_quiz": True,
+                    "verse_range": state["verse_range"]
+                }
+    
+    # Handle starting the quiz or other tool calls
+    if len(state["messages"]) >= 2 and isinstance(last_message, ToolMessage):
+        prev_message = state["messages"][-2]
+        if isinstance(prev_message, AIMessage) and prev_message.tool_calls:
+            tool_call = prev_message.tool_calls[0]
+            if tool_call["name"] == "generate_quiz_question":
+                # Start the quiz
+                quiz_data_str = last_message.content
+                quiz_data = json.loads(quiz_data_str)
+                verse_range = quiz_data["verse_range"]
+                question = quiz_data["quiz_question"]
+                options = "\n".join([f"{k}: {v}" for k, v in quiz_data["options"].items()])
+                verse_content = quiz_data["verse_content"]
+                message_to_user = (
+                    f"Based on the following verse(s):\n\n{verse_content}\n\n"
+                    f"Here's your quiz question:\n\n{question}\n\n{options}\n\n"
+                    "Please select your answer (A, B, C, or D)."
+                )
+                return {
+                    "messages": [AIMessage(content=message_to_user)],
+                    "in_quiz": True,
+                    "verse_range": verse_range,
+                    "quiz_score": 0,
+                    "quiz_total": 0,
+                    "quiz_question": quiz_data,
+                    "waiting_for_answer": True
+                }
+    
+    # Process regular questions or commands
+    messages = [system_message] + state["messages"]
     response = llm_with_tools.invoke(messages)
-    return {
-        "messages": [response],
-        "context": state.get("context",[])
-    }
+    return {"messages": [response]}
+
 
 tool_node = ToolNode(tool_belt)
 
 def should_continue(state):
     last_message = state["messages"][-1]
-
     if last_message.tool_calls:
         return "action"
-    
     return END
 
-
+# Build the graph
 uncompiled_graph = StateGraph(AgentState)
-
 uncompiled_graph.add_node("agent", call_mode)
 uncompiled_graph.add_node("action", tool_node)
-
 uncompiled_graph.set_entry_point("agent")
-
-uncompiled_graph.add_conditional_edges(
-    "agent",
-    should_continue
-)
-
+uncompiled_graph.add_conditional_edges("agent", should_continue)
 uncompiled_graph.add_edge("action", "agent")
-
-# Compile the graph.
 compiled_graph = uncompiled_graph.compile()
 
+# Chainlit integration
+import chainlit as cl
+from langchain_core.messages import SystemMessage
 
-#user interface
 @cl.on_chat_start
-async def on_chat_start():
-   cl.user_session.set("graph", compiled_graph)
-
+async def start():
+    system_message = SystemMessage(content="Welcome to the Bible Study Tool!")
+    initial_state = {
+        "messages": [system_message],
+        "in_quiz": False,
+        "quiz_question": None,
+        "verse_range": None,
+        "quiz_score": 0,
+        "quiz_total": 0,
+        "waiting_for_answer": False
+    }
+    cl.user_session.set("state", initial_state)
+    await cl.Message(content="Welcome to the Bible Study Tool! Ask me anything about Genesis or type 'start quiz on <verse range>' (e.g., 'start quiz on Genesis 1:1-5') for a trivia challenge.").send()
 
 
 @cl.on_message
-async def handle(message: cl.Message):
-    graph = cl.user_session.get("graph")
-    state = {"messages": [HumanMessage(content=message.content)]}
-    response = await graph.ainvoke(state)
-    await cl.Message(content=response["messages"][-1].content).send()
+async def main(message: cl.Message):
+    state = cl.user_session.get("state")
+    current_messages = len(state["messages"])
+    state["messages"].append(HumanMessage(content=message.content))
+    result = compiled_graph.invoke(state)
+    cl.user_session.set("state", result)
+    new_messages = result["messages"][current_messages + 1:]
+    for msg in new_messages:
+        if isinstance(msg, AIMessage):
+            await cl.Message(content=msg.content).send()

pyproject.toml

@@ -17,4 +17,5 @@ dependencies = [
     "unstructured>=0.14.8",
     "langchain-huggingface>=0.1.2",
     "websockets>=15.0",
+    "rank-bm25>=0.2.2",
 ]

uv.lock

@@ -224,6 +224,7 @@ dependencies = [
     { name = "langchain-qdrant" },
     { name = "langgraph" },
     { name = "pandas" },
+    { name = "rank-bm25" },
     { name = "unstructured" },
     { name = "websockets" },
 ]
@@ -240,6 +241,7 @@ requires-dist = [
     { name = "langchain-qdrant", specifier = ">=0.2.0" },
     { name = "langgraph", specifier = ">=0.2.67" },
     { name = "pandas", specifier = ">=2.2.3" },
+    { name = "rank-bm25", specifier = ">=0.2.2" },
     { name = "unstructured", specifier = ">=0.14.8" },
     { name = "websockets", specifier = ">=15.0" },
 ]
@@ -2529,6 +2531,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5f/26/89ebaee5fcbd99bf1c0a627a9447b440118b2d31dea423d074cb0481be5c/qdrant_client-1.13.2-py3-none-any.whl", hash = "sha256:db97e759bd3f8d483a383984ba4c2a158eef56f2188d83df7771591d43de2201", size = 306637 },
 ]
 
+[[package]]
+name = "rank-bm25"
+version = "0.2.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fc/0a/f9579384aa017d8b4c15613f86954b92a95a93d641cc849182467cf0bb3b/rank_bm25-0.2.2.tar.gz", hash = "sha256:096ccef76f8188563419aaf384a02f0ea459503fdf77901378d4fd9d87e5e51d", size = 8347 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/21/f691fb2613100a62b3fa91e9988c991e9ca5b89ea31c0d3152a3210344f9/rank_bm25-0.2.2-py3-none-any.whl", hash = "sha256:7bd4a95571adadfc271746fa146a4bcfd89c0cf731e49c3d1ad863290adbe8ae", size = 8584 },
+]
+
 [[package]]
 name = "rapidfuzz"
 version = "3.12.1"