Remove pdf parsing for now

aimakerspace/openai_utils/embedding.py

@@ -21,6 +21,8 @@ class EmbeddingModel:
         self.embeddings_model_name = embeddings_model_name
 
     async def async_get_embeddings(self, list_of_text: List[str]) -> List[List[float]]:
+        if not list_of_text:
+            raise(ValueError("Cannot embed nonexistent text."))
         embedding_response = await self.async_client.embeddings.create(
             input=list_of_text, model=self.embeddings_model_name
         )

aimakerspace/qa_pipeline.py

@@ -1,11 +1,30 @@
+from typing import List, Tuple
+
 from rank_bm25 import BM25Plus
-from langchain.vectorstores import Qdrant
+
+from .vectordatabase import VectorDatabase
+from .openai_utils.chatmodel import ChatOpenAI
 
 from .openai_utils.prompts import (
+    UserRolePrompt,
     SystemRolePrompt
 )
-from .vectordatabase import VectorDatabase
-from .openai_utils.chatmodel import ChatOpenAI
+
+system_template = """\
+Use the provided context to answer the user's question. Answer in one paragraph and provide lots 
+of details based on the context. If you are certain the context is not relevant, apologize and say you don't 
+have enough information to answer."""
+system_role_prompt = SystemRolePrompt(system_template)
+
+user_prompt_template = """\
+Context:
+{context}
+
+The question to answer is:
+{question}
+"""
+user_role_prompt = UserRolePrompt(user_prompt_template)
+
 
 # Utility function for reranking
 def bm25plus_rerank(corpus, query, initial_ranking, top_n=3):
@@ -18,12 +37,6 @@ def bm25plus_rerank(corpus, query, initial_ranking, top_n=3):
     ranked_indices = [initial_ranking[i] for i in bm25_scores.argsort()[::-1]]    
     return ranked_indices[:top_n]
 
-def search_by_text(qdrant: Qdrant, query_text: str, k: int, return_as_text: bool = False) -> List[Tuple[str, float]]:
-    results = qdrant.similarity_search_with_score(query_text, k)
-    if return_as_text:
-        return [result[0].page_content for result in results]
-    return [(result[0].page_content, result[1]) for result in results]
-
 
 class RetrievalAugmentedQAPipeline:
     def __init__(self, llm: ChatOpenAI(), vector_db_retriever) -> None:
@@ -31,10 +44,7 @@ class RetrievalAugmentedQAPipeline:
         self.vector_db_retriever = vector_db_retriever
 
     async def arun_pipeline(self, user_query: str):
-        if type(self.vector_db_retriever == "Qdrant"):
-            context_list = search_by_text(self.vector_db_retriever,user_query, k=4)
-        else:
-            context_list = self.vector_db_retriever.search_by_text(user_query, k=4)
+        context_list = self.vector_db_retriever.search_by_text(user_query, k=4)
 
         context_prompt = ""
         for context in context_list:
@@ -56,10 +66,8 @@ class RerankedQAPipeline(RetrievalAugmentedQAPipeline):
     async def arun_pipeline(self, user_query: str, rerank: bool=False) -> str:
         # Retrieve the top 10 results. Either return the top 3, or rerank with BM25 and then return 
         # the new top 3
-        if type(self.vector_db_retriever == "Qdrant"):
-            context_list = search_by_text(self.vector_db_retriever,user_query, k=10)
-        else:
-            context_list = self.vector_db_retriever.search_by_text(user_query, k=10)
+        context_list = self.vector_db_retriever.search_by_text(user_query, k=10)
+
         # Convert from tuples to strings
         context_list_str = [context_list[i][0] for i in range(len(context_list))]
 
@@ -72,10 +80,13 @@ class RerankedQAPipeline(RetrievalAugmentedQAPipeline):
             reranked_indices = bm25plus_rerank(context_list_str, user_query, initial_ranking, top_n=n)
             reranked_contexts = [context_list_str[i] for i in reranked_indices]
 
-        context_prompt = "\n\n".join(context for context in reranked_contexts) + "\n\n"
+        context_prompt = ""
+        for context in context_list:
+            context_prompt += context[0] + "\n"
+
+        formatted_system_prompt = system_role_prompt.create_message()
 
-        formatted_system_prompt = SystemRolePrompt(system_prompt).create_message() if system_prompt else rag_prompt.create_message()
-        formatted_user_prompt = user_prompt.create_message(user_query=user_query, context=context_prompt)
+        formatted_user_prompt = user_role_prompt.create_message(question=user_query, context=context_prompt)
 
         async def generate_response():
             async for chunk in self.llm.astream([formatted_system_prompt, formatted_user_prompt]):

app.py

@@ -1,39 +1,20 @@
-import os
 from typing import List
 import tempfile
 
 import chainlit as cl
 from chainlit.types import AskFileResponse
-from PyPDF2 import PdfReader
+import fitz
 
-from langchain.vectorstores import Qdrant
-from langchain.embeddings import OpenAIEmbeddings
+from langchain_community.embeddings import OpenAIEmbeddings
 
 from aimakerspace.text_utils import CharacterTextSplitter, TextFileLoader
-from aimakerspace.openai_utils.prompts import (
-    UserRolePrompt,
-    SystemRolePrompt
-)
 from aimakerspace.openai_utils.embedding import EmbeddingModel
 from aimakerspace.vectordatabase import VectorDatabase
 from aimakerspace.openai_utils.chatmodel import ChatOpenAI
 from aimakerspace.qa_pipeline import RerankedQAPipeline
 
-system_template = """\
-Use the following context to answer a users question. If you cannot find the answer in the context, say you don't know the answer."""
-system_role_prompt = SystemRolePrompt(system_template)
-
-user_prompt_template = """\
-Context:
-{context}
-
-Question:
-{question}
-"""
-user_role_prompt = UserRolePrompt(user_prompt_template)
-
 text_splitter = CharacterTextSplitter()
-EmbeddingModel = OpenAIEmbeddings()
+embedding_model = OpenAIEmbeddings()
 
 def process_text_file(file: AskFileResponse):
 
@@ -49,33 +30,18 @@ def process_text_file(file: AskFileResponse):
     return texts
 
 def process_pdf(file: AskFileResponse) -> list[str]:
-    
-    # Create a temporary file to store the PDF content
     with tempfile.NamedTemporaryFile(mode="wb", delete=False, suffix=".pdf") as temp_file:
         temp_file_path = temp_file.name
         temp_file.write(file.content)
+        temp_file.flush()
 
-    # Read the PDF content
-    with open(temp_file_path, "rb") as f:
-        pdf_reader = PdfReader(f)
-        text = ""
-        for page in pdf_reader.pages:
-            text += page.extract_text()
-
-    # Assuming you have a text splitter similar to the one used for text files
-    texts = text_splitter.split_texts([text])
-    return texts
+    text = ""
+    with fitz.open(temp_file_path) as doc:
+        for page in doc:
+            text += page.get_text().strip()
 
-# Function to build the vector database from a list of texts
-async def build_qdrant_vector_database(list_of_text: List[str]) -> Qdrant:
-    embeddings = await embedding_model.async_get_embeddings(list_of_text)
-    qdrant = Qdrant.from_texts(
-        texts=list_of_text,
-        embeddings=[embedding.tolist() for embedding in embeddings],
-        embedding=embedding_model,
-        collection_name="vectors"
-    )
-    return qdrant
+    text_list = text_splitter.split_texts(text)  
+    return text_list
 
 @cl.on_chat_start
 async def on_chat_start():
@@ -84,9 +50,9 @@ async def on_chat_start():
     # Wait for the user to upload a file
     while files == None:
         files = await cl.AskFileMessage(
-            content="Please upload a Text or PDF File file to begin!",
-            accept=["text/plain","application/pdf"],
-            max_size_mb=5,
+            content="Please upload a Text File file to begin!",
+            accept=["text/plain"],
+            max_size_mb=20,
             timeout=180,
         ).send()
 
@@ -98,29 +64,30 @@ async def on_chat_start():
     await msg.send()
 
     # load the file
-    if "pdf" not in file.name.lower():
-        texts = process_text_file(file)
-    else: texts = process_pdf(file)
-
-    print(f"Processing {len(texts)} text chunks")
-
-    # Create a dict vector store
-    qdrant_db = await build_qdrant_vector_database(texts)
-    
-    chat_openai = ChatOpenAI()
-
-    # Create a chain
-    retrieval_augmented_qa_pipeline = RerankedQAPipeline(
-        vector_db_retriever=qdrant_db,
-        llm=chat_openai,
-        True
-    )
-    
-    # Let the user know that the system is ready
-    msg.content = f"Processing `{file.name}` done. You can now ask questions!"
-    await msg.update()
+    texts = process_text_file(file)
+
+    if not texts:
+        await cl.Message(content=f"Error: Could not extract any text from input file").send()
+    else:
+        print(f"Processing {len(texts)} text chunks")
+
+        # Create a dict vector store
+        vector_db = VectorDatabase()
+        vector_db = await vector_db.abuild_from_list(texts)
+        
+        chat_openai = ChatOpenAI()
+
+        # Create a chain
+        retrieval_augmented_qa_pipeline = RerankedQAPipeline(
+            vector_db_retriever=vector_db,
+            llm=chat_openai,
+        )
+        
+        # Let the user know that the system is ready
+        msg.content = f"Processing `{file.name}` done. You can now ask questions!"
+        await msg.update()
 
-    cl.user_session.set("chain", retrieval_augmented_qa_pipeline)
+        cl.user_session.set("chain", retrieval_augmented_qa_pipeline)
 
 
 @cl.on_message
@@ -128,7 +95,7 @@ async def main(message):
     chain = cl.user_session.get("chain")
 
     msg = cl.Message(content="")
-    result = await chain.arun_pipeline(message.content)
+    result = await chain.arun_pipeline(message.content,rerank=True)
 
     async for stream_resp in result["response"]:
         await msg.stream_token(stream_resp)

requirements.txt

@@ -2,5 +2,8 @@ numpy
 chainlit==0.7.700
 openai
 rank_bm25
-PyPDF2
-langchain>=0.2
+pymupdf
+langchain>=0.2
+langchain-community
+tiktoken
+langchain-openai