versions pinned

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11.9
 RUN useradd -m -u 1000 user
 USER user
 ENV HOME=/home/user \

aimakerspace/openai_utils/embedding.py

@@ -7,11 +7,12 @@ import asyncio
 
 
 class EmbeddingModel:
-    def __init__(self, embeddings_model_name: str = "text-embedding-3-small"):
+    def __init__(self, embeddings_model_name: str = "text-embedding-3-small", dimensions: int = None):
         load_dotenv()
         self.openai_api_key = os.getenv("OPENAI_API_KEY")
         self.async_client = AsyncOpenAI()
         self.client = OpenAI()
+        self.dimensions = dimensions
 
         if self.openai_api_key is None:
             raise ValueError(
@@ -22,28 +23,35 @@ class EmbeddingModel:
 
     async def async_get_embeddings(self, list_of_text: List[str]) -> List[List[float]]:
         embedding_response = await self.async_client.embeddings.create(
-            input=list_of_text, model=self.embeddings_model_name
+            input=list_of_text, model=self.embeddings_model_name, dimensions=self.dimensions
         )
 
         return [embeddings.embedding for embeddings in embedding_response.data]
 
+    async def async_get_embeddings_openai(self, list_of_text: List[str]) :
+        embedding_response = await self.async_client.embeddings.create(
+            input=list_of_text, model=self.embeddings_model_name, dimensions=self.dimensions
+        )
+
+        return embedding_response
+
     async def async_get_embedding(self, text: str) -> List[float]:
         embedding = await self.async_client.embeddings.create(
-            input=text, model=self.embeddings_model_name
+            input=text, model=self.embeddings_model_name, dimensions=self.dimensions
         )
 
         return embedding.data[0].embedding
 
     def get_embeddings(self, list_of_text: List[str]) -> List[List[float]]:
         embedding_response = self.client.embeddings.create(
-            input=list_of_text, model=self.embeddings_model_name
+            input=list_of_text, model=self.embeddings_model_name, dimensions=self.dimensions
         )
 
         return [embeddings.embedding for embeddings in embedding_response.data]
 
     def get_embedding(self, text: str) -> List[float]:
         embedding = self.client.embeddings.create(
-            input=text, model=self.embeddings_model_name
+            input=text, model=self.embeddings_model_name, dimensions=self.dimensions
         )
 
         return embedding.data[0].embedding

aimakerspace/text_utils.py

@@ -1,6 +1,8 @@
+# aimakerspace.text_utils.py
+
 import os
 from typing import List
-
+from langchain_community.document_loaders import PyPDFLoader
 
 class TextFileLoader:
     def __init__(self, path: str, encoding: str = "utf-8"):
@@ -35,8 +37,39 @@ class TextFileLoader:
         self.load()
         return self.documents
 
+class PdfFileLoader:
+    def __init__(self, path: str):
+        self.documents = []
+        self.path = path
+
+    def load(self):
+        if os.path.isdir(self.path):
+            self.load_directory()
+        elif os.path.isfile(self.path) and self.path.endswith(".pdf"):
+            self.load_file()
+        else:
+            raise ValueError(
+                "Provided path is neither a valid directory nor a .pdf file."
+            )
+
+    def load_file(self):
+        pdf_loader = PyPDFLoader(self.path)
+        pdf_pages =  pdf_loader.load_and_split()   #  Defaults to RecursiveCharacterTextSplitter.
+        self.documents = [page.page_content for page in pdf_pages] 
+
+    def load_directory(self):
+        for root, _, files in os.walk(self.path):
+            for file in files:
+                if file.endswith(".pdf"):
+                    pdf_loader = PyPDFLoader(file.path)
+                    pdf_pages =  pdf_loader.load_and_split()
+                    self.documents.append([page.page_content for page in pdf_pages])
+
+    def load_documents(self):
+        self.load()
+        return self.documents
 
-class CharacterTextSplitter:
+class CharacterTextSplitter():
     def __init__(
         self,
         chunk_size: int = 1000,
@@ -60,18 +93,3 @@ class CharacterTextSplitter:
         for text in texts:
             chunks.extend(self.split(text))
         return chunks
-
-
-if __name__ == "__main__":
-    loader = TextFileLoader("data/KingLear.txt")
-    loader.load()
-    splitter = CharacterTextSplitter()
-    chunks = splitter.split_texts(loader.documents)
-    print(len(chunks))
-    print(chunks[0])
-    print("--------")
-    print(chunks[1])
-    print("--------")
-    print(chunks[-2])
-    print("--------")
-    print(chunks[-1])

aimakerspace/vectordatabase.py

@@ -1,9 +1,13 @@
+from enum import Enum
 import numpy as np
 from collections import defaultdict
 from typing import List, Tuple, Callable
 from aimakerspace.openai_utils.embedding import EmbeddingModel
 import asyncio
+from qdrant_client import models, QdrantClient
+from qdrant_client.models import PointStruct,VectorParams,Distance
 
+collection_name = "embedding_collection"
 
 def cosine_similarity(vector_a: np.array, vector_b: np.array) -> float:
     """Computes the cosine similarity between two vectors."""
@@ -13,10 +17,62 @@ def cosine_similarity(vector_a: np.array, vector_b: np.array) -> float:
     return dot_product / (norm_a * norm_b)
 
 
+def euclidean_distance(vector_a: np.array, vector_b: np.array) -> float:
+    """Computes the Euclidean distance between two vectors."""
+    return np.sqrt(np.sum((vector_a - vector_b) ** 2))
+
+
+def manhattan_distance(vector_a: np.array, vector_b: np.array) -> float:
+    """Computes the Manhattan distance between two vectors."""
+    return np.sum(np.abs(vector_a - vector_b))
+
+
+def minkowski_distance(vector_a: np.array, vector_b: np.array, p: float) -> float:
+    """
+    Computes the Minkowski distance between two vectors.
+
+    Parameters:
+    vector_a (np.array): First vector.
+    vector_b (np.array): Second vector.
+    p (float): The order of the norm. For example, p=1 gives Manhattan distance, p=2 gives Euclidean distance.
+
+    Returns:
+    float: Minkowski distance between vector_a and vector_b.
+    """
+    # Ensure the input vectors are NumPy arrays
+    vector_a = np.asarray(vector_a)
+    vector_b = np.asarray(vector_b)
+
+    # Compute Minkowski distance
+    distance = np.sum(np.abs(vector_a - vector_b) ** p) ** (1 / p)
+    return distance
+
+
+class DistanceMeasure(Enum):
+    COSINE_SIMILARITY = cosine_similarity
+    EUCLIDEAN_DISTANCE = euclidean_distance
+    MANHATTAN_DISTANCE = manhattan_distance
+    MINKOWSKI_DISTANCE = minkowski_distance
+
+
+class VectorDatabaseOptions(Enum):
+    DICTIONARY = "dictionary"
+    QDRANT = "qdrant"
+
+
 class VectorDatabase:
-    def __init__(self, embedding_model: EmbeddingModel = None):
-        self.vectors = defaultdict(np.array)
+    def __init__(
+        self,
+        vector_db_options: VectorDatabaseOptions,
+        embedding_model: EmbeddingModel = None,
+    ):
+        self.vectors = None
+        self.vector_db_options = vector_db_options
         self.embedding_model = embedding_model or EmbeddingModel()
+        if vector_db_options == VectorDatabaseOptions.DICTIONARY:
+            self.vectors = defaultdict(np.array)
+        if vector_db_options == VectorDatabaseOptions.QDRANT:
+            self.qdrant_client = QdrantClient(":memory:")
 
     def insert(self, key: str, vector: np.array) -> None:
         self.vectors[key] = vector
@@ -41,16 +97,39 @@ class VectorDatabase:
         return_as_text: bool = False,
     ) -> List[Tuple[str, float]]:
         query_vector = self.embedding_model.get_embedding(query_text)
-        results = self.search(query_vector, k, distance_measure)
-        return [result[0] for result in results] if return_as_text else results
+        if self.vector_db_options == VectorDatabaseOptions.DICTIONARY:
+            results = self.search(query_vector, k, distance_measure)
+            return [result[0] for result in results] if return_as_text else results
+        if self.vector_db_options == VectorDatabaseOptions.QDRANT:
+            search_result = self.qdrant_client.search(collection_name,query_vector=query_vector)
+            return [(point.payload["text"],point.score) for point in search_result]
 
     def retrieve_from_key(self, key: str) -> np.array:
         return self.vectors.get(key, None)
 
     async def abuild_from_list(self, list_of_text: List[str]) -> "VectorDatabase":
-        embeddings = await self.embedding_model.async_get_embeddings(list_of_text)
-        for text, embedding in zip(list_of_text, embeddings):
-            self.insert(text, np.array(embedding))
+        if self.vector_db_options == VectorDatabaseOptions.DICTIONARY:
+            embeddings = await self.embedding_model.async_get_embeddings(list_of_text)
+            for text, embedding in zip(list_of_text, embeddings):
+                self.insert(text, np.array(embedding))
+        if self.vector_db_options == VectorDatabaseOptions.QDRANT:
+            embeddings_response = await self.embedding_model.async_get_embeddings_openai(list_of_text)
+            points = [
+                PointStruct(
+                    id=idx,
+                    vector=data.embedding,
+                    payload={"text": text},
+                )
+                for idx, (data, text) in enumerate(zip(embeddings_response.data, list_of_text))
+            ]
+            self.qdrant_client.create_collection(
+                collection_name,
+                vectors_config=VectorParams(
+                    size=self.embedding_model.dimensions,
+                    distance=Distance.COSINE,
+                ),
+            )
+            self.qdrant_client.upsert(collection_name, points)
         return self
 
 

app.py

@@ -1,20 +1,27 @@
 import os
+from openai import AsyncOpenAI
 from typing import List
 from chainlit.types import AskFileResponse
-from aimakerspace.text_utils import CharacterTextSplitter, TextFileLoader
+from chainlit.cli import run_chainlit
+from aimakerspace.text_utils import CharacterTextSplitter, PdfFileLoader, TextFileLoader
 from aimakerspace.openai_utils.prompts import (
     UserRolePrompt,
     SystemRolePrompt,
     AssistantRolePrompt,
 )
 from aimakerspace.openai_utils.embedding import EmbeddingModel
-from aimakerspace.vectordatabase import VectorDatabase
+from aimakerspace.vectordatabase import VectorDatabase, VectorDatabaseOptions
 from aimakerspace.openai_utils.chatmodel import ChatOpenAI
 import chainlit as cl
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
+
+# Instrument the OpenAI client
+# cl.instrument_openai()
+
+##### Prompt Templates #####
 system_template = """\
 Use the following context to answer a users question. If you cannot find the answer in the context, say you don't know the answer."""
-system_role_prompt = SystemRolePrompt(system_template)
 
 user_prompt_template = """\
 Context:
@@ -23,100 +30,130 @@ Context:
 Question:
 {question}
 """
-user_role_prompt = UserRolePrompt(user_prompt_template)
-
-class RetrievalAugmentedQAPipeline:
-    def __init__(self, llm: ChatOpenAI(), vector_db_retriever: VectorDatabase) -> None:
-        self.llm = llm
-        self.vector_db_retriever = vector_db_retriever
-
-    async def arun_pipeline(self, user_query: str):
-        context_list = self.vector_db_retriever.search_by_text(user_query, k=4)
-
-        context_prompt = ""
-        for context in context_list:
-            context_prompt += context[0] + "\n"
-
-        formatted_system_prompt = system_role_prompt.create_message()
-
-        formatted_user_prompt = user_role_prompt.create_message(question=user_query, context=context_prompt)
-
-        async def generate_response():
-            async for chunk in self.llm.astream([formatted_system_prompt, formatted_user_prompt]):
-                yield chunk
 
-        return {"response": generate_response(), "context": context_list}
-
-text_splitter = CharacterTextSplitter()
+system_role_prompt = SystemRolePrompt(system_template)
+user_role_prompt = UserRolePrompt(user_prompt_template)
 
+### Text Chunking ###
+
+# text_splitter = CharacterTextSplitter()
+text_splitter = RecursiveCharacterTextSplitter(
+    separators=[
+        "\n\n",
+        "\n",
+        " ",
+        ".",
+        ",",
+        "\u200b",  # Zero-width space
+        "\uff0c",  # Fullwidth comma
+        "\u3001",  # Ideographic comma
+        "\uff0e",  # Fullwidth full stop
+        "\u3002",  # Ideographic full stop
+        "",
+    ],
+)
 
-def process_text_file(file: AskFileResponse):
+def process_text_file(file: AskFileResponse) -> List[str]:
     import tempfile
 
-    with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as temp_file:
+    with tempfile.NamedTemporaryFile(
+        mode="w", delete=False, suffix=".txt"
+    ) as temp_file:
         temp_file_path = temp_file.name
 
-    with open(temp_file_path, "wb") as f:
-        f.write(file.content)
+    with open(file.path, "r", encoding="utf-8") as f:
+        text = f.read()
+
+    with open(temp_file_path, "w") as f:
+        f.write(text)
 
     text_loader = TextFileLoader(temp_file_path)
     documents = text_loader.load_documents()
-    texts = text_splitter.split_texts(documents)
+    texts = []
+    for doc in documents:
+        texts.append(text_splitter.split_text(doc))
     return texts
 
+def process_pdf_file(file: AskFileResponse) -> List[str]:
+    pdf_loader = PdfFileLoader(file.path)
+    texts = pdf_loader.load_documents() # Also handles splitting the text in this case pages
+    return texts
+
+async def send_new_message(content, elemets=None):
+    msg = cl.Message(content,elements=elemets)
+    await msg.send()
+    return msg
+
 
 @cl.on_chat_start
 async def on_chat_start():
+    print("On Chat Start")
+    # await send_new_message("Welcome to the Chat with Files app!")
+    msg = cl.Message(content="Welcome to the Chat with Files app!")
+    await msg.send()
+    print("After First message")
+
     files = None
 
     # Wait for the user to upload a file
     while files == None:
+
         files = await cl.AskFileMessage(
-            content="Please upload a Text File file to begin!",
-            accept=["text/plain"],
-            max_size_mb=2,
+            content="Please upload a text file to begin!",
+            accept=["text/plain", "application/pdf"],
+            max_size_mb=10,
+            max_files=4,
             timeout=180,
         ).send()
-
-    file = files[0]
-
-    msg = cl.Message(
-        content=f"Processing `{file.name}`...", disable_human_feedback=True
-    )
+    texts : List[str] = []
+    for file in files:
+        if file.type == "application/pdf":
+            texts.extend(process_pdf_file(file))
+        if file.type == "text/plain":
+            texts.extend(process_text_file(file))
+
+    # await send_new_message(content=f"Processing `{file.name}`...")
+    msg = cl.Message(content=f"Processing `{file.name}`...")
     await msg.send()
 
-    # load the file
-    texts = process_text_file(file)
-
     print(f"Processing {len(texts)} text chunks")
-
+    
     # Create a dict vector store
-    vector_db = VectorDatabase()
+    vector_db_options =VectorDatabaseOptions.QDRANT
+    embedding_model = EmbeddingModel(embeddings_model_name= "text-embedding-3-small",dimensions=1000)
+    vector_db = VectorDatabase(vector_db_options,embedding_model)
     vector_db = await vector_db.abuild_from_list(texts)
-    
+
     chat_openai = ChatOpenAI()
 
     # Create a chain
-    retrieval_augmented_qa_pipeline = RetrievalAugmentedQAPipeline(
-        vector_db_retriever=vector_db,
-        llm=chat_openai
+    retrieval_augmented_qa_pipeline = RetrievalAugmentedQAPipeline(system_role_prompt, user_role_prompt,
+        vector_db_retriever=vector_db, llm=chat_openai
     )
-    
+
     # Let the user know that the system is ready
-    msg.content = f"Processing `{file.name}` done. You can now ask questions!"
-    await msg.update()
+    msg = cl.Message(content=f"Processing `{file.name}` done. You can now ask questions!")
+    await msg.send()
 
     cl.user_session.set("chain", retrieval_augmented_qa_pipeline)
 
 
 @cl.on_message
-async def main(message):
-    chain = cl.user_session.get("chain")
+async def main(message: cl.Message):
+    msg = cl.Message(content="on message")
+    await msg.send()
+
+    chain :RetrievalAugmentedQAPipeline = cl.user_session.get("chain")
 
     msg = cl.Message(content="")
     result = await chain.arun_pipeline(message.content)
 
-    async for stream_resp in result["response"]:
+    async for stream_resp in result.get('response'):
         await msg.stream_token(stream_resp)
 
-    await msg.send()
+    await msg.send()
+    cl.user_session.set("chain", chain)
+
+
+if __name__ == "__main__":
+    run_chainlit(__file__)

requirements.txt

@@ -1,7 +1,7 @@
-numpy
-chainlit==0.7.700
-openai
+numpy==1.26.4
+chainlit==0.7.700   # 1.1.402 
+openai==1.3.5
+qdrant-client==1.11.0
 langchain-text-splitters
-pypdf
 langchain-community
-qdrant-client
+pypdf