loading file into chat

app.py

@@ -1,13 +1,17 @@
 import os
+import logging
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.embeddings.openai import OpenAIEmbeddings
 import chainlit as cl
+from src.utils import get_docSearch
+from src.model import load_chain
+
+
+
 
 
 
-text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
-embeddings = OpenAIEmbeddings()
 
 welcome_message = """ Upload your file here"""
 
@@ -25,3 +29,21 @@ async def start():
     file = files[0]
     msg = cl.Message(content=f"Processing `{type(files)}` {file.name}....")
     await msg.send()
+
+    docsearch = get_docSearch(file)
+    
+
+    chain = load_chain(docsearch)
+
+    logging.info(f"Model loaded successfully")
+
+    
+    ## let the user know when system is ready
+
+    msg.content = f"{file.name} processed. You begin asking questions"
+
+    await msg.update()
+
+    cl.user_session.set("chain", chain)
+
+

requirements.txt

@@ -1,4 +1,6 @@
 langchain
 openai
 python-dotenv
-chainlit
+chainlit
+chromadb
+tiktoken

src/config.py

@@ -0,0 +1,5 @@
+class Config:
+    temperature = 0
+    streaming = True
+    chain_type = "stuff"
+    max_token_limit = 4098

src/model.py

@@ -0,0 +1,24 @@
+from langchain.chains import RetrievalQAWithSourcesChain
+from langchain.chat_models import ChatOpenAI
+import logging
+
+
+
+from src.config import Config
+
+
+
+
+
+def load_model():
+    model = ChatOpenAI(temperature=Config.temperature,
+                   streaming=Config.streaming)
+    return model
+
+
+def load_chain(docsearch):
+    model = load_model()
+    chain = RetrievalQAWithSourcesChain.from_chain_type(model,
+                                                        chain_type=Config.chain_type,
+                                                        retriever=docsearch.as_retriever(max_tokens_limit=Config.max_token_limit))
+    return chain

src/utils.py

@@ -1,8 +1,37 @@
 from chainlit.types import AskFileResponse
 from langchain.document_loaders import TextLoader
+from langchain.document_loaders import PyPDFDirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.embeddings import OpenAIEmbeddings
+
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+embeddings = OpenAIEmbeddings()
 
 def process_file(file: AskFileResponse):
-    pass
+    import tempfile 
+
+    if file.type == "text/plain":
+        Loader = TextLoader
+    elif file.type == "application/pdf":
+        Loader = PyPDFDirectoryLoader
+
+    with tempfile.NamedTemporaryFile() as tempfile:
+        tempfile.write(file.content)
+        loader = Loader(tempfile.name)
+        documents = loader.load()
+        # text_splitter = text_splitter()
+        docs = text_splitter.split_documents(documents)
+
+        for i, doc in enumerate(docs):
+            doc.metadata["source"] = f"source_{i}"
+        return docs
 
 def get_docSearch(file: AskFileResponse):
-    pass
+    docs = process_file(file)
+
+    ## save data in user session 
+    
+    docsearch = Chroma.from_documents(docs, embeddings)
+
+    return docsearch