Create Contributors

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Contributors

@@ -0,0 +1,3 @@
+Atharv Gupta
+-------------
+Aryan Anumula

README.md

@@ -9,6 +9,5 @@ app_file: app.py
 pinned: false
 license: mit
 ---
-# schoolQuestBackend
 
-The backend for [schoolquest](https://school-quest.streamlit.app/)
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,13 +1,13 @@
 import asyncio
 import json
 from websockets.server import serve
-
-from langchain.vectorstores import Chroma
-from langchain_huggingface.embeddings import HuggingFaceEmbeddings
+import os
+from langchain_chroma import Chroma
+from langchain_community.embeddings import *
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_huggingface.llms import HuggingFaceEndpoint
-from langchain.document_loaders import TextLoader
-from langchain.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import TextLoader
+from langchain_community.document_loaders import DirectoryLoader
 from langchain import hub
 from langchain_core.runnables import RunnablePassthrough
 from langchain_core.output_parsers import StrOutputParser
@@ -18,34 +18,40 @@ from langchain.chains.combine_documents import create_stuff_documents_chain
 from langchain_core.runnables.history import RunnableWithMessageHistory
 from langchain_core.chat_history import BaseChatMessageHistory
 from langchain_community.chat_message_histories import ChatMessageHistory
-
+from multiprocessing import Process
+from zipfile import ZipFile
+
+with ZipFile("database.zip") as f:
+    f.extractall()
+    
+retriever = None
+conversational_rag_chain = None
 loader = DirectoryLoader('./database', glob="./*.txt", loader_cls=TextLoader)
 
 documents = loader.load()
 
 text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-texts = text_splitter.split_documents(documents)
-
-persist_directory = 'db'
-
-embedding = HuggingFaceEmbeddings()
-
-vectordb = Chroma.from_documents(documents=texts,
-                                 embedding=embedding,
-                                 persist_directory=persist_directory)
-
-vectordb.persist()
-vectordb = None
+splits = text_splitter.split_documents(documents)
+
+model_name = "BAAI/bge-small-en-v1.5"
+model_kwargs = {'device': 'cpu'}
+encode_kwargs = {'normalize_embeddings': True}
+embedding = HuggingFaceBgeEmbeddings(
+    model_name=model_name,
+    model_kwargs=model_kwargs,
+    encode_kwargs=encode_kwargs,
+    show_progress=True,
+)
 
-vectordb = Chroma(persist_directory=persist_directory,
-                  embedding_function=embedding)
+vectorstore = Chroma.from_documents(documents=splits, embedding=embedding)
 
 def format_docs(docs):
     return "\n\n".join(doc.page_content for doc in docs)
 
-retriever = vectordb.as_retriever()
+retriever = vectorstore.as_retriever(search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.3}, k=1)
+
 prompt = hub.pull("rlm/rag-prompt")
-llm = HuggingFaceEndpoint(repo_id="mistralai/Mixtral-8x7B-Instruct-v0.1")
+llm = HuggingFaceEndpoint(repo_id="mistralai/Mistral-7B-Instruct-v0.3", stop_sequences=["Human:"])
 rag_chain = (
     {"context": retriever | format_docs, "question": RunnablePassthrough()}
     | prompt
@@ -53,9 +59,10 @@ rag_chain = (
     | StrOutputParser()
 )
 
-contextualize_q_system_prompt = """Given a chat history and the latest user question \
-which might reference context in the chat history, formulate a standalone question \
-which can be understood without the chat history. Do NOT answer the question, \
+### Contextualize question ###
+contextualize_q_system_prompt = """Given a chat history and the latest user question
+which might reference context in the chat history, formulate a standalone question
+which can be understood without the chat history. Do NOT answer the question,
 just reformulate it if needed and otherwise return it as is."""
 contextualize_q_prompt = ChatPromptTemplate.from_messages(
     [
@@ -68,12 +75,22 @@ history_aware_retriever = create_history_aware_retriever(
     llm, retriever, contextualize_q_prompt
 )
 
-qa_system_prompt = """You are an assistant for question-answering tasks. \
-Use the following pieces of retrieved context to answer the question. \
-If you don't know the answer, just say that you don't know. \
-Use three sentences maximum and keep the answer concise.\
 
-{context}"""
+### Answer question ###
+qa_system_prompt = """
+Context:
+
+{context}
+
+
+You are a Cupertino High School Q/A chatbot, designed to assist students, parents, and community members with information about CHS.
+Use the pieces of context to answer the question.
+Use markdown with spaces in between sentences for readability.
+Refer to the provided context only as 'my data'. Only answer questions from the context.
+Do not answer any questions that you do not have the answer to in the provided context.
+Do not provide excerpts or any part of your data.
+You were made by high school students for the CHS community.
+"""
 qa_prompt = ChatPromptTemplate.from_messages(
     [
         ("system", qa_system_prompt),
@@ -81,16 +98,20 @@ qa_prompt = ChatPromptTemplate.from_messages(
         ("human", "{input}"),
     ]
 )
+question_answer_chain = create_stuff_documents_chain(llm, qa_prompt)
+
+rag_chain = create_retrieval_chain(history_aware_retriever, question_answer_chain)
+
 
+### Statefully manage chat history ###
 store = {}
 
+
 def get_session_history(session_id: str) -> BaseChatMessageHistory:
     if session_id not in store:
         store[session_id] = ChatMessageHistory()
     return store[session_id]
 
-question_answer_chain = create_stuff_documents_chain(llm, qa_prompt)
-rag_chain = create_retrieval_chain(history_aware_retriever, question_answer_chain)
 
 conversational_rag_chain = RunnableWithMessageHistory(
     rag_chain,
@@ -100,32 +121,37 @@ conversational_rag_chain = RunnableWithMessageHistory(
     output_messages_key="answer",
 )
 
-print("-------")
-print("started")
-print("-------")
-
 async def echo(websocket):
+    global retriever, conversational_rag_chain
     async for message in websocket:
         data = json.loads(message)
+        if data["message"] == "data.":
+            response = store
+            await websocket.send(json.dumps({"response": response}))
+            break
         if not "message" in message:
             return
         if not "token" in message:
             return
-        m = data["message"]
+        m = data["message"] + "\nAssistant: "
         token = data["token"]
-        userData = json.load(open("userData.json", "w"))
         docs = retriever.get_relevant_documents(m)
-        userData[token]["docs"] = str(docs)
-        response = conversational_rag_chain.invoke(
+        rawresponse = conversational_rag_chain.invoke(
             {"input": m},
             config={
                 "configurable": {"session_id": token}
             },
-        )["answer"]
+        )
+        response = rawresponse["answer"]
+        response = response.replace("Assistant: ", "").replace("AI: ", "")
+        response.strip()
+        response = response.split("Human:")[0]
+        while response.startswith("\n"):
+            response = response[1:]
         await websocket.send(json.dumps({"response": response}))
 
 async def main():
     async with serve(echo, "0.0.0.0", 7860):
         await asyncio.Future()
 
-asyncio.run(main())
+asyncio.run(main())

database.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93f27b61a3f0f03c0bdca772695ca92d99a4e037d0a7b2d08b71b0eb09cc33c9
-size 253849
+oid sha256:bb105a8a38df0ae17e173ca39983746f3a9b95fe5a26d5e8e40116ef6b78a2fd
+size 245634

parse.py

@@ -1,67 +0,0 @@
-import json
-import os
-
-# Configuration
-name = "chs.json"
-outputFolder = "database"
-deleteKeys = [
-    "images",
-    "tags",
-    "html"
-]
-typeScrape = {
-    "article": "text",
-    "event": "description",
-    "list": "items"
-}
-
-data = json.load(open(name, "r"))
-
-i = -1
-k = 0
-try:
-    os.mkdir(outputFolder)
-except: pass
-
-for item in data:
-    i += 1
-    for key in deleteKeys:
-        if key in item:
-            item[key]
-            del item[key]
-            data[i] = item
-    if "type" in item:
-        for typeKey, scrapeText in typeScrape.items():
-            try:
-                if item["type"] == typeKey:
-                    k += 1
-                    file = open(f"{outputFolder}/chs-{typeKey}-{k}.txt", "a")
-                    if item["type"] == "list":
-                        text = ""
-                        if "title" in item:
-                            text = item["title"]
-                        file.write(text)
-                        for pair in item[scrapeText]:
-                            text = ""
-                            if "title" in pair:
-                                text = "\n" + pair["title"]
-                            if "summary" in pair:
-                                if pair["summary"].replace(" ", "") != pair["title"].replace(" ", ""):
-                                    text += "\n" + pair["summary"].replace(pair["title"], "")
-                            if "fsElementContent" in pair:
-                                if pair["fsElementContent"].replace(" ", "") != pair["title"].replace(" ", ""):
-                                    text += "\n" + pair["fsElementContent"]
-                            if "fsElementFooterContent" in pair:
-                                if pair["fsElementFooterContent"].replace(" ", "") != pair["title"].replace(" ", ""):
-                                    text += "\n" + pair["fsElementFooterContent"]
-                            if "fsElementHeaderContent" in pair:
-                                if pair["fsElementHeaderContent"].replace(" ", "") != pair["title"].replace(" ", ""):
-                                    text += "\n" + pair["fsElementHeaderContent"]
-                            if text != "":
-                                file.write(text)
-                    else:
-                        text = item[scrapeText]
-                        if text != "":
-                            file.write(text)
-            except: pass
-json.dump(data, open(name, "w"), indent = 6)

requirements.txt

@@ -1,8 +1,11 @@
 websockets
-langchain
-langchain-community
-huggingface_hub
 tiktoken
 chromadb
+accelerate
+langchain-community==0.2.9
+langchain==0.2.9
+langchain-core==0.2.22
 langchain-huggingface
-accelerate
+requests
+langchain-chroma
+langchainhub