Upload 3 files

First version with working UI. Need some improvements in the results.

app.py

@@ -0,0 +1,78 @@
+from langchain_openai import ChatOpenAI
+from langchain.chains import ConversationChain
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.prompts import (
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+    ChatPromptTemplate,
+    MessagesPlaceholder
+)
+import streamlit as st
+from utils import find_match, query_refiner, get_conversation_string
+
+from dotenv import load_dotenv
+import os
+load_dotenv()
+
+st.subheader("Aido-We assist Universities for recruiting International students")
+
+if 'responses' not in st.session_state:
+    st.session_state['responses'] = ["How can I assist you?"]
+
+if 'requests' not in st.session_state:
+    st.session_state['requests'] = []
+
+llm = ChatOpenAI(model_name="gpt-4o-mini", api_key=os.getenv('OPENAI_API_KEY'))
+
+if 'buffer_memory' not in st.session_state:
+    st.session_state.buffer_memory = ConversationBufferWindowMemory(k=3, return_messages=True)
+
+system_msg_template = SystemMessagePromptTemplate.from_template(template="""Answer the question as truthfully as possible using the provided context, 
+and if the answer is not contained within the text below, say 'I don't know'""")
+
+human_msg_template = HumanMessagePromptTemplate.from_template(template="{input}")
+
+prompt_template = ChatPromptTemplate.from_messages(
+    [system_msg_template, MessagesPlaceholder(variable_name="history"), human_msg_template])
+
+conversation = ConversationChain(memory=st.session_state.buffer_memory, prompt=prompt_template, llm=llm, verbose=True)
+
+# container for chat history
+response_container = st.container()
+# container for text box
+textcontainer = st.container()
+
+with textcontainer:
+    # Replace the single-line text input with a text area that expands
+    query = st.text_area(
+        "Query: ",
+        key="input",
+        height=100,  # Initial height
+        max_chars=None,  # No character limit
+        help="Type your question here.",
+        placeholder="●	What are some concerns students from Algeria have about studying in the USA?"
+    )
+
+    # Add a submit button to control when the query is processed
+    submit_button = st.button("Submit")
+
+    if submit_button and query:
+        with st.spinner("typing..."):
+            conversation_string = get_conversation_string()
+            refined_query = query_refiner(conversation_string, query)
+            st.subheader("Refined Query:")
+            st.write(refined_query)
+            context = find_match(refined_query)
+            response = conversation.predict(input=f"Context:\n {context} \n\n Query:\n{query}")
+        st.session_state.requests.append(query)
+        st.session_state.responses.append(response)
+
+with response_container:
+    if st.session_state['responses']:
+        for i in range(len(st.session_state['responses'])):
+            # Using Streamlit's native chat message functionality instead of streamlit_chat
+            with st.chat_message("assistant"):
+                st.write(st.session_state['responses'][i])
+            if i < len(st.session_state['requests']):
+                with st.chat_message("user"):
+                    st.write(st.session_state["requests"][i])

indexing.py

@@ -0,0 +1,62 @@
+from langchain_community.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from pinecone import Pinecone, ServerlessSpec
+#from langchain_community.vectorstores import Pinecone
+from dotenv import load_dotenv
+import os
+from langchain_pinecone import PineconeVectorStore
+load_dotenv()
+
+
+
+directory = "D:/Projects/Aido/data"
+
+def load_docs(directory):
+    loader = DirectoryLoader(directory)
+    documents = loader.load()
+    return documents
+
+documents = load_docs(directory)
+print(f"Number of documents in the dataset: {len(documents)}")
+
+def split_docs(documents,chunk_size=400,chunk_overlap=150):
+  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+  docs = text_splitter.split_documents(documents)
+  return docs
+
+docs = split_docs(documents)
+print(f"There are total of {len(docs)} chunks derived from {len(documents)} document" )
+
+embeddings = OpenAIEmbeddings(model="text-embedding-ada-002")
+
+
+pc= Pinecone(api_key=os.getenv('PINECONE_API_KEY') ) # next to api key in console
+
+index_name = "aido"
+if index_name not in pc.list_indexes().names():
+    pc.create_index(
+        name=index_name,
+        dimension=1536,  # dimensionality of text-embedding-ada-002
+        metric="cosine"
+    )
+pinecone_index = pc.Index(index_name)
+
+
+index = PineconeVectorStore.from_documents(
+    docs,
+    embeddings,
+    index_name=index_name
+)
+
+def get_similiar_docs(query,k=3,score=False):
+  if score:
+    similar_docs = index.similarity_search_with_score(query,k=k)
+  else:
+    similar_docs = index.similarity_search(query,k=k)
+  return similar_docs
+
+
+query = "What do students from Albino doubtful on their return on investment when considering studying in the USA?"
+similar_docs = get_similiar_docs(query)
+print(similar_docs)

utils.py

@@ -0,0 +1,65 @@
+from langchain_openai import OpenAIEmbeddings
+from pinecone import Pinecone
+import streamlit as st
+from openai import OpenAI
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Initialize OpenAI client
+client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
+
+# Initialize embeddings
+embeddings = OpenAIEmbeddings(model="text-embedding-ada-002", api_key=os.getenv('OPENAI_API_KEY'))
+
+# Initialize Pinecone
+pc = Pinecone(api_key=os.getenv('PINECONE_API_KEY'))
+
+# Check if index exists and connect to it
+index_name = "aido-hybrid"
+if index_name not in pc.list_indexes().names():
+    print("Creating a new Pinecone index...")
+    pc.create_index(
+        name=index_name,
+        dimension=1536,  # dimensionality of text-embedding-ada-002
+        metric="cosine"
+    )
+
+# Connect to the existing Pinecone index
+index = pc.Index(index_name)
+
+
+def find_match(input):
+    # Get embeddings for the input query
+    input_em = embeddings.embed_query(input)
+
+    # Query Pinecone
+    result = index.query(vector=input_em, top_k=5, include_metadata=True)
+
+    # Return the top 2 matches
+    return result['matches'][0]['metadata']['text'] + "\n" + result['matches'][1]['metadata']['text']
+
+
+def query_refiner(conversation, query):
+    # Using the new ChatCompletion API instead of the deprecated Completion API
+    response = client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system",
+             "content": "You are a helpful assistant that refines user queries based on conversation context."},
+            {"role": "user",
+             "content": f"Given the following user query and conversation log, formulate a question that would be the most relevant to provide the user with an answer from a knowledge base.\n\nCONVERSATION LOG: \n{conversation}\n\nQuery: {query}\n\nRefined Query:"}
+        ],
+        temperature=0.7,
+        max_tokens=256
+    )
+    return response.choices[0].message.content
+
+
+def get_conversation_string():
+    conversation_string = ""
+    for i in range(len(st.session_state['responses']) - 1):
+        conversation_string += "Human: " + st.session_state['requests'][i] + "\n"
+        conversation_string += "Bot: " + st.session_state['responses'][i + 1] + "\n"
+    return conversation_string