Learning Path Index Files

First Commit of the Learning Path Index Context Based Search Project

.env_template

@@ -0,0 +1,4 @@
+# This file won't become part of the git history as long as it exists in
+# the .gitignore file, and it should stay like that
+OPENAI_API_KEY=<yourOpenAIAPI key>
+PINECONE_API_KEY=<yourPineCone key>

.gitignore

@@ -0,0 +1,8 @@
+# Ignore all files with .env extension in any directory
+**/*.env
+
+# Ignore all .env files in the root directory and its subdirectories
+.env
+
+__*/**
+faiss_learning_path_index/

faiss_index.py

@@ -0,0 +1,31 @@
+import os
+
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.document_loaders import TextLoader
+from langchain.vectorstores import FAISS
+from langchain.chains import RetrievalQA
+from langchain.llms import OpenAI
+
+def faiss_index():
+    current_directory = os.getcwd()
+    data_path = current_directory + "\\final_project\\Learning_Pathway_Index.csv"
+    loader = TextLoader(data_path)
+    documents = loader.load()
+    text_splitter = CharacterTextSplitter(
+        chunk_size=1000, chunk_overlap=30, separator="\n"
+    )
+    docs = text_splitter.split_documents(documents=documents)
+
+    embeddings = OpenAIEmbeddings()
+    vectorstore = FAISS.from_documents(docs, embeddings)
+    vectorstore.save_local("faiss_learning_path_index")
+
+    new_vectorstore = FAISS.load_local("faiss_learning_path_index", embeddings)
+    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=new_vectorstore.as_retriever())
+    res = qa.run("Give me Machine Learning Course with 10 or 20 min duration.")
+    print(res)
+
+
+if __name__ == "__main__":
+    faiss_index()

interface.py

@@ -0,0 +1,24 @@
+import streamlit as st
+
+# Define your Streamlit app and return the input variable   
+def app():
+    # Add a title to your app
+    st.title("KaggleX Learning Path Index Search")
+
+    # Add some text to your app
+    st.write("Embark your Learning Path Journey with right search !!")
+
+    # Add a text input to your app
+    user_input = st.text_input("Enter your course query here")
+
+    # Store the input in a variable
+    my_variable = user_input
+    # Display the stored variable
+    # st.write(f"The stored variable is: {my_variable}")
+    
+    return my_variable
+
+# Run your Streamlit app
+# if __name__ == "__main__":
+#     var = app()
+#     print(var)

main.py

@@ -0,0 +1,182 @@
+import os
+from dotenv import load_dotenv
+from datetime import datetime
+import time
+from langchain.llms import OpenAI
+from langchain.document_loaders import TextLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+from langchain.llms import OpenAI
+from langchain.vectorstores import FAISS
+from langchain.prompts import PromptTemplate
+
+from interface import app
+import streamlit as st
+# Define GenerateLearningPathIndexEmbeddings class: 
+#  - Load .csv file
+#  - Chunk text
+#    - Chunk size = 1000 characters
+#    - Chunk overlap = 30 characters
+#  - Create FAISS vector store from chunked text and OpenAI embeddings
+#  - Get FAISS vector store
+# This class is used to generate the FAISS vector store from the .csv file.
+class GenerateLearningPathIndexEmbeddings:
+    def __init__(self, csv_filename):
+        load_dotenv()  # Load .env file
+        self.openai_api_key = os.getenv("OPENAI_API_KEY")
+        self.data_path = os.path.join(os.getcwd(), csv_filename)
+        self.our_custom_data = None
+        self.openai_embeddings = None
+        self.faiss_vectorstore = None
+
+        self.load_csv_data()
+        self.get_openai_embeddings()
+        self.create_faiss_vectorstore_with_csv_data_and_openai_embeddings()
+           
+    def load_csv_data(self):
+        # Load your dataset (e.g., CSV, JSON, etc.)
+        print(' -- Started loading .csv file for chunking purposes.')
+        loader = TextLoader(self.data_path)
+        document = loader.load()
+        text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=30, separator="\n")
+        self.our_custom_data = text_splitter.split_documents(document)
+        print(f' -- Finished spitting (i.e. chunking) text (i.e. documents) from the .csv file (i.e. {self.data_path}).')
+        
+    def get_openai_embeddings(self):
+        self.openai_embeddings = OpenAIEmbeddings(openai_api_key=self.openai_api_key, request_timeout=60)
+        
+    def create_faiss_vectorstore_with_csv_data_and_openai_embeddings(self):
+        faiss_vectorstore_foldername = "faiss_learning_path_index"
+        if not os.path.exists(faiss_vectorstore_foldername):
+            print(' -- Creating a new FAISS vector store from chunked text and OpenAI embeddings.')
+            vectorstore = FAISS.from_documents(self.our_custom_data, self.openai_embeddings)
+            vectorstore.save_local(faiss_vectorstore_foldername)
+            print(f' -- Saved the newly created FAISS vector store at "{faiss_vectorstore_foldername}".')
+        else:
+            print(f' -- WARNING: Found existing FAISS vector store at "{faiss_vectorstore_foldername}", loading from cache.')
+            print(f' -- NOTE: Delete the FAISS vector store at "{faiss_vectorstore_foldername}", if you wish to regenerate it from scratch for the next run.')
+        self.faiss_vectorstore = FAISS.load_local(
+            "faiss_learning_path_index", self.openai_embeddings
+        )
+
+    def get_faiss_vector_store(self):
+        return self.faiss_vectorstore
+
+
+# https://discuss.streamlit.io/t/how-to-check-if-code-is-run-inside-streamlit-and-not-e-g-ipython/23439/7
+def running_inside_streamlit():
+    """
+    Function to check whether python code is run within streamlit
+
+    Returns
+    -------
+    use_streamlit : boolean
+        True if code is run within streamlit, else False
+    """
+    try:
+        from streamlit.runtime.scriptrunner import get_script_run_ctx
+        if not get_script_run_ctx():
+            use_streamlit = False
+        else:
+            use_streamlit = True
+    except ModuleNotFoundError:
+        use_streamlit = False
+    return use_streamlit
+
+
+# Define GenAI class:
+#  - Create prompt template
+#  - Create GenAI project
+#  - Get response for query
+# This class is used to get the response for a query from the GenAI project.
+# The GenAI project is created from the FAISS vector store.
+class GenAILearningPathIndex:
+    def __init__(self, faiss_vectorstore):
+        load_dotenv()  # Load .env file
+        self.openai_api_key = os.getenv("OPENAI_API_KEY")
+        self.faiss_vectorstore = faiss_vectorstore
+
+        prompt_template = \
+            """
+                Use the following template to answer the question at the end, 
+                from the Learning Path Index csv file,
+                display top 4 results in a tablular format and it 
+                should look like this:
+                | Learning Pathway | duration  | link | Module
+                | --- | --- | --- | --- |
+                | ... | ... | ... | ... |
+                it must contain a link for each line of the result in a table,
+                consider the duration and Module information mentioned in the question,
+                If you don't know the answer, don't make an entry in the table,
+                {context}
+                Question: {question}
+            """
+        PROMPT = PromptTemplate(template=prompt_template, input_variables=["context","question"])
+        # The chain_type_kwargs are passed to the chain_type when it is created.
+        self.chain_type_kwargs = {"prompt": PROMPT}
+        # Create the GenAI project 
+        self.llm = OpenAI(temperature=1.0, openai_api_key=self.openai_api_key)
+    # Get response for query
+    # The response is returned as a string.   
+       
+    def get_response_for(self, query: str):
+        qa = RetrievalQA.from_chain_type(
+            llm=self.llm, chain_type="stuff", 
+            retriever=self.faiss_vectorstore.as_retriever(),
+            chain_type_kwargs=self.chain_type_kwargs
+        )
+        return qa.run(query)
+
+def get_formatted_time(current_time = time.time()):
+    return datetime.utcfromtimestamp(current_time).strftime('%Y-%m-%d %H:%M:%S')
+
+#   Load the model
+@st.cache_data
+def load_model():
+    start_time = time.time()
+    print(f"\nStarted loading custom embeddings (created from .csv file) at {get_formatted_time(start_time)}")
+    learningPathIndexEmbeddings = GenerateLearningPathIndexEmbeddings("Learning_Pathway_Index.csv")
+    faiss_vectorstore = learningPathIndexEmbeddings.get_faiss_vector_store()
+    end_time = time.time()
+    print(f"Finished loading custom embeddings (created from .csv file) at {get_formatted_time(end_time)}")
+    print(f"Custom embeddings (created from .csv file) took about {end_time - start_time} seconds to load.")
+    return faiss_vectorstore
+
+#  Query the model
+def query_gpt_model(query: str):
+    start_time = time.time()
+    print(f"\nQuery processing start time: {get_formatted_time(start_time)}")
+    genAIproject = GenAILearningPathIndex(faiss_vectorstore)
+    answer = genAIproject.get_response_for(query)
+    end_time = time.time()
+    print(f"\nQuery processing finish time: {get_formatted_time(end_time)}")
+    print(f"\nAnswer (took about {end_time - start_time} seconds)")
+    return answer
+
+
+if __name__=='__main__':
+    faiss_vectorstore = load_model()
+
+    if running_inside_streamlit():
+        print("\nStreamlit environment detected. \nTo run a CLI interactive version just run `python main.py` in the CLI.\n")
+        query_from_stream_list = app()
+        if query_from_stream_list:
+            answer = query_gpt_model(query_from_stream_list)
+            st.write(answer)
+    else:
+        print("\nCommand-line interactive environment detected.\n")
+        while True:
+            query = input("\nEnter a query: ")
+            if query == "exit":
+                break
+            if query.strip() == "":
+                continue
+
+            if query:
+                answer = query_gpt_model(query)
+
+                print("\n\n> Question:")
+                print(query)
+                print(answer)

openai_faiss_exmpl.py

@@ -0,0 +1,44 @@
+import pandas as pd
+import openai
+import faiss
+import os
+
+# Load your custom CSV data
+data = pd.read_csv( os.getcwd() + "\\Learning_Pathway_Index.csv")
+
+# Initialize and populate FAISS index
+vector_dimension = 768  # For example, if you use a GPT-3 model with 768-dimensional embeddings
+index = faiss.IndexFlatL2(vector_dimension)
+vectors = []  # List to store vector representations of data
+
+for text in data['text_column']:
+    # Vectorize the text using a pre-trained model (e.g., GPT-3)
+    # Replace 'YOUR_OPENAI_API_KEY' with your actual API key
+    openai.api_key = os.getenv("OPENAI_API_KEY")
+    response = openai.Completion.create(
+        engine="text-davinci-002",
+        prompt=text,
+        max_tokens=50  # Adjust the token limit as needed
+    )
+    vector = response.choices[0].embedding
+    vectors.append(vector)
+
+# Convert the list of vectors to a numpy array
+vectors = np.array(vectors).astype('float32')
+
+# Add vectors to the FAISS index
+index.add(vectors)
+
+# Accept user questions using OpenAI
+user_question = input("Ask a question: ")
+
+# Vectorize the user's question
+user_vector = vectorize_user_question(user_question)  # Implement this function
+
+# Search for similar items in the FAISS index
+k = 5  # Number of similar items to retrieve
+distances, indices = index.search(user_vector, k)
+
+# Retrieve and display the similar items
+similar_items = data.iloc[indices[0]]
+print(similar_items)

requirements.txt

@@ -0,0 +1,7 @@
+langchain==0.0.216
+streamlit==1.27.2
+tqdm==4.65.0
+# Pre-requisites: [sudo] apt install libopenblas-base libomp-dev
+# See https://github.com/onfido/faiss_prebuilt
+faiss-cpu==1.7.4
+faiss-gpu==1.7.2