Upload 8 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Solar[[:space:]]Eclipse[[:space:]]Information.pdf filter=lfs diff=lfs merge=lfs -text
+Solar[[:space:]]Eclipse[[:space:]]Table.pdf filter=lfs diff=lfs merge=lfs -text

Solar Eclipse Information.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bb79220fd02ad5e690cdbdedd4e8adfe14af816ada1b3f788b29784d8d243e6
+size 1640826

Solar Eclipse Table.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64e43d97461076f828796edbda176477ebc9b2726b6f3e58237dfcaa1a7754c8
+size 10125700

agents2.py

@@ -0,0 +1,176 @@
+from api_interface import API_Interface
+
+ai = API_Interface(chunk_size=1200)
+
+class Obnoxious_Agent:
+    """Checks if a query is obnoxious."""
+    def check_query(self, query, need_answer=False) -> bool:
+        """Checks if a query is obnoxious. Responds with True or False."""
+        prompt = f"""
+        Is this query obnoxious? Answer using 'Yes' or 'No', and explain.
+        
+        Query: {query}
+        """
+        messages = [{"role": "user", "content": prompt}]
+        response = ai.client_chat(messages)
+        if need_answer:
+            print(response)
+        return response.lower().find("yes") != -1
+
+
+class Query_Agent:
+    """Retrieves relevant documents from the vector store."""
+        
+    def query_vector_store(self, query, k_docs=3, k_tables=4):
+        (docs, scores), (tables, tscores) = ai.query_pinecone_vector_store(query, top_k_docs=k_docs, top_k_tbls=k_tables)
+        return docs, scores, tables, tscores
+
+
+class Answering_Agent:
+    """Generates responses for user queries."""
+    def __init__(self, mode="concise"):
+        self.mode = mode
+
+    def check_mode(self, query):
+        prompt = f"""
+        If the query asks for a change in speaking style, which most closely matches the user's needs - "concise" or "talkative"?
+        If the query does not ask for a change in speaking style, respond with "N/A"?
+        
+        Query: {query}
+        """
+        messages = [{"role": "user", "content": prompt}]
+        response = ai.client_chat(messages)
+
+        new_mode = False
+        if response.lower().find("concise") != -1:
+            self.mode = "concise"
+            new_mode = True
+        elif response.lower().find("talkative") != -1:
+            self.mode = "talktative"
+            new_mode = True
+        return new_mode
+    
+    def generate_response(self, query:str, documents:list, conv_history:list, context:str=""):
+        if context != "":
+            context = f"\nYou are given the following conversation context: {context}\n\n"
+        
+        """Generates response for user query given relevant documents and previous conversation.
+        Returns both the raw response and the extended conversation history used to generate the response."""
+        new_messages = [{"role": "developer", "content": f"""
+                         You are given the following information:
+                         
+                        {documents}
+                        {context}
+                         
+                         Do not answer any unrelated questions in the query.
+                         Answer the following query in a {self.mode} manner:
+                         
+                         {query}
+                         """}
+                        ]
+        
+        conv_history.extend(new_messages)
+        response = ai.client_chat(new_messages)
+        
+        return response, conv_history # return conv_history in case it's useful
+    
+    def requires_context(self, query, need_answer=False) -> bool:
+        prompt = f"""
+        Does this query require further context? Answer using "Yes" or "No", then explain.
+        
+        Query: {query}
+        """
+        messages = [{"role": "user", "content": prompt}]
+        response = ai.client_chat(messages)
+        if need_answer:
+            print(response)
+        return response.lower().find("yes") != -1
+
+
+class Relevant_Documents_Agent:
+    """Determines if documents are relevant to the query."""
+
+    def get_relevance(self, query, documents, need_answer = False) -> bool:
+        relevances = [self.__get_doc_relevance(query, doc, need_answer) for doc in documents]
+        print("\n\n", relevances)
+        return any(relevances)
+            
+    def __get_doc_relevance(self, query, document, need_answer=False)->bool:
+        prompt = f"""Does any portion of the following text or context share the
+        same or similar topic as the following query, and is the query related
+        to solar eclipses, numerical data, or astronomy? Answer with "Yes" or "No", and explain why or why not.
+        
+        Text: {document.page_content}
+        
+        Query: {query}"""
+        
+        messages = [{"role": "user", "content": prompt}]
+        response = ai.client_chat(messages)
+        if need_answer:
+            print("Relevance prompt:", prompt[250:1000].replace("\n", " <> "))
+            print("Relevant response:", response, "\n\n\n")
+        
+        return response.lower().find("yes") != -1
+
+class Greeting_Agent:
+    def check_greeting(self, query):
+        prompt = f"""
+        Would the following statement be regarded as a general
+        greeting or friendly conversation opener? Answer using 'Yes' or 'No'.
+        
+        Statement: {query}
+        """
+        messages = [{"role": "user", "content": prompt}]
+        response = ai.client_chat(messages)
+        return response.lower().find("yes") != -1
+    def get_greeting_response(self, query):
+        messages = [{"role": "user", "content": query}]
+        response = ai.client_chat(messages)
+        return response
+        
+class Head_Agent:
+    def __init__(self):
+        self.setup_sub_agents()
+        
+
+    def setup_sub_agents(self):
+        self.a_obnoxious = Obnoxious_Agent()
+        self.a_query = Query_Agent()
+        self.a_answering = Answering_Agent()
+        self.a_relevant = Relevant_Documents_Agent()
+        self.a_greeting = Greeting_Agent()
+        
+    def generate_response(self, query, conv_history:list=None):
+        conv_history = conv_history or []
+        if self.a_obnoxious.check_query(query):
+            return "Please do not ask obnoxious questions."
+        
+        if self.a_greeting.check_greeting(query):
+            return self.a_greeting.get_greeting_response(query)
+        
+        ret = ""
+        
+        if self.a_answering.check_mode(query):
+            ret = "I have updated my communication style to better suit your needs!\n\n"
+        
+        context = ""
+        for msg in conv_history[-8:-1]:
+            if msg["role"] != "developer":
+                context += f"{msg["role"]}: {msg["content"]}\n"
+        
+        no_rel = ("No relevant information found. "
+                  "Please refine your query or ask another question pertaining to solar eclipses on Earth.")
+        
+        
+        documents, scores, tables, tscores = self.a_query.query_vector_store(context)
+        context_plus = "This is our prior conversation for additional context:\n\n" + context + f"Latest user query: {query}"
+        docs_is_c_relevant = self.a_relevant.get_relevance(context_plus, documents+tables, need_answer=True)
+        print(scores, "\n", tscores, "\n\n", context_plus, "\n", "#"*50)
+        
+        if not docs_is_c_relevant:
+            return no_rel
+        print("Using context to generate response.")
+        r, h = self.a_answering.generate_response(query, documents+tables, conv_history, context)
+        
+        ret += r
+        return ret

api_interface.py

@@ -0,0 +1,261 @@
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.vectorstores import Pinecone
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_pinecone import PineconeVectorStore
+from pinecone import Pinecone
+from tqdm import tqdm
+from openai import OpenAI
+import string
+import pickle
+import os
+import time
+
+from langchain_community.document_loaders import PyMuPDFLoader
+
+class API_Interface:
+    def __init__(self, chunk_size:int = 1500, embed_model:str = "text-embedding-3-small", chat_model:str = "gpt-3.5-turbo"):
+        self.chunk_size = chunk_size
+        self.embed_model = embed_model
+        self.chat_model = chat_model
+        
+        with open("open_ai_key.txt") as infile:
+            OPEN_AI_KEY = infile.readline().strip()
+        with open("pinecone_key.txt") as infile:
+            PINECONE_KEY = infile.readline().strip()
+            
+        self.__client = OpenAI(api_key=OPEN_AI_KEY)
+        self.__pc = Pinecone(api_key=PINECONE_KEY)
+        self.__index = self.__pc.Index('eep596mp2')
+        
+        print("Chunking documents.")
+        self.chunked_texts, self.chunked_pnums = self.__chunk_document()
+        self.table_texts, self.table_pnums = self.__chunk_tables()
+        print("Initializing vector store.")
+        self.namespace, self.vectorstore = self.__init_vectorstore(OPEN_AI_KEY)
+        print("Initializing table store.")
+        self.tablespace, self.tablestore = self.__init_tablestore(OPEN_AI_KEY)
+        
+    def __chunk_document(self) -> tuple[list[str], list[int]]:
+        loader = PyMuPDFLoader(file_path = "Solar Eclipse Information.pdf", mode = "page")
+        docs = loader.load()
+
+        page_texts = [page.page_content for page in docs]  # Extract page_content
+        page_numbers = [page.metadata["page"] for page in docs]  # Extract metadata["page"]
+
+        splitter = RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=500)
+
+        chunked_texts, chunk_page_numbers = [], []
+        previous_page_tail = ""
+
+        for text, pnum in zip(page_texts, page_numbers):
+            chunks = splitter.split_text(previous_page_tail + " " + text)
+            chunked_texts.extend(chunks[:-1])
+            chunk_page_numbers.extend([pnum]*(len(chunks)-1))
+            previous_page_tail = chunks[-1]
+        chunked_texts.append(chunks[-1])
+        chunk_page_numbers.append(pnum)
+        
+        
+        return chunked_texts, chunk_page_numbers
+    
+    def __chunk_tables(self):
+        tabler = PyMuPDFLoader(file_path = "Solar Eclipse Table.pdf", mode = "page")
+        tables = tabler.load()
+        
+        HEADER = "Catalog Number, Canon Plate, Calendar Date, Terrestrial Dynamical Time of Greatest Eclipse, UT - TD (s), Luna Number, Saros Number, Eclipse Type, QLE, Gamma, Eclipse Magnitude, Latitude, Longitude, Sun Altitude, Sun Azimuth, Path Width (km), Central Line Duration"
+
+        table_texts = []
+        # print(tables[0].page_content)
+        c = tables[0].page_content
+        i = c.find("km")
+        # print(c[i+3:])
+        for page in tables:
+            c = page.page_content
+            i = c.find("km")
+            values = c[i+3:].split("\n")
+            text = ""
+            idv = 0
+            dates = [None, None]
+            partial_flag = False
+            for val in values:
+                if idv == 2:
+                    year = val
+                    if dates[0] is None:
+                        dates[0] = year
+                    else:
+                        dates[1] = year
+                if idv % 16 == 4:
+                    val = val.replace(" ", "   ")
+                text += val + "  "
+                idv += 1
+                if val.startswith("P"):
+                    partial_flag = True
+                if val.endswith("W") or val.endswith("E"):
+                    if partial_flag:
+                        idv = -2
+                    else:
+                        idv = -4
+                if idv == 0:
+                    text += "\n"
+                    partial_flag = False
+            table_texts.append(f"Solar eclipses between {dates[0]} and {dates[1]}:\n\n" + HEADER + "\n" + text)
+        table_numbers = [page.metadata["page"] for page in tables]
+            
+        return table_texts, table_numbers
+        
+
+    def __init_vectorstore(self, OPEN_AI_KEY):
+        
+        NAMESPACE = f"ns_eclipse_{self.chunk_size}"
+        
+        _ns = self.__index.describe_index_stats()['namespaces'].get(NAMESPACE)
+        if _ns is not None and _ns.get('vector_count') in (None, 0):
+            self.__index.delete(delete_all=True, namespace=NAMESPACE)
+            _ns = None
+            
+        if _ns is None:
+            print("... generating embeddings.")      
+            self.__generate_embeddings()
+
+            records = []
+            for i, (text, pnum, embedding) in enumerate(zip(self.chunked_texts, self.chunked_pnums, self.embeddings)):
+                records.append({
+                    "id": f"chunk{i}",
+                    "values": embedding,
+                    "metadata": {
+                        "text": text,
+                        "page_number": pnum
+                    }
+                })
+
+            print(len(records))
+            batch_size = 180
+            print("... upsertting records.")
+            for b in tqdm(range((len(records)-1)//batch_size+1)):
+                self.__index.upsert(records[b*batch_size:(b+1)*batch_size], namespace=NAMESPACE)
+                # print(b+1, "/", (len(records)-1)//batch_size+1)
+            
+            while self.__index.describe_index_stats()['namespaces'].get(NAMESPACE) is None:
+                time.sleep(1)
+
+        print("Index stats:", self.__index.describe_index_stats())
+        openaiembs = OpenAIEmbeddings(api_key=OPEN_AI_KEY, model=self.embed_model)
+        vectorstore = PineconeVectorStore(self.__index, embedding=openaiembs)
+        
+        return NAMESPACE, vectorstore
+    
+    def __init_tablestore(self, OPEN_AI_KEY):
+        
+        NAMESPACE = f"ts_eclipse"
+        
+        _ns = self.__index.describe_index_stats()['namespaces'].get(NAMESPACE)
+        if _ns is not None and _ns.get('vector_count') in (None, 0):
+            self.__index.delete(delete_all=True, namespace=NAMESPACE)
+            _ns = None
+            
+        if _ns is None:
+            print("... generating table embeddings.")
+            self.__generate_table_embeddings()
+
+            records = []
+            for i, (text, pnum, embedding) in enumerate(zip(self.table_texts, self.table_pnums, self.tmbeddings)):
+                records.append({
+                    "id": f"chunk{i}",
+                    "values": embedding,
+                    "metadata": {
+                        "text": text,
+                        "page_number": pnum
+                    }
+                })
+
+            print(len(records))
+            batch_size = 180
+            print("... upsertting records.")
+            for b in tqdm(range((len(records)-1)//batch_size+1)):
+                self.__index.upsert(records[b*batch_size:(b+1)*batch_size], namespace=NAMESPACE)
+                # print(b+1, "/", (len(records)-1)//batch_size+1)
+            
+            while self.__index.describe_index_stats()['namespaces'].get(NAMESPACE) is None:
+                time.sleep(1)
+
+        print("Index stats:", self.__index.describe_index_stats())
+        openaiembs = OpenAIEmbeddings(api_key=OPEN_AI_KEY, model=self.embed_model)
+        vectorstore = PineconeVectorStore(self.__index, embedding=openaiembs)
+        
+        return NAMESPACE, vectorstore
+    
+    def __generate_embeddings(self) -> None:
+        """ Generates self.embeddings """
+        EMBED_PATH = f"eclipse_text_embeddings_{self.chunk_size}.pkl"
+        
+        def get_embedding(text):
+            text = text.replace("\n", " ")
+            # text = text.replace(string.punctuation, "")
+            response = self.__client.embeddings.create(input = [text], model=self.embed_model)
+            return response.data[0].embedding
+
+        if not os.path.exists(EMBED_PATH):
+            self.embeddings = []
+            for text in tqdm(self.chunked_texts):
+                self.embeddings.append(get_embedding(text))
+
+            with open(EMBED_PATH, "wb") as outfile:
+                pickle.dump(self.embeddings, outfile)
+        else:
+            print("--- found existing embeddings file. Shortcutting.")
+            with open(EMBED_PATH, "rb") as infile:
+                self.embeddings:list[list[float]] = pickle.load(infile)
+                
+        
+    def __generate_table_embeddings(self) -> None:
+        """ Generates self.tmbeddings """
+        TABLE_PATH = f"eclipse_table_embeddings.pkl"
+        HEADER = """Catalog Number, Canon Plate, Calendar Date, Terrestrial Dynamical Time of Greatest Eclipse, UT - TD (s), Luna Number, Saros Number, Eclipse Type, QLE, Gamma, Eclipse Magnitude, Latitude, Longitude, Sun Altitude, Sun Azimuth, Path Width (km), Central Line Duration"""
+        
+        def get_embedding(text):
+            text = text.replace("\n", " ")
+            response = self.__client.embeddings.create(input = [text], model=self.embed_model)
+            return response.data[0].embedding
+
+        if not os.path.exists(TABLE_PATH):
+            self.tmbeddings = []
+            
+            for table in tqdm(self.table_texts):
+                self.tmbeddings.append(get_embedding(table))
+
+            with open(TABLE_PATH, "wb") as outfile:
+                pickle.dump(self.tmbeddings, outfile)
+        else:
+            print("--- found existing embeddings file. Shortcutting.")
+            with open(TABLE_PATH, "rb") as infile:
+                self.tmbeddings:list[list[float]] = pickle.load(infile)
+
+
+    def query_pinecone_vector_store(self, query:str, top_k_docs:int = 5, top_k_tbls:int = 5,
+                                    namespace:str = None, tablespace:str = None):
+        namespace = namespace or self.namespace
+        tablespace = tablespace or self.tablespace
+        assert namespace in self.__index.describe_index_stats().get('namespaces')
+        assert tablespace in self.__index.describe_index_stats().get('namespaces')
+        
+        response = self.vectorstore.similarity_search_with_relevance_scores(query=query,
+                                                        k=top_k_docs,
+                                                        namespace=namespace)
+        tesponse = self.vectorstore.similarity_search_with_relevance_scores(query=query,
+                                                        k=top_k_tbls,
+                                                        namespace=tablespace)
+        
+        return [tuple(zip(*response)), tuple(zip(*tesponse))]
+    
+    
+    def client_chat(self, messages, model=None):
+        model = model or self.chat_model
+        response = self.__client.chat.completions.create(messages=messages, model=model)
+        return response.choices[0].message.content
+
+if __name__ == "__main__":
+    tester = API_Interface()
+    my_query = "What is the backpropogation algorithm?"
+    response = tester.query_pinecone_vector_store(my_query)
+    for doc in response:
+        print(doc.metadata["page_number"], doc.page_content, "\n\n")

app.py

@@ -0,0 +1,29 @@
+import streamlit as st
+
+st.set_page_config(layout="wide")
+st.title("Mini Project 2 Part 3: Streamlit Chatbot")
+if "messages" not in st.session_state:
+    # ... (initialize messages)
+    st.session_state.messages = []
+
+with st.spinner("Please wait while the chat bot is loading!"):
+    from agents2 import *
+
+
+# Display existing chat messages
+for message in st.session_state.messages:
+    if message["role"] != "developer":
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+# Wait for user input
+a_h = Head_Agent()
+if prompt := st.chat_input("What would you like to chat about?"):
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    
+    with st.chat_message("user"):
+        st.markdown(prompt)
+    with st.chat_message("assistant"):
+        ai_message = a_h.generate_response(prompt, st.session_state.messages)
+        st.markdown(ai_message)
+
+    st.session_state.messages.append({"role": "assistant", "content": ai_message})

eclipse_table_embeddings.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34dc5992431812d25a06c722e5edde59aa3fac59f7309389f5798f665801e401
+size 2752555

eclipse_text_embeddings_1200.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e49f944552529449a2eb875e803a4864e28d6445598eb32551325e5d74854fd0
+size 3126019

eclipse_text_embeddings_1500.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b96bed648622c3e801011cfa8e586c9bd18447dfd5af698c28d4240f3c0c87
+size 2185453