Upload 6 files

.env

@@ -0,0 +1,3 @@
+openai_api_key = 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
+INDEX=chatbot
+PINECONE_API_KEY=8d2d8b7b-7d1a-4cc6-aff3-cf9d64c326d0

.gitignore

@@ -0,0 +1,2 @@
+content
+.env

app.py

@@ -0,0 +1,149 @@
+import gradio as gr
+import os
+import time
+
+from langchain.docstore.document import Document
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import PromptTemplate
+from pinecone import Pinecone
+from langchain_pinecone import PineconeVectorStore
+from langchain_openai import OpenAIEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+from langchain_openai import ChatOpenAI
+from langchain_community.vectorstores import Chroma
+from langchain.docstore.document import Document
+
+from openai import OpenAI
+from dotenv import load_dotenv
+import os, random, json
+from bs4 import BeautifulSoup
+
+load_dotenv()
+
+openai_api_key = os.getenv("OPENAI_API_KEY")
+pinecone_index = os.getenv("INDEX")
+pinecone_api_key = os.getenv("PINECONE_API_KEY")
+
+metadata_list = ['fullname', 'mediator email', 'mediator profile on mediate.com', 'mediator Biography']
+metadata_value = ['Name', "Email", "Profile", "Biography"]
+
+embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
+openai_client = OpenAI(api_key=openai_api_key)
+def search(message, history):
+    tools = [
+            {
+                "type": "function", 
+                "function": {
+                    "name": "mediator_search",
+                    "description": "Extract how many mediators user want to search.",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "mediator": {
+                                "type": "number",
+                                "description": "The number of mediators that user want to search",
+                                "default": 1
+                            }
+                        },
+                        "required": ["mediator"]
+                    }
+                }
+            }
+        ]
+    
+    response = openai_client.chat.completions.create(
+            model="gpt-4-1106-preview",
+            messages=[
+                    {"role": "system", "content": "Please extract how many mediators users want to search."},
+                    {"role": "user", "content": message}
+                ],
+                tools=tools,
+    )
+
+    number_str = response.choices[0].message.tool_calls[0].function.arguments
+    
+    mediator_num = json.loads(number_str)['mediator']
+
+    print(mediator_num)
+    template = """"""
+    prompt = "You are a professional mediator information analyzer. You have to write why the following context is related to human's message. Please write 3 or 4 sentences."
+    
+    end = """Context: {context}
+        Chat history: {chat_history}
+        Human: {human_input}
+        Your Response as Chatbot:"""
+    
+    template += prompt + end
+
+    prompt = PromptTemplate(
+        input_variables=["chat_history", "human_input", "context"], 
+        template=template
+        )
+
+    memory = ConversationBufferMemory(memory_key="chat_history", input_key="human_input")
+
+    print(message)
+    start_time = time.time()
+
+    pc = Pinecone(api_key=pinecone_api_key)
+
+    embeddings = OpenAIEmbeddings(api_key=openai_api_key)
+    
+    index = pc.Index(pinecone_index)
+
+    results = index.query(
+        vector=embeddings.embed_query(message),
+        top_k=800,
+        include_metadata=True
+    )
+    end_time = time.time()
+
+    print("Search Time =>", end_time-start_time)
+    
+    new_docs = []
+    new_data = []
+    for result in results['matches']:
+        if result['score'] > 0.75:
+            data = {}
+            for metadata in metadata_list:                
+                data[metadata] = BeautifulSoup(result['metadata'][metadata], "html.parser").get_text()
+            new_data.append(data)
+    
+    print(len(new_data))
+    random.shuffle(new_data)
+
+    answer = ""
+    for index, new_datum in enumerate(new_data):
+        if index < mediator_num:
+            answer += f"{index+1}\n"
+            content = ""
+            for metadata_index, metadata in enumerate(metadata_list):
+                content += f"{metadata_value[metadata_index]}: {new_datum[metadata]} \n"
+                answer += f"{metadata_value[metadata_index]}: {new_datum[metadata]} \n"
+
+            answer += "\n\n"
+            new_doc = Document(page_content=answer)
+            new_docs.append(new_doc)
+        else:
+            break
+
+    chat_openai = ChatOpenAI(model='gpt-4-1106-preview', 
+            openai_api_key=openai_api_key)
+
+    # print(new_docs)
+    chain = load_qa_chain(chat_openai, chain_type="stuff",  prompt=prompt, memory=memory)
+    start_time = time.time()
+    output = chain({"input_documents": new_docs, "human_input": message}, return_only_outputs=False)
+    end_time = time.time()
+
+    print("Query Time =>", end_time-start_time)
+    
+    answer += f"Why appropriate: {output['output_text']}"
+    return answer
+
+chatbot = gr.Chatbot(avatar_images=["user.png", "bot.jpg"], height=600)
+
+demo = gr.ChatInterface(fn=search, title="Mediate.com Chatbot Prototype", multimodal=False, retry_btn=None, clear_btn=None, undo_btn=None, chatbot=chatbot)
+
+if __name__ == "__main__":
+    demo.launch(debug=True)

embed.py

@@ -0,0 +1,98 @@
+import streamlit as st
+from pathlib import Path
+
+from langchain_openai import OpenAIEmbeddings
+from langchain.document_loaders.base import BaseLoader
+from langchain.docstore.document import Document
+from langchain_pinecone import PineconeVectorStore
+
+from typing import Dict, List, Optional
+from dotenv import load_dotenv
+import os, csv
+
+load_dotenv()
+
+openai_api_key = os.getenv("OPENAI_API_KEY")
+pinecone_index = os.getenv("INDEX")
+pinecone_api_key = os.getenv("PINECONE_API_KEY")
+
+embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
+
+class MetaDataCSVLoader(BaseLoader):
+    def __init__(
+        self,
+        file_path: str,
+        source_column: Optional[str] = None,
+        metadata_columns: Optional[List[str]] = None,   
+        content_columns: Optional[List[str]] =None ,  
+        csv_args: Optional[Dict] = None,
+        encoding: Optional[str] = None,
+    ):
+        self.file_path = file_path
+        self.source_column = source_column
+        self.encoding = encoding
+        self.csv_args = csv_args or {}
+        self.content_columns= content_columns
+        self.metadata_columns = metadata_columns
+
+    def load(self) -> List[Document]:
+        docs = []
+        with open(self.file_path, newline="", encoding=self.encoding) as csvfile:
+            csv_reader = csv.DictReader(csvfile, **self.csv_args)  # type: ignore
+            for i, row in enumerate(csv_reader):
+                if self.content_columns: 
+                    content = "\n".join(f"{k.strip()}: {v.strip()}" for k, v in row.items() if k in self.content_columns)
+                else: 
+                    content = "\n".join(f"{k.strip()}: {v.strip()}" for k, v in row.items())
+                try:
+                    source = (
+                        row[self.source_column]
+                        if self.source_column is not None
+                        else self.file_path
+                    )
+                except KeyError:
+                    raise ValueError(
+                        f"Source column '{self.source_column}' not found in CSV file."
+                    )
+                metadata = {"source": source, "row": i}
+                # ADDED TO SAVE METADATA
+                if self.metadata_columns:
+                    for k, v in row.items():
+                        if k in self.metadata_columns:
+                            metadata[k] = v
+                # END OF ADDED CODE
+                doc = Document(page_content=content, metadata=metadata)
+                docs.append(doc)
+
+        return docs
+
+
+csv_file_uploaded = st.file_uploader(label="Upload your CSV File here")
+
+if csv_file_uploaded is not None:
+    def save_file_to_folder(uploadedFile):
+        save_folder = 'content'
+        save_path = Path(save_folder, uploadedFile.name)
+        with open(save_path, mode='wb') as w:
+            w.write(uploadedFile.getvalue())
+
+        if save_path.exists():
+            st.success(f'File {uploadedFile.name} is successfully saved!')
+
+            with open(os.path.join('content/', csv_file_uploaded.name), 'r') as file:
+
+                csv_reader = csv.reader(file)
+
+                # Read the headers from the CSV file
+                headers = next(csv_reader)
+
+            filtered_headers= list(filter(lambda x: x != '', headers))
+
+            loader = MetaDataCSVLoader(os.path.join('content/', csv_file_uploaded.name), 
+                metadata_columns=filtered_headers, encoding = "utf-8")
+            data = loader.load()
+
+            # Pinecone.from_documents(data, embeddings, index_name=index_name)
+            PineconeVectorStore.from_documents(data, embeddings, index_name=pinecone_index)
+
+    save_file_to_folder(csv_file_uploaded)