Spaces:
Sleeping
Sleeping
File size: 3,162 Bytes
1c798e7 7d8a70b ce3f912 1c798e7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
from langchain.document_loaders import DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import SentenceTransformerEmbeddings
from sentence_transformers import SentenceTransformer
import pinecone
from langchain.vectorstores import Pinecone
from langchain.document_loaders import PyPDFLoader
import tempfile
import streamlit as st
import openai
# To create embeddings on hard disk
@st.cache_resource()
def get_embeddings_model():
model = SentenceTransformer('all-MiniLM-L6-v2')
embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
return model, embeddings
model, embeddings = get_embeddings_model()
def ingest(
uploaded_document,
pinecone_api_key,
pinecone_env,
pinecone_index_namespace,
chunk_size=500,
chunk_overlap=20
):
with tempfile.NamedTemporaryFile(delete=False) as tf:
tf.write(uploaded_document.getbuffer())
file_path = tf.name
loader = PyPDFLoader(file_path)
documents = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
docs = text_splitter.split_documents(documents)
# embeddings = get_embeddings_model()
pinecone.init(
api_key=pinecone_api_key,
environment=pinecone_env
)
index_name = pinecone_index_namespace
try:
index = Pinecone.from_documents(docs, embeddings, index_name=index_name)
st.success('Document uploaded to Pinecone database successfully')
except Exception as error_message:
st.error(error_message)
# # To create embeddings on hard disk
# # !pip install chromadb
# # from langchain.vectorstores import Chroma
# # persist_directory = './data/embeddings'
# # vStore = Chroma.from_documents(docs, embeddings, persist_directory=persist_directory)
def query_refiner(conversation, query):
response = openai.Completion.create(
model="text-davinci-003",
prompt=f"Given the following user query and conversation log, formulate a question that would be the most relevant to provide the user with an answer from a knowledge base.\n\nCONVERSATION LOG: \n{conversation}\n\nQuery: {query}\n\nRefined Query:",
temperature=0.7,
max_tokens=256,
top_p=1,
frequency_penalty=0,
presence_penalty=0
)
return response['choices'][0]['text']
def find_match(input, pinecone_api_key, pinecone_env, pinecone_index_namespace):
pinecone.init(
api_key=pinecone_api_key,
environment=pinecone_env
)
index = pinecone.Index(pinecone_index_namespace)
input_em = model.encode(input).tolist()
result = index.query(input_em, top_k=2, includeMetadata=True)
return result['matches'][0]['metadata']['text']+"\n"+result['matches'][1]['metadata']['text']
def get_conversation_string():
conversation_string = ""
for i in range(len(st.session_state['responses'])-1):
conversation_string += "Human: "+st.session_state['requests'][i] + "\n"
conversation_string += "Bot: "+ st.session_state['responses'][i+1] + "\n"
return conversation_string |