Spaces:
Runtime error
Runtime error
File size: 2,320 Bytes
5227dfc bc388c1 5227dfc bc388c1 5227dfc bc388c1 5227dfc bc388c1 5227dfc bc388c1 5227dfc bc388c1 5227dfc 1409f1c 5227dfc 1409f1c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Pinecone
from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
import pinecone
import asyncio
from langchain.document_loaders.sitemap import SitemapLoader
#Step 1: Loading data from website
def get_website_data(sitemap_url):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loader = SitemapLoader(
sitemap_url
)
docs = loader.load()
return docs
#Step 2:Split data into smaller chunks
def split_data(docs):
text_splitter = RecursiveCharacterTextSplitter(
chunk_size = 1000,
chunk_overlap = 200,
length_function = len,
)
docs_chunks = text_splitter.split_documents(docs)
return docs_chunks
#Step3: Embedding this Function to create embeddings instance
def create_embeddings():
embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
return embeddings
#Step 3: Push data to Pinecone
def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs):
pinecone.init(
api_key=pinecone_apikey,
environment=pinecone_environment
)
index_name = pinecone_index_name
index = Pinecone.from_documents(docs, embeddings, index_name=index_name)
return index
#Step 4 & 5 pull index data from Pinecone
def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings):
pinecone.init(
api_key=pinecone_apikey,
environment=pinecone_environment
)
index_name = pinecone_index_name
index = Pinecone.from_existing_index(index_name, embeddings)
return index
#Step 4 & 5 Fetch the top relevent documents from our vector store - Pinecone Index
def get_similar_docs(index,query,k=2):
similar_docs = index.similarity_search(query, k=k)
return similar_docs
def ask_and_get_answer(vector_store, q, k=3):
from langchain.chains import RetrievalQA
from langchain.chat_models import ChatOpenAI
llm = ChatOpenAI(model='gpt-3.5-turbo', temperature=1)
retriever = vector_store.as_retriever(search_type='similarity', search_kwargs={'k': k})
chain = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever)
answer = chain.run(q)
return answer
|