Spaces:
Sleeping
Sleeping
File size: 1,272 Bytes
7a4d005 cefe798 7a4d005 cefe798 7a4d005 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | from dotenv import load_dotenv
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import WebBaseLoader
from langchain_community.vectorstores import Chroma
from langchain_huggingface import HuggingFaceEndpointEmbeddings
load_dotenv()
urls = [
"https://lilianweng.github.io/posts/2023-06-23-agent/",
"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/",
"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/",
]
docs = [WebBaseLoader(urls).load() for url in urls]
docs_list = [item for sublist in docs for item in sublist]
text_splitter = RecursiveCharacterTextSplitter()
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
chunk_size=800, chunk_overlap=0
)
splits = text_splitter.split_documents(docs_list)
#embedding = OllamaEmbeddings(model="nomic-embed-text")
embedding = HuggingFaceEndpointEmbeddings(model="sentence-transformers/all-MiniLM-L6-v2")
vectorstore = Chroma.from_documents(
documents=splits,
collection_name="rag-chroma",
embedding=embedding,
persist_directory="./.chroma"
)
retriever = Chroma(
collection_name="rag-chroma",
persist_directory="./.chroma",
embedding_function=embedding,
).as_retriever()
|