Spaces:
Runtime error
Runtime error
File size: 2,001 Bytes
532a759 ea3c0cf 532a759 ea3c0cf 532a759 ea3c0cf 532a759 ea3c0cf 532a759 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | import json
import logging
from fastapi.encoders import jsonable_encoder
from langchain.chains import RetrievalQA
from langchain.document_loaders import DirectoryLoader, TextLoader
from langchain.text_splitter import CharacterTextSplitter
from qdrant_client.http.models import Distance, VectorParams
from edu_assistant.utils.common_utils import init_local_logging
from edu_assistant.utils.langchain_utils import load_llm, load_vectorstore
from edu_assistant.utils.qdrant_utils import load_qdrant_client
init_local_logging(logging.DEBUG)
def create_collection(collection_name: str):
client = load_qdrant_client()
client.recreate_collection(
collection_name=collection_name,
vectors_config=VectorParams(size=1536, distance=Distance.DOT),
)
collection_info = client.get_collection(collection_name=collection_name)
print(collection_info)
def delete_collection(collection_name: str):
client = load_qdrant_client()
client.delete_collection(collection_name=collection_name)
def add_docs(path: str, collection_name: str):
vs = load_vectorstore(collection_name=collection_name)
loader = DirectoryLoader(path=path, glob="*.txt", loader_cls=TextLoader)
documents = loader.load()
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=50)
docs = text_splitter.split_documents(documents)
for doc in docs:
vs.add_documents([doc])
def qa(collection_name: str, question: str):
chain = RetrievalQA.from_llm(
llm=load_llm(),
retriever=load_vectorstore(collection_name=collection_name).as_retriever(k=1),
return_source_documents=True,
)
result = chain(question)
print(json.dumps(jsonable_encoder(result), ensure_ascii=False, indent=4))
if __name__ == "__main__":
name = "example"
path = "examples/docs"
question = "C++有哪些数据类型修饰符？"
# delete_collection(name)
# create_collection(name)
# add_docs(path, name)
qa(name, question)
|