Spaces:
Configuration error
Configuration error
File size: 2,914 Bytes
cd7bbf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | import os
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from dotenv import load_dotenv
from langchain_groq import ChatGroq
from langchain import hub
from langchain_chroma import Chroma
from langchain_community.document_loaders import DirectoryLoader
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough
from langchain_huggingface.embeddings import HuggingFaceEmbeddings
from langchain_text_splitters import RecursiveCharacterTextSplitter
class Watcher:
def __init__(self, directory_to_watch, callback):
self.DIRECTORY_TO_WATCH = directory_to_watch
self.observer = Observer()
self.callback = callback
def run(self):
event_handler = Handler(self.callback)
self.observer.schedule(event_handler, self.DIRECTORY_TO_WATCH, recursive=True)
self.observer.start()
try:
while True:
time.sleep(5)
except KeyboardInterrupt:
self.observer.stop()
self.observer.join()
class Handler(FileSystemEventHandler):
def __init__(self, callback):
self.callback = callback
def on_any_event(self, event):
if event.is_directory:
return None
elif event.event_type == 'created' or event.event_type == 'modified' or event.event_type == 'deleted':
self.callback()
def update_database():
docs = DirectoryLoader(
path="./data/files",
silent_errors=True,
show_progress=True,
use_multithreading=True,
).load_documents()
if os.path.exists("./chroma_db"):
vectorstore = Chroma.from_directory("./chroma_db")
else:
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
splits = text_splitter.split_documents(docs)
vectorstore = Chroma.from_documents(
documents=splits,
embedding=embeddings,
persist_directory="./chroma_db",
)
retriever = vectorstore.as_retriever()
prompt = hub.pull("rlm/rag-prompt")
def format_docs(docs):
return "\n\n".join(doc.page_content for doc in docs)
rag_chain = (
{"context": retriever | format_docs, "question": RunnablePassthrough()}
| prompt
| llm
| StrOutputParser()
)
response = rag_chain.invoke("Summarize the abstract of Importance of AI in evaluatiing climate change and food safety risk paper")
print(response)
model_name = "BAAI/bge-small-en"
model_kwargs = {"device": "cpu"}
encode_kwargs = {"normalize_embeddings": True}
embeddings = HuggingFaceEmbeddings()
load_dotenv()
os.environ["LANGCHAIN_TRACING_V2"] = "true"
llm = ChatGroq(model="llama3-8b-8192")
if __name__ == "__main__":
watcher = Watcher(directory_to_watch="./data/files", callback=update_database)
watcher.run()
|