import os
from langchain_community.document_loaders import PyPDFDirectoryLoader
from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma
from dotenv import load_dotenv
from langchain_text_splitters import CharacterTextSplitter

load_dotenv()
persist_directory = 'db'
raw_documents = PyPDFDirectoryLoader("data").load()  # Load PDFs from folder

if not raw_documents:
    raise ValueError("No documents found. Ensure the document loading is correct.")

# split it into chunks
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
documents = text_splitter.split_documents(raw_documents)

db = Chroma.from_documents(documents, OpenAIEmbeddings(), persist_directory=persist_directory)  # Initialize the database
print("Documents loaded into the database successfully.")