Spaces:
Sleeping
Sleeping
File size: 1,691 Bytes
cf796c5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | from pathlib import Path
from langchain_community.document_loaders import DirectoryLoader, TextLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_community.embeddings import HuggingFaceEmbeddings
DOCS_PATH = Path("data/documents")
STORE_PATH = "data/vectorstore"
EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
def build():
"""Build vector store from policy documents"""
print("Building vector store...")
if not DOCS_PATH.exists():
print(f"❌ Documents directory not found: {DOCS_PATH}")
print("Please create policy documents in data/documents/")
return
# Load docs
loader = DirectoryLoader(
DOCS_PATH,
glob="**/*.md",
loader_cls=TextLoader,
show_progress=True
)
docs = loader.load()
if not docs:
print(f"❌ No documents found in {DOCS_PATH}")
return
print(f"Loaded {len(docs)} documents")
# Chunk — 512 tokens, 50 overlap preserves sentence boundaries
splitter = RecursiveCharacterTextSplitter(
chunk_size=512,
chunk_overlap=50,
separators=["\n\n", "\n", ".", " "]
)
chunks = splitter.split_documents(docs)
print(f"Created {len(chunks)} chunks")
# Embed and persist
embeddings = HuggingFaceEmbeddings(model_name=EMBED_MODEL)
vectorstore = Chroma.from_documents(
chunks,
embeddings,
persist_directory=STORE_PATH,
collection_name="olist_policies"
)
print(f"✅ Vector store built at {STORE_PATH} ({len(chunks)} vectors)")
if __name__ == "__main__":
build()
|