File size: 3,390 Bytes
a71ea0a b2be33e a71ea0a c4cef6d 3f60e32 a71ea0a 588cdee a71ea0a 66c4741 a71ea0a 588cdee a71ea0a 66c4741 a71ea0a 66c4741 a71ea0a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | """
Standalone script to ingest PDFs and URLs into the vector store.
Run this script periodically to update your document knowledge base.
"""
import os
from pathlib import Path
from ingestion import DocumentIngestion
# Configuration
PDF_FOLDER = "data/pdfs" # Folder containing PDF files
URLS = [
# Add your URLs here, one per line
"https://inspection.canada.ca/en/food-labels/organic-products/operating-manual",
"https://inspection.canada.ca/en/food-labels/organic-products/import-requirements",
"https://inspection.canada.ca/en/food-labels/labelling/industry/organic-claims",
"https://cog.ca/faqs/"
]
# Optional: map PDF filenames to their publicly hosted URLs so references are hyperlinked.
# Keys are bare filenames (no path), values are the public URL for that PDF.
PDF_URLS = {
"Organic production systems - General principles and management standards.pdf": "https://publications.gc.ca/collections/collection_2026/ongc-cgsb/P29-32-310-2026-eng.pdf",
# "another-doc.pdf": "https://example.com/another-doc.pdf",
}
def main():
"""Main ingestion function."""
print("=" * 60)
print("Document Ingestion Script")
print("=" * 60)
# Initialize ingestion system
print("\nInitializing document ingestion system...")
ingestion = DocumentIngestion(embedding_model="all-mpnet-base-v2")
# Collect PDF files
pdf_paths = []
if os.path.exists(PDF_FOLDER):
pdf_files = list(Path(PDF_FOLDER).glob("*.pdf"))
pdf_paths = [str(f) for f in pdf_files]
print(f"\nFound {len(pdf_paths)} PDF file(s) in {PDF_FOLDER}:")
for pdf in pdf_paths:
print(f" - {os.path.basename(pdf)}")
else:
print(f"\nPDF folder '{PDF_FOLDER}' not found. Creating it...")
os.makedirs(PDF_FOLDER, exist_ok=True)
print(f"Please add PDF files to {PDF_FOLDER} and run again.")
# Filter out empty URLs
urls = [url.strip() for url in URLS if url.strip()]
if urls:
print(f"\nFound {len(urls)} URL(s) to process:")
for url in urls:
print(f" - {url}")
else:
print("\nNo URLs configured. Add URLs to the URLS list in this script.")
if not pdf_paths and not urls:
print("\n[ERROR] No documents to process. Please add PDFs or URLs.")
return
# Process documents
print("\n" + "=" * 60)
print("Processing documents...")
print("=" * 60)
try:
documents = ingestion.process_documents(pdf_paths=pdf_paths, urls=urls, pdf_urls=PDF_URLS)
print(f"\n[SUCCESS] Successfully processed {len(documents)} document chunks")
# Build vector store
print("\nBuilding vector store...")
ingestion.build_vector_store()
# Save vector store
print("\nSaving vector store...")
ingestion.save("data/vector_store")
print("\n" + "=" * 60)
print("[SUCCESS] Ingestion complete!")
print("=" * 60)
print(f"\nTotal document chunks: {len(documents)}")
print(f"Vector store saved to: data/vector_store")
print("\nYou can now run 'py app.py' to start the chatbot.")
except Exception as e:
print(f"\n[ERROR] Error during ingestion: {str(e)}")
import traceback
traceback.print_exc()
return
if __name__ == "__main__":
main()
|