Change sample data to train data folder

app.py

@@ -51,7 +51,7 @@ logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICA
 DATA_DIR = Path("/data") if Path("/data").is_dir() else Path("./data")
 
 CHROMA_PERSIST_DIR = str(DATA_DIR / "chroma_db")
-SAMPLE_DOCS_DIR = DATA_DIR / "sample_documents"
+TRAIN_DOCS_DIR = Path("./train_data")
 COLLECTION_NAME = "rag_documents"
 CHUNK_SIZE = 512
 CHUNK_OVERLAP = 50
@@ -375,30 +375,29 @@ def rag_query(query: str, top_k: int = TOP_K_RESULTS) -> dict:
 
 
 # ---------------------------------------------------------------------------
-# Step 4: Ingest Sample Documents on Startup
+# Step 4: Ingest Train Documents (on-demand)
 # ---------------------------------------------------------------------------
-# Load sample documents so the demo works out of the box.
 
 
-def ingest_sample_documents():
-    """Load and embed sample documents into the vector store on first run."""
+def ingest_train_documents():
+    """Load and embed training documents into the vector store."""
     if collection.count() > 0:
         logger.info("Vector store already has documents, skipping ingestion.")
         return
 
-    if not SAMPLE_DOCS_DIR.exists():
-        logger.warning("No sample_documents directory found.")
+    if not TRAIN_DOCS_DIR.exists():
+        logger.warning(f"No train_data directory found at: {TRAIN_DOCS_DIR}")
         return
 
-    # Ingest plain text files
-    for file_path in SAMPLE_DOCS_DIR.glob("*.txt"):
+    # Ingest plain text files (recursively in subdirectories)
+    for file_path in TRAIN_DOCS_DIR.rglob("*.txt"):
         logger.info(f"Ingesting text file: {file_path.name}")
         text = file_path.read_text(encoding="utf-8")
         chunks = chunk_text(text, source=file_path.name)
         add_documents_to_vectorstore(chunks)
 
-    # Ingest PDF files (e.g., ebook-a-compact-guide-to-pretraining-custom-llms.pdf)
-    for file_path in SAMPLE_DOCS_DIR.glob("*.pdf"):
+    # Ingest PDF files (recursively in subdirectories)
+    for file_path in TRAIN_DOCS_DIR.rglob("*.pdf"):
         logger.info(f"Ingesting PDF file: {file_path.name}")
         text = extract_text_from_pdf(file_path)
         if text.strip():
@@ -407,10 +406,7 @@ def ingest_sample_documents():
         else:
             logger.warning(f"No extractable text found in: {file_path.name}")
 
-    logger.info(f"Sample document ingestion complete. Total chunks: {collection.count()}")
-
-
-ingest_sample_documents()
+    logger.info(f"Train document ingestion complete. Total chunks: {collection.count()}")
 
 
 # ---------------------------------------------------------------------------