Update ingest logic

ingest.py

@@ -1,18 +1,38 @@
 import os
-from huggingface_hub import snapshot_download
-from config import HF_DATASET_ID, KB_DIR
+from datasets import load_dataset
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_chroma import Chroma
+from config import KB_DIR, HF_DATASET_REPO, EMBEDDING_MODEL, CHROMA_DIR
 
-def download_kb():
-    os.makedirs(KB_DIR, exist_ok=True)
+os.makedirs(KB_DIR, exist_ok=True)
 
-    snapshot_download(
-        repo_id=HF_DATASET_ID,
-        repo_type="dataset",
-        local_dir=KB_DIR,
-        local_dir_use_symlinks=False
-    )
+print("⬇️ Downloading PDFs from HF Dataset...")
+dataset = load_dataset(HF_DATASET_REPO, split="train")
 
-    print("✅ Knowledge base downloaded")
+pdf_paths = []
+for row in dataset:
+    path = os.path.join(KB_DIR, row["file_name"])
+    with open(path, "wb") as f:
+        f.write(row["file"])
+    pdf_paths.append(path)
 
-if __name__ == "__main__":
-    download_kb()
+print("📄 Loading documents...")
+docs = []
+for p in pdf_paths:
+    docs.extend(PyPDFLoader(p).load())
+
+splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=100)
+splits = splitter.split_documents(docs)
+
+print("🧠 Creating embeddings...")
+embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
+
+Chroma.from_documents(
+    splits,
+    embedding=embeddings,
+    persist_directory=CHROMA_DIR
+)
+
+print("✅ Ingestion complete")