update

ingest.py

@@ -6,53 +6,68 @@ from langchain_community.document_loaders import Docx2txtLoader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_chroma import Chroma
-from config import KB_DIR, HF_DATASET_REPO, EMBEDDING_MODEL, CHROMA_DIR
+from config import KB_DIR, HF_DATASET_REPO, EMBEDDING_MODEL, CHROMA_DIR, CHUNK_SIZE, CHUNK_OVERLAP
 
 def run_ingestion():
+    # 1. Clean directories
     if os.path.exists(KB_DIR): shutil.rmtree(KB_DIR)
     if os.path.exists(CHROMA_DIR): shutil.rmtree(CHROMA_DIR)
     os.makedirs(KB_DIR, exist_ok=True)
 
-    print(f"⬇️ Loading raw files from {HF_DATASET_REPO}...")
+    print(f"⬇️ Loading files from {HF_DATASET_REPO}...")
     
-    # We load only the file paths to avoid the specialized PDF decoder errors
-    # This works for any file extension in your repo
-    dataset = load_dataset(HF_DATASET_REPO, split="train", ignore_verifications=True)
+    # Use standard load without extra flags that cause ValueErrors
+    dataset = load_dataset(HF_DATASET_REPO, split="train")
     
     docs = []
+    # Loop through the rows to find paths to files
     for i, row in enumerate(dataset):
-        # In a folder dataset, the 'file' or extension-named column contains path info
-        file_item = row.get("docx") or row.get("file") or row.get("pdf")
+        # We check common keys used by HF for file paths
+        file_info = row.get("docx") or row.get("file") or row.get("pdf")
         
         src_path = None
-        if isinstance(file_item, dict): src_path = file_item.get("path")
-        elif isinstance(file_item, str): src_path = file_item
+        if isinstance(file_info, dict): 
+            src_path = file_info.get("path")
+        elif isinstance(file_info, str): 
+            src_path = file_info
 
         if src_path and os.path.exists(src_path):
             ext = os.path.splitext(src_path)[1].lower()
+            
+            # ONLY process .docx files to avoid the PDF error
             if ext == ".docx":
                 dest_path = os.path.join(KB_DIR, f"doc_{i}.docx")
                 shutil.copy(src_path, dest_path)
+                
                 try:
                     loader = Docx2txtLoader(dest_path)
                     docs.extend(loader.load())
-                    print(f"✅ Extracted docx: doc_{i}")
+                    print(f"✅ Successfully loaded: doc_{i}.docx")
                 except Exception as e:
-                    print(f"❌ Error parsing doc_{i}: {e}")
-        else:
-            print(f"⏭️ Skipping non-docx or missing path at row {i}")
+                    print(f"❌ Loader error on doc_{i}: {e}")
+            else:
+                print(f"⏭️ Skipping non-docx file: {src_path}")
 
     if not docs:
-        print("❌ CRITICAL: No .docx documents were loaded.")
+        print("❌ CRITICAL: No .docx documents found. Ensure your dataset has .docx files.")
         return
 
-    # Chunk and Embed
-    splits = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100).split_documents(docs)
+    # 2. Chunking
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE, 
+        chunk_overlap=CHUNK_OVERLAP
+    )
+    splits = splitter.split_documents(docs)
+
+    # 3. Embedding and Storage
     print(f"🧠 Indexing {len(splits)} chunks...")
-    
     embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
-    Chroma.from_documents(documents=splits, embedding=embeddings, persist_directory=CHROMA_DIR)
-    print(f"✅ Knowledge base initialized successfully.")
+    Chroma.from_documents(
+        documents=splits, 
+        embedding=embeddings, 
+        persist_directory=CHROMA_DIR
+    )
+    print(f"✅ Knowledge base initialized at {CHROMA_DIR}")
 
 if __name__ == "__main__":
     run_ingestion()