update

ingest.py

@@ -10,43 +10,37 @@ def run_ingestion():
     os.makedirs(KB_DIR, exist_ok=True)
 
     print(f"⬇️ Loading dataset from {HF_DATASET_REPO}...")
-    dataset = load_dataset(HF_DATASET_REPO, split="train")
+    # decode(False) prevents the library from turning bytes into pdfplumber objects
+    dataset = load_dataset(HF_DATASET_REPO, split="train").with_format("binary").decode(False)
     
-    # Debug: Print column names to logs
-    print(f"📊 Dataset columns: {dataset.column_names}")
-
     pdf_paths = []
     for i, row in enumerate(dataset):
-        # Flexible column mapping
-        fname = row.get("file_name") or row.get("filename") or f"document_{i}.pdf"
-        pdf_data = row.get("file") or row.get("pdf")
-
-        if pdf_data is None:
-            print(f"⚠️ Skipping row {i}: No PDF data found.")
+        # Determine filename and raw data column
+        fname = row.get("filename") or row.get("file_name") or f"doc_{i}.pdf"
+        # Access the raw 'bytes' from the 'pdf' column
+        pdf_feature = row.get("pdf")
+        
+        if pdf_feature is None:
             continue
 
         path = os.path.join(KB_DIR, fname)
         with open(path, "wb") as f:
-            # Handle HF dataset format (bytes vs dict)
-            if isinstance(pdf_data, dict) and "bytes" in pdf_data:
-                f.write(pdf_data["bytes"])
+            if isinstance(pdf_feature, dict) and "bytes" in pdf_feature:
+                f.write(pdf_feature["bytes"])
             else:
-                f.write(pdf_data)
+                f.write(pdf_feature)
         pdf_paths.append(path)
 
-    print(f"📄 Processing {len(pdf_paths)} PDFs...")
+    print(f"📄 Processing {len(pdf_paths)} documents...")
     docs = []
     for p in pdf_paths:
-        try:
-            loader = PyPDFLoader(p)
-            docs.extend(loader.load())
-        except Exception as e:
-            print(f"❌ Error loading {p}: {e}")
+        loader = PyPDFLoader(p)
+        docs.extend(loader.load())
 
     splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=100)
     splits = splitter.split_documents(docs)
 
-    print("🧠 Creating embeddings and Vector DB...")
+    print("🧠 Initializing Vector DB...")
     embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
 
     Chroma.from_documents(
@@ -54,7 +48,7 @@ def run_ingestion():
         embedding=embeddings,
         persist_directory=CHROMA_DIR
     )
-    print(f"✅ Ingestion complete. DB saved to {CHROMA_DIR}")
+    print(f"✅ Ingestion complete. Data saved to {CHROMA_DIR}")
 
 if __name__ == "__main__":
     run_ingestion()

rag.py

@@ -7,36 +7,32 @@ from config import EMBEDDING_MODEL, LLM_MODEL, CHROMA_DIR
 # 1. Initialize Embeddings
 embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
 
-# 2. Load Vector DB (Load only, do not recreate)
+# 2. Load Vector DB (Load only)
 if os.path.exists(CHROMA_DIR) and os.listdir(CHROMA_DIR):
     vectordb = Chroma(
         persist_directory=CHROMA_DIR,
         embedding_function=embeddings
     )
-    print(f"✅ Vector DB loaded successfully")
+    print(f"✅ Vector DB loaded from {CHROMA_DIR}")
 else:
-    print(f"❌ Vector DB NOT found at {CHROMA_DIR}")
     vectordb = None
+    print("❌ Vector DB directory is missing or empty")
 
-# 3. LLM Pipeline (CPU Safe)
-qa_pipeline = pipeline(
-    task="text-generation",
-    model=LLM_MODEL,
-    max_new_tokens=256
-)
+# 3. LLM Pipeline
+qa_pipeline = pipeline("text-generation", model=LLM_MODEL, max_new_tokens=256)
 
 def ask_rag_with_status(question: str):
     if vectordb is None:
-        return "Knowledge base is empty. Technical error during build.", "NO_KB"
+        return "The knowledge base is not initialized.", "ERROR"
 
     docs = vectordb.similarity_search(question, k=3)
     if not docs:
-        return "I couldn't find any relevant information in the documents.", "NO_MATCH"
+        return "No relevant information found.", "NO_MATCH"
 
     context = "\n\n".join(d.page_content for d in docs)
-    prompt = f"Use the context to answer.\n\nContext:\n{context}\n\nQuestion:\n{question}\n\nAnswer:"
+    prompt = f"Context:\n{context}\n\nQuestion: {question}\n\nAnswer:"
     
     result = qa_pipeline(prompt)
     answer = result[0]["generated_text"].split("Answer:")[-1].strip()
 
-    return answer, ["Context retrieved", "Response generated"]
+    return answer, ["Context retrieved", "OK"]