update

Dockerfile

@@ -1,15 +1,22 @@
 FROM python:3.10-slim
+
 WORKDIR /app
+
+# Install system dependencies for git and PDF processing
 RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
 
+# Install Python requirements
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
+# Copy project files
 COPY app.py rag.py ingest.py config.py ./ 
 COPY frontend ./frontend
 
-# This will now succeed because requirements.txt has langchain-chroma
+# CRITICAL: Build the knowledge base during the Docker build process
 RUN python ingest.py
 
+# Hugging Face Spaces standard port
 EXPOSE 7860
+
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

ingest.py

@@ -2,37 +2,59 @@ import os
 from datasets import load_dataset
 from langchain_community.document_loaders import PyPDFLoader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_chroma import Chroma
 from config import KB_DIR, HF_DATASET_REPO, EMBEDDING_MODEL, CHROMA_DIR
 
-os.makedirs(KB_DIR, exist_ok=True)
-
-print("⬇️ Downloading PDFs from HF Dataset...")
-dataset = load_dataset(HF_DATASET_REPO, split="train")
-
-pdf_paths = []
-for row in dataset:
-    path = os.path.join(KB_DIR, row["file_name"])
-    with open(path, "wb") as f:
-        f.write(row["file"])
-    pdf_paths.append(path)
-
-print("📄 Loading documents...")
-docs = []
-for p in pdf_paths:
-    docs.extend(PyPDFLoader(p).load())
-
-splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=100)
-splits = splitter.split_documents(docs)
-
-print("🧠 Creating embeddings...")
-embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
-
-Chroma.from_documents(
-    splits,
-    embedding=embeddings,
-    persist_directory=CHROMA_DIR
-)
-
-print("✅ Ingestion complete")
+def run_ingestion():
+    os.makedirs(KB_DIR, exist_ok=True)
+
+    print(f"⬇️ Loading dataset from {HF_DATASET_REPO}...")
+    dataset = load_dataset(HF_DATASET_REPO, split="train")
+    
+    # Debug: Print column names to logs
+    print(f"📊 Dataset columns: {dataset.column_names}")
+
+    pdf_paths = []
+    for i, row in enumerate(dataset):
+        # Flexible column mapping
+        fname = row.get("file_name") or row.get("filename") or f"document_{i}.pdf"
+        pdf_data = row.get("file") or row.get("pdf")
+
+        if pdf_data is None:
+            print(f"⚠️ Skipping row {i}: No PDF data found.")
+            continue
+
+        path = os.path.join(KB_DIR, fname)
+        with open(path, "wb") as f:
+            # Handle HF dataset format (bytes vs dict)
+            if isinstance(pdf_data, dict) and "bytes" in pdf_data:
+                f.write(pdf_data["bytes"])
+            else:
+                f.write(pdf_data)
+        pdf_paths.append(path)
+
+    print(f"📄 Processing {len(pdf_paths)} PDFs...")
+    docs = []
+    for p in pdf_paths:
+        try:
+            loader = PyPDFLoader(p)
+            docs.extend(loader.load())
+        except Exception as e:
+            print(f"❌ Error loading {p}: {e}")
+
+    splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=100)
+    splits = splitter.split_documents(docs)
+
+    print("🧠 Creating embeddings and Vector DB...")
+    embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
+
+    Chroma.from_documents(
+        documents=splits,
+        embedding=embeddings,
+        persist_directory=CHROMA_DIR
+    )
+    print(f"✅ Ingestion complete. DB saved to {CHROMA_DIR}")
+
+if __name__ == "__main__":
+    run_ingestion()

rag.py

@@ -1,35 +1,42 @@
-# rag.py
 import os
 from transformers import pipeline
 from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_chroma import Chroma  # This requires langchain-chroma package
+from langchain_chroma import Chroma
 from config import EMBEDDING_MODEL, LLM_MODEL, CHROMA_DIR
 
 # 1. Initialize Embeddings
 embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
 
-# 2. Load Vector DB
+# 2. Load Vector DB (Load only, do not recreate)
 if os.path.exists(CHROMA_DIR) and os.listdir(CHROMA_DIR):
     vectordb = Chroma(
         persist_directory=CHROMA_DIR,
         embedding_function=embeddings
     )
-    print(f"✅ Vector DB loaded")
+    print(f"✅ Vector DB loaded successfully")
 else:
+    print(f"❌ Vector DB NOT found at {CHROMA_DIR}")
     vectordb = None
-    print(f"⚠️ Vector DB not found")
 
-# 3. LLM Pipeline
-qa_pipeline = pipeline(task="text-generation", model=LLM_MODEL, max_new_tokens=256)
+# 3. LLM Pipeline (CPU Safe)
+qa_pipeline = pipeline(
+    task="text-generation",
+    model=LLM_MODEL,
+    max_new_tokens=256
+)
 
 def ask_rag_with_status(question: str):
     if vectordb is None:
-        return "Knowledge base is empty.", "NO_KB"
+        return "Knowledge base is empty. Technical error during build.", "NO_KB"
 
     docs = vectordb.similarity_search(question, k=3)
-    context = "\n\n".join(d.page_content for d in docs)
+    if not docs:
+        return "I couldn't find any relevant information in the documents.", "NO_MATCH"
 
-    prompt = f"Use the context to answer.\nContext:\n{context}\nQuestion:\n{question}\nAnswer:"
-    result = qa_pipeline(prompt)
+    context = "\n\n".join(d.page_content for d in docs)
+    prompt = f"Use the context to answer.\n\nContext:\n{context}\n\nQuestion:\n{question}\n\nAnswer:"
     
-    return result[0]["generated_text"].split("Answer:")[-1].strip(), "OK"
+    result = qa_pipeline(prompt)
+    answer = result[0]["generated_text"].split("Answer:")[-1].strip()
+
+    return answer, ["Context retrieved", "Response generated"]