fix

ragg/ingest_all.py

@@ -1,54 +1,62 @@
-# ingest_all.py
+# ragg/ingest_all.py
 import os
-from rag_backend import ingest_pdfs_from_folder, get_embeddings
-# from ragg.rag_backend import ingest_pdfs_from_folder, get_embeddings
+import sys
 from langchain_community.vectorstores import Chroma
 
+# Allow running both as a module and as a script
+if __package__ in (None, ""):
+    # When you run: python ragg/ingest_all.py
+    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+    from ragg.rag_backend import ingest_pdfs_from_folder, get_embeddings
+    import ragg.rag_backend as rag_backend
+else:
+    # When you run: python -m ragg.ingest_all  (recommended)
+    from ragg.rag_backend import ingest_pdfs_from_folder, get_embeddings
+    import ragg.rag_backend as rag_backend
+
+IS_PROD = os.getenv("ENV", "dev").lower() == "prod"
+HERE = os.path.dirname(__file__)
+PDFS_ROOT = os.path.abspath(os.path.join(HERE, "..", "pdfs"))
+# Hugging Face shows cwd as /app → persist under /app/chroma
+CHROMA_BASE = "/app/chroma" if IS_PROD else "chroma"
+
 def ingest_all_levels():
     """
-    Ingest all level-based PDFs (low, mid, high) into separate Chroma vector databases.
-    Each folder (../pdfs/low, ../pdfs/mid, ../pdfs/high) should contain its own PDFs.
+    Ingest PDFs from ../pdfs/{low,mid,high} into separate Chroma DBs.
     """
-    pdf_sets = ["low", "mid", "high"]
-    print("\n🚀 Starting ingestion for all PDF levels...\n")
-
-    for name in pdf_sets:
-        folder_path = os.path.join("..", "pdfs", name)
-        if not os.path.exists(folder_path):
-            print(f"⚠️ Skipping '{name}' — folder not found at {folder_path}")
+    print("\n🚀 Starting ingestion\n")
+    print(f"📂 PDFs root: {PDFS_ROOT}")
+    print(f"💾 Chroma base: {CHROMA_BASE} (ENV={'prod' if IS_PROD else 'dev'})\n")
+
+    for name in ["low", "mid", "high"]:
+        folder_path = os.path.join(PDFS_ROOT, name)
+        if not os.path.isdir(folder_path):
+            print(f"⚠️ Skip {name}: {folder_path} not found")
             continue
 
-        print(f"📘 Ingesting PDF set: {name}")
-
-        # ✅ Prepare a dedicated Chroma folder for this level
-        # chroma_dir = os.path.join("chroma", name)
-        chroma_dir = os.path.join("/home/user/chroma", name) if os.getenv("ENV") == "prod" else os.path.join("chroma", name)
+        chroma_dir = os.path.join(CHROMA_BASE, name)
         os.makedirs(chroma_dir, exist_ok=True)
 
-        # ✅ Monkey patch: temporarily override get_vectorstore() for this ingestion
+        # Monkey-patch vectorstore factory used by your backend for this run
         def get_vectorstore_for_level():
-            print(f"🔹 Initializing Chroma vectorstore at: {chroma_dir}")
+            print(f"🔹 Chroma at: {chroma_dir}")
             return Chroma(
                 persist_directory=chroma_dir,
                 embedding_function=get_embeddings()
             )
-            # Print number of documents in the vector store
-            print(f"📦 Number of documents in {name} Chroma store: {len(vectorstore)}")
-            return vectorstore
-
 
-        # ✅ Temporarily replace the function used in rag_backend
-        import rag_backend as rag_backend
         rag_backend.get_vectorstore = get_vectorstore_for_level
 
-        # ✅ Ingest PDFs for this level
-        result = ingest_pdfs_from_folder(folder_path, subject="English", grade="5", chapter=name)
-        print(f"✅ Done for '{name}': {result}")
-        print(f"📦 Stored in: {chroma_dir}\n")
-         # ✅ After ingestion, print chunks from the Chroma vector store
-
-    print("🎯 All available PDFs processed successfully.\n")
-
+        try:
+            res = ingest_pdfs_from_folder(
+                folder_path, subject="English", grade="5", chapter=name
+            )
+            print(f"✅ {name}: {res}")
+            print(f"📦 Stored in: {chroma_dir}\n")
+        except Exception as e:
+            import traceback
+            print(f"❌ {name}: {e}")
+            print(traceback.format_exc())
 
 if __name__ == "__main__":
     ingest_all_levels()