Add debugging logging

Dockerfile

@@ -19,6 +19,10 @@ COPY app/ ./app/
 COPY data/processed_data/chunks.pkl ./data/processed_data/
 COPY data/processed_data/embedded_docs.pkl ./data/processed_data/
 
+# Enable more verbose logging
+ENV PYTHONUNBUFFERED=1
+
 # Set the entry point to run the Streamlit app
+# Use debug.py to troubleshoot if the main app fails
 EXPOSE 8501
-CMD ["streamlit", "run", "app/app.py", "--server.address=0.0.0.0", "--server.port=8501"] 
+CMD ["streamlit", "run", "app/app.py", "--server.address=0.0.0.0", "--server.port=8501", "--logger.level=debug"] 

app/app.py

@@ -103,26 +103,54 @@ def find_processed_data():
 @st.cache_resource
 def initialize_vectorstore():
     """Initialize the vectorstore from processed data"""
-    processed_data_path = find_processed_data()
-    st.write(f"Using processed data from: {processed_data_path}")
-    
-    # Load chunks for reference
-    chunks_path = os.path.join(processed_data_path, "chunks.pkl")
-    with open(chunks_path, "rb") as f:
-        chunks = pickle.load(f)
-    
-    # Load embedded docs
-    embedded_docs_path = os.path.join(processed_data_path, "embedded_docs.pkl")
-    with open(embedded_docs_path, "rb") as f:
-        embedded_docs = pickle.load(f)
-    
-    # Initialize embedding model
-    embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
-    
-    # Create custom vectorstore
-    vectorstore = CustomVectorStore(embedded_docs, embedding_model)
-    
-    return vectorstore, chunks
+    try:
+        processed_data_path = find_processed_data()
+        st.write(f"Using processed data from: {processed_data_path}")
+        
+        # Load chunks for reference
+        chunks_path = os.path.join(processed_data_path, "chunks.pkl")
+        st.write(f"Attempting to load chunks from: {chunks_path}")
+        try:
+            with open(chunks_path, "rb") as f:
+                chunks = pickle.load(f)
+            st.write(f"Successfully loaded chunks: {len(chunks)} items")
+        except Exception as e:
+            st.error(f"Error loading chunks.pkl: {str(e)}")
+            chunks = []
+        
+        # Load embedded docs
+        embedded_docs_path = os.path.join(processed_data_path, "embedded_docs.pkl")
+        st.write(f"Attempting to load embedded docs from: {embedded_docs_path}")
+        try:
+            with open(embedded_docs_path, "rb") as f:
+                embedded_docs = pickle.load(f)
+            st.write(f"Successfully loaded embedded docs: {len(embedded_docs)} items")
+        except Exception as e:
+            st.error(f"Error loading embedded_docs.pkl: {str(e)}")
+            embedded_docs = []
+        
+        if not chunks or not embedded_docs:
+            st.warning("Using empty vectorstore as fallback due to loading errors")
+            # Return empty vectorstore as fallback
+            embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
+            vectorstore = CustomVectorStore([], embedding_model)
+            return vectorstore, []
+            
+        # Initialize embedding model
+        try:
+            embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
+            st.write("Successfully initialized OpenAI embeddings model")
+        except Exception as e:
+            st.error(f"Error initializing OpenAI embeddings model: {str(e)}")
+            raise
+        
+        # Create custom vectorstore
+        vectorstore = CustomVectorStore(embedded_docs, embedding_model)
+        
+        return vectorstore, chunks
+    except Exception as e:
+        st.error(f"Error in vectorstore initialization: {str(e)}")
+        raise
 
 # Define prompts
 RAG_PROMPT = """

app/debug.py

@@ -0,0 +1,67 @@
+import os
+import streamlit as st
+import pickle
+from pathlib import Path
+
+st.title("Debug App")
+
+# Check environment variables
+st.write("## Environment Variables")
+if os.environ.get("OPENAI_API_KEY"):
+    st.write("✅ OPENAI_API_KEY is set")
+else:
+    st.error("❌ OPENAI_API_KEY is not set")
+
+# Try to find data directory
+st.write("## Data Directory")
+possible_paths = [
+    "data/processed_data",
+    "app/data/processed_data",
+    "/data/processed_data", 
+    "/app/data/processed_data"
+]
+
+for path in possible_paths:
+    if os.path.exists(path):
+        st.write(f"✅ Found data directory at: {path}")
+        
+        # Check for pickle files
+        chunks_path = os.path.join(path, "chunks.pkl")
+        if os.path.exists(chunks_path):
+            st.write(f"✅ Found chunks.pkl: {os.path.getsize(chunks_path) / (1024*1024):.2f} MB")
+            
+            # Try to load 
+            try:
+                with open(chunks_path, "rb") as f:
+                    chunks = pickle.load(f)
+                st.write(f"✅ Successfully loaded chunks: {len(chunks)} items")
+            except Exception as e:
+                st.error(f"❌ Error loading chunks.pkl: {str(e)}")
+        else:
+            st.error(f"❌ chunks.pkl not found in {path}")
+            
+        embedded_docs_path = os.path.join(path, "embedded_docs.pkl")
+        if os.path.exists(embedded_docs_path):
+            st.write(f"✅ Found embedded_docs.pkl: {os.path.getsize(embedded_docs_path) / (1024*1024):.2f} MB")
+            
+            # Try to load 
+            try:
+                with open(embedded_docs_path, "rb") as f:
+                    embedded_docs = pickle.load(f)
+                st.write(f"✅ Successfully loaded embedded_docs: {len(embedded_docs)} items")
+            except Exception as e:
+                st.error(f"❌ Error loading embedded_docs.pkl: {str(e)}")
+        else:
+            st.error(f"❌ embedded_docs.pkl not found in {path}")
+            
+        break
+else:
+    st.error("❌ Could not find data directory")
+
+st.write("## System Info")
+import sys
+st.write(f"Python version: {sys.version}")
+st.write(f"Working directory: {os.getcwd()}")
+st.write(f"Directory contents: {os.listdir('.')}")
+
+st.write("Debug complete.")