Add debugging code about the preprocessed data

.gitignore

@@ -1,4 +1,35 @@
-notebook_version/
+# Ignore PDF files
 *.pdf
-*.pkl
-processed_data/qdrant_vectorstore/*
+
+# Keep processed data
+!processed_data/
+!processed_data/document_chunks.pkl
+!processed_data/qdrant_vectorstore/
+
+# Ignore notebook version folder
+notebook_version/
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Jupyter
+.ipynb_checkpoints

streamlit_app.py

@@ -32,11 +32,50 @@ print("Loaded .env file")
 if not os.environ.get("OPENAI_API_KEY"):
     os.environ["OPENAI_API_KEY"] = os.environ.get("OPENAI_API_KEY_BACKUP", "")
 
+# Debugging: Print current directory and its contents
+print(f"Current directory: {os.getcwd()}")
+print(f"Directory contents: {os.listdir('.')}")
+
+# Find the processed data directory
+# Try multiple possible paths
+possible_paths = [
+    "processed_data",
+    "/app/processed_data",
+    "../processed_data",
+    "./processed_data",
+    "/home/user/app/processed_data"
+]
+
+# Find the first path that exists
+for path in possible_paths:
+    print(f"Checking path: {path}")
+    if os.path.exists(path):
+        PROCESSED_DATA_DIR = Path(path)
+        print(f"Found processed data at: {path}")
+        print(f"Contents: {os.listdir(path)}")
+        break
+else:
+    # Default if none found
+    PROCESSED_DATA_DIR = Path("processed_data")
+    print(f"Using default processed data path: {PROCESSED_DATA_DIR}")
+    
+    # Create directory if it doesn't exist (for logging)
+    if not os.path.exists(PROCESSED_DATA_DIR):
+        os.makedirs(PROCESSED_DATA_DIR, exist_ok=True)
+        print(f"Created directory: {PROCESSED_DATA_DIR}")
+
 # Paths to pre-processed data
-PROCESSED_DATA_DIR = Path("processed_data")
 CHUNKS_FILE = PROCESSED_DATA_DIR / "document_chunks.pkl"
 QDRANT_DIR = PROCESSED_DATA_DIR / "qdrant_vectorstore"
 
+# Print paths for debugging
+print(f"CHUNKS_FILE path: {CHUNKS_FILE}")
+print(f"CHUNKS_FILE exists: {os.path.exists(CHUNKS_FILE)}")
+print(f"QDRANT_DIR path: {QDRANT_DIR}")
+print(f"QDRANT_DIR exists: {os.path.exists(QDRANT_DIR)}")
+if os.path.exists(QDRANT_DIR):
+    print(f"QDRANT_DIR contents: {os.listdir(QDRANT_DIR)}")
+
 # Define prompts exactly as in the notebook
 RAG_PROMPT = """
 CONTEXT:
@@ -72,6 +111,7 @@ evaluate_prompt = PromptTemplate.from_template(EVALUATE_RESPONSE_PROMPT)
 @st.cache_resource
 def load_document_chunks():
     """Load pre-processed document chunks from disk."""
+    print(f"Attempting to load document chunks from {CHUNKS_FILE}")
     if not os.path.exists(CHUNKS_FILE):
         print(f"WARNING: Chunks file not found at {CHUNKS_FILE}")
         print(f"Working directory contents: {os.listdir('.')}")
@@ -83,116 +123,185 @@ def load_document_chunks():
         with open(CHUNKS_FILE, 'rb') as f:
             chunks = pickle.load(f)
             print(f"Successfully loaded {len(chunks)} document chunks")
+            # Print first chunk to verify data
+            if chunks:
+                print(f"First chunk metadata: {chunks[0].metadata}")
             return chunks
     except Exception as e:
         print(f"Error loading document chunks: {str(e)}")
+        import traceback
+        traceback.print_exc()
         return []
 
 @st.cache_resource
 def get_chat_model():
     """Get the chat model for initial RAG."""
+    print("Initializing chat model...")
+    # Try multiple approaches to initialize the model
     try:
-        # Use direct OpenAI client to avoid proxy issues
-        openai_client = OpenAI()
-        
-        # Create a wrapper that mimics LangChain's interface
-        class SimpleOpenAIWrapper:
-            def invoke(self, messages):
-                # Convert LangChain messages to OpenAI format
-                openai_messages = []
-                for msg in messages:
-                    role = "user"
-                    if hasattr(msg, "type"):
-                        role = "assistant" if msg.type == "ai" else "user"
-                    openai_messages.append({
-                        "role": role,
-                        "content": msg.content
-                    })
-                
-                # Call API directly
-                response = openai_client.chat.completions.create(
-                    model="gpt-4.1-mini",
-                    messages=openai_messages,
-                    temperature=0
-                )
-                
-                # Create response object with content attribute
-                class SimpleResponse:
-                    def __init__(self, content):
-                        self.content = content
-                
-                return SimpleResponse(response.choices[0].message.content)
-        
-        return SimpleOpenAIWrapper()
-    except Exception as e:
-        print(f"Error creating OpenAI wrapper: {str(e)}")
+        # Approach 1: Direct OpenAI client
+        print("Trying direct OpenAI client approach")
         try:
-            # Last resort fallback to basic LangChain with minimal config
-            return ChatOpenAI(model="gpt-4.1-mini", temperature=0)
-        except Exception as e2:
-            print(f"Fallback also failed: {str(e2)}")
+            # Use direct OpenAI client to avoid proxy issues
+            openai_client = OpenAI()
             
-            # Create dummy that returns a fixed response
-            class DummyModel:
+            # Create a wrapper that mimics LangChain's interface
+            class SimpleOpenAIWrapper:
                 def invoke(self, messages):
-                    class DummyResponse:
-                        def __init__(self):
-                            self.content = "I apologize, but I'm unable to process your query right now. Please try again later."
-                    return DummyResponse()
+                    print("Invoking SimpleOpenAIWrapper...")
+                    # Convert LangChain messages to OpenAI format
+                    openai_messages = []
+                    for msg in messages:
+                        role = "user"
+                        if hasattr(msg, "type"):
+                            role = "assistant" if msg.type == "ai" else "user"
+                        openai_messages.append({
+                            "role": role,
+                            "content": msg.content
+                        })
+                    
+                    # Log what we're sending to OpenAI
+                    print(f"Sending {len(openai_messages)} messages to OpenAI API")
+                    
+                    # Call API directly
+                    response = openai_client.chat.completions.create(
+                        model="gpt-4.1-mini",
+                        messages=openai_messages,
+                        temperature=0
+                    )
+                    
+                    # Create response object with content attribute
+                    class SimpleResponse:
+                        def __init__(self, content):
+                            self.content = content
+                    
+                    result = SimpleResponse(response.choices[0].message.content)
+                    print(f"Got response from OpenAI (length: {len(result.content)})")
+                    return result
+            
+            print("Successfully created SimpleOpenAIWrapper")
+            return SimpleOpenAIWrapper()
+        except Exception as e:
+            print(f"Direct OpenAI client approach failed: {str(e)}")
+            import traceback
+            traceback.print_exc()
+            raise
+            
+    except Exception as outer_e:
+        print(f"First approach failed: {str(outer_e)}")
+        
+        # Approach 2: Standard LangChain
+        try:
+            print("Trying standard LangChain approach")
+            model = ChatOpenAI(model="gpt-4.1-mini", temperature=0)
+            print("Successfully created ChatOpenAI model")
+            return model
+        except Exception as e:
+            print(f"Standard LangChain approach failed: {str(e)}")
             
-            return DummyModel()
+            # Approach 3: Very minimal LangChain
+            try:
+                print("Trying minimal LangChain approach")
+                model = ChatOpenAI(model="gpt-3.5-turbo")
+                print("Successfully created minimal ChatOpenAI model")
+                return model
+            except Exception as e2:
+                print(f"Minimal LangChain also failed: {str(e2)}")
+                
+                # Last resort: Dummy implementation
+                print("Using dummy model as last resort")
+                class DummyModel:
+                    def invoke(self, messages):
+                        print("WARNING: Using dummy model that returns fixed responses")
+                        class DummyResponse:
+                            def __init__(self):
+                                self.content = "I apologize, but I'm unable to process your query right now due to a technical issue. The system administrators have been notified."
+                        return DummyResponse()
+                
+                return DummyModel()
 
 @st.cache_resource
 def get_agent_model():
     """Get the more powerful model for agent and evaluation."""
+    print("Initializing agent model...")
+    # Try multiple approaches to initialize the model
     try:
-        # Use same approach as get_chat_model
-        openai_client = OpenAI()
-        
-        class SimpleOpenAIWrapper:
-            def invoke(self, messages):
-                # Convert LangChain messages to OpenAI format
-                openai_messages = []
-                for msg in messages:
-                    role = "user"
-                    if hasattr(msg, "type"):
-                        role = "assistant" if msg.type == "ai" else "user"
-                    openai_messages.append({
-                        "role": role,
-                        "content": msg.content
-                    })
-                
-                # Call API directly with a more powerful model
-                response = openai_client.chat.completions.create(
-                    model="gpt-4.1",
-                    messages=openai_messages,
-                    temperature=0
-                )
-                
-                class SimpleResponse:
-                    def __init__(self, content):
-                        self.content = content
-                
-                return SimpleResponse(response.choices[0].message.content)
+        # Approach 1: Direct OpenAI client
+        print("Trying direct OpenAI client approach for agent model")
+        try:
+            # Use direct OpenAI client to avoid proxy issues
+            openai_client = OpenAI()
+            
+            # Create a wrapper that mimics LangChain's interface
+            class SimpleOpenAIWrapper:
+                def invoke(self, messages):
+                    print("Invoking agent SimpleOpenAIWrapper...")
+                    # Convert LangChain messages to OpenAI format
+                    openai_messages = []
+                    for msg in messages:
+                        role = "user"
+                        if hasattr(msg, "type"):
+                            role = "assistant" if msg.type == "ai" else "user"
+                        openai_messages.append({
+                            "role": role,
+                            "content": msg.content
+                        })
+                    
+                    # Log what we're sending to OpenAI
+                    print(f"Sending {len(openai_messages)} messages to OpenAI API (agent)")
+                    
+                    # Call API directly with a more powerful model
+                    response = openai_client.chat.completions.create(
+                        model="gpt-4.1",
+                        messages=openai_messages,
+                        temperature=0
+                    )
+                    
+                    class SimpleResponse:
+                        def __init__(self, content):
+                            self.content = content
+                    
+                    result = SimpleResponse(response.choices[0].message.content)
+                    print(f"Got agent response from OpenAI (length: {len(result.content)})")
+                    return result
+            
+            print("Successfully created agent SimpleOpenAIWrapper")
+            return SimpleOpenAIWrapper()
+        except Exception as e:
+            print(f"Direct OpenAI client approach for agent failed: {str(e)}")
+            import traceback
+            traceback.print_exc()
+            raise
+            
+    except Exception as outer_e:
+        print(f"First agent approach failed: {str(outer_e)}")
         
-        return SimpleOpenAIWrapper()
-    except Exception as e:
-        print(f"Error creating agent model: {str(e)}")
+        # Approach 2: Standard LangChain
         try:
-            # Fallback
-            return ChatOpenAI(model="gpt-4.1", temperature=0)
-        except Exception as e2:
-            print(f"Agent model fallback also failed: {str(e2)}")
-            # Final fallback to gpt-3.5-turbo
+            print("Trying standard LangChain approach for agent")
+            model = ChatOpenAI(model="gpt-4.1", temperature=0)
+            print("Successfully created agent ChatOpenAI model")
+            return model
+        except Exception as e:
+            print(f"Standard LangChain approach for agent failed: {str(e)}")
+            
+            # Approach 3: Very minimal LangChain with fallback model
             try:
-                return ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
-            except:
-                # Create dummy that returns a fixed response
+                print("Trying minimal LangChain approach for agent")
+                model = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
+                print("Successfully created minimal agent ChatOpenAI model")
+                return model
+            except Exception as e2:
+                print(f"Minimal LangChain for agent also failed: {str(e2)}")
+                
+                # Last resort: Dummy implementation
+                print("Using dummy agent model as last resort")
                 class DummyModel:
                     def invoke(self, messages):
+                        print("WARNING: Using dummy agent model that returns fixed responses")
                         class DummyResponse:
                             def __init__(self):
-                                self.content = "I apologize, but I'm unable to process your query right now. Please try again later."
+                                self.content = "I apologize, but I'm unable to process complex queries right now due to a technical issue."
                         return DummyResponse()
                 
                 return DummyModel()
@@ -200,97 +309,157 @@ def get_agent_model():
 @st.cache_resource
 def get_embedding_model():
     """Get the embedding model."""
+    print("Initializing embedding model...")
     try:
-        # Create an OpenAI client directly
-        openai_client = OpenAI()
-        
-        # Create a wrapper class that matches the interface LangChain expects
-        class SimpleEmbeddings:
-            def embed_query(self, text):
-                try:
-                    response = openai_client.embeddings.create(
-                        model="text-embedding-3-small",
-                        input=text
-                    )
-                    return response.data[0].embedding
-                except Exception as e:
-                    print(f"Error in embed_query: {str(e)}")
-                    # Return a dummy embedding of the right size
-                    return [0.0] * 1536  # Standard size for embeddings
-            
-            def embed_documents(self, texts):
-                try:
-                    if not texts:
-                        return []
-                    
-                    # Embed each text individually to avoid batch size issues
-                    return [self.embed_query(text) for text in texts]
-                except Exception as e:
-                    print(f"Error in embed_documents: {str(e)}")
-                    # Return dummy embeddings
-                    return [[0.0] * 1536 for _ in range(len(texts))]
-        
-        return SimpleEmbeddings()
-    except Exception as e:
-        print(f"Error initializing embedding model: {str(e)}")
-        # Last resort fallback
+        # Approach 1: Direct OpenAI client
+        print("Trying direct OpenAI client approach for embeddings")
         try:
-            return OpenAIEmbeddings(model="text-embedding-3-small")
-        except Exception as e2:
-            print(f"Embedding fallback also failed: {str(e2)}")
+            # Create an OpenAI client directly
+            openai_client = OpenAI()
             
-            # Return a dummy embeddings class
-            class DummyEmbeddings:
+            # Create a wrapper class that matches the interface LangChain expects
+            class SimpleEmbeddings:
                 def embed_query(self, text):
-                    return [0.0] * 1536
+                    print(f"Embedding query text (length: {len(text)})")
+                    try:
+                        response = openai_client.embeddings.create(
+                            model="text-embedding-3-small",
+                            input=text
+                        )
+                        print("Successfully got embedding from OpenAI API")
+                        return response.data[0].embedding
+                    except Exception as e:
+                        print(f"Error in embed_query: {str(e)}")
+                        import traceback
+                        traceback.print_exc()
+                        # Return a dummy embedding of the right size
+                        print("WARNING: Returning dummy embedding vector")
+                        return [0.0] * 1536  # Standard size for embeddings
                 
                 def embed_documents(self, texts):
-                    return [[0.0] * 1536 for _ in range(len(texts))]
+                    print(f"Embedding {len(texts)} documents")
+                    try:
+                        if not texts:
+                            return []
+                        
+                        # Embed each text individually to avoid batch size issues
+                        results = []
+                        for i, text in enumerate(texts):
+                            print(f"Embedding document {i+1}/{len(texts)}")
+                            results.append(self.embed_query(text))
+                        return results
+                    except Exception as e:
+                        print(f"Error in embed_documents: {str(e)}")
+                        import traceback
+                        traceback.print_exc()
+                        # Return dummy embeddings
+                        print("WARNING: Returning dummy document embeddings")
+                        return [[0.0] * 1536 for _ in range(len(texts))]
+            
+            print("Successfully created SimpleEmbeddings")
+            return SimpleEmbeddings()
+        except Exception as e:
+            print(f"Direct OpenAI client approach for embeddings failed: {str(e)}")
+            import traceback
+            traceback.print_exc()
+            raise
+            
+    except Exception as outer_e:
+        print(f"First embedding approach failed: {str(outer_e)}")
+        
+        # Approach 2: Standard LangChain OpenAIEmbeddings
+        try:
+            print("Trying standard LangChain approach for embeddings")
+            embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+            print("Successfully created OpenAIEmbeddings")
+            return embeddings
+        except Exception as e:
+            print(f"Standard OpenAIEmbeddings failed: {str(e)}")
             
-            return DummyEmbeddings()
+            # Approach 3: Very minimal OpenAIEmbeddings
+            try:
+                print("Trying minimal OpenAIEmbeddings")
+                embeddings = OpenAIEmbeddings()
+                print("Successfully created minimal OpenAIEmbeddings")
+                return embeddings
+            except Exception as e2:
+                print(f"Minimal OpenAIEmbeddings failed: {str(e2)}")
+                
+                # Last resort: Dummy implementation
+                print("Using dummy embeddings as last resort")
+                class DummyEmbeddings:
+                    def embed_query(self, text):
+                        print("WARNING: Using dummy embeddings")
+                        return [0.0] * 1536
+                    
+                    def embed_documents(self, texts):
+                        print("WARNING: Using dummy document embeddings")
+                        return [[0.0] * 1536 for _ in range(len(texts))]
+                
+                return DummyEmbeddings()
 
 @st.cache_resource
 def setup_qdrant_client():
     """Set up the Qdrant client."""
+    print(f"Attempting to setup Qdrant client with path: {QDRANT_DIR}")
     # Check if Qdrant dir exists
     if not os.path.exists(QDRANT_DIR):
         print(f"WARNING: Qdrant directory not found: {QDRANT_DIR}")
         print(f"Contents of {PROCESSED_DATA_DIR}: {os.listdir(PROCESSED_DATA_DIR) if os.path.exists(PROCESSED_DATA_DIR) else 'Not found'}")
     
     try:
+        print("Trying to create QdrantClient with path parameter")
         client = QdrantClient(path=str(QDRANT_DIR))
-        print("Successfully created Qdrant client")
+        print("Successfully created Qdrant client with path parameter")
         
         # Verify client works by getting collections
         try:
             collection_name = "kohavi_ab_testing_pdf_collection"
+            print(f"Trying to get collections from Qdrant")
             collections = client.get_collections()
-            print(f"Available collections: {collections}")
+            print(f"Available collections: {collections.collections}")
             
             # Check if our collection exists
             collection_exists = False
             for collection in collections.collections:
                 if collection.name == collection_name:
                     collection_exists = True
+                    print(f"Found our collection: {collection_name}")
                     break
             
             if not collection_exists:
                 print(f"WARNING: Collection '{collection_name}' not found!")
         except Exception as e:
             print(f"Warning: Could not get collections: {str(e)}")
+            import traceback
+            traceback.print_exc()
         
         return client
     except Exception as e:
         print(f"Error creating QdrantClient with path: {str(e)}")
+        import traceback
+        traceback.print_exc()
         
         # Try alternative parameter
         try:
+            print("Trying to create QdrantClient with location parameter")
             client = QdrantClient(location=str(QDRANT_DIR))
             print("Successfully created QdrantClient with location parameter")
             return client
         except Exception as e2:
             print(f"Alternative initialization failed: {str(e2)}")
-            raise
+            
+            # Try in-memory as last resort (for testing)
+            try:
+                print("FALLBACK: Creating in-memory QdrantClient")
+                client = QdrantClient(":memory:")
+                print("Created in-memory QdrantClient as fallback")
+                return client
+            except Exception as e3:
+                print(f"Even in-memory Qdrant failed: {str(e3)}")
+                import traceback
+                traceback.print_exc()
+                raise
 
 def rag_chain_node(query):
     """
@@ -307,36 +476,69 @@ def rag_chain_node(query):
         
         # Get embedding for the query
         embedding_model = get_embedding_model()
+        print("Getting embedding for query...")
         query_embedding = embedding_model.embed_query(query)
+        print(f"Generated embedding of length: {len(query_embedding)}")
         
         # Get documents
-        print("Retrieving documents...")
+        print("Loading document chunks...")
         chunks = load_document_chunks()
+        print(f"Loaded {len(chunks)} document chunks")
         
         # Map of document IDs to actual documents
         docs_by_id = {i: doc for i, doc in enumerate(chunks)}
         
         # Search for relevant documents
-        search_results = client.search(
-            collection_name=collection_name,
-            query_vector=query_embedding,
-            limit=5
-        )
+        print(f"Searching collection '{collection_name}' for documents...")
+        try:
+            # First try using query_points (preferred method)
+            print("Trying query_points method first...")
+            search_results = client.query_points(
+                collection_name=collection_name,
+                query_vector=query_embedding,
+                limit=5
+            )
+            print(f"Found {len(search_results)} results using query_points method")
+        except Exception as e1:
+            print(f"query_points method failed: {str(e1)}")
+            
+            # Fall back to search method
+            print("Falling back to search method...")
+            try:
+                search_results = client.search(
+                    collection_name=collection_name,
+                    query_vector=query_embedding,
+                    limit=5
+                )
+                print(f"Found {len(search_results)} results using search method")
+            except Exception as e2:
+                print(f"Both query methods failed: {str(e2)}")
+                import traceback
+                traceback.print_exc()
+                raise
         
         # Convert search results to documents
         docs = []
+        print("Processing search results...")
         for result in search_results:
             doc_id = result.id
             if doc_id in docs_by_id:
                 docs.append(docs_by_id[doc_id])
+                print(f"Added doc with ID {doc_id}")
+            else:
+                print(f"Warning: Doc ID {doc_id} not found in loaded chunks")
     except Exception as e:
         print(f"Error in document retrieval: {str(e)}")
+        import traceback
+        traceback.print_exc()
         return "I'm having trouble retrieving relevant information. Please try again later.", []
     
     # 2. Extract sources from the documents
     sources = []
+    print(f"Extracting sources from {len(docs)} documents...")
     for doc in docs:
         source_path = doc.metadata.get("source", "")
+        print(f"Processing source: {source_path}")
         filename = source_path.split("/")[-1] if "/" in source_path else source_path
         
         # Remove .pdf extension if present
@@ -348,6 +550,7 @@ def rag_chain_node(query):
             "page": doc.metadata.get("page", "unknown"),
             "type": "pdf"
         })
+        print(f"Added source: {filename}, Page: {doc.metadata.get('page', 'unknown')}")
     
     # 3. Use the RAG chain to generate an answer
     if not docs:
@@ -356,6 +559,7 @@ def rag_chain_node(query):
     
     # Create context from documents
     context = "\n\n".join([doc.page_content for doc in docs])
+    print(f"Created context of length: {len(context)}")
     
     # Format the prompt with context and query
     formatted_prompt = rag_prompt.format(context=context, question=query)
@@ -363,10 +567,16 @@ def rag_chain_node(query):
     # Send to the model and parse the output
     print("Generating answer...")
     chat_model = get_chat_model()
-    response = chat_model.invoke(formatted_prompt)
-    response_text = response.content
-    
-    return response_text, sources
+    try:
+        response = chat_model.invoke(formatted_prompt)
+        response_text = response.content
+        print(f"Generated response of length: {len(response_text)}")
+        return response_text, sources
+    except Exception as e:
+        print(f"Error generating response: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return "I encountered an error while generating a response. Please try again.", sources
 
 def evaluate_response(query, response):
     """

verify_data.py

@@ -0,0 +1,29 @@
+import os
+from pathlib import Path
+
+# Check various possible locations
+possible_paths = [
+    "processed_data",
+    "/app/processed_data",
+    "../processed_data",
+    "./processed_data"
+]
+
+for path in possible_paths:
+    chunks_file = Path(path) / "document_chunks.pkl"
+    qdrant_dir = Path(path) / "qdrant_vectorstore"
+    
+    print(f"Checking path: {path}")
+    print(f"  Exists?: {os.path.exists(path)}")
+    
+    if os.path.exists(path):
+        print(f"  Contents: {os.listdir(path)}")
+        print(f"  Chunks file exists?: {os.path.exists(chunks_file)}")
+        print(f"  Qdrant dir exists?: {os.path.exists(qdrant_dir)}")
+        
+        if os.path.exists(qdrant_dir):
+            print(f"  Qdrant contents: {os.listdir(qdrant_dir)}")
+
+# Show current working directory and its contents
+print(f"Current directory: {os.getcwd()}")
+print(f"Contents: {os.listdir('.')}")