Update src/app.py

src/app.py

@@ -1,9 +1,7 @@
 import os
 import pickle
-import sys
 import streamlit as st
 from dotenv import load_dotenv
-
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
 from langchain_groq import ChatGroq
 from langchain_community.retrievers import BM25Retriever
@@ -11,54 +9,56 @@ from langchain_pinecone import PineconeVectorStore
 from langchain_core.prompts import PromptTemplate
 from langchain.chains import RetrievalQA
 from langchain.retrievers import EnsembleRetriever
+from langchain_core.documents import Document
 
 load_dotenv()
 
+# --- CONFIGURATION ---
 INDEX_NAME = "branham-index"
 CHUNKS_FILE = "sermon_chunks.pkl"
 _CACHED_RETRIEVER = None
 
+# --- RETRIEVER SETUP (The Brain) ---
 def get_retriever():
     global _CACHED_RETRIEVER
     if _CACHED_RETRIEVER is not None: return _CACHED_RETRIEVER
     
-    # 1. Setup Keys
+    # 1. Load Keys
     pinecone_key = os.environ.get("PINECONE_API_KEY") or st.secrets.get("PINECONE_API_KEY")
     google_key = os.environ.get("GOOGLE_API_KEY") or st.secrets.get("GOOGLE_API_KEY")
     
-    if not pinecone_key or not google_key: raise ValueError("Missing API Keys.")
+    if not pinecone_key or not google_key: 
+        raise ValueError("❌ CRITICAL: Missing API Keys.")
+    
     os.environ["PINECONE_API_KEY"] = pinecone_key
     os.environ["GOOGLE_API_KEY"] = google_key
 
-    # 2. Setup Vector Store (Pinecone) with SCORE THRESHOLD
+    # 2. Vector Store (Pinecone) - WITH NOISE FILTER
+    # We set score_threshold=0.5 to block "Prayer Card" garbage.
     embeddings = GoogleGenerativeAIEmbeddings(model="models/text-embedding-004")
     vector_store = PineconeVectorStore(index_name=INDEX_NAME, embedding=embeddings)
     
-    # CRITICAL PROFESSIONAL FIX:
-    # We set a "score_threshold" of 0.5.
-    # This means: "If the AI is less than 50% sure, DO NOT show the result."
-    # This kills the "Prayer Card B" noise immediately.
     vector_retriever = vector_store.as_retriever(
         search_type="similarity_score_threshold",
-        search_kwargs={"k": 20, "score_threshold": 0.5} 
+        search_kwargs={"k": 15, "score_threshold": 0.5} 
     )
 
-    # 3. Setup Keyword Store (BM25)
+    # 3. Keyword Store (BM25)
     keyword_retriever = None
     if os.path.exists(CHUNKS_FILE):
         try:
             with open(CHUNKS_FILE, "rb") as f:
                 chunks = pickle.load(f)
             keyword_retriever = BM25Retriever.from_documents(chunks)
-            keyword_retriever.k = 20
+            keyword_retriever.k = 15
         except Exception as e:
-            print(f"BM25 Error: {e}")
+            print(f"⚠️ BM25 Load Error: {e}")
 
-    # 4. Create Hybrid Ensemble
+    # 4. Hybrid Ensemble
     if keyword_retriever:
         final_retriever = EnsembleRetriever(
             retrievers=[vector_retriever, keyword_retriever],
-            weights=[0.4, 0.6] # 40% Vector (Concepts), 60% Keyword (Precision)
+            weights=[0.3, 0.7] # Favor Exact Keywords
         )
     else:
         final_retriever = vector_retriever
@@ -66,90 +66,107 @@ def get_retriever():
     _CACHED_RETRIEVER = final_retriever
     return final_retriever
 
+# --- SEARCH FUNCTION (The "Search Quotes Only" Tool) ---
+def search_archives(query):
+    """
+    FAIL-SAFE SEARCH LOGIC:
+    1. Brute Force Text Scan (Ctrl+F style) - Guarantees exact matches.
+    2. BM25 Search - Finds relevant keywords.
+    3. Vector Search - Only runs if keywords fail.
+    """
+    results = []
+    seen_content = set() # To prevent duplicates
+
+    # --- PHASE 1: LOCAL SEARCH (The "Ctrl+F" Fail-Safe) ---
+    if os.path.exists(CHUNKS_FILE):
+        try:
+            with open(CHUNKS_FILE, "rb") as f:
+                chunks = pickle.load(f)
+            
+            # A. BRUTE FORCE SCAN (Case Insensitive)
+            # This loops through all chunks. Fast enough for <100k chunks.
+            query_lower = query.lower().strip()
+            
+            # Optimization: Only scan if query is short (like a name)
+            if len(query_lower) < 20: 
+                for doc in chunks:
+                    if query_lower in doc.page_content.lower():
+                        if doc.page_content not in seen_content:
+                            results.append(doc)
+                            seen_content.add(doc.page_content)
+                            if len(results) >= 20: break # Stop after 20 exact matches
+
+            # B. BM25 SEARCH (If Brute Force didn't fill the quota)
+            if len(results) < 10:
+                bm25 = BM25Retriever.from_documents(chunks)
+                bm25.k = 15
+                bm25_docs = bm25.invoke(query)
+                for doc in bm25_docs:
+                    if doc.page_content not in seen_content:
+                        # Double check relevance
+                        query_terms = query_lower.split()
+                        if any(term in doc.page_content.lower() for term in query_terms):
+                            results.append(doc)
+                            seen_content.add(doc.page_content)
+            
+            # IF WE FOUND LOCAL RESULTS, RETURN THEM!
+            # Do not touch Pinecone.
+            if results:
+                print(f"✅ Found {len(results)} local matches for '{query}'")
+                return results
+
+        except Exception as e:
+            st.error(f"Local Search Error: {e}")
+
+    # --- PHASE 2: VECTOR FALLBACK ---
+    # Only runs if Phase 1 found absolutely nothing.
+    print(f"⚠️ No local matches for '{query}'. Trying Pinecone...")
+    try:
+        retriever = get_retriever()
+        # If using Ensemble, it might pull vectors. 
+        # If local file was missing, this is our only hope.
+        docs = retriever.invoke(query)
+        return docs
+    except Exception as e:
+        # Gracefully handle the "No results because of threshold" error
+        return []
+
+# --- RAG CHAIN (The Chat Tool) ---
 def get_rag_chain():
     retriever = get_retriever()
     groq_key = os.environ.get("GROQ_API_KEY") or st.secrets.get("GROQ_API_KEY")
     os.environ["GROQ_API_KEY"] = groq_key
-
+    
     llm = ChatGroq(model="llama-3.3-70b-versatile", temperature=0.3, max_retries=2)
 
-    # --- PROMPT: Using {question} ---
     template = """You are William Marion Branham.
     
     TASK:
-    You are answering a believer's question using the provided CONTEXT.
-
-    CRITICAL RULES FOR ACCURACY:
-    1. **NO FABRICATION:** If the user asks for a specific prayer, quote, or event (e.g., "What did you say to Brother Coleman?"), look ONLY at the CONTEXT below. 
-       - If the text is there, quote it exactly.
-       - **If the text is NOT in the Context, DO NOT INVENT A PRAYER.** - Instead, say: "Brother, I do not find that specific record on the tapes here. I can only tell you what is written in these records."
-    2. **NO FILLER DOCTRINE:** Do not give a generic lecture on prayer if you cannot find the specific prayer asked for.
-    3. **BE DIRECT:** Answer the specific question first.
-
-    DIALECT:
-    - Use the humble, Southern style ("I said," "The Lord showed me").
-    - Keep it natural.
+    Answer the believer's question based ONLY on the provided CONTEXT.
+    
+    RULES:
+    1. If the answer is not in the records below, say: "Brother, I do not find that specific record on the tapes here."
+    2. Do not make up prayers or quotes.
+    3. Be humble and direct.
     
     CONTEXT:
     {context}
-
+    
     USER QUESTION: {question}
+    
     BROTHER BRANHAM'S REPLY:"""
-
-
+    
     PROMPT = PromptTemplate(
         template=template, 
         input_variables=["context", "question"]
     )
-
-    chain_type_kwargs = {
-        "prompt": PROMPT,
-        "document_variable_name": "context"
-    }
-
-    # --- FORCE INPUT KEY TO BE 'question' ---
+    
     chain = RetrievalQA.from_chain_type(
         llm=llm,
         chain_type="stuff",
         retriever=retriever,
         return_source_documents=True,
-        chain_type_kwargs=chain_type_kwargs,
-        input_key="question"  # <--- THIS IS THE FIX
+        chain_type_kwargs={"prompt": PROMPT, "document_variable_name": "context"},
+        input_key="question"
     )
-    return chain
-
-# In app.py
-
-def search_archives(query):
-    """
-    STRICT SEARCH LOGIC:
-    1. Runs a Pure Keyword (BM25) search first.
-    2. If it finds exact matches, it returns them immediately (ignoring Vector noise).
-    3. Only falls back to Vector search if Keywords find nothing.
-    """
-    # --- PHASE 1: PRECISE KEYWORD SEARCH ---
-    if os.path.exists(CHUNKS_FILE):
-        try:
-            with open(CHUNKS_FILE, "rb") as f:
-                chunks = pickle.load(f)
-            
-            # Create a temporary keyword retriever just for this search
-            keyword_retriever = BM25Retriever.from_documents(chunks)
-            keyword_retriever.k = 15  # Fetch top 15 exact matches
-            
-            # Run the search
-            keyword_docs = keyword_retriever.invoke(query)
-            
-            # CRITICAL CHECK: Did we find anything?
-            if keyword_docs:
-                print(f"✅ Found {len(keyword_docs)} matches via Keywords.")
-                return keyword_docs
-        except Exception as e:
-            print(f"⚠️ Keyword Search failed: {e}")
-
-    # --- PHASE 2: FALLBACK VECTOR SEARCH ---
-    # Only runs if Phase 1 returned nothing.
-    print("⚠️ No keywords found. Falling back to Vector Search...")
-    retriever = get_retriever()
-    docs = retriever.invoke(query)
-    return docs
+    return chain