Update src/app.py

src/app.py

@@ -12,6 +12,8 @@ from langchain.chains import RetrievalQA
 from langchain_core.documents import Document
 from langchain_core.retrievers import BaseRetriever
 from langchain_core.callbacks import CallbackManagerForRetrieverRun
+from langchain_community.retrievers import BM25Retriever  # Added for upgraded local retriever
+from sentence_transformers import CrossEncoder  # Added for reranking; install via pip if needed (but note env limits)
 
 load_dotenv()
 
@@ -25,29 +27,23 @@ if not os.path.exists(CHUNKS_FILE):
     print(f"⚠️ WARNING: Pickle file not found at: {CHUNKS_FILE}")
 else:
     print(f"✅ SUCCESS: Pickle file found at: {CHUNKS_FILE}")
-    
+
 def search_archives(query):
-    """
-    This function strictly scans the local file.
-    It does NOT use Pinecone.
-    It returns ALL matches found.
-    """
+    """ This function strictly scans the local file. It does NOT use Pinecone. It returns ALL matches found. """
     status_log = []
     results = []
-    
     if os.path.exists(CHUNKS_FILE):
         try:
             # 1. Load the Data
             with open(CHUNKS_FILE, "rb") as f:
                 chunks = pickle.load(f)
-            
             status_log.append(f"📂 Scanning {len(chunks)} local paragraphs...")
             query_lower = query.lower().strip()
-            
+
             # 2. Find ALL Matches (No Limit)
             # We check every single chunk.
             results = [doc for doc in chunks if query_lower in doc.page_content.lower()]
-            
+
             # 3. Safety Check
             # If we find > 1000 results, we show the first 1000 to keep the browser from freezing.
             total_found = len(results)
@@ -56,9 +52,7 @@ def search_archives(query):
                 status_log.append(f"⚠️ Found {total_found} matches! Showing first 1000 to prevent crash.")
             else:
                 status_log.append(f"✅ Found {total_found} exact matches.")
-                
             return results, status_log
-
         except Exception as e:
             status_log.append(f"❌ Local Load Error: {e}")
             return [], status_log
@@ -68,7 +62,6 @@ def search_archives(query):
 
 # --- RAG CHAIN (The Chat Tool - POWERED BY GEMINI) ---
 def get_rag_chain():
-    
     class SmartRetriever(BaseRetriever):
         def _get_relevant_documents(
             self, query: str, *, run_manager: CallbackManagerForRetrieverRun = None
@@ -76,93 +69,105 @@ def get_rag_chain():
             print(f"🧠 Chat is thinking about: '{query}'")
             final_docs = []
             seen_content = set()
-            
-            # --- PHASE A: LOCAL LOOKUP (The "Vacuum") ---
-            # Suck up everything that even vaguely matches keywords
+
+            # --- PHASE A: LOCAL LOOKUP (Upgraded to BM25 for ranked relevance) ---
             if os.path.exists(CHUNKS_FILE):
                 try:
                     with open(CHUNKS_FILE, "rb") as f:
                         chunks = pickle.load(f)
-                    
-                    keywords = [w.lower() for w in query.split() if len(w) > 3]
-                    
-                    if keywords:
-                        local_matches = []
-                        for doc in chunks:
-                            content_lower = doc.page_content.lower()
-                            # If ANY keyword is present, grab it.
-                            if any(k in content_lower for k in keywords):
-                                local_matches.append(doc)
-                        
-                        # Gemini is powerful: Send top 40 matches
-                        for doc in local_matches[:40]:
-                            if doc.page_content not in seen_content:
-                                final_docs.append(doc)
-                                seen_content.add(doc.page_content)
-                                
-                        print(f"✅ Vacuumed {len(final_docs)} local matches.")
+                    # Upgrade: Use BM25Retriever instead of crude keyword matching
+                    keyword_retriever = BM25Retriever.from_documents(chunks)
+                    keyword_retriever.k = 40  # Top 40 for initial pull
+                    local_matches = keyword_retriever.invoke(query)
+                    for doc in local_matches:
+                        if doc.page_content not in seen_content:
+                            final_docs.append(doc)
+                            seen_content.add(doc.page_content)
+                    print(f"✅ Vacuumed {len(final_docs)} local matches with BM25.")
                 except Exception as e:
                     print(f"⚠️ Local Search Warning: {e}")
-            
-            # --- PHASE B: CLOUD LOOKUP (The "Net") ---
+
+            # --- PHASE B: CLOUD LOOKUP (With metadata filtering for 1963–1965 priority) ---
             print("☁️ Checking Cloud...")
             try:
                 embeddings = GoogleGenerativeAIEmbeddings(model="models/text-embedding-004")
                 vector_store = PineconeVectorStore(index_name=INDEX_NAME, embedding=embeddings)
-                
-                # Grab top 20 concepts
-                retriever = vector_store.as_retriever(search_kwargs={"k": 20}) 
+                # Assume chunks have metadata like {'year': 1963}; filter for priority
+                # Note: If your Pinecone index doesn't have 'year' metadata, add it during ingestion
+                retriever = vector_store.as_retriever(
+                    search_kwargs={
+                        "k": 20,
+                        "filter": {"year": {"$gte": 1963}}  # Prioritize 1963–1965
+                    }
+                )
                 cloud_docs = retriever.invoke(query)
-                
                 for doc in cloud_docs:
                     if doc.page_content not in seen_content:
                         final_docs.append(doc)
                         seen_content.add(doc.page_content)
-                        
                 print(f"✅ Added {len(cloud_docs)} cloud matches.")
             except Exception as e:
                 print(f"❌ Cloud Error: {e}")
 
+            # --- NEW: Reranking with Cross-Encoder for quality ---
+            if final_docs:
+                try:
+                    reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+                    pairs = [[query, doc.page_content] for doc in final_docs]
+                    scores = reranker.predict(pairs)
+                    ranked_docs = [doc for _, doc in sorted(zip(scores, final_docs), reverse=True)]
+                    final_docs = ranked_docs[:15]  # Limit to top 15 to avoid overload
+                    print(f"✅ Reranked and limited to {len(final_docs)} docs.")
+                except Exception as e:
+                    print(f"⚠️ Reranking Warning: {e} (Install sentence-transformers if needed)")
+
             return final_docs
 
-    # 2. SETUP LLM (Gemini 2.5 Flash)
+    # 2. SETUP LLM (Upgraded to gemini-1.5-pro for better depth)
     google_key = os.environ.get("GOOGLE_API_KEY") or st.secrets.get("GOOGLE_API_KEY")
     os.environ["GOOGLE_API_KEY"] = google_key
-    
-    # Using gemini-2.5-flash
+    # Upgraded model for stronger handling of details/complexity
     llm = ChatGoogleGenerativeAI(
-        model="gemini-2.5-flash-lite", 
+        model="gemini-2.5-flash",  # Upgrade from flash-lite
         temperature=0.3,
         convert_system_message_to_human=True
     )
 
-    # 3. PROMPT
-    template = """You are William Marion Branham ai. 
-    
-    INSTRUCTIONS: 
-    - You are a helpful evangelist. Answer the question comprehensively using the Context below. 
-    - Use a humble, 1950s Southern preaching dialect. 
-    - VARY YOUR OPENINGS. Do not always start with "My dear brother." Speak naturally. 
-    - If the Context mentions the topic but isn't complete, use your general knowledge of the Message to fill in the gaps. 
-    - IGNORE irrelevant "noise" in the search results (e.g. tape gaps, random prayer lines).
-    - NEVER generate new revelations, interpretations, or content beyond the CONTEXT; if insufficient, note limitations and suggest consulting original sermons.
-    - Promote ethical AI use: Emphasize that this is a tool for study, not a substitute for prayerful listening to the tapes or Holy Spirit-led understanding.
-    - Minimize biases: Present doctrines factually from CONTEXT, prioritizing 1963–1965 sermons as the mature unveiling for progressive revelations, treating earlier ones as foundational shadows without forcing harmony.
-    
-    CONTEXT: {context} 
-    
-    USER QUESTION: {question} 
-    
-    BROTHER BRANHAM'S REPLY:"""
-    
+    # 3. PROMPT (Updated per suggestions: Neutral role, no dialect, rigid structure, no contradictions)
+    template = """
+You are a doctrinal study assistant for William Branham's Message teachings (1947–1965). Your purpose is to provide accurate, scripture-centered expositions based EXCLUSIVELY on the retrieved sermon excerpts in the CONTEXT below.
+
+CRITICAL RULES (follow exactly):
+- NEVER impersonate William Branham or speak in first person as him.
+- NEVER claim prophetic authority or present anything as new revelation.
+- Base EVERY statement directly on the provided CONTEXT (direct quotes and teachings from the sermons).
+- If the CONTEXT does not sufficiently cover the question, provide a partial exposition from what is available and suggest specific relevant sermons (e.g., "Consult 'The First Seal' sermon from March 18, 1963").
+- Cite sources naturally (e.g., "As taught in the sermon 'The Seven Seals' (65-1127B)...").
+- Prioritize teachings from later sermons (1963–1965) as the mature, further unveiling of the mystery when doctrines develop over time. Treat earlier statements as partial or progressive if they appear to differ.
+- IGNORE irrelevant "noise" in the search results (e.g., tape gaps, random prayer lines).
+- Promote ethical AI use: Emphasize that this is a tool for study, not a substitute for prayerful listening to the tapes or Holy Spirit-led understanding.
+- Minimize biases: Present doctrines factually from CONTEXT, without forcing harmony.
+
+STYLE AND STRUCTURE:
+- Use symbolic, scripture-centered language and Message terminology naturally (e.g., Seals, Thunders, Capstone, Bride, restoration, end-time mystery, spoken Word, rapturing faith).
+- Adopt a declarative, confident, instructional tone, avoiding preaching, emotional appeals, or direct address.
+- NO greetings, fluff, or conversational fillers. Start immediately with the exposition.
+- For lists or detailed interpretations, use bullet points or numbered lists.
+CONTEXT (direct excerpts from Brother Branham's sermons):
+{context}
+
+QUESTION: {question}
+
+DOCTRINAL EXPOSITION:
+"""
     PROMPT = PromptTemplate(template=template, input_variables=["context", "question"])
-    
+
     chain = RetrievalQA.from_chain_type(
         llm=llm,
-        chain_type="stuff",
-        retriever=SmartRetriever(), 
+        chain_type="refine",  # Changed from "stuff" to "refine" for iterative processing
+        retriever=SmartRetriever(),
         return_source_documents=True,
         chain_type_kwargs={"prompt": PROMPT, "document_variable_name": "context"},
         input_key="question"
     )
-    return chain
+    return chain