Optimize comparison search: bypass redundant KG search, avoid thread-safety ChromaDB init lock error, and prevent false positive tier lighting up in dev console

backend/main.py

@@ -391,7 +391,9 @@ async def chat_stream(session_id: str, query: str, plan_tier: str = "Unknown"):
 
                 for step in new_steps:
                     yield emit({"type": "substep", "node": node, "intent": intent, "step": step, "all_steps": current_steps})
-                    await asyncio.sleep(0.05)
+                    # Skip artificial delay for raw graph DB edge logs and entity listings to prevent stream lagging
+                    if not step.startswith(("[GraphDB-Edge]", "[GraphDB]")):
+                        await asyncio.sleep(0.05)
 
                 final_answer = state.get("answer", "")
                 yield emit({

frontend/dev_console.html

@@ -1242,6 +1242,7 @@ function activateFromStep(step, lgNode) {
           return;
         }
         if (id === 'graphdb' && currentIntent === 'POLICY_QUESTION') return;
+        if (['bronze', 'silver', 'gold'].includes(id) && currentIntent !== 'COMPARISON') return;
         setNode(id, 'active');
         const fromNode = lgNode === 'retrieve' ? 'retrieve_agent' : lgNode;
         activateEdge(fromNode, id);
@@ -1392,6 +1393,7 @@ async function run() {
 
     if (type === 'substep') {
       activateFromStep(step||'', node||'');
+      const isGraphEdge = step && step.startsWith('[GraphDB-Edge]');
       const tagMap = {
         retrieve:'tag-retrieve', synthesize:'tag-synthesize',
         query_guard:'tag-guard', query_decomposer:'tag-classify',
@@ -1399,7 +1401,9 @@ async function run() {
         confidence_scorer:'tag-critique',
       };
       const tagClass = tagMap[node] || 'tag-classify';
-      addLog('→', 'STEP', tagClass, step||'');
+      if (!isGraphEdge) {
+        addLog('→', 'STEP', tagClass, step||'');
+      }
 
       if (step && step.includes('🧠 Multi-Query Variant')) {
         const variantText = step.split(':').slice(1).join(':').trim();

orchestration/tools.py

@@ -22,7 +22,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from langchain_core.tools import tool
 from langchain_core.documents import Document
 
-from retrieval.retriever import get_hybrid_retriever
+from retrieval.retriever import get_hybrid_retriever, get_base_ensemble_retriever
 from retrieval.graph_retriever import GraphRetriever
 
 # ── Lazy singletons ────────────────────────────────────────────────────────────
@@ -189,11 +189,13 @@ def plan_comparison_search(query: str, tier: str) -> str:
     prioritises documents whose metadata or content mentions that tier.
     Enhanced: uses ChromaDB metadata pre-filter for plan_tier before hybrid search.
     """
-    hybrid, _ = _get_retrievers()
+    # Use base ensemble retriever without multi-query and graph wrappers for plan comparison searches
+    # to avoid redundant graph lookups/GPT rephrasings and dramatically reduce latency.
+    base_ensemble = get_base_ensemble_retriever()
 
     # Tier-augmented query forces relevant docs up the ranking
     tier_query = f"{tier} plan {query}"
-    docs = hybrid.invoke(tier_query)
+    docs = base_ensemble.invoke(tier_query)
 
     # Guarantee tier-specific documents via metadata pre-filter (high precision boost)
     try:

retrieval/retriever.py

@@ -148,6 +148,8 @@ def _get_all_documents(vectorstore: Chroma) -> list[Document]:
 # Main Pipeline Builder
 # ══════════════════════════════════════════════════════════════
 
+_base_ensemble_retriever = None
+
 def get_hybrid_retriever(
     ensemble_weights: list[float] | None = None,
     retriever_k: int | None = None,
@@ -265,10 +267,12 @@ def get_hybrid_retriever(
                 
             return merged_docs
 
+    global _base_ensemble_retriever
     ensemble_retriever = LoggingEnsembleRetriever(
         retrievers=[logging_bm25_retriever, logging_vector_retriever],
         weights=weights,
     )
+    _base_ensemble_retriever = ensemble_retriever
     console.print(f"    ✅ Ensemble retriever ready (weights: BM25={weights[0]}, Vector={weights[1]})")
 
     # ── Stage 4: Knowledge Graph Integration ────────────────────
@@ -366,3 +370,16 @@ def get_bm25_only_retriever(k: int | None = None):
     vectorstore = _load_vectorstore()
     all_docs    = _get_all_documents(vectorstore)
     return BM25Retriever.from_documents(all_docs, k=k or RERANKER_TOP_N)
+
+
+import threading
+_retriever_lock = threading.Lock()
+
+def get_base_ensemble_retriever():
+    """Get the cached base ensemble retriever without multi-query, graph, or reranking wrappers."""
+    global _base_ensemble_retriever
+    if _base_ensemble_retriever is None:
+        with _retriever_lock:
+            if _base_ensemble_retriever is None:
+                get_hybrid_retriever()
+    return _base_ensemble_retriever