Update src/qa.py

src/qa.py

@@ -215,20 +215,27 @@ REASONING_PROMPT = (
 # ==========================================================
 from vectorstore import build_faiss_index
 
-def retrieve_chunks(query: str, index, chunks: list, top_k: int = 5,
-                    min_similarity: float = 0.3, candidate_multiplier: int = 3,
+def retrieve_chunks(query: str, index, chunks: list, top_k: int = 7,
+                    min_similarity: float = 0.4, candidate_multiplier: int = 3,
                     embeddings: list = None):
+    """
+    Retrieves the most relevant chunks using FAISS similarity + reranking.
+    Includes bullet-aware similarity boost and a fallback mechanism if
+    similarity threshold isn't met — ensuring predictable, complete retrieval.
+    """
     if not index or not chunks:
         print("⚠️ No FAISS index or chunks provided — returning empty result.")
         return []
 
     try:
+        # --- Encode query
         q_emb = _query_model.encode(
             [f"query: {query.strip()}"],
             convert_to_numpy=True,
             normalize_embeddings=True
         )[0]
 
+        # --- Rebuild index if mismatch occurs
         if hasattr(index, "d") and q_emb.shape[0] != index.d:
             print(f"⚠️ FAISS dimension mismatch: index={index.d}, query={q_emb.shape[0]}")
             if embeddings:
@@ -237,41 +244,56 @@ def retrieve_chunks(query: str, index, chunks: list, top_k: int = 5,
             else:
                 return []
 
+        # --- Retrieve top candidate chunks
         num_candidates = max(top_k * candidate_multiplier, top_k + 2)
         distances, indices = index.search(np.array([q_emb]).astype("float32"), num_candidates)
         candidate_indices = [int(i) for i in indices[0] if i >= 0]
-        candidate_indices = list(dict.fromkeys(candidate_indices))
+        candidate_indices = list(dict.fromkeys(candidate_indices))  # remove duplicates
 
+        # --- Re-rank using cosine similarity
         doc_embs = _query_model.encode(
             [f"passage: {chunks[i]}" for i in candidate_indices],
             convert_to_numpy=True,
             normalize_embeddings=True,
         )
         sims = cosine_similarity([q_emb], doc_embs)[0]
+
         boosted_sims = []
         for idx, sim in zip(candidate_indices, sims):
             text = chunks[idx].strip()
             if re.match(r"^[-•\d]+[\.\s]", text):
-                sim += 0.05
+                sim += 0.05  # slight boost for procedural bullets
             boosted_sims.append((idx, sim))
 
         ranked = sorted(boosted_sims, key=lambda x: x[1], reverse=True)
+
+        # --- Filter based on similarity threshold
         filtered = [idx for idx, sim in ranked if sim >= min_similarity][:top_k]
 
+        # --- Fallback: if no matches above threshold, pick top_k anyway
+        if not filtered:
+            print(f"⚠️ No chunks ≥ {min_similarity:.2f} — using top {top_k} ranked chunks instead.")
+            filtered = [idx for idx, sim in ranked[:top_k]]
+
+        # --- Neighbor continuity: include nearby chunks
         neighbors = set()
         for idx in filtered:
             for n in [idx - 1, idx + 1]:
                 if 0 <= n < len(chunks):
                     neighbors.add(n)
         filtered = sorted(set(filtered) | neighbors)
+
+        # --- Return final chunk set
         final_chunks = [chunks[i] for i in filtered]
-        print(f"✅ Retrieved {len(final_chunks)} chunks (bullet-aware + continuity).")
+        avg_sim = np.mean([s for _, s in ranked[:top_k]])
+        print(f"✅ Retrieved {len(final_chunks)} chunks | avg_sim={avg_sim:.3f} | threshold={min_similarity:.2f}")
         return final_chunks
 
     except Exception as e:
         print(f"⚠️ Retrieval error: {repr(e)}")
         return []
 
+
 # ==========================================================
 # 8️⃣ Answer Generation (Lazy GPT-4o Initialization)
 # ==========================================================