fix: temporal queries now prioritize live results end-to-end

Ranker (hybrid_result_ranker.py):
- Temporal composite score: freshness=50%, live_boost=25%, relevance=15%, quality=10%
- General composite score: relevance=40%, freshness=30%, quality=20%, live_boost=10%
- Temporal candidate pool: 12 live + 8 DB (was 20 mixed) sent to reranker
- Live results now dominate reranker input for temporal queries

RAG pipeline (rag_chat_use_case.py):
- Adaptive rerank threshold: 0.25 for BGE, -0.05 for Jina API scores
- Adaptive keyword overlap threshold: 0.35 for BGE, 0.02 for Jina API
- Temporal final pool: ceil(top_k/2) live + rest DB (was 1 non-English guarantee)
- Final log shows live vs DB count explicitly

src/core/ranking/hybrid_result_ranker.py

@@ -71,64 +71,64 @@ class HybridResultRanker:
     
     def merge_and_rank(
         self,
-        db_results: List[Any],  # SearchResult objects from Qdrant
-        live_results: List[Dict[str, Any]],  # Dicts from DuckDuckGo
-        strategy,  # SearchStrategy object
+        db_results: List[Any],
+        live_results: List[Dict[str, Any]],
+        strategy,
         query: str,
         final_top_n: int = 10
     ) -> List[Dict[str, Any]]:
         """
         Merge results from database and live sources, then rerank.
-        
-        Args:
-            db_results: SearchResult objects from Qdrant
-            live_results: Normalized dicts from DuckDuckGo
-            strategy: SearchStrategy with weights
-            query: Original user query
-            final_top_n: Number of final results to return
-        
-        Returns:
-            List of top-N ranked results
+
+        For temporal queries (live_weight >= 0.7): live results are pre-boosted
+        so they appear in the top candidates sent to the reranker.
         """
-        # 1. Normalize both result types to common format
+        is_temporal = strategy.live_weight >= 0.7
+
         all_results = []
-        
-        # Normalize database results
         for r in db_results:
             all_results.append(self._normalize_db_result(r, strategy.db_weight))
-        
-        # Normalize live results
         for r in live_results:
             all_results.append(self._normalize_live_result(r, strategy.live_weight))
-        
+
         if not all_results:
             logger.warning("No results to rank")
             return []
-        
+
         logger.info(
             f"Merging {len(db_results)} DB + {len(live_results)} live = "
-            f"{len(all_results)} total results"
+            f"{len(all_results)} total results "
+            f"(temporal_boost={'ON' if is_temporal else 'OFF'})"
         )
-        
-        # 2. Deduplicate by URL and title similarity
+
         unique_results = self._deduplicate(all_results)
         logger.info(f"After deduplication: {len(unique_results)} unique results")
-        
-        # 3. Apply composite scoring
+
+        # Composite scoring — temporal queries boost live results heavily
         for r in unique_results:
-            r["composite_score"] = self._calculate_composite_score(r)
-        
-        # 4. Pre-sort by composite score
+            r["composite_score"] = self._calculate_composite_score(r, is_temporal)
+
         unique_results.sort(key=lambda x: x["composite_score"], reverse=True)
-        
-        # 5. Use existing BGE reranker for final ranking
-        # Take top 20 candidates for reranking (balance quality vs speed)
-        top_candidates = unique_results[:20]
-        
+
+        # For temporal queries: guarantee at least half the reranker candidates
+        # are live results so the reranker can actually compare them
+        if is_temporal and live_results:
+            live_pool = [r for r in unique_results if r.get("source_type") == "live"]
+            db_pool   = [r for r in unique_results if r.get("source_type") != "live"]
+            # Take up to 12 live + 8 DB = 20 candidates
+            top_candidates = (live_pool[:12] + db_pool[:8])
+            # Re-sort by composite score so reranker gets best candidates first
+            top_candidates.sort(key=lambda x: x["composite_score"], reverse=True)
+            logger.info(
+                f"Temporal candidate pool: {len(live_pool[:12])} live + "
+                f"{len(db_pool[:8])} DB = {len(top_candidates)} total"
+            )
+        else:
+            top_candidates = unique_results[:20]
+
         if not top_candidates:
             return []
-        
-        # Reranker expects list of dicts with "content" key
+
         try:
             reranked = self.reranker.rerank(
                 query=query,
@@ -240,36 +240,46 @@ class HybridResultRanker:
         
         return unique
     
-    def _calculate_composite_score(self, result: Dict[str, Any]) -> float:
+    def _calculate_composite_score(self, result: Dict[str, Any], is_temporal: bool = False) -> float:
         """
         Calculate composite score from multiple factors.
-        
-        Factors:
-        - Relevance (vector/base score): 40%
-        - Freshness (recency): 30%
-        - Source quality (reputation): 20%
-        - Source type boost (live vs DB): 10%
-        
-        Args:
-            result: Normalized result dict
-        
-        Returns:
-            Composite score (0.0 to 1.0+)
+
+        Weights for TEMPORAL queries (live_weight >= 0.7):
+          - Freshness:      50%  (recency is critical)
+          - Source type:    25%  (live results strongly preferred)
+          - Relevance:      15%
+          - Source quality: 10%
+
+        Weights for GENERAL/HISTORICAL queries:
+          - Relevance:      40%
+          - Freshness:      30%
+          - Source quality: 20%
+          - Source type:    10%
         """
-        relevance = result.get("score", 0.5)
-        freshness = result.get("freshness_score", 0.5)
+        relevance     = result.get("score", 0.5)
+        freshness     = result.get("freshness_score", 0.5)
         source_quality = result.get("source_quality", 0.6)
-        
-        # Boost live results slightly (they're fresher by definition)
-        source_type_boost = 1.0 if result.get("source_type") == "live" else 0.5
-        
-        composite = (
-            relevance * 0.4 +
-            freshness * 0.3 +
-            source_quality * 0.2 +
-            source_type_boost * 0.1
-        )
-        
+        is_live       = result.get("source_type") == "live"
+
+        if is_temporal:
+            # Temporal: freshness + live source dominate
+            source_type_boost = 1.0 if is_live else 0.2
+            composite = (
+                freshness      * 0.50 +
+                source_type_boost * 0.25 +
+                relevance      * 0.15 +
+                source_quality * 0.10
+            )
+        else:
+            # General/historical: relevance + quality dominate
+            source_type_boost = 1.0 if is_live else 0.5
+            composite = (
+                relevance      * 0.40 +
+                freshness      * 0.30 +
+                source_quality * 0.20 +
+                source_type_boost * 0.10
+            )
+
         return composite
     
     def _calculate_freshness(self, published_at: Any) -> float:

src/core/use_cases/rag_chat_use_case.py

@@ -633,19 +633,25 @@ JSON:"""
         logger.info(f"[RAG] After blocked-source filter: {len(quality_docs)} docs")
 
         # ── Relevance threshold — drop docs the reranker scored too low ───────
-        # Raised from 0.15 → 0.25 based on live testing.
-        # The airport article (bbc_swahili) was scoring ~0.18 on GERD queries
-        # because it mentioned "Addis Ababa" in a rankings list.
-        # 0.25 cuts it while keeping genuinely relevant multilingual content.
-        RERANK_THRESHOLD = 0.25
+        # Threshold is adaptive based on reranker type:
+        # - BGE (normalized 0-1): threshold 0.25
+        # - Jina API (raw logits, range ~-0.2 to +0.5): threshold -0.05
+        # We detect which reranker was used by checking the score range.
+        top_score = max((d.get("rerank_score", 0) for d in quality_docs), default=0)
+        if top_score > 0.5:
+            # BGE-style normalized scores
+            RERANK_THRESHOLD = 0.25
+        else:
+            # Jina API raw logit scores — much lower range
+            RERANK_THRESHOLD = -0.05
+
         above_threshold = [d for d in quality_docs if d.get("rerank_score", 1.0) >= RERANK_THRESHOLD]
         if above_threshold:
             quality_docs = above_threshold
-            logger.info(f"[RAG] {len(quality_docs)} docs above rerank threshold {RERANK_THRESHOLD}")
+            logger.info(f"[RAG] {len(quality_docs)} docs above rerank threshold {RERANK_THRESHOLD:.2f} (top_score={top_score:.3f})")
         else:
-            # All scores low — keep top 3 anyway rather than returning nothing
             quality_docs = quality_docs[:3]
-            logger.info(f"[RAG] All docs below threshold — keeping top 3 by rerank score")
+            logger.info(f"[RAG] All docs below threshold {RERANK_THRESHOLD:.2f} — keeping top 3")
 
         # ── Keyword overlap filter — soft filter, keeps docs with ANY query term ─
         # Only drops docs with ZERO overlap AND low rerank score.
@@ -668,8 +674,10 @@ JSON:"""
                     return True
                 # Also keep docs with high rerank/vector score even without exact match
                 # (semantic match via embeddings is valid)
+                # Adaptive threshold: BGE uses 0.35, Jina API uses 0.02
                 score = doc.get("rerank_score") or doc.get("score", 0)
-                return score >= 0.35
+                score_threshold = 0.35 if top_score > 0.5 else 0.02
+                return score >= score_threshold
 
             overlapping = [d for d in quality_docs if _has_overlap(d)]
             if overlapping:
@@ -684,13 +692,35 @@ JSON:"""
                 )[:5]
                 logger.info(f"[RAG] No keyword overlap — keeping top 5 by score ({len(quality_docs)} docs)")
 
-        # Guarantee at least 1 non-English result if available
-        non_english = [d for d in quality_docs if d.get("metadata", {}).get("_search_lang", "en") != "en"]
-        if non_english:
-            final_pool = [non_english[0]] + [d for d in quality_docs if d is not non_english[0]]
+        # ── Final pool: guarantee live results for temporal queries ──────────
+        # For temporal queries (live_weight >= 0.7): ensure at least half the
+        # final docs are live results so the LLM gets fresh content.
+        # For other queries: guarantee at least 1 non-English result.
+        is_temporal_query = (
+            use_hybrid and strategy is not None and
+            getattr(strategy, 'live_weight', 0) >= 0.7
+        )
+
+        if is_temporal_query:
+            live_docs = [d for d in quality_docs if d.get("source_type") == "live" or d.get("is_live")]
+            db_docs   = [d for d in quality_docs if not (d.get("source_type") == "live" or d.get("is_live"))]
+            # Take ceil(top_k/2) live + rest from DB
+            live_slots = max(1, (top_k + 1) // 2)
+            db_slots   = top_k - min(live_slots, len(live_docs))
+            final_pool = live_docs[:live_slots] + db_docs[:db_slots]
             final_pool = final_pool[:top_k]
+            logger.info(
+                f"[RAG] Temporal final pool: {len(live_docs[:live_slots])} live + "
+                f"{len(db_docs[:db_slots])} DB = {len(final_pool)} docs"
+            )
         else:
-            final_pool = quality_docs[:top_k]
+            # Guarantee at least 1 non-English result if available
+            non_english = [d for d in quality_docs if d.get("metadata", {}).get("_search_lang", "en") != "en"]
+            if non_english:
+                final_pool = [non_english[0]] + [d for d in quality_docs if d is not non_english[0]]
+                final_pool = final_pool[:top_k]
+            else:
+                final_pool = quality_docs[:top_k]
 
         # Deduplicate by doc_id
         seen: set = set()
@@ -702,8 +732,13 @@ JSON:"""
             seen.add(did)
             deduped_final.append(d)
 
+        live_count = sum(1 for d in deduped_final if d.get("source_type") == "live" or d.get("is_live"))
         langs_in_result = list({d.get("metadata", {}).get("_search_lang", "en") for d in deduped_final})
-        logger.info(f"[RAG] Final {len(deduped_final)} docs — languages: {langs_in_result}")
+        logger.info(
+            f"[RAG] Final {len(deduped_final)} docs — "
+            f"{live_count} live, {len(deduped_final)-live_count} DB — "
+            f"languages: {langs_in_result}"
+        )
 
         # ── Step 8: Token limitation ──────────────────────────────────────────
         return self._limit_context(query, deduped_final)