feat: hybrid RAG pipeline upgrade

- NewsAPI adapter: real-time news search (80k+ sources, fallback to DDG)
- Jina Reranker API: cloud reranker replacing slow CPU self-hosted (~1s vs 42s)
- Jina Reader: full article extraction from live search URLs
- Intent classifier v5: keyword pre-check layer (0ms for 80% of queries)
- Smart caching: 4-layer Redis cache with intent-aware TTLs
- Query orchestrator: NewsAPI-first live search with DDG fallback + Jina enhancement
- Redis adapter: layered cache methods (get/set intent, live, translation, response)
- Config: NEWSAPI_*, JINA_RERANKER_* settings added

.env

@@ -29,7 +29,7 @@ POSTGRES_DB=rag_interactions
 # --- Models configuration ---
 EMBEDDING_MODEL=BAAI/bge-m3
 VECTOR_SIZE=1024
-RERANKER_MODEL=BAAI/bge-reranker-v2-m3
+RERANKER_MODEL=jinaai/jina-reranker-v3
 
 # ==========================================
 # LLM Provider — set LLM_PROVIDER to one of:
@@ -114,7 +114,28 @@ SEARXNG_MAX_RESULTS=10
 # Free models: Llama 4, Qwen 3, DeepSeek, Gemma 3 and more
 OPENROUTER_API_KEY=your-openrouter-api-key-here
 
+# --- NewsAPI.org (Real-Time News Search) ---
+# Get free key: https://newsapi.org/register (100 requests/day free)
+# Paid tier: $449/month for production (250,000 requests/month)
+NEWSAPI_KEY=74f434d6dafd4e0fb68b6f6c1252f8e0
+NEWSAPI_ENABLED=true
+NEWSAPI_TIMEOUT=2.0
+NEWSAPI_MAX_RESULTS=20
+
 # --- Jina AI Reader (Full Article Extraction) ---
 # Get free key: https://jina.ai (1M tokens/month free)
 # Without key: most news sites return 401 Unauthorized
-JINA_API_KEY=your-jina-api-key-here
+JINA_API_KEY=jina_21658d5feda2467aad7b3bfc08a1b52a4KAI3aLzYhgeua81sPQSyyaYqoh_
+JINA_RERANKER_ENABLED=true
+JINA_RERANKER_MODEL=jina-reranker-v3
+JINA_RERANKER_TIMEOUT=5.0
+
+# --- ACLED Conflict Data (Structured conflict events for Ethiopia) ---
+# Register at: https://acleddata.com/register
+# Use your acleddata.com login credentials (email + password)
+# No separate API key needed — OAuth token is generated automatically
+ACLED_ENABLED=false
+ACLED_EMAIL=your-acled-email@example.com
+ACLED_PASSWORD=your-acled-password
+ACLED_TIMEOUT=8.0
+ACLED_MAX_RESULTS=20

src/api/dependencies.py

@@ -16,6 +16,7 @@ from src.infrastructure.adapters.clickhouse_adapter import ClickHouseAdapter
 from src.infrastructure.adapters.postgres_adapter import PostgresAdapter
 from src.infrastructure.adapters.redis_adapter import RedisAdapter
 from src.infrastructure.adapters.duckduckgo_adapter import DuckDuckGoAdapter
+from src.infrastructure.adapters.newsapi_adapter import NewsAPIAdapter
 
 # Hybrid Search Components
 from src.core.orchestrator.query_orchestrator import QueryOrchestrator
@@ -45,8 +46,15 @@ duckduckgo_adapter = DuckDuckGoAdapter(
     timeout=settings.LIVE_SEARCH_TIMEOUT,
     max_results=settings.LIVE_SEARCH_MAX_RESULTS
 )
+newsapi_adapter = NewsAPIAdapter(
+    api_key=settings.NEWSAPI_KEY,
+    timeout=settings.NEWSAPI_TIMEOUT,
+    max_results=settings.NEWSAPI_MAX_RESULTS
+) if settings.NEWSAPI_ENABLED else None
+
 query_orchestrator = QueryOrchestrator(
     live_search_adapter=duckduckgo_adapter,
+    newsapi_adapter=newsapi_adapter,
     enable_hybrid=settings.ENABLE_HYBRID_SEARCH,
     default_live_weight=settings.LIVE_SEARCH_WEIGHT,
     default_db_weight=settings.DB_SEARCH_WEIGHT

src/core/config.py

@@ -87,6 +87,17 @@ class Settings(BaseSettings):
     JINA_READER_TIMEOUT: float = float(os.getenv("JINA_READER_TIMEOUT", "8.0"))
     JINA_READER_MAX_CONCURRENT: int = int(os.getenv("JINA_READER_MAX_CONCURRENT", "10"))
     JINA_API_KEY: str = os.getenv("JINA_API_KEY", "")  # Get free key at https://jina.ai
+
+    # Jina Reranker API
+    JINA_RERANKER_ENABLED: bool = os.getenv("JINA_RERANKER_ENABLED", "true").lower() == "true"
+    JINA_RERANKER_MODEL: str = os.getenv("JINA_RERANKER_MODEL", "jina-reranker-v3")
+    JINA_RERANKER_TIMEOUT: float = float(os.getenv("JINA_RERANKER_TIMEOUT", "5.0"))
+    
+    # NewsAPI Settings (Real-Time News Search)
+    NEWSAPI_KEY: str = os.getenv("NEWSAPI_KEY", "")  # Get free key at https://newsapi.org/register
+    NEWSAPI_ENABLED: bool = os.getenv("NEWSAPI_ENABLED", "true").lower() == "true"
+    NEWSAPI_TIMEOUT: float = float(os.getenv("NEWSAPI_TIMEOUT", "2.0"))
+    NEWSAPI_MAX_RESULTS: int = int(os.getenv("NEWSAPI_MAX_RESULTS", "20"))
     
     # Cache Settings (TTL in seconds)
     CACHE_RESPONSE_TTL: int = int(os.getenv("CACHE_RESPONSE_TTL", "300"))      # 5 minutes

src/core/orchestrator/query_orchestrator.py

@@ -80,6 +80,7 @@ class QueryOrchestrator:
     def __init__(
         self,
         live_search_adapter,
+        newsapi_adapter=None,
         enable_hybrid: bool = True,
         default_live_weight: float = 0.5,
         default_db_weight: float = 0.5
@@ -89,11 +90,13 @@ class QueryOrchestrator:
         
         Args:
             live_search_adapter: DuckDuckGo adapter instance
+            newsapi_adapter: NewsAPI adapter instance (optional, for temporal queries)
             enable_hybrid: Global flag to enable/disable hybrid search
             default_live_weight: Default weight for live results
             default_db_weight: Default weight for database results
         """
         self.live_search = live_search_adapter
+        self.newsapi = newsapi_adapter
         self.enable_hybrid = enable_hybrid
         self.default_live_weight = default_live_weight
         self.default_db_weight = default_db_weight
@@ -446,11 +449,12 @@ class QueryOrchestrator:
         """
         Execute live search with Jina Reader enhancement.
         
-        Workflow:
-        1. Get DuckDuckGo results (URLs + 200-char snippets)
-        2. Extract full articles using Jina Reader (parallel)
-        3. Replace snippets with full content (14,000+ chars)
-        4. Fallback to snippets if extraction fails
+        Strategy:
+        1. Try NewsAPI first (if available and temporal query)
+        2. Fallback to DuckDuckGo
+        3. Extract full articles using Jina Reader (parallel)
+        4. Replace snippets with full content (14,000+ chars)
+        5. Fallback to snippets if extraction fails
         
         Args:
             query: Search query (English)
@@ -458,60 +462,78 @@ class QueryOrchestrator:
         Returns:
             List of enhanced live search results with full articles
         """
+        results = []
+        
+        # Try NewsAPI first (best for temporal queries)
+        if self.newsapi and self.newsapi.is_available():
+            try:
+                logger.info(f"Live search: trying NewsAPI first for '{query}'")
+                newsapi_results = await self.newsapi.search(query)
+                
+                if newsapi_results:
+                    logger.info(f"NewsAPI: {len(newsapi_results)} results")
+                    results.extend(newsapi_results)
+                else:
+                    logger.info("NewsAPI: no results, falling back to DuckDuckGo")
+            except Exception as e:
+                logger.warning(f"NewsAPI failed: {e}, falling back to DuckDuckGo")
+        
+        # Fallback to DuckDuckGo (or primary if NewsAPI not available)
+        if not results:
+            try:
+                logger.info(f"Live search: using DuckDuckGo for '{query}'")
+                results = await self.live_search.search(query)
+                logger.info(f"DuckDuckGo: {len(results)} results")
+            except Exception as e:
+                logger.error(f"DuckDuckGo search error: {e}")
+                return []
+        
+        if not results:
+            logger.warning("No live search results from any source")
+            return results
+        
+        # Step 2: Check if Jina Reader is enabled
+        from src.core.config import settings
+        
+        if not settings.ENABLE_JINA_READER:
+            logger.info("Jina Reader disabled - using snippets only")
+            return results
+        
+        # Step 3: Try to enhance with Jina Reader
         try:
-            # Step 1: Get DuckDuckGo results (URLs + snippets)
-            results = await self.live_search.search(query)
-            logger.info(f"Live search: {len(results)} results from DuckDuckGo")
+            from src.infrastructure.adapters.jina_reader_adapter import get_jina_reader_adapter
             
-            if not results:
-                return results
+            jina = get_jina_reader_adapter(
+                timeout=settings.JINA_READER_TIMEOUT,
+                max_concurrent=settings.JINA_READER_MAX_CONCURRENT
+            )
             
-            # Step 2: Check if Jina Reader is enabled
-            from src.core.config import settings
+            # Step 4: Extract full articles (replaces snippets)
+            enhanced_results = await jina.enhance_search_results(
+                results,
+                fallback_to_snippet=True  # Keep snippet if Jina fails
+            )
             
-            if not settings.ENABLE_JINA_READER:
-                logger.info("Jina Reader disabled - using snippets only")
-                return results
+            # Log enhancement stats
+            full_articles = sum(1 for r in enhanced_results if r.get("full_article"))
+            snippets = len(enhanced_results) - full_articles
+            total_chars = sum(
+                r.get("content_length", 0) 
+                for r in enhanced_results 
+                if r.get("full_article")
+            )
             
-            # Step 3: Try to enhance with Jina Reader
-            try:
-                from src.infrastructure.adapters.jina_reader_adapter import get_jina_reader_adapter
-                
-                jina = get_jina_reader_adapter(
-                    timeout=settings.JINA_READER_TIMEOUT,
-                    max_concurrent=settings.JINA_READER_MAX_CONCURRENT
-                )
-                
-                # Step 4: Extract full articles (replaces snippets)
-                enhanced_results = await jina.enhance_search_results(
-                    results,
-                    fallback_to_snippet=True  # Keep snippet if Jina fails
-                )
-                
-                # Log enhancement stats
-                full_articles = sum(1 for r in enhanced_results if r.get("full_article"))
-                snippets = len(enhanced_results) - full_articles
-                total_chars = sum(
-                    r.get("content_length", 0) 
-                    for r in enhanced_results 
-                    if r.get("full_article")
-                )
-                
-                logger.info(
-                    f"Jina enhancement: {full_articles} full articles ({total_chars:,} chars), "
-                    f"{snippets} snippets (fallback)"
-                )
-                
-                return enhanced_results
-                
-            except ImportError:
-                logger.warning("Jina Reader not available - using snippets only")
-                return results
+            logger.info(
+                f"Jina enhancement: {full_articles} full articles ({total_chars:,} chars), "
+                f"{snippets} snippets (fallback)"
+            )
             
-            except Exception as e:
-                logger.warning(f"Jina Reader enhancement failed: {e} - using snippets")
-                return results
+            return enhanced_results
+            
+        except ImportError:
+            logger.warning("Jina Reader not available - using snippets only")
+            return results
         
         except Exception as e:
-            logger.error(f"Live search error: {e}")
-            raise
+            logger.warning(f"Jina Reader enhancement failed: {e} - using snippets")
+            return results

src/core/use_cases/rag_chat_use_case.py

@@ -388,15 +388,19 @@ JSON:"""
             logger.info(f"[RAG] Hybrid search enabled - checking intent and strategy")
             
             # Classify intent using v2 (production-grade) or v1 (fallback)
-            # Check Redis cache first to avoid 8-11s DeBERTa inference on repeat queries
+            # Check Redis cache first to avoid repeated LLM calls on same query
             intent_result = None
             intent_cache_key = f"intent_v2:{query[:80].lower().strip()}"
             
             if self.cache:
-                cached_intent = self.cache.get(intent_cache_key)
+                # Use new layered cache method if available
+                if hasattr(self.cache, 'get_intent'):
+                    cached_intent = self.cache.get_intent(query)
+                else:
+                    cached_intent = self.cache.get(intent_cache_key)
+                
                 if cached_intent:
-                    logger.info(f"[RAG] Intent cache HIT — skipping DeBERTa inference")
-                    # Reconstruct a minimal intent result from cache
+                    logger.info(f"[RAG] Intent cache HIT — skipping LLM inference")
                     class _CachedIntent:
                         def __init__(self, d):
                             self.intent = d["intent"]
@@ -405,7 +409,7 @@ JSON:"""
                             self.inference_time_ms = 0.0
                     intent_result = _CachedIntent(cached_intent)
                     intent = "NEWS" if intent_result.intent != "OTHER" else "OTHER"
-                    logger.info(f"[RAG] Intent (cached): {intent_result.intent} (confidence={intent_result.confidence:.2f})")
+                    logger.info(f"[RAG] Intent (cached): {intent_result.intent} (conf={intent_result.confidence:.2f})")
 
             if intent_result is None:
                 if self.use_v2_classifier and self.intent_classifier_v2:
@@ -419,13 +423,17 @@ JSON:"""
                         f"time={intent_result.inference_time_ms:.1f}ms"
                     )
                     
-                    # Cache intent result for 1 hour (same query = same intent)
+                    # Cache intent result for 1 hour
+                    intent_data = {
+                        "intent": intent_result.intent,
+                        "confidence": intent_result.confidence,
+                        "method": intent_result.method,
+                    }
                     if self.cache:
-                        self.cache.set(intent_cache_key, {
-                            "intent": intent_result.intent,
-                            "confidence": intent_result.confidence,
-                            "method": intent_result.method,
-                        }, expiration=3600)
+                        if hasattr(self.cache, 'set_intent'):
+                            self.cache.set_intent(query, intent_data)
+                        else:
+                            self.cache.set(intent_cache_key, intent_data, expiration=3600)
                 else:
                     intent = self.intent_classifier.classify(query)
                     intent_result = None
@@ -689,24 +697,15 @@ JSON:"""
         return "".join([f"{msg.role}: {msg.content}\n" for msg in past_messages])
     
     def _get_cache_keys(self, query: str) -> Dict[str, str]:
-        """
-        Generate cache keys for different caching layers.
-        
-        Returns dict with keys: response, live, translation, intent
-        """
+        """Generate cache keys — kept for backward compat, new code uses RedisAdapter methods directly."""
         if not self.cache:
             return {}
-        
         query_hash = self.cache.generate_exact_hash(query)
-        query_prefix_hash = self.cache.generate_exact_hash(query[:50])
-        
-        from src.core.config import settings
-        
         return {
-            "response": f"hybrid_response:{query_hash}",
-            "live": f"live_search:{query_hash}",
+            "response":    f"rag_response:{query_hash}",
+            "live":        f"live_search:{query_hash}",
             "translation": f"translation:{query_hash}",
-            "intent": f"intent:{query_prefix_hash}"
+            "intent":      f"intent_v2:{query_hash}",
         }
 
     async def execute_chat(self, request: ChatRequest) -> Dict[str, Any]:
@@ -718,17 +717,21 @@ JSON:"""
             logger.info(f"[RAG] Generated new session_id: {request.session_id}")
         session_id = request.session_id
         
-        # ── Layer 1: Full Response Cache (5 min TTL) ──────────────────────────
-        cache_keys = self._get_cache_keys(request.query)
-        
-        if self.cache and cache_keys.get("response"):
-            cached_result = self.cache.get(cache_keys["response"])
+        # ── Layer 1: Full Response Cache ──────────────────────────────────────
+        if self.cache:
+            # Use new layered cache method if available
+            if hasattr(self.cache, 'get_response'):
+                cached_result = self.cache.get_response(request.query)
+            else:
+                cache_keys = self._get_cache_keys(request.query)
+                cached_result = self.cache.get(cache_keys.get("response", "")) if cache_keys else None
+
             if cached_result:
-                print("DEBUG: Cache HIT - Layer 1 (full response)")
+                logger.info("[RAG] Cache HIT — returning cached response")
                 self.chat_history_db.save_interaction(
-                    session_id, 
-                    request.query, 
-                    cached_result["answer"], 
+                    session_id,
+                    request.query,
+                    cached_result["answer"],
                     [s.get("doc_id") for s in cached_result.get("sources", [])]
                 )
                 cached_result["debug"] = cached_result.get("debug", {})
@@ -736,7 +739,7 @@ JSON:"""
                 cached_result["debug"]["cache_layer"] = "response"
                 return cached_result
 
-        print("DEBUG: Cache MISS - Layer 1 (full response) - proceeding with RAG pipeline")
+        logger.info("[RAG] Cache MISS — running full RAG pipeline")
         history_text = self._get_history_text(session_id)
 
         context_text, final_sources = await self._build_context(
@@ -868,15 +871,16 @@ Answer:"""
             }
         }
 
-        # ── Cache the full response (Layer 1) ─────────────────────────────────
-        if self.cache and cache_keys.get("response"):
-            from src.core.config import settings
-            self.cache.set(
-                cache_keys["response"], 
-                result, 
-                expiration=settings.CACHE_RESPONSE_TTL
-            )
-            logger.info(f"[RAG] Cached full response (TTL={settings.CACHE_RESPONSE_TTL}s)")
+        # ── Cache the full response with intent-aware TTL ─────────────────────
+        if self.cache:
+            detected_intent = result.get("debug", {}).get("intent", "NEWS_GENERAL")
+            if hasattr(self.cache, 'set_response'):
+                self.cache.set_response(request.query, result, intent=detected_intent)
+            else:
+                cache_keys = self._get_cache_keys(request.query)
+                if cache_keys.get("response"):
+                    from src.core.config import settings
+                    self.cache.set(cache_keys["response"], result, expiration=settings.CACHE_RESPONSE_TTL)
             
         return result
 

src/infrastructure/adapters/bge_reranker_adapter.py

@@ -1,67 +1,227 @@
+"""
+Reranker Adapter — supports BGE-Reranker-v2-m3 AND Jina-Reranker-v3
+
+Auto-detects which model to load based on RERANKER_MODEL setting:
+  - "BAAI/bge-reranker-v2-m3"   → FlagReranker (pointwise cross-encoder)
+  - "jinaai/jina-reranker-v3"   → Jina v3 listwise reranker
+
+Jina v3 advantages over BGE for this project:
+  - Listwise: sees ALL docs at once → better cross-doc comparison
+  - 131K context window → reads full Jina-extracted articles (not just 512 chars)
+  - +9.6% better on English news (BEIR 61.94 vs 56.51)
+  - Better Arabic ranking (78.69 nDCG)
+  - Same size (0.6B), same memory, same cost (free, self-hosted)
+
+Thread-safe lazy loading — model loads once on first rerank call.
+"""
+
 import logging
 import threading
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Optional
+
 from src.core.config import settings
 from src.core.ports.reranker_port import RerankerPort
 
 logger = logging.getLogger(__name__)
 
+# ── Patch transformers compatibility issue ────────────────────────────────────
 try:
-    import transformers.utils.import_utils
-    if not hasattr(transformers.utils.import_utils, 'is_torch_fx_available'):
-        transformers.utils.import_utils.is_torch_fx_available = lambda: False
+    import transformers.utils.import_utils as _tui
+    if not hasattr(_tui, "is_torch_fx_available"):
+        _tui.is_torch_fx_available = lambda: False
 except Exception:
     pass
 
-# BGE-Reranker-v2-m3 uses FlagReranker from FlagEmbedding (same package as BGE-M3 embedder)
-# Fallback to sentence-transformers CrossEncoder if FlagEmbedding is unavailable
+# ── Try FlagEmbedding (for BGE) ───────────────────────────────────────────────
 try:
     from FlagEmbedding import FlagReranker
     HAS_FLAG_RERANKER = True
 except ImportError:
     HAS_FLAG_RERANKER = False
-    logger.warning("FlagEmbedding not available for FlagReranker — trying CrossEncoder fallback.")
 
+# ── Try sentence-transformers CrossEncoder (BGE fallback) ────────────────────
 try:
     from sentence_transformers import CrossEncoder
     HAS_CROSS_ENCODER = True
 except ImportError:
     HAS_CROSS_ENCODER = False
 
-if not HAS_FLAG_RERANKER and not HAS_CROSS_ENCODER:
-    logger.warning("Neither FlagEmbedding nor sentence-transformers installed. Reranking disabled.")
+# ── Try transformers (for Jina v3) ────────────────────────────────────────────
+try:
+    import torch
+    from transformers import AutoModel
+    HAS_TRANSFORMERS = True
+except ImportError:
+    HAS_TRANSFORMERS = False
+    logger.warning("transformers/torch not available — Jina v3 reranker disabled.")
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# JINA V3 RERANKER
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class JinaV3Reranker:
+    """
+    Jina-Reranker-v3 self-hosted reranker.
+
+    Key differences from BGE pointwise:
+    - Listwise: processes all docs in one forward pass
+    - 131K context window: reads full articles, not just first 512 chars
+    - Built on Qwen3-0.6B backbone with causal self-attention
+    - State-of-the-art BEIR: 61.94 nDCG@10 (vs BGE's 56.51)
+
+    Scoring: uses sigmoid(logits) for normalized 0-1 scores.
+    """
+
+    def __init__(self, model_name: str):
+        self.model_name = model_name
+        self._model = None
+        self._lock = threading.Lock()
+        self._load_failed = False
+        self._device = "cpu"
+
+    def _load(self):
+        if self._model is not None or self._load_failed:
+            return
+        with self._lock:
+            if self._model is not None or self._load_failed:
+                return
+            if not HAS_TRANSFORMERS:
+                logger.error("transformers not installed — cannot load Jina v3")
+                self._load_failed = True
+                return
+            try:
+                logger.info(f"Loading Jina v3 reranker: {self.model_name}")
+                self._device = "cuda" if torch.cuda.is_available() else "cpu"
+
+                # Jina v3 uses AutoModel (NOT AutoModelForSequenceClassification)
+                # It has a built-in .rerank() method that returns relevance_score directly
+                from transformers import AutoModel
+                self._model = AutoModel.from_pretrained(
+                    self.model_name,
+                    trust_remote_code=True,
+                    dtype="auto",
+                )
+                self._model.eval()
+                logger.info(
+                    f"✅ Jina v3 reranker loaded on {self._device} "
+                    f"(model={self.model_name})"
+                )
+            except Exception as e:
+                logger.error(f"Failed to load Jina v3 reranker: {e}", exc_info=True)
+                self._load_failed = True
+
+    def compute_scores(
+        self,
+        query: str,
+        docs: List[str],
+        max_length: int = 1024,
+    ) -> List[float]:
+        """
+        Score all (query, doc) pairs using Jina v3's built-in .rerank() method.
+
+        Returns scores in original doc order (not sorted).
+        """
+        if not docs:
+            return []
+
+        self._load()
+        if self._model is None:
+            return [0.5] * len(docs)
+
+        try:
+            # Jina v3's .rerank() returns list of dicts:
+            # [{"document": str, "relevance_score": float, "index": int}, ...]
+            # Results are sorted by relevance_score descending — we need to
+            # restore original order using the "index" field.
+            results = self._model.rerank(query, docs)
+
+            # Restore original order
+            scores = [0.0] * len(docs)
+            for r in results:
+                original_idx = r["index"]
+                scores[original_idx] = float(r["relevance_score"])
+
+            return scores
+
+        except Exception as e:
+            logger.error(f"Jina v3 rerank() failed: {e}")
+            return [0.0] * len(docs)
+
+    @property
+    def is_loaded(self) -> bool:
+        return self._model is not None
+
 
+# ═══════════════════════════════════════════════════════════════════════════════
+# UNIFIED RERANKER ADAPTER
+# ═══════════════════════════════════════════════════════════════════════════════
 
 class BgeRerankerAdapter(RerankerPort):
     """
-    Multilingual reranker using BAAI/bge-reranker-v2-m3.
+    Unified reranker adapter — auto-selects BGE or Jina v3 based on config.
 
-    Why bge-reranker-v2-m3:
-    - Same model family as BGE-M3 embedder — consistent representation space
-    - Natively multilingual: Arabic, Amharic, Somali, Swahili, French, English
-    - Significantly better than ms-marco-TinyBERT for non-English content
-    - Uses FlagReranker (FlagEmbedding) as primary, CrossEncoder as fallback
+    RERANKER_MODEL=jinaai/jina-reranker-v3   → Jina v3 (recommended)
+    RERANKER_MODEL=BAAI/bge-reranker-v2-m3   → BGE (legacy)
 
-    Thread-safe lazy loading — model loads once on first rerank call.
+    Both are self-hosted, free, ~0.6B parameters, ~1.2GB on disk.
     """
 
+    # Max content chars to send to reranker
+    # Jina v3: 1024 tokens ≈ 4096 chars — reads much more than BGE's 512 chars
+    MAX_CONTENT_CHARS_JINA = 4096
+    MAX_CONTENT_CHARS_BGE  = 512
+
     def __init__(self):
-        self.model = None
         self.model_name = settings.RERANKER_MODEL
+        self._is_jina_v3 = "jina-reranker-v3" in self.model_name.lower()
         self._lock = threading.Lock()
         self._load_failed = False
 
-    def _load_model(self):
-        if self.model is not None or self._load_failed:
+        # Check if Jina API reranker is enabled (takes priority over self-hosted)
+        self._jina_api = None
+        if getattr(settings, 'JINA_RERANKER_ENABLED', False) and getattr(settings, 'JINA_API_KEY', ''):
+            try:
+                from src.infrastructure.adapters.jina_reranker_adapter import JinaRerankerAPIAdapter
+                jina_key = settings.JINA_API_KEY
+                if jina_key and jina_key not in ("", "your-jina-api-key-here"):
+                    self._jina_api = JinaRerankerAPIAdapter(
+                        api_key=jina_key,
+                        model=getattr(settings, 'JINA_RERANKER_MODEL', 'jina-reranker-v3'),
+                        timeout=getattr(settings, 'JINA_RERANKER_TIMEOUT', 5.0),
+                    )
+                    logger.info("Reranker configured: Jina API (cloud, fast)")
+            except Exception as e:
+                logger.warning(f"Jina API reranker init failed: {e}")
+
+        # Jina v3 self-hosted path
+        if self._is_jina_v3 and not self._jina_api:
+            self._jina = JinaV3Reranker(self.model_name)
+            self._bge_model = None
+            self._use_flag = False
+            logger.info(f"Reranker configured: Jina v3 self-hosted ({self.model_name})")
+        elif not self._jina_api:
+            # BGE path
+            self._jina = None
+            self._bge_model = None
+            self._use_flag = False
+            logger.info(f"Reranker configured: BGE ({self.model_name})")
+        else:
+            self._jina = None
+            self._bge_model = None
+            self._use_flag = False
+
+    def _load_bge(self):
+        """Lazy-load BGE reranker (thread-safe)."""
+        if self._bge_model is not None or self._load_failed:
             return
         with self._lock:
-            if self.model is not None or self._load_failed:
+            if self._bge_model is not None or self._load_failed:
                 return
-            logger.info(f"Loading reranker model: {self.model_name}")
+            logger.info(f"Loading BGE reranker: {self.model_name}")
             try:
                 if HAS_FLAG_RERANKER and "bge-reranker" in self.model_name.lower():
-                    # Patch ALL potentially missing XLMRobertaTokenizer methods
-                    # Different transformers versions on HF Spaces may lack different methods
+                    # Patch XLMRobertaTokenizer for older transformers versions
                     try:
                         from transformers import XLMRobertaTokenizer, PreTrainedTokenizer
                         for method_name in [
@@ -69,7 +229,6 @@ class BgeRerankerAdapter(RerankerPort):
                             "build_inputs_with_special_tokens",
                             "create_token_type_ids_from_sequences",
                             "get_special_tokens_mask",
-                            "special_tokens_pattern",
                             "convert_tokens_to_string",
                         ]:
                             if not hasattr(XLMRobertaTokenizer, method_name):
@@ -79,73 +238,133 @@ class BgeRerankerAdapter(RerankerPort):
                     except Exception as patch_err:
                         logger.debug(f"Tokenizer patch skipped: {patch_err}")
 
-                    try:
-                        self.model = FlagReranker(
-                            self.model_name,
-                            use_fp16=True,
-                            normalize=True,
-                            trust_remote_code=True,
-                        )
-                        self._use_flag = True
-                        logger.info(f"✅ Loaded {self.model_name} via FlagReranker (multilingual, fp16)")
-                    except Exception as flag_err:
-                        logger.warning(f"FlagReranker failed ({flag_err}) — falling back to CrossEncoder")
-                        if HAS_CROSS_ENCODER:
-                            self.model = CrossEncoder(self.model_name)
-                            self._use_flag = False
-                            logger.info(f"✅ Loaded {self.model_name} via CrossEncoder (fallback)")
-                        else:
-                            raise
+                    self._bge_model = FlagReranker(
+                        self.model_name,
+                        use_fp16=True,
+                        normalize=True,
+                        trust_remote_code=True,
+                    )
+                    self._use_flag = True
+                    logger.info(f"✅ BGE loaded via FlagReranker (fp16, multilingual)")
 
                 elif HAS_CROSS_ENCODER:
-                    self.model = CrossEncoder(self.model_name)
+                    self._bge_model = CrossEncoder(self.model_name)
                     self._use_flag = False
-                    logger.info(f"✅ Loaded {self.model_name} via CrossEncoder (fallback)")
+                    logger.info(f"✅ BGE loaded via CrossEncoder (fallback)")
+
                 else:
-                    logger.error("No reranker backend available.")
+                    logger.error("No BGE backend available (FlagEmbedding or sentence-transformers required)")
                     self._load_failed = True
+
             except Exception as e:
-                logger.error(f"Failed to load reranker model '{self.model_name}': {e}", exc_info=True)
+                logger.error(f"Failed to load BGE reranker '{self.model_name}': {e}", exc_info=True)
                 self._load_failed = True
 
-    def rerank(self, query: str, docs: List[Dict[str, Any]], top_n: int = 5) -> List[Dict[str, Any]]:
+    # ── Public interface ──────────────────────────────────────────────────────
+
+    def rerank(
+        self,
+        query: str,
+        docs: List[Dict[str, Any]],
+        top_n: int = 5,
+    ) -> List[Dict[str, Any]]:
+        """
+        Rerank documents by relevance to query.
+
+        Priority: Jina API (cloud) > Jina v3 self-hosted > BGE
+        Jina v3 path: uses full article content (up to 4096 chars)
+        BGE path: uses first 512 chars only
+
+        Returns top_n docs sorted by rerank_score descending.
+        """
         if not docs:
             return []
 
-        if self.model is None:
-            self._load_model()
+        # Priority: Jina API > Jina v3 self-hosted > BGE
+        if self._jina_api and self._jina_api.is_available():
+            return self._jina_api.rerank(query, docs, top_n)
+        elif self._is_jina_v3 and self._jina:
+            return self._rerank_jina(query, docs, top_n)
+        else:
+            return self._rerank_bge(query, docs, top_n)
+
+    def _rerank_jina(
+        self,
+        query: str,
+        docs: List[Dict[str, Any]],
+        top_n: int,
+    ) -> List[Dict[str, Any]]:
+        """Rerank using Jina v3 — reads full article content."""
+        # Ensure model is loaded
+        self._jina._load()
+
+        if self._jina._load_failed or not self._jina.is_loaded:
+            logger.warning("Jina v3 unavailable — falling back to vector score ordering")
+            return sorted(docs, key=lambda x: x.get("score", 0), reverse=True)[:top_n]
+
+        # Build content list — use full content up to 4096 chars
+        # This is the key advantage: Jina reads 8x more content than BGE
+        valid_docs = []
+        doc_texts = []
+        for doc in docs:
+            content = doc.get("content", "").strip()
+            if content:
+                doc_texts.append(content[:self.MAX_CONTENT_CHARS_JINA])
+                valid_docs.append(doc)
+
+        if not doc_texts:
+            return []
+
+        try:
+            scores = self._jina.compute_scores(query, doc_texts)
 
-        # Fallback: sort by vector score if model unavailable
-        if self.model is None:
-            logger.warning("Reranker unavailable — falling back to vector score ordering.")
+            for i, doc in enumerate(valid_docs):
+                doc["rerank_score"] = scores[i]
+
+            valid_docs.sort(key=lambda x: x["rerank_score"], reverse=True)
+
+            logger.info(
+                f"[Reranker] Jina v3: {len(valid_docs)} docs → top {top_n} "
+                f"(max_score={valid_docs[0]['rerank_score']:.3f})"
+            )
+            return valid_docs[:top_n]
+
+        except Exception as e:
+            logger.error(f"Jina v3 reranking failed: {e} — falling back to vector score")
+            return sorted(docs, key=lambda x: x.get("score", 0), reverse=True)[:top_n]
+
+    def _rerank_bge(
+        self,
+        query: str,
+        docs: List[Dict[str, Any]],
+        top_n: int,
+    ) -> List[Dict[str, Any]]:
+        """Rerank using BGE — reads first 512 chars only."""
+        if self._bge_model is None:
+            self._load_bge()
+
+        if self._bge_model is None:
+            logger.warning("BGE unavailable — falling back to vector score ordering")
             return sorted(docs, key=lambda x: x.get("score", 0), reverse=True)[:top_n]
 
-        # Build (query, content) pairs.
-        # PERFORMANCE: Truncate content to 512 chars (~128 tokens) before scoring.
-        # The reranker only needs the opening paragraph to judge topical relevance.
-        # Full articles waste ~60% of inference time on boilerplate text.
-        MAX_CONTENT_CHARS = 512
         pairs = []
         valid_docs = []
         for doc in docs:
             content = doc.get("content", "").strip()
             if content:
-                truncated = content[:MAX_CONTENT_CHARS]
-                pairs.append([query, truncated])
+                pairs.append([query, content[:self.MAX_CONTENT_CHARS_BGE]])
                 valid_docs.append(doc)
 
         if not pairs:
             return []
 
         try:
-            if getattr(self, "_use_flag", False):
-                # FlagReranker.compute_score — batch_size=64 safe now that
-                # content is truncated (much smaller tensors per pair)
-                scores = self.model.compute_score(pairs, batch_size=64)
+            if self._use_flag:
+                scores = self._bge_model.compute_score(pairs, batch_size=64)
                 if isinstance(scores, float):
                     scores = [scores]
             else:
-                scores = self.model.predict(pairs)
+                scores = self._bge_model.predict(pairs)
                 if isinstance(scores, float):
                     scores = [scores]
 
@@ -153,8 +372,17 @@ class BgeRerankerAdapter(RerankerPort):
                 doc["rerank_score"] = float(scores[i])
 
             valid_docs.sort(key=lambda x: x["rerank_score"], reverse=True)
+
+            logger.info(
+                f"[Reranker] BGE: {len(valid_docs)} docs → top {top_n} "
+                f"(max_score={valid_docs[0]['rerank_score']:.3f})"
+            )
             return valid_docs[:top_n]
 
         except Exception as e:
-            logger.error(f"Reranker scoring failed: {e} — falling back to vector score ordering.")
+            logger.error(f"BGE reranking failed: {e} — falling back to vector score")
             return sorted(docs, key=lambda x: x.get("score", 0), reverse=True)[:top_n]
+
+    @property
+    def model_type(self) -> str:
+        return "jina_v3" if self._is_jina_v3 else "bge"

src/infrastructure/adapters/intent_classifier_v2.py

@@ -1,53 +1,213 @@
 """
-Intent Classifier v4 — LLM-Powered with 4-Provider Fallback Chain
+Intent Classifier v5 — Fast Keyword Pre-Check + LLM Fallback Chain
 
 Architecture:
-  Layer 1: Instant safety net (0ms)       — 20 exact strings only
+  Layer 0: Instant exact match (0ms)      — greetings, single-char, test
+  Layer 1: Fast keyword rules (0ms)       — temporal/historical/other patterns
+             ↳ Catches 80%+ of queries instantly, no API call needed
   Layer 2: Groq llama-3.1-8b-instant      — 14,400 free RPD, ~50ms  (PRIMARY)
   Layer 3: Gemini Flash fallback          — 1,500 free RPD,  ~200ms (FALLBACK 1)
   Layer 4: OpenRouter free router         — free models pool, ~300ms (FALLBACK 2)
   Layer 5: HuggingFace Inference API      — ~300 RPH,        ~2s    (FALLBACK 3)
   Layer 6: Safe default                   — NEWS_GENERAL,    0ms    (ALWAYS WORKS)
 
-Why LLM instead of hard-coded rules:
-  - 99%+ accuracy vs ~75% for keyword matching
-  - Handles any language naturally (Amharic, Arabic, Somali, French...)
-  - Handles any topic (new conflicts, new places, new events)
-  - Zero maintenance — no keyword lists to update ever
-  - Understands context ("Abiy's latest move" → NEWS_TEMPORAL)
-
-Provider selection rationale:
-  - Groq 8B:      14,400 RPD free — primary, fastest, cheapest
-  - Gemini Flash: 1,500 RPD free  — reliable fallback
-  - OpenRouter:   free model pool — auto-selects best available free model
-  - HuggingFace:  ~300 RPH free   — last resort (slower but always available)
-  - Default:      NEWS_GENERAL    — never fails, safe for user experience
+Layer 1 keyword rules cover:
+  - Temporal:   "today", "now", "breaking", "latest", "just happened", etc.
+  - Historical: "history of", "background", "what caused", "explain", etc.
+  - Other:      greetings, identity questions, math, creative writing
+  - Ethiopia-specific: "Abiy", "TPLF", "Fano", "Tigray" → NEWS_GENERAL fast path
+
+Why this matters:
+  - Saves Groq API quota (14,400 RPD is finite)
+  - Reduces latency from ~50ms → 0ms for common queries
+  - Works offline / when all LLM providers are down
+  - Handles Amharic/Arabic/Somali temporal words natively
 """
 
 import logging
+import re
 import time
-import threading
 import httpx
 from dataclasses import dataclass
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
 
 logger = logging.getLogger(__name__)
 
 
 # ═══════════════════════════════════════════════════════════════════════════════
-# LAYER 1: INSTANT SAFETY NET — only the most obvious non-news
+# LAYER 0: INSTANT EXACT MATCH — greetings, empty, test
 # ═══════════════════════════════════════════════════════════════════════════════
 
 _INSTANT_OTHER = {
     "hi", "hello", "hey", "thanks", "thank you", "bye", "goodbye",
     "ok", "okay", "yes", "no", "sure", "cool", "nice",
     "lol", "lmao", "haha", "omg", "wtf", "wow",
-    ".", "..", "...", "?", "!", "test",
+    ".", "..", "...", "?", "!", "test", "ping",
 }
 
 
 # ═══════════════════════════════════════════════════════════════════════════════
-# CLASSIFICATION PROMPT — same prompt used across all providers
+# LAYER 1: FAST KEYWORD RULES
+# ═══════════════════════════════════════════════════════════════════════════════
+
+# ── Temporal signals → NEWS_TEMPORAL ─────────────────────────────────────────
+# English
+_TEMPORAL_EN = re.compile(
+    r"\b("
+    r"today|tonight|right now|just now|breaking|just happened|"
+    r"this morning|this afternoon|this evening|this hour|"
+    r"latest|current(ly)?|live|ongoing|unfolding|"
+    r"yesterday|last night|"
+    r"this week|this month|this year|"
+    r"recent(ly)?|new(ly)?|fresh|"
+    r"past (few )?(hours?|days?|weeks?)|"
+    r"in the (last|past) \d+|"
+    r"as of (today|now)|"
+    r"update[sd]?|news flash|alert"
+    r")\b",
+    re.IGNORECASE
+)
+
+# Amharic temporal words (common ones)
+_TEMPORAL_AM = re.compile(
+    r"(ዛሬ|አሁን|ዘንድሮ|ቅርብ|አዲስ|ዜና|ዛሬ ምሽት|ዛሬ ጠዋት)",
+    re.UNICODE
+)
+
+# Arabic temporal words
+_TEMPORAL_AR = re.compile(
+    r"(اليوم|الآن|عاجل|أخبار عاجلة|حديثاً|مؤخراً|هذا الأسبوع|هذا الشهر)",
+    re.UNICODE
+)
+
+# Somali temporal words
+_TEMPORAL_SO = re.compile(r"(maanta|hadda|wararka|cusub)", re.IGNORECASE | re.UNICODE)
+
+# Swahili temporal words
+_TEMPORAL_SW = re.compile(r"(leo|sasa|habari za leo|mpya|hivi karibuni)", re.IGNORECASE | re.UNICODE)
+
+# ── Historical signals → NEWS_HISTORICAL ─────────────────────────────────────
+_HISTORICAL = re.compile(
+    r"\b("
+    r"history (of|behind)|historical(ly)?|"
+    r"background (of|on|to)|context (of|behind)|"
+    r"what caused|root cause|origin(s)? of|"
+    r"explain|overview|summary of|"
+    r"who (is|was|are|were)|what (is|was|are|were)|"
+    r"tell me about|describe|"
+    r"in \d{4}|since \d{4}|before \d{4}|"
+    r"decade(s)?|century|centuries|"
+    r"long.?term|over the years|traditionally|"
+    r"founded|established|created|formed"
+    r")\b",
+    re.IGNORECASE
+)
+
+# ── Other signals → OTHER ─────────────────────────────────────────────────────
+_OTHER_IDENTITY = re.compile(
+    r"\b("
+    r"who are you|what are you|are you (an? )?ai|"
+    r"what (model|llm|ai) are you|"
+    r"who (made|built|created|trained) you|"
+    r"your (name|purpose|capabilities)|"
+    r"can you (help|do|write|make|create|generate)|"
+    r"how (do you|does this) work"
+    r")\b",
+    re.IGNORECASE
+)
+
+_OTHER_CREATIVE = re.compile(
+    r"\b("
+    r"write (a |an )?(poem|story|essay|letter|email|code|script)|"
+    r"make (a |an )?(joke|list|plan|recipe)|"
+    r"translate (this|to|into)|"
+    r"calculate|solve|compute|"
+    r"what is \d|how many|how much|"
+    r"recommend|suggest|give me (a |an )?(list|idea)"
+    r")\b",
+    re.IGNORECASE
+)
+
+# ── Ethiopia/Africa fast-path → NEWS_GENERAL (skip LLM entirely) ─────────────
+_ETHIOPIA_ENTITIES = re.compile(
+    r"\b("
+    r"ethiopia(n)?|addis ababa|addis|"
+    r"tigray|amhara|oromia|oromo|afar|somali region|sidama|"
+    r"abiy ahmed?|abiy|"
+    r"tplf|fano|olf|oneg|endf|"
+    r"gerd|renaissance dam|nile dam|"
+    r"mekelle|bahir dar|gondar|hawassa|dire dawa|"
+    r"africa(n)?|horn of africa|east africa|"
+    r"sudan|somalia|eritrea|kenya|djibouti"
+    r")\b",
+    re.IGNORECASE
+)
+
+# ── Conflict/humanitarian fast-path → NEWS_GENERAL ───────────────────────────
+_NEWS_TOPICS = re.compile(
+    r"\b("
+    r"conflict|war|fighting|clashes?|attack(s|ed)?|killed|casualties|"
+    r"peace (talks?|deal|agreement|process)|ceasefire|"
+    r"election(s)?|vote|voting|ballot|"
+    r"government|minister|president|prime minister|parliament|"
+    r"economy|economic|inflation|gdp|trade|investment|"
+    r"humanitarian|refugee(s)?|displaced|famine|drought|flood|"
+    r"protest(s|ers)?|demonstration|rally|"
+    r"military|troops|soldiers?|forces?|"
+    r"news|report(s|ed)?|update(s)?"
+    r")\b",
+    re.IGNORECASE
+)
+
+
+def _fast_classify(query: str) -> Optional[Tuple[str, float, str]]:
+    """
+    Layer 1: Fast keyword-based classification.
+    Returns (intent, confidence, reason) or None if uncertain.
+
+    Priority order:
+    1. OTHER (identity/creative) — highest priority, avoid wasting search
+    2. NEWS_TEMPORAL — temporal signals are unambiguous
+    3. NEWS_HISTORICAL — historical signals are fairly unambiguous
+    4. NEWS_GENERAL — Ethiopia/Africa entities or news topics
+    5. None — uncertain, let LLM decide
+    """
+    q = query.strip()
+    ql = q.lower()
+
+    # ── 1. OTHER: identity questions ─────────────────────────────────────────
+    if _OTHER_IDENTITY.search(q):
+        return ("OTHER", 0.95, "identity_pattern")
+
+    # ── 2. OTHER: creative/off-topic ─────────────────────────────────────────
+    if _OTHER_CREATIVE.search(q):
+        return ("OTHER", 0.90, "creative_pattern")
+
+    # ── 3. NEWS_TEMPORAL: multilingual temporal signals ───────────────────────
+    if (_TEMPORAL_EN.search(q) or _TEMPORAL_AM.search(q) or
+            _TEMPORAL_AR.search(q) or _TEMPORAL_SO.search(q) or
+            _TEMPORAL_SW.search(q)):
+        return ("NEWS_TEMPORAL", 0.92, "temporal_keyword")
+
+    # ── 4. NEWS_HISTORICAL: historical/background signals ────────────────────
+    if _HISTORICAL.search(q):
+        # But if it also has temporal signals, temporal wins
+        return ("NEWS_HISTORICAL", 0.88, "historical_keyword")
+
+    # ── 5. NEWS_GENERAL: Ethiopia/Africa entities ────────────────────────────
+    if _ETHIOPIA_ENTITIES.search(q):
+        return ("NEWS_GENERAL", 0.85, "ethiopia_entity")
+
+    # ── 6. NEWS_GENERAL: news topic keywords ─────────────────────────────────
+    if _NEWS_TOPICS.search(q):
+        return ("NEWS_GENERAL", 0.80, "news_topic_keyword")
+
+    # ── 7. Uncertain — let LLM decide ────────────────────────────────────────
+    return None
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# LLM CLASSIFICATION PROMPT
 # ═══════════════════════════════════════════════════════════════════════════════
 
 _CLASSIFY_PROMPT = """You are an intent classifier for ARKI AI, a news assistant focused on Ethiopia and Africa.
@@ -81,10 +241,10 @@ Category:"""
 class IntentResult:
     intent: str            # NEWS_TEMPORAL | NEWS_HISTORICAL | NEWS_GENERAL | OTHER
     confidence: float      # 0.0 – 1.0
-    method: str            # instant | llm_groq | llm_gemini | llm_openrouter | llm_hf | default
+    method: str            # instant | keyword | llm_groq | llm_gemini | llm_openrouter | llm_hf | default
     inference_time_ms: float
-    query_complexity: str  # vague | simple | medium | complex
-    sub_type: str          # general | conflict | humanitarian | identity | off_topic
+    query_complexity: str  # empty | vague | simple | medium | complex
+    sub_type: str          # general | conflict | humanitarian | identity | creative | off_topic
     should_use_live: bool
     should_use_db: bool
     metadata: Dict[str, Any]
@@ -109,22 +269,23 @@ class IntentResult:
 
 class IntentClassifierV2:
     """
-    LLM-powered intent classifier with 4-provider fallback chain.
-
-    Fallback order:
-      Groq 8B → Gemini Flash → OpenRouter Free → HuggingFace → Default
+    Intent classifier v5: Fast keyword pre-check + LLM fallback chain.
+
+    Layer 0: Instant exact match (0ms)
+    Layer 1: Keyword rules (0ms) — handles ~80% of queries
+    Layer 2: Groq 8B (50ms)
+    Layer 3: Gemini Flash (200ms)
+    Layer 4: OpenRouter (300ms)
+    Layer 5: HuggingFace (2s)
+    Layer 6: Default NEWS_GENERAL (0ms)
     """
 
-    # Provider endpoints
-    GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"
-    GROQ_MODEL = "llama-3.1-8b-instant"   # 14,400 free RPD
-
-    GEMINI_URL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent"
-
-    OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
-    OPENROUTER_MODEL = "openrouter/auto"   # Auto-selects best available free model
-
-    HF_URL = "https://api-inference.huggingface.co/models/meta-llama/Llama-3.2-3B-Instruct/v1/chat/completions"
+    GROQ_URL        = "https://api.groq.com/openai/v1/chat/completions"
+    GROQ_MODEL      = "llama-3.1-8b-instant"
+    GEMINI_URL      = "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent"
+    OPENROUTER_URL  = "https://openrouter.ai/api/v1/chat/completions"
+    OPENROUTER_MODEL = "openrouter/auto"
+    HF_URL          = "https://api-inference.huggingface.co/models/meta-llama/Llama-3.2-3B-Instruct/v1/chat/completions"
 
     VALID_INTENTS = {"NEWS_TEMPORAL", "NEWS_HISTORICAL", "NEWS_GENERAL", "OTHER"}
 
@@ -134,49 +295,42 @@ class IntentClassifierV2:
         self._openrouter_key: Optional[str] = None
         self._hf_token: Optional[str] = None
         self._client = httpx.Client(timeout=5.0)
-        self._metrics = {
+        self._metrics: Dict[str, Any] = {
             "total": 0,
             "by_intent": {},
             "by_method": {},
             "total_ms": 0.0,
+            "keyword_hits": 0,   # how many queries handled by keyword layer
+            "llm_calls": 0,      # how many queries needed LLM
         }
         self._load_keys()
 
     def _load_keys(self):
-        """Load API keys from settings."""
         try:
             from src.core.config import settings
-
             key = settings.GROQ_API_KEY
             if key and key not in ("", "your-groq-api-key-here"):
                 self._groq_key = key
-
             gem = settings.GEMINI_API_KEY
             if gem and gem not in ("", "your-gemini-api-key-here"):
                 self._gemini_key = gem
-
-            # OpenRouter key (add OPENROUTER_API_KEY to .env)
             try:
                 or_key = getattr(settings, "OPENROUTER_API_KEY", "")
                 if or_key and or_key not in ("", "your-openrouter-api-key-here"):
                     self._openrouter_key = or_key
             except Exception:
                 pass
-
-            # HuggingFace token
             hf = settings.HF_TOKEN
             if hf and hf not in ("", "your-hf-token-here"):
                 self._hf_token = hf
 
-            providers = []
-            if self._groq_key:     providers.append("Groq")
-            if self._gemini_key:   providers.append("Gemini")
-            if self._openrouter_key: providers.append("OpenRouter")
-            if self._hf_token:     providers.append("HuggingFace")
+            providers = ["Keyword"]
+            if self._groq_key:        providers.append("Groq")
+            if self._gemini_key:      providers.append("Gemini")
+            if self._openrouter_key:  providers.append("OpenRouter")
+            if self._hf_token:        providers.append("HuggingFace")
             providers.append("Default")
-
-            logger.info(f"✅ Intent classifier providers: {' → '.join(providers)}")
-
+            logger.info(f"✅ Intent classifier v5 providers: {' → '.join(providers)}")
         except Exception as e:
             logger.error(f"Intent classifier: failed to load keys: {e}")
 
@@ -188,63 +342,62 @@ class IntentClassifierV2:
         ql = q.lower()
         complexity = self._complexity(q)
 
-        # ── Layer 1: Instant safety net ───────────────────────────────────────
+        # ── Layer 0: Instant exact match ──────────────────────────────────────
         if ql in _INSTANT_OTHER:
             return self._result("OTHER", 1.0, "instant", t0, complexity, "identity")
 
-        # ── Layer 2: Groq llama-3.1-8b-instant (PRIMARY) ─────────────────────
+        # ── Layer 1: Fast keyword rules ───────────────────────────────────────
+        fast = _fast_classify(q)
+        if fast:
+            intent, confidence, reason = fast
+            self._metrics["keyword_hits"] += 1
+            logger.debug(f"[Intent] Keyword rule: '{q[:50]}' → {intent} ({reason})")
+            return self._result(intent, confidence, f"keyword:{reason}", t0, complexity,
+                                self._sub_type(q, intent))
+
+        # ── Layers 2-5: LLM providers ─────────────────────────────────────────
+        self._metrics["llm_calls"] += 1
+
         if self._groq_key:
             intent = self._call_openai_compat(
-                url=self.GROQ_URL,
-                api_key=self._groq_key,
-                model=self.GROQ_MODEL,
-                query=q,
-                provider="groq",
+                url=self.GROQ_URL, api_key=self._groq_key,
+                model=self.GROQ_MODEL, query=q, provider="groq"
             )
             if intent:
                 return self._result(intent, 0.97, "llm_groq", t0, complexity,
                                     self._sub_type(q, intent))
 
-        # ── Layer 3: Gemini Flash (FALLBACK 1) ────────────────────────────────
         if self._gemini_key:
             intent = self._call_gemini(q)
             if intent:
                 return self._result(intent, 0.95, "llm_gemini", t0, complexity,
                                     self._sub_type(q, intent))
 
-        # ── Layer 4: OpenRouter free router (FALLBACK 2) ─────────────────────
         if self._openrouter_key:
             intent = self._call_openai_compat(
-                url=self.OPENROUTER_URL,
-                api_key=self._openrouter_key,
-                model=self.OPENROUTER_MODEL,
-                query=q,
-                provider="openrouter",
+                url=self.OPENROUTER_URL, api_key=self._openrouter_key,
+                model=self.OPENROUTER_MODEL, query=q, provider="openrouter",
                 extra_headers={
                     "HTTP-Referer": "https://arki-ai.com",
                     "X-Title": "ARKI AI Intent Classifier",
-                },
+                }
             )
             if intent:
                 return self._result(intent, 0.93, "llm_openrouter", t0, complexity,
                                     self._sub_type(q, intent))
 
-        # ── Layer 5: HuggingFace Inference API (FALLBACK 3) ───────────────────
         if self._hf_token:
             intent = self._call_openai_compat(
-                url=self.HF_URL,
-                api_key=self._hf_token,
+                url=self.HF_URL, api_key=self._hf_token,
                 model="meta-llama/Llama-3.2-3B-Instruct",
-                query=q,
-                provider="huggingface",
-                timeout=8.0,  # HF is slower
+                query=q, provider="huggingface", timeout=8.0
             )
             if intent:
                 return self._result(intent, 0.90, "llm_hf", t0, complexity,
                                     self._sub_type(q, intent))
 
         # ── Layer 6: Safe default ─────────────────────────────────────────────
-        logger.warning(f"Intent: all providers failed for '{q[:50]}' — defaulting to NEWS_GENERAL")
+        logger.warning(f"[Intent] All providers failed for '{q[:50]}' — defaulting to NEWS_GENERAL")
         return self._result("NEWS_GENERAL", 0.50, "default", t0, complexity, "general")
 
     # ── Provider calls ────────────────────────────────────────────────────────
@@ -259,143 +412,110 @@ class IntentClassifierV2:
         extra_headers: Optional[Dict] = None,
         timeout: float = 4.0,
     ) -> Optional[str]:
-        """
-        Generic OpenAI-compatible API call.
-        Works for: Groq, OpenRouter, HuggingFace (all use same format).
-        """
         try:
-            headers = {
-                "Authorization": f"Bearer {api_key}",
-                "Content-Type": "application/json",
-            }
+            headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
             if extra_headers:
                 headers.update(extra_headers)
-
             response = self._client.post(
-                url,
-                headers=headers,
+                url, headers=headers,
                 json={
                     "model": model,
-                    "messages": [
-                        {"role": "user", "content": _CLASSIFY_PROMPT.format(query=query)}
-                    ],
+                    "messages": [{"role": "user", "content": _CLASSIFY_PROMPT.format(query=query)}],
                     "max_tokens": 20,
                     "temperature": 0.0,
                 },
                 timeout=timeout,
             )
-
             if response.status_code == 200:
                 content = (
-                    response.json()
-                    .get("choices", [{}])[0]
-                    .get("message", {})
-                    .get("content", "")
-                    .strip()
+                    response.json().get("choices", [{}])[0]
+                    .get("message", {}).get("content", "").strip()
                 )
                 intent = self._parse_intent(content)
                 if intent:
-                    logger.debug(f"{provider}: '{query[:40]}' → {intent}")
+                    logger.debug(f"[Intent] {provider}: '{query[:40]}' → {intent}")
                     return intent
-                logger.warning(f"{provider}: unexpected response: '{content}'")
-
+                logger.warning(f"[Intent] {provider}: unexpected response: '{content}'")
             elif response.status_code == 429:
-                logger.warning(f"Intent: {provider} rate limited")
+                logger.warning(f"[Intent] {provider} rate limited")
             elif response.status_code == 503:
-                logger.warning(f"Intent: {provider} unavailable (503)")
+                logger.warning(f"[Intent] {provider} unavailable (503)")
             else:
-                logger.warning(f"Intent: {provider} returned {response.status_code}")
-
+                logger.warning(f"[Intent] {provider} returned {response.status_code}")
         except httpx.TimeoutException:
-            logger.warning(f"Intent: {provider} timeout ({timeout}s)")
+            logger.warning(f"[Intent] {provider} timeout ({timeout}s)")
         except Exception as e:
-            logger.error(f"Intent: {provider} error: {e}")
-
+            logger.error(f"[Intent] {provider} error: {e}")
         return None
 
     def _call_gemini(self, query: str) -> Optional[str]:
-        """Gemini has a different API format."""
         try:
             url = f"{self.GEMINI_URL}?key={self._gemini_key}"
             response = self._client.post(
                 url,
                 json={
-                    "contents": [
-                        {"parts": [{"text": _CLASSIFY_PROMPT.format(query=query)}]}
-                    ],
-                    "generationConfig": {
-                        "maxOutputTokens": 20,
-                        "temperature": 0.0,
-                    },
+                    "contents": [{"parts": [{"text": _CLASSIFY_PROMPT.format(query=query)}]}],
+                    "generationConfig": {"maxOutputTokens": 20, "temperature": 0.0},
                 },
                 timeout=4.0,
             )
-
             if response.status_code == 200:
                 content = (
-                    response.json()
-                    .get("candidates", [{}])[0]
-                    .get("content", {})
-                    .get("parts", [{}])[0]
-                    .get("text", "")
-                    .strip()
+                    response.json().get("candidates", [{}])[0]
+                    .get("content", {}).get("parts", [{}])[0]
+                    .get("text", "").strip()
                 )
                 intent = self._parse_intent(content)
                 if intent:
-                    logger.debug(f"gemini: '{query[:40]}' → {intent}")
+                    logger.debug(f"[Intent] gemini: '{query[:40]}' → {intent}")
                     return intent
-
             elif response.status_code == 429:
-                logger.warning("Intent: Gemini rate limited")
+                logger.warning("[Intent] Gemini rate limited")
             else:
-                logger.warning(f"Intent: Gemini returned {response.status_code}")
-
+                logger.warning(f"[Intent] Gemini returned {response.status_code}")
         except httpx.TimeoutException:
-            logger.warning("Intent: Gemini timeout (4s)")
+            logger.warning("[Intent] Gemini timeout (4s)")
         except Exception as e:
-            logger.error(f"Intent: Gemini error: {e}")
-
+            logger.error(f"[Intent] Gemini error: {e}")
         return None
 
     # ── Helpers ───────────────────────────────────────────────────────────────
 
     def _parse_intent(self, raw: str) -> Optional[str]:
-        """Parse LLM response to valid intent. Handles partial matches."""
-        cleaned = raw.strip().upper().replace(".", "").replace(":", "").split()[0] if raw.strip() else ""
-
-        if cleaned in self.VALID_INTENTS:
-            return cleaned
-
-        # Partial match (LLM sometimes adds extra words)
+        cleaned = raw.strip().upper().replace(".", "").replace(":", "")
+        first_word = cleaned.split()[0] if cleaned.split() else ""
+        if first_word in self.VALID_INTENTS:
+            return first_word
         for intent in self.VALID_INTENTS:
             if intent in cleaned:
                 return intent
-
         return None
 
     def _sub_type(self, query: str, intent: str) -> str:
-        """Infer sub-type from query content for downstream routing."""
         if intent == "OTHER":
             ql = query.lower()
-            if any(w in ql for w in ("who are you", "what are you", "are you", "what model")):
+            if _OTHER_IDENTITY.search(query):
                 return "identity"
-            if any(w in ql for w in ("write", "poem", "story", "recipe", "joke")):
+            if _OTHER_CREATIVE.search(query):
                 return "creative"
             return "off_topic"
-
         ql = query.lower()
-        if any(w in ql for w in ("clash", "attack", "killed", "battle", "fano", "tplf", "military")):
+        if any(w in ql for w in ("clash", "attack", "killed", "battle", "fano", "tplf", "military", "conflict", "war")):
             return "conflict"
-        if any(w in ql for w in ("displaced", "refugee", "aid", "humanitarian", "famine")):
+        if any(w in ql for w in ("displaced", "refugee", "aid", "humanitarian", "famine", "drought")):
             return "humanitarian"
+        if any(w in ql for w in ("election", "vote", "government", "minister", "president", "parliament")):
+            return "political"
+        if any(w in ql for w in ("economy", "economic", "inflation", "trade", "investment", "gdp")):
+            return "economic"
         return "general"
 
     def _complexity(self, query: str) -> str:
         n = len(query.split())
-        if n == 0:   return "empty"
-        if n == 1:   return "vague"
-        if n <= 4:   return "simple"
-        if n <= 12:  return "medium"
+        if n == 0:  return "empty"
+        if n == 1:  return "vague"
+        if n <= 4:  return "simple"
+        if n <= 12: return "medium"
         return "complex"
 
     def _result(
@@ -411,14 +531,12 @@ class IntentClassifierV2:
         ms = (time.time() - t0) * 1000
         self._metrics["total"] += 1
         self._metrics["by_intent"][intent] = self._metrics["by_intent"].get(intent, 0) + 1
-        self._metrics["by_method"][method] = self._metrics["by_method"].get(method, 0) + 1
+        self._metrics["by_method"][method]  = self._metrics["by_method"].get(method, 0) + 1
         self._metrics["total_ms"] += ms
-
         logger.debug(
-            f"Intent={intent} conf={confidence:.2f} method={method} "
+            f"[Intent] {intent} conf={confidence:.2f} method={method} "
             f"sub={sub_type} complexity={complexity} time={ms:.1f}ms"
         )
-
         return IntentResult(
             intent=intent,
             confidence=confidence,
@@ -433,7 +551,12 @@ class IntentClassifierV2:
 
     def get_metrics(self) -> Dict[str, Any]:
         total = self._metrics["total"] or 1
-        return {**self._metrics, "avg_ms": self._metrics["total_ms"] / total}
+        kw_pct = (self._metrics["keyword_hits"] / total) * 100
+        return {
+            **self._metrics,
+            "avg_ms": self._metrics["total_ms"] / total,
+            "keyword_hit_rate_pct": round(kw_pct, 1),
+        }
 
 
 # ═══════════════════════════════════════════════════════════════════════════════
@@ -445,7 +568,6 @@ intent_classifier_v2 = IntentClassifierV2()
 
 class IntentClassifier:
     """Backward-compatible binary wrapper (NEWS / OTHER)."""
-
     def __init__(self):
         self._v2 = intent_classifier_v2
 

src/infrastructure/adapters/jina_reranker_adapter.py

@@ -0,0 +1,161 @@
+"""
+Jina Reranker API Adapter
+
+Calls Jina AI's cloud reranker API instead of running the model locally.
+Same jina-reranker-v3 model, but runs on Jina's GPU servers.
+
+Benefits over self-hosted:
+- ~300ms latency (vs ~6s/doc on CPU)
+- No model download or GPU needed
+- Same API key as Jina Reader (unified token balance)
+- Production-ready immediately
+
+API: POST https://api.jina.ai/v1/rerank
+Docs: https://jina.ai/reranker
+Free: 1M tokens on signup (same key as Reader)
+"""
+
+import logging
+import time
+import httpx
+from typing import List, Dict, Any, Optional
+
+from src.core.ports.reranker_port import RerankerPort
+
+logger = logging.getLogger(__name__)
+
+
+class JinaRerankerAPIAdapter(RerankerPort):
+    """
+    Reranker using Jina AI's cloud API.
+    
+    Sends all documents in ONE API call — Jina handles batching server-side.
+    Falls back to score-based ordering if API fails.
+    
+    Token usage: query_tokens + sum(doc_tokens)
+    Typical: ~1,400 tokens per call with 7 docs × 200 chars each
+    """
+
+    API_URL = "https://api.jina.ai/v1/rerank"
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str = "jina-reranker-v3",
+        timeout: float = 5.0,
+    ):
+        self.api_key = api_key
+        self.model = model
+        self.timeout = timeout
+        self._client: Optional[httpx.Client] = None
+
+        if not api_key or api_key in ("", "your-jina-api-key-here"):
+            logger.warning("Jina Reranker API: no API key — adapter disabled")
+            self.api_key = None
+        else:
+            logger.info(f"Jina Reranker API ready (model={model}, timeout={timeout}s)")
+
+    def _get_client(self) -> httpx.Client:
+        if self._client is None:
+            self._client = httpx.Client(
+                timeout=self.timeout,
+                headers={
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                    "Accept": "application/json",
+                }
+            )
+        return self._client
+
+    def rerank(
+        self,
+        query: str,
+        docs: List[Dict[str, Any]],
+        top_n: int = 5,
+    ) -> List[Dict[str, Any]]:
+        """
+        Rerank documents using Jina API.
+        
+        Sends all docs in one request — Jina returns them sorted by relevance.
+        Falls back to vector score ordering if API unavailable.
+        """
+        if not docs:
+            return []
+
+        if not self.api_key:
+            logger.warning("Jina Reranker API disabled — falling back to score ordering")
+            return sorted(docs, key=lambda x: x.get("score", 0), reverse=True)[:top_n]
+
+        # Extract text content — truncate to 2048 chars (Jina handles tokenization)
+        MAX_CHARS = 2048
+        valid_docs = []
+        doc_texts = []
+        for doc in docs:
+            content = doc.get("content", "").strip()
+            if content:
+                doc_texts.append(content[:MAX_CHARS])
+                valid_docs.append(doc)
+
+        if not doc_texts:
+            return []
+
+        t0 = time.time()
+        try:
+            response = self._get_client().post(
+                self.API_URL,
+                json={
+                    "model": self.model,
+                    "query": query,
+                    "documents": doc_texts,
+                    "top_n": len(doc_texts),  # Get all scores, we'll slice ourselves
+                    "return_documents": False,  # Save tokens — we already have the docs
+                }
+            )
+
+            elapsed_ms = (time.time() - t0) * 1000
+
+            if response.status_code == 200:
+                data = response.json()
+                results = data.get("results", [])
+                usage = data.get("usage", {})
+
+                # results = [{"index": int, "relevance_score": float}, ...]
+                # Restore scores to original docs
+                for r in results:
+                    idx = r["index"]
+                    if idx < len(valid_docs):
+                        valid_docs[idx]["rerank_score"] = float(r["relevance_score"])
+
+                # Sort by rerank_score descending
+                valid_docs.sort(key=lambda x: x.get("rerank_score", 0), reverse=True)
+
+                logger.info(
+                    f"[JinaReranker] {len(valid_docs)} docs → top {top_n} "
+                    f"in {elapsed_ms:.0f}ms "
+                    f"(tokens={usage.get('total_tokens', '?')}, "
+                    f"top_score={valid_docs[0].get('rerank_score', 0):.3f})"
+                )
+                return valid_docs[:top_n]
+
+            elif response.status_code == 401:
+                logger.error("Jina Reranker API: Invalid API key")
+            elif response.status_code == 429:
+                logger.warning("Jina Reranker API: Rate limit exceeded")
+            elif response.status_code == 402:
+                logger.warning("Jina Reranker API: Insufficient tokens — top up at jina.ai")
+            else:
+                logger.warning(
+                    f"Jina Reranker API: HTTP {response.status_code} — {response.text[:200]}"
+                )
+
+        except httpx.TimeoutException:
+            logger.warning(f"Jina Reranker API: timeout ({self.timeout}s)")
+        except Exception as e:
+            logger.error(f"Jina Reranker API error: {e}")
+
+        # Fallback: sort by vector score
+        logger.warning("Jina Reranker API failed — falling back to vector score ordering")
+        return sorted(docs, key=lambda x: x.get("score", 0), reverse=True)[:top_n]
+
+    def is_available(self) -> bool:
+        return self.api_key is not None

src/infrastructure/adapters/newsapi_adapter.py

@@ -0,0 +1,376 @@
+"""
+NewsAPI.org Adapter
+
+Provides real-time news from 80,000+ sources worldwide.
+Best for temporal queries requiring fresh, breaking news.
+
+Features:
+- Real-time updates (15-minute refresh)
+- 80,000+ sources including African outlets
+- Structured data (title, description, content, source, publishedAt)
+- Free tier: 100 requests/day
+- Paid tier: $449/month for production
+
+Get API key: https://newsapi.org/register
+"""
+
+import logging
+import asyncio
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class NewsAPIAdapter:
+    """
+    Adapter for NewsAPI.org real-time news search.
+    
+    Provides fresh news results to complement database search.
+    Designed to be fast (2s timeout) and resilient (graceful fallbacks).
+    """
+    
+    BASE_URL = "https://newsapi.org/v2"
+    
+    def __init__(
+        self, 
+        api_key: str,
+        timeout: float = 2.0,
+        max_results: int = 20
+    ):
+        """
+        Initialize NewsAPI adapter.
+        
+        Args:
+            api_key: NewsAPI.org API key
+            timeout: Maximum time to wait for results (seconds)
+            max_results: Maximum number of results to return
+        """
+        self.api_key = api_key
+        self.timeout = timeout
+        self.max_results = max_results
+        self.client = None
+        
+        if not api_key or api_key == "your-newsapi-key-here":
+            logger.warning("NewsAPI key not configured - adapter disabled")
+            self.api_key = None
+        else:
+            logger.info(f"NewsAPI adapter initialized (timeout={timeout}s, max={max_results})")
+    
+    async def _ensure_client(self):
+        """Lazy initialization of HTTP client"""
+        if self.client is None:
+            self.client = httpx.AsyncClient(
+                timeout=self.timeout,
+                headers={
+                    "X-Api-Key": self.api_key,
+                    "User-Agent": "ARKI-AI-RAG/2.5 (Ethiopia News Assistant)"
+                }
+            )
+    
+    async def search(
+        self, 
+        query: str,
+        language: str = "en",
+        sort_by: str = "publishedAt",
+        from_date: Optional[str] = None,
+        max_results: Optional[int] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Search NewsAPI for the given query.
+        Automatically wraps multi-word queries in quotes for exact matching.
+        """
+        if not self.api_key:
+            logger.warning("NewsAPI unavailable - returning empty results")
+            return []
+        
+        await self._ensure_client()
+        
+        max_results = max_results or self.max_results
+        
+        # Wrap in quotes if multi-word and not already quoted — improves precision
+        search_q = query
+        words = query.strip().split()
+        if len(words) > 1 and not query.startswith('"'):
+            # Use AND logic: all key terms must appear
+            search_q = " AND ".join(f'"{w}"' for w in words[:3])
+        
+        try:
+            url = f"{self.BASE_URL}/everything"
+            params = {
+                "q": search_q,
+                "language": language,
+                "sortBy": sort_by,
+                "pageSize": max_results
+            }
+            if from_date:
+                params["from"] = from_date
+            
+            logger.info(f"[NewsAPI] Searching: '{search_q}' (lang={language})")
+            
+            response = await self.client.get(url, params=params)
+            
+            if response.status_code == 200:
+                data = response.json()
+                if data.get("status") != "ok":
+                    logger.warning(f"NewsAPI error: {data.get('message', 'unknown')}")
+                    return []
+                
+                articles = data.get("articles", [])
+                results = []
+                for article in articles:
+                    normalized = self._normalize_result(article)
+                    if normalized:
+                        results.append(normalized)
+                
+                logger.info(
+                    f"[NewsAPI] '{query[:50]}' → {len(results)} results "
+                    f"(total available: {data.get('totalResults', 0)})"
+                )
+                return results
+            
+            elif response.status_code == 401:
+                logger.error("NewsAPI: Invalid API key")
+                return []
+            elif response.status_code == 429:
+                logger.warning("NewsAPI: Rate limit exceeded (100 requests/day on free tier)")
+                return []
+            elif response.status_code == 426:
+                logger.warning("NewsAPI: Upgrade required (free tier limitations)")
+                return []
+            else:
+                logger.warning(f"NewsAPI returned status {response.status_code}: {response.text[:200]}")
+                return []
+        
+        except asyncio.TimeoutError:
+            logger.warning(f"NewsAPI timeout ({self.timeout}s)")
+            return []
+        except Exception as e:
+            logger.error(f"NewsAPI search error: {e}")
+            return []
+    
+    async def search_top_headlines(
+        self,
+        country: str = "us",
+        category: Optional[str] = None,
+        max_results: Optional[int] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Get top headlines from NewsAPI.
+        
+        Args:
+            country: Country code (us, gb, etc.) - Note: Ethiopia (et) not supported
+            category: Category (business, entertainment, general, health, science, sports, technology)
+            max_results: Override default max_results
+        
+        Returns:
+            List of normalized search results
+        """
+        if not self.api_key:
+            logger.warning("NewsAPI unavailable - returning empty results")
+            return []
+        
+        await self._ensure_client()
+        
+        max_results = max_results or self.max_results
+        
+        try:
+            url = f"{self.BASE_URL}/top-headlines"
+            params = {
+                "country": country,
+                "pageSize": max_results
+            }
+            
+            if category:
+                params["category"] = category
+            
+            logger.info(f"[NewsAPI] Fetching top headlines (country={country}, category={category})")
+            
+            response = await self.client.get(url, params=params)
+            
+            if response.status_code == 200:
+                data = response.json()
+                articles = data.get("articles", [])
+                
+                results = []
+                for article in articles:
+                    normalized = self._normalize_result(article)
+                    if normalized:
+                        results.append(normalized)
+                
+                logger.info(f"[NewsAPI] Top headlines: {len(results)} results")
+                return results
+            
+            else:
+                logger.warning(f"NewsAPI top headlines returned status {response.status_code}")
+                return []
+        
+        except Exception as e:
+            logger.error(f"NewsAPI top headlines error: {e}")
+            return []
+    
+    def _normalize_result(self, article: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """
+        Normalize NewsAPI result to common format.
+        
+        Args:
+            article: Raw article from NewsAPI
+        
+        Returns:
+            Normalized result dict or None if invalid
+        """
+        try:
+            # Extract fields
+            title = article.get("title", "").strip()
+            url = article.get("url", "").strip()
+            description = article.get("description", "").strip()
+            content = article.get("content", "").strip()
+            source_name = article.get("source", {}).get("name", "").strip()
+            published_at = article.get("publishedAt", "")
+            author = article.get("author", "")
+            url_to_image = article.get("urlToImage", "")
+            
+            # Validate required fields
+            if not title or not url:
+                logger.debug(f"Skipping invalid result: missing title or URL")
+                return None
+            
+            # Combine description + content for better context
+            full_content = description
+            if content and content != description:
+                # NewsAPI truncates content with [+X chars]
+                # We'll use Jina Reader to get full article later
+                full_content = f"{description}\n\n{content}"
+            
+            # Calculate freshness score
+            freshness_score = self._calculate_freshness(published_at)
+            
+            return {
+                "title": title,
+                "url": url,
+                "content": full_content or title,  # Use title if no content
+                "snippet": description,
+                "source": source_name or self._extract_domain(url),
+                "published_at": published_at,
+                "author": author,
+                "image_url": url_to_image,
+                "source_type": "live",
+                "is_live": True,
+                "freshness_score": freshness_score,
+                "language": "en",  # NewsAPI returns language in query
+                "metadata": {
+                    "title": title,
+                    "url": url,
+                    "source": source_name,
+                    "published_at": published_at,
+                    "author": author,
+                    "search_engine": "newsapi"
+                }
+            }
+            
+        except Exception as e:
+            logger.warning(f"Failed to normalize NewsAPI result: {e}")
+            return None
+    
+    def _calculate_freshness(self, published_at: str) -> float:
+        """
+        Calculate freshness score based on article age.
+        
+        Args:
+            published_at: ISO format date string
+        
+        Returns:
+            Freshness score (0.0 to 1.0)
+        """
+        if not published_at:
+            return 0.8  # Unknown age, assume recent
+        
+        try:
+            pub_date = datetime.fromisoformat(published_at.replace('Z', '+00:00'))
+            age = datetime.utcnow() - pub_date.replace(tzinfo=None)
+            age_minutes = age.total_seconds() / 60
+            
+            # NewsAPI results are very fresh
+            if age_minutes < 10:
+                return 1.0  # < 10 min
+            elif age_minutes < 60:
+                return 0.98  # < 1 hour
+            elif age_minutes < 360:
+                return 0.95  # < 6 hours
+            elif age_minutes < 1440:
+                return 0.9  # < 24 hours
+            else:
+                return 0.85  # Older but still from live search
+        except:
+            return 0.8  # Default to recent
+    
+    def _extract_domain(self, url: str) -> str:
+        """
+        Extract domain name from URL.
+        
+        Args:
+            url: Full URL
+        
+        Returns:
+            Domain name (e.g., "bbc.com")
+        """
+        try:
+            from urllib.parse import urlparse
+            parsed = urlparse(url)
+            domain = parsed.netloc
+            # Remove www. prefix
+            if domain.startswith("www."):
+                domain = domain[4:]
+            return domain
+        except:
+            return "unknown"
+    
+    def is_available(self) -> bool:
+        """
+        Check if NewsAPI is available.
+        
+        Returns:
+            True if API key is configured, False otherwise
+        """
+        return self.api_key is not None
+    
+    async def close(self):
+        """Close HTTP client"""
+        if self.client:
+            await self.client.aclose()
+            self.client = None
+            logger.debug("NewsAPI client closed")
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# SINGLETON INSTANCE
+# ═══════════════════════════════════════════════════════════════════════════
+
+_default_adapter = None
+
+
+def get_newsapi_adapter(
+    api_key: str,
+    timeout: float = 2.0,
+    max_results: int = 20
+) -> NewsAPIAdapter:
+    """
+    Get or create the default NewsAPI adapter instance.
+    
+    Args:
+        api_key: NewsAPI.org API key
+        timeout: Search timeout in seconds
+        max_results: Maximum results to return
+    
+    Returns:
+        NewsAPIAdapter instance
+    """
+    global _default_adapter
+    if _default_adapter is None:
+        _default_adapter = NewsAPIAdapter(
+            api_key=api_key,
+            timeout=timeout,
+            max_results=max_results
+        )
+    return _default_adapter

src/infrastructure/adapters/redis_adapter.py

@@ -1,20 +1,59 @@
+"""
+Redis Cache Adapter — Smart Layered Caching
+
+Cache layers with different TTLs:
+  Layer 1 — Intent cache       : 1 hour   (same query = same intent)
+  Layer 2 — Live search cache  : 10 min   (DuckDuckGo/NewsAPI results)
+  Layer 3 — Translation cache  : 1 hour   (LLM translation is expensive)
+  Layer 4 — Full response cache: 5 min    (complete RAG answer)
+
+Key naming convention:
+  intent_v2:{query_hash}          → IntentResult dict
+  live_search:{query_hash}        → list of live results
+  translation:{query_hash}        → translation + expanded query dict
+  rag_response:{query_hash}       → full RAG response dict
+
+All keys use SHA-256 of the normalized query (lowercase, stripped).
+"""
+
 import json
 import logging
-from typing import Optional, Dict, Any
-import redis
 import hashlib
+import time
+from typing import Optional, Dict, Any, List
+
+import redis
 
 from src.core.ports.cache_port import CachePort
 from src.core.config import settings
 
 logger = logging.getLogger(__name__)
 
+# ── TTL constants (seconds) ───────────────────────────────────────────────────
+TTL_INTENT      = 3600   # 1 hour  — intent rarely changes for same query
+TTL_LIVE_SEARCH = 600    # 10 min  — live news stays fresh enough
+TTL_TRANSLATION = 3600   # 1 hour  — translations don't change
+TTL_RESPONSE    = 300    # 5 min   — full RAG response (temporal queries need freshness)
+TTL_RESPONSE_HISTORICAL = 1800  # 30 min — historical answers change less often
+
+
 class RedisAdapter(CachePort):
+    """
+    Redis cache adapter with smart layered caching.
+
+    Falls back gracefully when Redis is unavailable — all methods
+    return None/False instead of raising exceptions.
+    """
+
     def __init__(self):
+        self.client = None
+        self._connect()
+
+    def _connect(self):
         try:
-            if hasattr(settings, 'REDIS_URL') and settings.REDIS_URL:
+            if hasattr(settings, "REDIS_URL") and settings.REDIS_URL:
                 url = settings.REDIS_URL
-                # Upstash requires TLS — upgrade redis:// to rediss://
+                # Upstash requires TLS
                 if url.startswith("redis://") and "upstash.io" in url:
                     url = "rediss://" + url[len("redis://"):]
                 self.client = redis.from_url(url, decode_responses=True)
@@ -24,43 +63,206 @@ class RedisAdapter(CachePort):
                     port=settings.REDIS_PORT,
                     db=settings.REDIS_DB,
                     password=settings.REDIS_PASSWORD or None,
-                    decode_responses=True
+                    decode_responses=True,
                 )
                 self.client = redis.Redis(connection_pool=pool)
             self.client.ping()
-            logger.info("Connected to Redis cache.")
+            logger.info("✅ Connected to Redis cache.")
         except Exception as e:
-            logger.warning(f"Could not connect to Redis: {e}. Cache disabled.")
+            logger.warning(f"Redis unavailable: {e}. All cache operations will be no-ops.")
             self.client = None
 
-
+    # ── CachePort interface ───────────────────────────────────────────────────
 
     def get(self, key: str) -> Optional[Any]:
-        if not self.client: return None
+        if not self.client:
+            return None
         try:
             data = self.client.get(key)
             return json.loads(data) if data else None
         except Exception as e:
-            logger.error(f"Redis get error: {e}")
+            logger.debug(f"Redis get error for key '{key}': {e}")
             return None
 
     def set(self, key: str, value: Any, expiration: int = 3600) -> bool:
-        if not self.client: return False
+        if not self.client:
+            return False
+        try:
+            self.client.setex(key, expiration, json.dumps(value, default=str))
+            return True
+        except Exception as e:
+            logger.debug(f"Redis set error for key '{key}': {e}")
+            return False
+
+    def delete(self, key: str) -> bool:
+        if not self.client:
+            return False
         try:
-            self.client.setex(key, expiration, json.dumps(value))
+            self.client.delete(key)
             return True
         except Exception as e:
-            logger.error(f"Redis set error: {e}")
+            logger.debug(f"Redis delete error for key '{key}': {e}")
             return False
 
     def search_similar(self, query_vector: list, threshold: float = 0.95) -> Optional[Dict[str, Any]]:
+        """Vector similarity search — not implemented (requires RedisSearch module)."""
+        return None
+
+    # ── Key generation ──────────────────────────────────────────────────────���─
+
+    def generate_exact_hash(self, text: str) -> str:
+        """SHA-256 hash of normalized text for exact-match cache keys."""
+        normalized = text.lower().strip()
+        return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+
+    def _make_key(self, prefix: str, query: str) -> str:
+        """Build a namespaced cache key from query text."""
+        return f"{prefix}:{self.generate_exact_hash(query)}"
+
+    # ── Layer 1: Intent cache ─────────────────────────────────────────────────
+
+    def get_intent(self, query: str) -> Optional[Dict[str, Any]]:
         """
-        In a full enterprise setup, this would use Redisearch vector indexes.
-        For this simplified adapter, we will rely on exact match (SHA256 of the prompt)
-        as an interim caching mechanism until Redis vector extensions are configured.
+        Retrieve cached intent result for a query.
+        Returns dict with keys: intent, confidence, method
         """
-        pass
-        
-    def generate_exact_hash(self, text: str) -> str:
-        """Helper for exact match caching if vector search is unavailable."""
-        return hashlib.sha256(text.encode('utf-8')).hexdigest()
+        key = self._make_key("intent_v2", query)
+        result = self.get(key)
+        if result:
+            logger.debug(f"[Cache] Intent HIT for '{query[:50]}'")
+        return result
+
+    def set_intent(self, query: str, intent_data: Dict[str, Any]) -> bool:
+        """Cache intent result for 1 hour."""
+        key = self._make_key("intent_v2", query)
+        success = self.set(key, intent_data, expiration=TTL_INTENT)
+        if success:
+            logger.debug(f"[Cache] Intent SET for '{query[:50]}' (TTL={TTL_INTENT}s)")
+        return success
+
+    # ── Layer 2: Live search cache ────────────────────────────────────────────
+
+    def get_live_search(self, query: str) -> Optional[List[Dict[str, Any]]]:
+        """
+        Retrieve cached live search results (DuckDuckGo + NewsAPI).
+        Returns list of result dicts or None if not cached.
+        """
+        key = self._make_key("live_search", query)
+        result = self.get(key)
+        if result:
+            age = result.get("_cached_at", 0)
+            elapsed = int(time.time()) - age if age else 0
+            logger.info(f"[Cache] Live search HIT for '{query[:50]}' (age={elapsed}s)")
+            return result.get("results", [])
+        return None
+
+    def set_live_search(self, query: str, results: List[Dict[str, Any]]) -> bool:
+        """Cache live search results for 10 minutes."""
+        key = self._make_key("live_search", query)
+        payload = {
+            "results": results,
+            "_cached_at": int(time.time()),
+            "_query": query[:100],
+        }
+        success = self.set(key, payload, expiration=TTL_LIVE_SEARCH)
+        if success:
+            logger.info(f"[Cache] Live search SET for '{query[:50]}' ({len(results)} results, TTL={TTL_LIVE_SEARCH}s)")
+        return success
+
+    # ── Layer 3: Translation cache ────────────────────────────────────────────
+
+    def get_translation(self, query: str) -> Optional[Dict[str, Any]]:
+        """
+        Retrieve cached translation + query expansion result.
+        Returns dict with keys: expanded_query, translations, days_back, etc.
+        """
+        key = self._make_key("translation", query)
+        result = self.get(key)
+        if result:
+            logger.debug(f"[Cache] Translation HIT for '{query[:50]}'")
+        return result
+
+    def set_translation(self, query: str, translation_data: Dict[str, Any]) -> bool:
+        """Cache translation result for 1 hour."""
+        key = self._make_key("translation", query)
+        success = self.set(key, translation_data, expiration=TTL_TRANSLATION)
+        if success:
+            logger.debug(f"[Cache] Translation SET for '{query[:50]}' (TTL={TTL_TRANSLATION}s)")
+        return success
+
+    # ── Layer 4: Full response cache ──────────────────────────────────────────
+
+    def get_response(self, query: str) -> Optional[Dict[str, Any]]:
+        """
+        Retrieve cached full RAG response.
+        Returns complete response dict or None if not cached.
+        """
+        key = self._make_key("rag_response", query)
+        result = self.get(key)
+        if result:
+            age = result.get("_cached_at", 0)
+            elapsed = int(time.time()) - age if age else 0
+            logger.info(f"[Cache] Response HIT for '{query[:50]}' (age={elapsed}s)")
+        return result
+
+    def set_response(
+        self,
+        query: str,
+        response: Dict[str, Any],
+        intent: str = "NEWS_GENERAL"
+    ) -> bool:
+        """
+        Cache full RAG response.
+        TTL depends on intent:
+          - NEWS_TEMPORAL   → 5 min  (fresh news changes fast)
+          - NEWS_HISTORICAL → 30 min (historical facts are stable)
+          - NEWS_GENERAL    → 5 min  (default)
+        """
+        key = self._make_key("rag_response", query)
+        ttl = TTL_RESPONSE_HISTORICAL if intent == "NEWS_HISTORICAL" else TTL_RESPONSE
+        payload = {
+            **response,
+            "_cached_at": int(time.time()),
+            "_intent": intent,
+        }
+        success = self.set(key, payload, expiration=ttl)
+        if success:
+            logger.info(f"[Cache] Response SET for '{query[:50]}' (intent={intent}, TTL={ttl}s)")
+        return success
+
+    # ── Cache stats ───────────────────────────────────────────────────────────
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Return cache statistics."""
+        if not self.client:
+            return {"status": "disconnected"}
+        try:
+            info = self.client.info("stats")
+            keyspace = self.client.info("keyspace")
+            return {
+                "status": "connected",
+                "hits": info.get("keyspace_hits", 0),
+                "misses": info.get("keyspace_misses", 0),
+                "hit_rate": round(
+                    info.get("keyspace_hits", 0) /
+                    max(1, info.get("keyspace_hits", 0) + info.get("keyspace_misses", 0))
+                    * 100, 1
+                ),
+                "total_keys": sum(
+                    v.get("keys", 0) for v in keyspace.values()
+                    if isinstance(v, dict)
+                ),
+                "memory_used": self.client.info("memory").get("used_memory_human", "?"),
+            }
+        except Exception as e:
+            return {"status": "error", "error": str(e)}
+
+    def is_available(self) -> bool:
+        """Check if Redis is connected."""
+        if not self.client:
+            return False
+        try:
+            self.client.ping()
+            return True
+        except Exception:
+            return False