fix(3-bugs): Jina 401, reranker tokenizer, intent classifier v1 still running

Fix 1 - Jina 401 Unauthorized (jina_reader_adapter.py):
- Jina now requires API key for most news sites
- Added Authorization: Bearer header when JINA_API_KEY is set
- Added JINA_API_KEY to config.py and .env template
- Logs warning if no key set (get free key at jina.ai, 1M tokens/month)

Fix 2 - Reranker tokenizer (bge_reranker_adapter.py):
- Error changed from prepare_for_model to build_inputs_with_special_tokens
- Now patches ALL potentially missing XLMRobertaTokenizer methods:
prepare_for_model, build_inputs_with_special_tokens,
create_token_type_ids_from_sequences, get_special_tokens_mask
- Iterates over method list instead of single hard-coded patch

Fix 3 - Intent classifier v1 still running (agent_router_use_case.py):
- AgentRouterUseCase was using old v1 binary classifier
- Now uses v2 LLM-powered classifier (Groq 8B -> Gemini -> OpenRouter -> HF)
- Falls back to v1 if v2 fails
- Replaced print(DEBUG) with logger.info([Router] prefix)

.env

@@ -113,3 +113,8 @@ SEARXNG_MAX_RESULTS=10
 # Get free key: https://openrouter.ai/keys (no credit card required)
 # Free models: Llama 4, Qwen 3, DeepSeek, Gemma 3 and more
 OPENROUTER_API_KEY=your-openrouter-api-key-here
+
+# --- Jina AI Reader (Full Article Extraction) ---
+# Get free key: https://jina.ai (1M tokens/month free)
+# Without key: most news sites return 401 Unauthorized
+JINA_API_KEY=your-jina-api-key-here

src/core/config.py

@@ -86,6 +86,7 @@ class Settings(BaseSettings):
     ENABLE_JINA_READER: bool = os.getenv("ENABLE_JINA_READER", "true").lower() == "true"
     JINA_READER_TIMEOUT: float = float(os.getenv("JINA_READER_TIMEOUT", "8.0"))
     JINA_READER_MAX_CONCURRENT: int = int(os.getenv("JINA_READER_MAX_CONCURRENT", "5"))
+    JINA_API_KEY: str = os.getenv("JINA_API_KEY", "")  # Get free key at https://jina.ai
     
     # Cache Settings (TTL in seconds)
     CACHE_RESPONSE_TTL: int = int(os.getenv("CACHE_RESPONSE_TTL", "300"))      # 5 minutes

src/core/use_cases/agent_router_use_case.py

@@ -63,9 +63,21 @@ class AgentRouterUseCase:
         self.chat_history_db = chat_history_db
 
     def _classify_intent(self, query: str) -> str:
-        intent = intent_classifier.classify(query)
-        print(f"DEBUG: Intent Classification: {intent} for query: '{query[:80]}'")
-        return intent
+        """Use v2 LLM-powered classifier (Groq 8B → Gemini → OpenRouter → HF → default)"""
+        try:
+            from src.infrastructure.adapters.intent_classifier_v2 import intent_classifier_v2
+            result = intent_classifier_v2.classify(query)
+            intent = "OTHER" if result.intent == "OTHER" else "NEWS"
+            logger.info(
+                f"[Router] Intent={result.intent} conf={result.confidence:.2f} "
+                f"method={result.method} time={result.inference_time_ms:.1f}ms"
+            )
+            return intent
+        except Exception as e:
+            logger.warning(f"[Router] v2 classifier failed: {e} — falling back to v1")
+            intent = intent_classifier.classify(query)
+            logger.info(f"[Router] Intent v1={intent} for query: '{query[:60]}'")
+            return intent
 
     def _detect_language(self, query: str) -> str:
         """

src/infrastructure/adapters/bge_reranker_adapter.py

@@ -60,16 +60,24 @@ class BgeRerankerAdapter(RerankerPort):
             logger.info(f"Loading reranker model: {self.model_name}")
             try:
                 if HAS_FLAG_RERANKER and "bge-reranker" in self.model_name.lower():
-                    # Patch tokenizer compatibility issue before loading
+                    # Patch tokenizer compatibility issues before loading
                     try:
                         import transformers
-                        from transformers import XLMRobertaTokenizer
-                        if not hasattr(XLMRobertaTokenizer, 'prepare_for_model'):
-                            XLMRobertaTokenizer.prepare_for_model = (
-                                transformers.PreTrainedTokenizer.prepare_for_model
-                            )
-                    except Exception:
-                        pass
+                        from transformers import XLMRobertaTokenizer, PreTrainedTokenizer
+                        # Patch all missing methods that different transformers versions may lack
+                        for method_name in [
+                            "prepare_for_model",
+                            "build_inputs_with_special_tokens",
+                            "create_token_type_ids_from_sequences",
+                            "get_special_tokens_mask",
+                        ]:
+                            if not hasattr(XLMRobertaTokenizer, method_name):
+                                base_method = getattr(PreTrainedTokenizer, method_name, None)
+                                if base_method:
+                                    setattr(XLMRobertaTokenizer, method_name, base_method)
+                                    logger.debug(f"Patched XLMRobertaTokenizer.{method_name}")
+                    except Exception as patch_err:
+                        logger.debug(f"Tokenizer patch skipped: {patch_err}")
                     
                     # FlagReranker: use_fp16=True halves memory, normalize=True gives [0,1] scores
                     # trust_remote_code=True fixes tokenizer compatibility issues

src/infrastructure/adapters/jina_reader_adapter.py

@@ -58,14 +58,28 @@ class JinaReaderAdapter:
         )
     
     async def _ensure_client(self):
-        """Lazy initialization of HTTP client"""
+        """Lazy initialization of HTTP client with optional API key auth"""
         if self.client is None:
+            headers = {
+                "User-Agent": "ARKI-AI-RAG/2.4 (Ethiopia News Assistant)",
+                "Accept": "text/plain, text/markdown",
+            }
+            # Add Jina API key if available (required for most sites)
+            try:
+                from src.core.config import settings
+                jina_key = getattr(settings, "JINA_API_KEY", "")
+                if jina_key and jina_key not in ("", "your-jina-api-key-here"):
+                    headers["Authorization"] = f"Bearer {jina_key}"
+                    logger.info("Jina Reader: using API key authentication")
+                else:
+                    logger.warning("Jina Reader: no API key set — most sites will return 401. Get free key at https://jina.ai")
+            except Exception:
+                pass
+
             self.client = httpx.AsyncClient(
                 timeout=self.timeout,
                 follow_redirects=True,
-                headers={
-                    "User-Agent": "ARKI-AI-RAG/2.4 (Ethiopia News Assistant)"
-                }
+                headers=headers
             )
     
     async def extract_article(self, url: str) -> Dict[str, Any]: