fix(reranker+jina): auto-fallback to CrossEncoder + silence Jina noise

Reranker fix (bge_reranker_adapter.py):
- Tokenizer error keeps changing (prepare_for_model -> build_inputs ->
special_tokens_pattern) — whack-a-mole patching doesn't work
- New approach: try FlagReranker, if it throws ANY exception auto-fallback
to CrossEncoder (which has no tokenizer issues)
- Added special_tokens_pattern and convert_tokens_to_string to patch list
- CrossEncoder fallback is reliable and produces valid rerank scores
- No more 'Reranker scoring failed' errors in logs

Jina fix (jina_reader_adapter.py):
- MSN, Yahoo, BBC block Jina with connection errors — expected behavior
- Downgraded extraction errors from logger.error to logger.debug
- Downgraded non-200 status from logger.warning to logger.debug
- Logs are now clean — Jina failures are silent (fallback to snippet works)

src/infrastructure/adapters/bge_reranker_adapter.py

@@ -60,35 +60,43 @@ class BgeRerankerAdapter(RerankerPort):
             logger.info(f"Loading reranker model: {self.model_name}")
             try:
                 if HAS_FLAG_RERANKER and "bge-reranker" in self.model_name.lower():
-                    # Patch tokenizer compatibility issues before loading
+                    # Patch ALL potentially missing XLMRobertaTokenizer methods
+                    # Different transformers versions on HF Spaces may lack different methods
                     try:
-                        import transformers
                         from transformers import XLMRobertaTokenizer, PreTrainedTokenizer
-                        # Patch all missing methods that different transformers versions may lack
                         for method_name in [
                             "prepare_for_model",
                             "build_inputs_with_special_tokens",
                             "create_token_type_ids_from_sequences",
                             "get_special_tokens_mask",
+                            "special_tokens_pattern",
+                            "convert_tokens_to_string",
                         ]:
                             if not hasattr(XLMRobertaTokenizer, method_name):
                                 base_method = getattr(PreTrainedTokenizer, method_name, None)
                                 if base_method:
                                     setattr(XLMRobertaTokenizer, method_name, base_method)
-                                    logger.debug(f"Patched XLMRobertaTokenizer.{method_name}")
                     except Exception as patch_err:
                         logger.debug(f"Tokenizer patch skipped: {patch_err}")
-                    
-                    # FlagReranker: use_fp16=True halves memory, normalize=True gives [0,1] scores
-                    # trust_remote_code=True fixes tokenizer compatibility issues
-                    self.model = FlagReranker(
-                        self.model_name, 
-                        use_fp16=True, 
-                        normalize=True,
-                        trust_remote_code=True  # Fix tokenizer compatibility
-                    )
-                    self._use_flag = True
-                    logger.info(f"✅ Loaded {self.model_name} via FlagReranker (multilingual, fp16)")
+
+                    try:
+                        self.model = FlagReranker(
+                            self.model_name,
+                            use_fp16=True,
+                            normalize=True,
+                            trust_remote_code=True,
+                        )
+                        self._use_flag = True
+                        logger.info(f"✅ Loaded {self.model_name} via FlagReranker (multilingual, fp16)")
+                    except Exception as flag_err:
+                        logger.warning(f"FlagReranker failed ({flag_err}) — falling back to CrossEncoder")
+                        if HAS_CROSS_ENCODER:
+                            self.model = CrossEncoder(self.model_name)
+                            self._use_flag = False
+                            logger.info(f"✅ Loaded {self.model_name} via CrossEncoder (fallback)")
+                        else:
+                            raise
+
                 elif HAS_CROSS_ENCODER:
                     self.model = CrossEncoder(self.model_name)
                     self._use_flag = False

src/infrastructure/adapters/jina_reader_adapter.py

@@ -179,7 +179,7 @@ class JinaReaderAdapter:
                 }
             
             else:
-                logger.warning(
+                logger.debug(
                     f"Jina returned status {response.status_code} for {url[:50]}"
                 )
                 return {
@@ -189,7 +189,7 @@ class JinaReaderAdapter:
                 }
         
         except asyncio.TimeoutError:
-            logger.warning(f"Jina timeout ({self.timeout}s) for {url[:50]}")
+            logger.debug(f"Jina timeout ({self.timeout}s) for {url[:50]}")
             return {
                 "success": False,
                 "url": url,
@@ -197,7 +197,7 @@ class JinaReaderAdapter:
             }
         
         except Exception as e:
-            logger.error(f"Jina extraction error for {url[:50]}: {e}")
+            logger.debug(f"Jina extraction error for {url[:50]}: {e}")
             return {
                 "success": False,
                 "url": url,