sync: 159 file da Baida98/AI@c35825ad (2026-08-16 08:23 UTC) [deploy-all]

models/ai_client.py

@@ -280,10 +280,31 @@ class AIClient:
             start = self._rr_indices.get(index_key, 0)
             selected.append(profiles[start % len(profiles)])
             self._rr_indices[index_key] = start + 1
-        if selected:
-            return selected
-        # All profiles are quarantined: probe the earliest-to-recover profile only.
-        return [min(providers, key=lambda p: self._breaker.get(p.identity, {}).get("open_until", 0.0))] if providers else []
+        return selected
+
+    def _inter_provider_fallback_pool(
+        self,
+        purpose: str,
+        excluded: set[str] | None = None,
+    ) -> list[ProviderConfig]:
+        """Select one healthy profile per provider, prioritizing the target purpose.
+
+        A provider whose complete profile group is open in the circuit breaker is
+        absent from this list; the next healthy provider becomes the automatic
+        fallback. This prevents retry storms against an exhausted pool.
+        """
+        excluded = excluded or set()
+        candidates = [
+            provider for provider in self.providers
+            if provider.name not in excluded and self._is_available(provider)
+        ]
+        candidates.sort(key=lambda provider: (
+            0 if provider.purpose == purpose else 1,
+            provider.tier,
+            provider.name,
+            provider.profile,
+        ))
+        return self._execution_pool(candidates, f"fallback:{purpose}")
 
     async def _fetch_one(self, provider: ProviderConfig, messages: list, temperature: float, max_tokens: int) -> Tuple[ProviderConfig, str, float]:
         client = self._client_for(provider)
@@ -352,18 +373,42 @@ class AIClient:
             raise ProviderUnavailableError([])
         
         # 3. Un solo profilo per endpoint e richiesta: round-robin evita che
-        # profili OpenRouter condividano quota e client, mentre provider diversi
-        # restano disponibili come ensemble/fallback.
+        # profili condividano quota e client, mentre provider diversi restano
+        # disponibili come ensemble/fallback.
         pool = self._execution_pool(pool, primary_purpose)
-        tasks = [self._fetch_one(p, messages, temperature, max_tokens) for p in pool]
-        results = await asyncio.gather(*tasks)
-        
-        best_r = self._judge_best_response(results, primary_purpose)
-        
+        results = []
+        if pool:
+            tasks = [self._fetch_one(p, messages, temperature, max_tokens) for p in pool]
+            results = await asyncio.gather(*tasks)
+
+        valid = [result for result in results if not result[1].startswith("ERROR:") and len(result[1]) > 10]
+        if valid:
+            best_r = self._judge_best_response(results, primary_purpose)
+        else:
+            # Il pool primario è interamente in rate limit, errore auth o timeout:
+            # prova un solo profilo per ogni provider sano, in ordine di purpose/tier.
+            excluded = {provider.name for provider, _response, _latency in results}
+            fallback_pool = self._inter_provider_fallback_pool(primary_purpose, excluded)
+            fallback_results = []
+            for fallback in fallback_pool:
+                result = await self._fetch_one(fallback, messages, temperature, max_tokens)
+                fallback_results.append(result)
+                if not result[1].startswith("ERROR:") and len(result[1]) > 10:
+                    _logger.info(
+                        "[fleet] inter-provider fallback succeeded on %s/%s",
+                        fallback.name,
+                        fallback.profile,
+                    )
+                    best_r = result[1]
+                    break
+            else:
+                failed_names = [provider.name for provider, _response, _latency in results + fallback_results]
+                raise ProviderUnavailableError(failed_names)
+
         # S-CACHE-1: Popolamento cache asincrono
         if not best_r.startswith("🔴"):
             asyncio.create_task(set_cached_response(messages, best_r))
-            
+
         return best_r
 
     def _judge_best_response(self, results: List[Tuple[ProviderConfig, str, float]], target_purpose: str) -> str:

tests/test_provider_profile_pool.py

@@ -1,7 +1,7 @@
 import json
 import os
 import unittest
-from unittest.mock import patch
+from unittest.mock import AsyncMock, patch
 
 from models.ai_client import AIClient, ProviderConfig, _PROVIDER_DEFS
 
@@ -55,6 +55,15 @@ class ProviderProfilePoolTests(unittest.TestCase):
         self.assertEqual([p.profile for p in profiles], ["primary", "backup"])
         self.assertNotIn("database-key", [p.api_key for p in profiles])
 
+    def test_inter_provider_pool_excludes_exhausted_provider(self):
+        client = AIClient()
+        openrouter = ProviderConfig(name="openrouter", api_key="or", base_url="https://openrouter.ai/api/v1", profile="a", purpose="coding", tier=1)
+        groq = ProviderConfig(name="groq", api_key="groq", base_url="https://api.groq.com/openai/v1", profile="a", purpose="reasoning", tier=0)
+        client.providers = [openrouter, groq]
+        client._breaker[openrouter.identity] = {"failures": 2, "open_until": 10**12}
+        selected = client._inter_provider_fallback_pool("coding", {"openrouter"})
+        self.assertEqual([provider.name for provider in selected], ["groq"])
+
     def test_profiles_have_distinct_client_cache_entries(self):
         client = AIClient()
         first, second = self._profiles()[:2]
@@ -75,5 +84,25 @@ class ProviderProfilePoolTests(unittest.TestCase):
         self.assertNotEqual(selected[0].profile, "b")
 
 
+class InterProviderFallbackChatTests(unittest.IsolatedAsyncioTestCase):
+    async def test_chat_falls_back_when_primary_pool_returns_errors(self):
+        client = AIClient()
+        openrouter = ProviderConfig(name="openrouter", api_key="or", base_url="https://openrouter.ai/api/v1", profile="a", purpose="coding", tier=1)
+        groq = ProviderConfig(name="groq", api_key="groq", base_url="https://api.groq.com/openai/v1", profile="a", purpose="reasoning", tier=0)
+        client.providers = [openrouter, groq]
+
+        async def fake_fetch(provider, messages, temperature, max_tokens):
+            if provider.name == "openrouter":
+                return provider, "ERROR: HTTP 429 rate limit", 0.0
+            return provider, "fallback answer from healthy provider", 0.2
+
+        with patch("models.ai_client.get_cached_response", new=AsyncMock(return_value=None)), \
+             patch("models.ai_client.set_cached_response", new=AsyncMock()), \
+             patch.object(client, "_fetch_one", side_effect=fake_fetch):
+            answer = await client.chat([{"role": "user", "content": "write code"}])
+
+        self.assertEqual(answer, "fallback answer from healthy provider")
+
+
 if __name__ == "__main__":
     unittest.main()