add fallback models

app.py

@@ -86,8 +86,13 @@ class MinimalistTheme(Base):
 class RAGChatbot:
     """Chatbot with RAG capabilities."""
     
-    # Default model: use one supported by HF Inference API (Mistral often requires enabled providers)
-    DEFAULT_CHAT_MODEL = "HuggingFaceH4/zephyr-7b-beta"
+    # Default and fallback models (try in order until one is supported by your Inference API providers)
+    DEFAULT_CHAT_MODEL = "microsoft/phi-2"
+    FALLBACK_CHAT_MODELS = [
+        "HuggingFaceH4/zephyr-7b-beta",
+        "Qwen/Qwen2-7B-Instruct",
+        "google/gemma-2-2b-it",
+    ]
 
     def __init__(
         self,
@@ -104,8 +109,12 @@ class RAGChatbot:
             vector_store_path: Path to saved vector store
         """
         self.model_name = model_name if model_name else self.DEFAULT_CHAT_MODEL
+        # Build list of models to try (primary first, then fallbacks not already primary)
+        self._models_to_try = [self.model_name] + [
+            m for m in self.FALLBACK_CHAT_MODELS if m != self.model_name
+        ]
 
-        # Initialize Inference API client
+        # Initialize Inference API client (no model in constructor so we can try multiple)
         hf_token = os.environ.get("HF_TOKEN")
         # Debug: report HF_TOKEN status (masked)
         if not hf_token:
@@ -117,13 +126,10 @@ class RAGChatbot:
             print(f"[DEBUG] HF_TOKEN: set (length={len(hf_token)}, masked={masked})")
             print("HF_TOKEN found. Inference API ready.")
         
-        print(f"[DEBUG] Initializing Inference API client for model: {self.model_name}")
+        print(f"[DEBUG] Inference API client (models to try: {self._models_to_try})")
         try:
-            self.inference_client = InferenceClient(
-                model=self.model_name,
-                token=hf_token
-            )
-            print("[DEBUG] Inference API client initialized successfully (using chat_completion for this model)")
+            self.inference_client = InferenceClient(token=hf_token)
+            print("[DEBUG] Inference API client initialized (model chosen per request with fallbacks)")
         except Exception as e:
             print(f"[DEBUG] Error initializing Inference API client: {type(e).__name__}: {e}")
             self.inference_client = None
@@ -144,28 +150,39 @@ class RAGChatbot:
         self.chat_history = []
     
     def _generate_with_chat(self, user_content: str, max_new_tokens: int = 512) -> str:
-        """Call the Inference API using chat/comversational endpoint (required for Mistral instruct)."""
-        print(f"[DEBUG] _generate_with_chat: model={self.model_name}, API=chat_completion, prompt_len={len(user_content)}, max_tokens={max_new_tokens}")
-        try:
-            response = self.inference_client.chat_completion(
-                model=self.model_name,
-                messages=[{"role": "user", "content": user_content}],
-                max_tokens=max_new_tokens,
-                temperature=0.7,
-            )
-            print(f"[DEBUG] chat_completion response type: {type(response).__name__}")
-            # ChatCompletion has choices[0].message.content
-            if response and response.choices and len(response.choices) > 0:
-                msg = response.choices[0].message
-                if hasattr(msg, "content") and msg.content:
-                    return msg.content.strip()
-            print("[DEBUG] chat_completion returned empty or unexpected structure")
-            return ""
-        except Exception as e:
-            print(f"[DEBUG] _generate_with_chat exception: {type(e).__name__}: {e}")
-            import traceback
-            traceback.print_exc()
-            raise
+        """Call the Inference API using chat_completion; try fallback models if current is not supported."""
+        last_error = None
+        for model in self._models_to_try:
+            print(f"[DEBUG] _generate_with_chat: trying model={model}, prompt_len={len(user_content)}, max_tokens={max_new_tokens}")
+            try:
+                response = self.inference_client.chat_completion(
+                    model=model,
+                    messages=[{"role": "user", "content": user_content}],
+                    max_tokens=max_new_tokens,
+                    temperature=0.7,
+                )
+                print(f"[DEBUG] chat_completion OK for model={model}, response type: {type(response).__name__}")
+                if response and response.choices and len(response.choices) > 0:
+                    msg = response.choices[0].message
+                    if hasattr(msg, "content") and msg.content:
+                        # Remember this model for next time
+                        self.model_name = model
+                        self._models_to_try = [model] + [m for m in self._models_to_try if m != model]
+                        return msg.content.strip()
+                print("[DEBUG] chat_completion returned empty or unexpected structure")
+            except Exception as e:
+                last_error = e
+                err_str = str(e).lower()
+                if "model_not_supported" in err_str or "not supported by any provider" in err_str:
+                    print(f"[DEBUG] Model {model} not available, trying next fallback.")
+                    continue
+                print(f"[DEBUG] _generate_with_chat exception for {model}: {type(e).__name__}: {e}")
+                import traceback
+                traceback.print_exc()
+                raise
+        if last_error is not None:
+            raise last_error
+        return ""
     
     def generate_response(self, query: str, use_rag: bool = True, num_results: int = 5) -> str:
         """
@@ -217,8 +234,8 @@ Answer:"""
                         err_str = str(api_error).lower()
                         if "model_not_supported" in err_str or "not supported by any provider" in err_str:
                             return (
-                                "The current chat model isn't available with your Inference API providers. "
-                                "Try using the default model (Zephyr) or enable a provider for your chosen model at "
+                                "None of the configured chat models are available with your Inference API providers. "
+                                "Enable a provider for at least one supported model (e.g. Phi-2, Zephyr, Qwen) at "
                                 "https://huggingface.co/settings/inference-api."
                             )
                         # Fallback: return formatted chunks with note