change to conversational model

app.py

@@ -1,6 +1,15 @@
 """
 Gradio app for Hugging Face chatbot with RAG capabilities.
 """
+import warnings
+
+# Suppress deprecation from dependencies (e.g. accelerate) until they use torch.distributed.ReduceOp
+warnings.filterwarnings(
+    "ignore",
+    message=".*torch.distributed.reduce_op.*ReduceOp.*",
+    category=FutureWarning,
+)
+
 import gradio as gr
 from gradio.themes.base import Base
 from gradio.themes.utils import colors, fonts, sizes
@@ -127,6 +136,21 @@ class RAGChatbot:
         
         self.chat_history = []
     
+    def _generate_with_chat(self, user_content: str, max_new_tokens: int = 512) -> str:
+        """Call the Inference API using chat/comversational endpoint (required for Mistral instruct)."""
+        response = self.inference_client.chat_completion(
+            model=self.model_name,
+            messages=[{"role": "user", "content": user_content}],
+            max_tokens=max_new_tokens,
+            temperature=0.7,
+        )
+        # ChatCompletion has choices[0].message.content
+        if response and response.choices and len(response.choices) > 0:
+            msg = response.choices[0].message
+            if hasattr(msg, "content") and msg.content:
+                return msg.content.strip()
+        return ""
+    
     def generate_response(self, query: str, use_rag: bool = True, num_results: int = 5) -> str:
         """
         Generate a response to the user query using RAG and Inference API.
@@ -166,16 +190,12 @@ Question: {query}
 
 Answer:"""
                     
-                    # Generate response using Inference API
+                    # Generate response using chat/comversational API (Mistral instruct uses this)
                     try:
-                        response = self.inference_client.text_generation(
-                            prompt,
-                            max_new_tokens=512,
-                            temperature=0.7,
-                            top_p=0.9,
-                            return_full_text=False
-                        )
-                        return response.strip()
+                        response_text = self._generate_with_chat(prompt, max_new_tokens=512)
+                        if response_text:
+                            return response_text
+                        raise ValueError("Empty response from model")
                     except Exception as api_error:
                         print(f"Error calling Inference API: {api_error}")
                         # Fallback: return formatted chunks with note
@@ -201,15 +221,10 @@ Answer:"""
 Question: {query}
 
 Answer:"""
-            
-            response = self.inference_client.text_generation(
-                prompt,
-                max_new_tokens=256,
-                temperature=0.7,
-                top_p=0.9,
-                return_full_text=False
-            )
-            return response.strip()
+            response_text = self._generate_with_chat(prompt, max_new_tokens=256)
+            if response_text:
+                return response_text
+            return "I couldn't generate a response. Please try again."
         except Exception as e:
             print(f"Error generating response: {e}")
             return f"I encountered an error while generating a response: {str(e)}. Please check your HF_TOKEN configuration."