feat: Add support for local model endpoints and enhance bot_response function to handle local vs. remote inference clients.

app.py

@@ -4,15 +4,22 @@ from huggingface_hub import InferenceClient
 MODELS = [
     "Qwen/Qwen2.5-72B-Instruct",
     "meta-llama/Llama-3.1-8B-Instruct",
+    "Local-Model-1",
+    "Local-Model-2",
 ]
 
-def bot_response(user_message, history, model_name, system_message, max_tokens, temperature, top_p, oauth_token: gr.OAuthToken | None):
+def bot_response(user_message, history, model_name, system_message, max_tokens, temperature, top_p, oauth_token: gr.OAuthToken | None, local_endpoint: str):
     if not user_message:
         yield history, ""
         return
         
     token = oauth_token.token if oauth_token else None
-    client = InferenceClient(token=token, model=model_name)
+    
+    # --- Logic switch: local vLLM vs Hugging Face Hub ---
+    if model_name.startswith("Local-"):
+        client = InferenceClient(base_url=local_endpoint, token="vllm-token")
+    else:
+        client = InferenceClient(token=token, model=model_name)
     
     # 1. Construct messages for API
     api_messages = [{"role": "system", "content": system_message}] + history + [{"role": "user", "content": user_message}]
@@ -48,6 +55,14 @@ with gr.Blocks() as demo:
     with gr.Sidebar():
         gr.Markdown("## Configuration")
         gr.LoginButton()
+        
+        # Local vLLM endpoint setting
+        local_endpoint = gr.Textbox(
+            value="http://localhost:8000/v1", 
+            label="Local vLLM Endpoint",
+            placeholder="http://127.0.0.1:8000/v1"
+        )
+        
         system_msg = gr.Textbox(value="You are a helpful assistant.", label="System Prompt")
         max_t = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max Tokens")
         temp = gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature")
@@ -62,7 +77,7 @@ with gr.Blocks() as demo:
             # Explicitly set type="messages"
             chatbot_a = gr.Chatbot(label="Model A Output", type="messages")
             msg_a = gr.Textbox(placeholder="Send message to Model A...", label="Model A Input")
-            btn_a = gr.Button("Send to A")
+            btn_a = gr.Button("Send to Model A")
 
         # --- Model B ---
         with gr.Column():
@@ -70,15 +85,15 @@ with gr.Blocks() as demo:
             # Explicitly set type="messages"
             chatbot_b = gr.Chatbot(label="Model B Output", type="messages")
             msg_b = gr.Textbox(placeholder="Send message to Model B...", label="Model B Input")
-            btn_b = gr.Button("Send to B")
+            btn_b = gr.Button("Send to Model B")
 
     # --- Bind Events ---
 
-    a_inputs = [msg_a, chatbot_a, model_a_name, system_msg, max_t, temp, top_p_val]
+    a_inputs = [msg_a, chatbot_a, model_a_name, system_msg, max_t, temp, top_p_val, local_endpoint]
     msg_a.submit(bot_response, a_inputs, [chatbot_a, msg_a])
     btn_a.click(bot_response, a_inputs, [chatbot_a, msg_a])
 
-    b_inputs = [msg_b, chatbot_b, model_b_name, system_msg, max_t, temp, top_p_val]
+    b_inputs = [msg_b, chatbot_b, model_b_name, system_msg, max_t, temp, top_p_val, local_endpoint]
     msg_b.submit(bot_response, b_inputs, [chatbot_b, msg_b])
     btn_b.click(bot_response, b_inputs, [chatbot_b, msg_b])