update app.py

app.py

@@ -6,6 +6,7 @@ import gradio as gr
 from huggingface_hub import InferenceClient
 
 
+# Default to GPT-OSS per your preference
 HF_MODEL = os.getenv("HF_MODEL", "openai/gpt-oss-20b")
 
 
@@ -78,28 +79,66 @@ def _oauth_or_env_token(hf_token: gr.OAuthToken | None):
     return os.getenv("HF_TOKEN")
 
 
+def _provider_for_model(model_id: str) -> str | None:
+    # Route provider explicitly when needed
+    if model_id.startswith("openai/"):
+        return "together"
+    return None
+
+
+def _provider_api_key(model_id: str) -> str | None:
+    prov = _provider_for_model(model_id)
+    if prov == "together":
+        return os.getenv("TOGETHER_API_KEY")
+    return None
+
+
+def _inference_params(hf_token: gr.OAuthToken | None, model_id: str):
+    provider = _provider_for_model(model_id)
+    api_key = _provider_api_key(model_id)
+    # Important: when using external providers (e.g., Together), do NOT pass HF OAuth/token,
+    # otherwise the router attempts a delegated call and may 403.
+    if provider:
+        token = None
+    else:
+        token = _oauth_or_env_token(hf_token)
+    return token, provider, api_key
+
+
 def bot_reply(history: List[Tuple[str, str]], temperature: float = 0.7, top_p: float = 0.95, max_tokens: int = 512, hf_token: gr.OAuthToken = None):
     last_user = history[-1][0] if history else ""
     messages = _build_messages(history[:-1], last_user)
-    client = InferenceClient(token=_oauth_or_env_token(hf_token), model=HF_MODEL)
+    token, provider, api_key = _inference_params(hf_token, HF_MODEL)
+    client = InferenceClient(token=token, model=HF_MODEL)
 
     acc = ""
-    for event in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = ""
-        try:
-            choices = event.choices
-            if len(choices) and getattr(choices[0], "delta", None) and choices[0].delta.content:
-                token = choices[0].delta.content
-        except Exception:
-            pass
-        acc += token
-        history[-1] = (history[-1][0], acc)
+    try:
+        for event in client.chat_completion(
+            messages,
+            max_tokens=max_tokens,
+            stream=True,
+            temperature=temperature,
+            top_p=top_p,
+            provider=provider,
+            api_key=api_key,
+        ):
+            token = ""
+            try:
+                choices = event.choices
+                if len(choices) and getattr(choices[0], "delta", None) and choices[0].delta.content:
+                    token = choices[0].delta.content
+            except Exception:
+                pass
+            acc += token
+            history[-1] = (history[-1][0], acc)
+            yield history
+    except Exception as e:
+        err = (
+            f"[Model error] {type(e).__name__}: {e}.\n"
+            "Tip: Click Sign in or set HF_TOKEN.\n"
+            "You can also set HF_MODEL to an HF-hosted chat model, e.g. 'meta-llama/Meta-Llama-3-8B-Instruct' or 'HuggingFaceH4/zephyr-7b-beta'."
+        )
+        history[-1] = (history[-1][0], err)
         yield history
 
 
@@ -108,7 +147,8 @@ def summarize(history: List[Tuple[str, str]], hf_token: gr.OAuthToken = None):
     if not transcript.strip():
         return {"note": "No conversation yet. Ask Maya some questions first."}
 
-    client = InferenceClient(token=_oauth_or_env_token(hf_token), model=HF_MODEL)
+    token, provider, api_key = _inference_params(hf_token, HF_MODEL)
+    client = InferenceClient(token=token, model=HF_MODEL)
     messages = [
         {"role": "system", "content": SUMMARY_SYSTEM_PROMPT},
         {
@@ -120,7 +160,18 @@ def summarize(history: List[Tuple[str, str]], hf_token: gr.OAuthToken = None):
         },
     ]
     # Non-streaming summarize for simplicity
-    event = client.chat_completion(messages, max_tokens=1024, stream=False, temperature=0.2, top_p=0.95)
+    try:
+        event = client.chat_completion(
+            messages,
+            max_tokens=1024,
+            stream=False,
+            temperature=0.2,
+            top_p=0.95,
+            provider=provider,
+            api_key=api_key,
+        )
+    except Exception as e:
+        return {"error": f"{type(e).__name__}: {e}"}
     # Extract content
     text = "{}"
     try: