switch from openrouter to HF inference providers

app.py

@@ -7,13 +7,16 @@ terminal from "/". The frontend uses Gradio's JS client to call /chat, so
 it picks up queuing, SSE streaming, and concurrency control for free.
 
 Reasoning is preserved across turns: the frontend re-sends prior assistant
-turns with their `reasoning_details`, so Laguna S 2.1 continues reasoning
-from where it left off on the previous call.
+turns with their content, so Laguna S 2.1 continues the conversation.
+
+Backend talks to Hugging Face's inference router
+(https://router.huggingface.co/v1) using a provider-routed model id
+(`poolside/Laguna-S-2.1:<provider>`). HF_TOKEN is set as a Space secret;
+the X-HF-Bill-To header ensures the request is billed to the org.
 """
 
 import json
 import os
-import time
 from typing import Iterator
 
 from openai import OpenAI
@@ -22,19 +25,24 @@ from gradio import Server
 from fastapi.responses import HTMLResponse
 
 # ---------------------------------------------------------------------------
-# Backend: OpenAI-compatible client pointed at OpenRouter.
+# Backend: OpenAI-compatible client pointed at HF inference router.
 # ---------------------------------------------------------------------------
-API_KEY = os.environ.get("OPENROUTER_API_KEY") or os.environ.get("API_KEY") or ""
-MODEL = os.environ.get("MODEL", "poolside/laguna-s-2.1:free")
+HF_TOKEN = os.environ.get("HF_TOKEN") or ""
+INFERENCE_BASE_URL = os.environ.get(
+    "INFERENCE_BASE_URL", "https://router.huggingface.co/v1"
+)
+# Provider-routed model id, e.g. "poolside/Laguna-S-2.1:featherless-ai".
+MODEL = os.environ.get("MODEL", "poolside/Laguna-S-2.1:featherless-ai")
 
 client = OpenAI(
-    base_url="https://openrouter.ai/api/v1",
-    api_key=API_KEY or "missing-key",
+    api_key=HF_TOKEN or "missing-key",
+    base_url=INFERENCE_BASE_URL,
+    default_headers={"X-HF-Bill-To": "poolside"},
 )
 
 # ---------------------------------------------------------------------------
-# ANSI palette for the terminal look. The frontend keeps this table so it
-# can render lines verbatim (no markdown parsing).
+# ANSI palette for the terminal look. The Chatbot renders content verbatim
+# (render_markdown=False), so we emit raw escape codes and style the container.
 # ---------------------------------------------------------------------------
 RESET = "\033[0m"
 BOLD = "\033[1m"
@@ -43,48 +51,9 @@ ITALIC = "\033[3m"
 GREEN = "\033[32m"
 CYAN = "\033[36m"
 YELLOW = "\033[33m"
-MAGENTA = "\033[35m"
 RED = "\033[31m"
 GREY = "\033[90m"
 BOLD_GREEN = BOLD + GREEN
-BOLD_CYAN = BOLD + CYAN
-
-
-def _extract_reasoning(reasoning_details) -> str:
-    """Read text out of OpenRouter reasoning_details (list/dict/str, nested)."""
-    if not reasoning_details:
-        return ""
-    chunks: list[str] = []
-
-    def push(val):
-        if isinstance(val, str):
-            chunks.append(val)
-        elif isinstance(val, list):
-            for v in val:
-                push(v)
-        elif isinstance(val, dict):
-            for k in ("summary", "text", "content", "reasoning"):
-                if k in val:
-                    push(val[k])
-
-    if isinstance(reasoning_details, list):
-        for p in reasoning_details:
-            push(p)
-    elif isinstance(reasoning_details, dict):
-        push(reasoning_details)
-    elif isinstance(reasoning_details, str):
-        chunks.append(reasoning_details)
-
-    return "\n".join(c for c in chunks if c).strip()
-
-
-def _one_line(text: str) -> str:
-    return (text or "").replace("\n", " ").strip()
-
-
-def _emit(payload: dict) -> dict:
-    """Strip non-JSON-safe bits before yielding (we only stream ASCII)."""
-    return {"event": payload.get("event", "delta"), "text": payload.get("text", "")}
 
 
 def _coerce_messages(messages):
@@ -96,8 +65,7 @@ def _coerce_messages(messages):
         return []
     if isinstance(messages, str):
         try:
-            import json as _json
-            decoded = _json.loads(messages)
+            decoded = json.loads(messages)
             return decoded if isinstance(decoded, list) else []
         except Exception:
             return []
@@ -118,22 +86,24 @@ def _coerce_bool(value, default=True):
     return bool(value) if value is not None else default
 
 
+def _delta(payload: dict) -> dict:
+    return {"event": "delta", "text": payload.get("text", "")}
+
+
 # ---------------------------------------------------------------------------
 # /chat endpoint. With gradio.Server, a generator-typed function streams via
-# SSE. We yield small text events (think: print-style lines the terminal
-# renders as they arrive) and finish with a `done` event carrying the new
-# assistant message — the frontend appends that to its in-memory history
-# along with the original `reasoning_details`.
+# SSE. We yield per-token deltas as they arrive from the poolside API, then
+# a final `done` event with the accumulated content so the frontend can
+# persist it for the next turn.
 # ---------------------------------------------------------------------------
 app = Server()
 
 
 @app.api()
 def chat(messages: list, reasoning: bool = True) -> Iterator[dict]:
-    """Reasoning-aware chat endpoint. Takes `messages` (list of
-    role/content/reasoning_details dicts) and an optional `reasoning` bool.
-    Streams `{"event":"delta","text":...}` lines, then a final `done` event
-    with the new assistant message so the frontend can persist it."""
+    """Streaming chat endpoint. Takes `messages` (list of role/content dicts)
+    and an optional `reasoning` bool. Streams `{"event":"delta","text":...}`
+    per token, then a final `done` event with the full assistant content."""
     messages = _coerce_messages(messages)
     reasoning = _coerce_bool(reasoning, default=True)
 
@@ -142,77 +112,63 @@ def chat(messages: list, reasoning: bool = True) -> Iterator[dict]:
     )
     user_text = (last_user or {}).get("content", "")
 
-    yield _emit({
-        "event": "delta",
-        "text": f"{GREY}$ {RESET}{GREEN}user{RESET} {DIM}»{RESET} {_one_line(user_text)}\n",
+    yield _delta({
+        "text": f"{GREY}$ {RESET}{GREEN}user{RESET} {DIM}»{RESET} {user_text.replace(chr(10), ' ').strip()}\n",
     })
-
-    yield _emit({
-        "event": "delta",
+    yield _delta({
         "text": (
-            f"{CYAN}laguna{RESET}{GREY}@{RESET}{YELLOW}openrouter{RESET}"
+            f"{CYAN}laguna{RESET}{GREY}@{RESET}{YELLOW}huggingface{RESET}"
             f"{GREY}:{RESET}{DIM}~{RESET} {GREY}{'thinking…' if reasoning else 'responding…'}{RESET}\n"
         ),
     })
 
+    full_text_parts: list[str] = []
+    in_answer = False
+
     try:
         response = client.chat.completions.create(
             model=MODEL,
             messages=messages,
-            extra_body={"reasoning": {"enabled": reasoning}},
+            stream=True,
         )
-        message = response.choices[0].message
+
+        for chunk in response:
+            if not chunk.choices:
+                continue
+            choice = chunk.choices[0]
+            delta = getattr(choice, "delta", None)
+            if delta is None:
+                continue
+            piece = getattr(delta, "content", None)
+            if piece is None:
+                continue
+
+            if not in_answer:
+                # First content chunk — open the answer box.
+                yield _delta({
+                    "text": f"{BOLD_GREEN}┌─ laguna ──────────────────────────{RESET}\n",
+                })
+                in_answer = True
+
+            full_text_parts.append(piece)
+            yield _delta({"text": piece})
     except Exception as e:
-        yield _emit({
-            "event": "delta",
+        yield _delta({
             "text": f"\n{BOLD}{RED}error{RESET} {DIM}»{RESET} {e}\n",
         })
-        yield {"event": "done", "text": "", "assistant": None}
-        return
-
-    reasoning_text = _extract_reasoning(getattr(message, "reasoning_details", None))
 
-    if reasoning_text:
-        yield _emit({
-            "event": "delta",
-            "text": f"{DIM}{ITALIC}┌─ reasoning ───────────────────────{RESET}\n",
+    if in_answer:
+        yield _delta({
+            "text": f"\n{BOLD_GREEN}└───────────────────────────────────{RESET}\n",
         })
-        for line in reasoning_text.splitlines() or [""]:
-            prefix = f"{DIM}{ITALIC}│ {RESET}" if line else f"{DIM}│{RESET} "
-            yield _emit({"event": "delta", "text": prefix + line + "\n"})
-            time.sleep(0.012)
-        yield _emit({
-            "event": "delta",
-            "text": f"{DIM}{ITALIC}└───────────────────────────────────{RESET}\n",
-        })
-
-    answer = (message.content or "").strip()
-    yield _emit({
-        "event": "delta",
-        "text": f"{BOLD_GREEN}┌─ laguna ──────────────────────────{RESET}\n",
-    })
-    for line in answer.splitlines() or [""]:
-        prefix = f"{BOLD_GREEN}│ {RESET}" if line else f"{GREEN}│{RESET} "
-        yield _emit({"event": "delta", "text": prefix + line + "\n"})
-        time.sleep(0.012)
-    yield _emit({
-        "event": "delta",
-        "text": f"{BOLD_GREEN}└───────────────────────────────────{RESET}\n",
-    })
-    yield _emit({"event": "delta", "text": f"{GREY}── done ──{RESET}\n"})
-
-    # The new assistant message — keep reasoning_details so the JS client
-    # can re-send it on the next turn, letting Laguna continue its reasoning.
-    rd = getattr(message, "reasoning_details", None)
-    rd_json = json.loads(json.dumps(rd, default=str)) if rd else None
+    yield _delta({"text": f"{GREY}── done ──{RESET}\n"})
 
     yield {
         "event": "done",
         "text": "",
         "assistant": {
             "role": "assistant",
-            "content": message.content or "",
-            "reasoning_details": rd_json,
+            "content": "".join(full_text_parts),
         },
     }
 
@@ -234,4 +190,4 @@ async def healthz():
 
 
 if __name__ == "__main__":
-    app.launch(show_error=True)
+    app.launch(show_error=True)

index.html

@@ -344,7 +344,7 @@
         </div>
 
         <div class="footline">
-          <span>model: <span class="grad" style="font-weight:600">poolside/laguna-s-2.1:free</span> · via openrouter · lossless chain-of-thought across turns via <code>reasoning_details</code></span>
+          <span>model: <span class="grad" style="font-weight:600">poolside/Laguna-S-2.1:featherless-ai</span> · via <code>router.huggingface.co/v1</code> · lossless chain-of-thought across turns via <code>reasoning_details</code></span>
           <span>built with gradio <code>Server</code> + @gradio/client</span>
         </div>
       </div>
@@ -371,7 +371,7 @@
 
     const banner = [
       `${ANSI.green}● ${ANSI.reset}Connected to agent server: ${ANSI.bold}Poolside v2.1${ANSI.reset}`,
-      `${ANSI.dim}  session id: robust-lake-${Math.floor(Math.random()*9000+1000)} · model poolside/laguna-s-2.1:free · openrouter${ANSI.reset}`,
+      `${ANSI.dim}  session id: robust-lake-${Math.floor(Math.random()*9000+1000)} · model poolside/Laguna-S-2.1:featherless-ai · huggingface router${ANSI.reset}`,
       "",
       `${ANSI.cyan}>${ANSI.reset} type a question below. the model ${ANSI.bold}reasons before it answers${ANSI.reset}, and remembers its thoughts on the next turn.`,
       "",