checkpoint: current behavior before latency work

backend/app/config.py

@@ -11,54 +11,77 @@ class ModelSpec:
     """Static description of a model available through OpenRouter."""
 
     def __init__(self, key: str, openrouter_id: str, display_name: str,
-                 cost_per_mtok_in: float, cost_per_mtok_out: float):
+                 cost_per_mtok_in: float, cost_per_mtok_out: float,
+                 fallback_id: str | None = None, specialty: str = ""):
         self.key = key
         self.openrouter_id = openrouter_id
         self.display_name = display_name
-        # USD per 1M tokens (approximate OpenRouter pricing, used for the
-        # auction cost term and the savings metric)
+        # One-line self-description injected into the bid prompt so the
+        # model bids according to its actual strengths
+        self.specialty = specialty
+        # USD per 1M tokens: pricing of the PAID model in this slot (the
+        # fallback when the primary is a free variant). Used for the auction
+        # cost term and the savings metric.
         self.cost_per_mtok_in = cost_per_mtok_in
         self.cost_per_mtok_out = cost_per_mtok_out
-
-    def estimate_cost(self, tokens_in: int, tokens_out: int) -> float:
+        # Paid model tried automatically when the free primary is
+        # rate-limited (OpenRouter `models` fallback routing)
+        self.fallback_id = fallback_id
+
+    def estimate_cost(self, tokens_in: int, tokens_out: int,
+                      served_model: str | None = None) -> float:
+        if served_model and served_model.endswith(":free"):
+            return 0.0
         return (tokens_in * self.cost_per_mtok_in
                 + tokens_out * self.cost_per_mtok_out) / 1_000_000
 
 
 # --- Tier 1: cheap bidders -------------------------------------------------
 TIER1_MODELS: dict[str, ModelSpec] = {
-    "gemma": ModelSpec(
-        key="gemma",
+    # Bidders/verifier run free-first with an automatic paid fallback when
+    # the free pool is rate-limited. Pricing fields = the paid fallback.
+    "gemini": ModelSpec(
+        key="gemini",
         openrouter_id="google/gemma-4-26b-a4b-it:free",
-        display_name="Gemma 4 26B",
-        cost_per_mtok_in=0.0,
-        cost_per_mtok_out=0.0,
+        fallback_id="google/gemini-2.5-flash-lite",
+        display_name="Gemma 4 / Gemini Lite",
+        cost_per_mtok_in=0.10,
+        cost_per_mtok_out=0.40,
+        specialty="a fast lightweight generalist: strong at general knowledge, "
+                  "summaries, and everyday questions; NOT a code specialist and "
+                  "weak at hard math proofs and complex multi-step reasoning — "
+                  "defer those to the specialists",
     ),
     "deepseek": ModelSpec(
         key="deepseek",
-        openrouter_id="deepseek/deepseek-chat",
+        openrouter_id="deepseek/deepseek-chat",  # no free variant; cheapest paid
         display_name="DeepSeek",
         cost_per_mtok_in=0.20,
         cost_per_mtok_out=0.80,
+        specialty="strongest at mathematical reasoning, logic puzzles, and "
+                  "quantitative problems; solid at code; average at niche "
+                  "world knowledge",
     ),
-    # Non-thinking instruct variant: thinking models burn the whole
-    # max_tokens budget on hidden reasoning and return empty content
     "qwen": ModelSpec(
         key="qwen",
-        openrouter_id="qwen/qwen3-next-80b-a3b-instruct:free",
-        display_name="Qwen3 Next 80B",
-        cost_per_mtok_in=0.0,
-        cost_per_mtok_out=0.0,
+        openrouter_id="qwen/qwen3-coder:free",
+        fallback_id="qwen/qwen3-coder",
+        display_name="Qwen3 Coder",
+        cost_per_mtok_in=0.22,
+        cost_per_mtok_out=1.80,
+        specialty="a coding specialist: strongest at writing, debugging, and "
+                  "explaining code and software architecture; weaker at "
+                  "non-technical general knowledge",
     ),
 }
 
 # --- Verifier ---------------------------------------------------------------
 VERIFIER_MODEL = ModelSpec(
     key="verifier",
-    openrouter_id="google/gemma-4-31b-it:free",
-    display_name="Gemma 4 31B (Verifier)",
-    cost_per_mtok_in=0.0,
-    cost_per_mtok_out=0.0,
+    openrouter_id="openai/gpt-oss-120b",
+    display_name="GPT-OSS 120B (Verifier)",
+    cost_per_mtok_in=0.04,
+    cost_per_mtok_out=0.17,
 )
 
 # --- Tier 2: frontier escalation target -------------------------------------
@@ -93,6 +116,9 @@ class Settings(BaseSettings):
     min_auction_confidence: float = 0.75
     verification_threshold: float = 0.80
     disagreement_stddev: float = 0.22
+    # Skip the disagreement check when some bidder is at least this
+    # confident (specialists legitimately disagree with generalists)
+    disagreement_exempt_confidence: float = 0.85
 
     # Default historical accuracy for models with no track record yet
     default_historical_accuracy: float = 0.70
@@ -104,9 +130,19 @@ class Settings(BaseSettings):
     # (also keeps low-credit OpenRouter keys usable)
     max_answer_tokens: int = 2000
     max_bid_tokens: int = 300
+
+    # Conversation history caps per pipeline stage (turns are single
+    # messages, so 4 turns = 2 user/assistant exchanges)
+    history_max_turns_bid: int = 4
+    history_max_chars_bid: int = 1600
+    history_max_turns_answer: int = 12
+    history_max_chars_answer: int = 12000
+    history_max_turns_verify: int = 6
+    history_max_chars_verify: int = 4000
     # Frontier gets extra headroom since reasoning tokens count against
-    # the cap even at low effort
-    max_frontier_tokens: int = 4000
+    # the cap (medium effort thinks longer than low)
+    max_frontier_tokens: int = 16000
+    frontier_reasoning_effort: str = "medium"
 
 
 settings = Settings()

backend/app/llm.py

@@ -16,11 +16,13 @@ class LLMError(Exception):
 
 
 class LLMResponse:
-    def __init__(self, content: str, tokens_in: int, tokens_out: int, latency_ms: int):
+    def __init__(self, content: str, tokens_in: int, tokens_out: int,
+                 latency_ms: int, served_model: str = ""):
         self.content = content
         self.tokens_in = tokens_in
         self.tokens_out = tokens_out
         self.latency_ms = latency_ms
+        self.served_model = served_model  # which model actually answered
 
 
 _client: httpx.AsyncClient | None = None
@@ -48,21 +50,31 @@ async def close_client() -> None:
         _client = None
 
 
+def _build_messages(system: str, user: str,
+                    history: list[dict] | None) -> list[dict]:
+    return [
+        {"role": "system", "content": system},
+        *({"role": t["role"], "content": t["content"]} for t in (history or [])),
+        {"role": "user", "content": user},
+    ]
+
+
 async def chat(model: ModelSpec, system: str, user: str,
                timeout: float | None = None,
                max_tokens: int | None = None,
-               reasoning_effort: str | None = None) -> LLMResponse:
+               reasoning_effort: str | None = None,
+               history: list[dict] | None = None) -> LLMResponse:
     start = time.monotonic()
     body: dict = {
         "model": model.openrouter_id,
-        "messages": [
-            {"role": "system", "content": system},
-            {"role": "user", "content": user},
-        ],
+        "messages": _build_messages(system, user, history),
         "max_tokens": max_tokens or settings.max_answer_tokens,
     }
     if reasoning_effort:
         body["reasoning"] = {"effort": reasoning_effort}
+    if model.fallback_id:
+        # OpenRouter fallback routing: try free primary, then paid fallback
+        body["models"] = [model.openrouter_id, model.fallback_id]
 
     # Free-tier models often 429 transiently ("rate-limited upstream,
     # retry shortly"), so retry a couple of times honoring Retry-After.
@@ -94,9 +106,87 @@ async def chat(model: ModelSpec, system: str, user: str,
         tokens_in=usage.get("prompt_tokens", 0),
         tokens_out=usage.get("completion_tokens", 0),
         latency_ms=latency_ms,
+        served_model=data.get("model", model.openrouter_id),
     )
 
 
+async def chat_stream(model: ModelSpec, system: str, user: str,
+                      timeout: float | None = None,
+                      max_tokens: int | None = None,
+                      reasoning_effort: str | None = None,
+                      history: list[dict] | None = None):
+    """Streaming variant of chat().
+
+    Yields {"type": "delta", "text": ...} per token chunk, then a final
+    {"type": "final", "response": LLMResponse} with full content and usage.
+    """
+    body: dict = {
+        "model": model.openrouter_id,
+        "messages": _build_messages(system, user, history),
+        "max_tokens": max_tokens or settings.max_answer_tokens,
+        "stream": True,
+        "stream_options": {"include_usage": True},
+    }
+    if reasoning_effort:
+        body["reasoning"] = {"effort": reasoning_effort}
+    if model.fallback_id:
+        body["models"] = [model.openrouter_id, model.fallback_id]
+
+    start = time.monotonic()
+    parts: list[str] = []
+    tokens_in = tokens_out = 0
+    served = model.openrouter_id
+
+    attempts = 3
+    for attempt in range(attempts):
+        async with get_client().stream(
+            "POST", "/chat/completions", json=body,
+            timeout=timeout or settings.request_timeout_s,
+        ) as resp:
+            if resp.status_code == 429 and attempt < attempts - 1:
+                retry_after = min(float(resp.headers.get("Retry-After", 2)), 5.0)
+                await asyncio.sleep(retry_after)
+                continue
+            if resp.status_code != 200:
+                text = (await resp.aread()).decode(errors="replace")
+                raise LLMError(f"{model.openrouter_id}: HTTP {resp.status_code}: {text[:300]}")
+            async for line in resp.aiter_lines():
+                if not line.startswith("data: "):
+                    continue
+                payload = line[len("data: "):].strip()
+                if payload == "[DONE]":
+                    break
+                try:
+                    data = json.loads(payload)
+                except json.JSONDecodeError:
+                    continue
+                if "error" in data:
+                    raise LLMError(f"{model.openrouter_id}: {str(data['error'])[:300]}")
+                served = data.get("model", served)
+                usage = data.get("usage")
+                if usage:
+                    tokens_in = usage.get("prompt_tokens", tokens_in)
+                    tokens_out = usage.get("completion_tokens", tokens_out)
+                choices = data.get("choices") or []
+                if choices:
+                    piece = (choices[0].get("delta") or {}).get("content") or ""
+                    if piece:
+                        parts.append(piece)
+                        yield {"type": "delta", "text": piece}
+        break
+
+    yield {
+        "type": "final",
+        "response": LLMResponse(
+            content="".join(parts),
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            latency_ms=int((time.monotonic() - start) * 1000),
+            served_model=served,
+        ),
+    }
+
+
 def extract_json(text: str) -> dict[str, Any]:
     """Pull the first JSON object out of a model response.
 

backend/app/main.py

@@ -11,9 +11,13 @@ from fastapi import FastAPI, HTTPException  # noqa: E402
 from fastapi.middleware.cors import CORSMiddleware  # noqa: E402
 from fastapi.staticfiles import StaticFiles  # noqa: E402
 
+import json  # noqa: E402
+
+from fastapi.responses import StreamingResponse  # noqa: E402
+
 from .config import TIER1_MODELS, TIER2_MODEL, VERIFIER_MODEL, settings  # noqa: E402
 from .llm import close_client  # noqa: E402
-from .pipeline import run_query  # noqa: E402
+from .pipeline import run_query, run_query_stream  # noqa: E402
 from .schemas import MetricsSummary, QueryRequest, RunResult  # noqa: E402
 from .store import get_store  # noqa: E402
 
@@ -50,7 +54,24 @@ async def health():
 async def query(req: QueryRequest):
     if not settings.openrouter_api_key:
         raise HTTPException(status_code=503, detail="OPENROUTER_API_KEY is not set")
-    return await run_query(req.query)
+    return await run_query(req.query, [t.model_dump() for t in req.history])
+
+
+@app.post("/api/query/stream")
+async def query_stream(req: QueryRequest):
+    if not settings.openrouter_api_key:
+        raise HTTPException(status_code=503, detail="OPENROUTER_API_KEY is not set")
+
+    history = [t.model_dump() for t in req.history]
+
+    async def gen():
+        try:
+            async for event in run_query_stream(req.query, history):
+                yield json.dumps(event) + "\n"
+        except Exception as e:  # surface pipeline crashes to the client
+            yield json.dumps({"type": "error", "message": str(e)[:300]}) + "\n"
+
+    return StreamingResponse(gen(), media_type="application/x-ndjson")
 
 
 @app.get("/api/runs", response_model=list[RunResult])

backend/app/pipeline.py

@@ -9,6 +9,7 @@ verifier gates the draft; failures escalate to the frontier model.
 """
 
 import asyncio
+import re
 import statistics
 import time
 import uuid
@@ -25,6 +26,7 @@ from .store import get_store
 
 class RouterState(TypedDict, total=False):
     query: str
+    history: list[dict]
     bids: list[Bid]
     winner: Optional[str]
     draft_answer: Optional[str]
@@ -42,11 +44,13 @@ def _clamp(x: float) -> float:
     return max(0.0, min(1.0, float(x)))
 
 
-async def _get_bid(model_key: str, query: str, history: dict[str, float]) -> tuple[Bid, Optional[Usage]]:
+async def _get_bid(model_key: str, query: str, accuracy: dict[str, float],
+                   chat_history: list[dict]) -> tuple[Bid, Optional[Usage]]:
     spec = TIER1_MODELS[model_key]
-    hist = history.get(model_key, settings.default_historical_accuracy)
+    hist = accuracy.get(model_key, settings.default_historical_accuracy)
     try:
-        resp = await chat(spec, prompts.BID_SYSTEM, prompts.bid_user(query),
+        resp = await chat(spec, prompts.BID_SYSTEM,
+                          prompts.bid_user(query, chat_history, spec.specialty),
                           timeout=settings.bid_timeout_s,
                           max_tokens=settings.max_bid_tokens)
         data = extract_json(resp.content)
@@ -61,7 +65,7 @@ async def _get_bid(model_key: str, query: str, history: dict[str, float]) -> tup
         usage = Usage(
             model_key=model_key, model_name=spec.display_name, stage="bid",
             tokens_in=resp.tokens_in, tokens_out=resp.tokens_out,
-            cost_usd=spec.estimate_cost(resp.tokens_in, resp.tokens_out),
+            cost_usd=spec.estimate_cost(resp.tokens_in, resp.tokens_out, resp.served_model),
             latency_ms=resp.latency_ms,
         )
         return bid, usage
@@ -72,15 +76,21 @@ async def _get_bid(model_key: str, query: str, history: dict[str, float]) -> tup
 
 
 async def bid_collection(state: RouterState) -> RouterState:
-    history = await get_store().historical_accuracy()
+    accuracy = await get_store().historical_accuracy()
     results = await asyncio.gather(
-        *(_get_bid(key, state["query"], history) for key in TIER1_MODELS)
+        *(_get_bid(key, state["query"], accuracy, state.get("history", []))
+          for key in TIER1_MODELS)
     )
     bids = [b for b, _ in results]
     usages = [u for _, u in results if u is not None]
 
-    # Normalize per-model output cost to 0..1 for the auction's cost term
-    costs = {k: m.cost_per_mtok_out for k, m in TIER1_MODELS.items()}
+    # Normalize per-model output cost to 0..1 for the auction's cost term.
+    # Free-primary models are discounted: they only cost their fallback
+    # price when the free pool is congested (~30% of the time).
+    costs = {
+        k: m.cost_per_mtok_out * (0.3 if m.openrouter_id.endswith(":free") else 1.0)
+        for k, m in TIER1_MODELS.items()
+    }
     max_cost = max(costs.values()) or 1.0
     for bid in bids:
         bid.cost_factor = costs[bid.model_key] / max_cost
@@ -105,7 +115,11 @@ async def auction(state: RouterState) -> RouterState:
         return {"escalated": True,
                 "escalation_reason": f"Low auction confidence (max {max_conf:.2f} < {settings.min_auction_confidence})"}
 
-    if len(confidences) >= 2:
+    # Disagreement only matters when nobody is sure: with specialist
+    # bidders, a wide spread (coder bids 0.3 on a trivia question) is the
+    # system working, not a red flag — so skip the check when a model is
+    # highly confident.
+    if len(confidences) >= 2 and max_conf < settings.disagreement_exempt_confidence:
         spread = statistics.pstdev(confidences)
         if spread > settings.disagreement_stddev:
             return {"escalated": True,
@@ -118,7 +132,8 @@ async def auction(state: RouterState) -> RouterState:
 async def draft(state: RouterState) -> RouterState:
     spec = TIER1_MODELS[state["winner"]]
     try:
-        resp = await chat(spec, prompts.ANSWER_SYSTEM, state["query"])
+        resp = await chat(spec, prompts.ANSWER_SYSTEM, state["query"],
+                          history=state.get("history"))
     except LLMError as e:
         return {"escalated": True, "escalation_reason": f"Winner failed to answer: {str(e)[:150]}"}
     if not resp.content.strip():
@@ -133,15 +148,35 @@ async def draft(state: RouterState) -> RouterState:
     return {"draft_answer": resp.content, "usages": state["usages"] + [usage]}
 
 
+# Leaked chain-of-thought in a "final" answer means the model was struggling;
+# the verifier should never pass it even if the end value happens to be right.
+_THINKING_ARTIFACTS = re.compile(
+    r"(?im)^\s*(wait|hmm+|hold on)\b"
+    r"|\b(wait,? (?:no|but|that)|hmm+,|let me (?:recalculate|recheck|reconsider|try again|start over)"
+    r"|actually,? (?:no|wait|that'?s (?:wrong|not right))|scratch that|i made an? (?:error|mistake))\b"
+)
+
+
 async def verify(state: RouterState) -> RouterState:
     try:
         resp = await chat(VERIFIER_MODEL, prompts.VERIFY_SYSTEM,
-                          prompts.verify_user(state["query"], state["draft_answer"]))
+                          prompts.verify_user(state["query"], state["draft_answer"],
+                                              state.get("history")),
+                          reasoning_effort="medium")
         data = extract_json(resp.content)
         score = _clamp(data.get("score", 0))
+        # Enforce score = min(subscores) server-side; models sometimes
+        # report an optimistic overall despite a low dimension
+        subscores = [_clamp(data[k]) for k in
+                     ("correctness", "completeness", "commitment", "presentation")
+                     if k in data]
+        if subscores:
+            score = min(score, *subscores)
         verification = Verification(
             score=score,
-            passed=bool(data.get("pass", score >= settings.verification_threshold)),
+            # our (possibly stricter) score overrides the model's own verdict
+            passed=score >= settings.verification_threshold
+            and bool(data.get("pass", True)),
             feedback=str(data.get("feedback", ""))[:500],
         )
     except (LLMError, ValueError) as e:
@@ -150,6 +185,18 @@ async def verify(state: RouterState) -> RouterState:
                                     feedback=f"Verifier error: {str(e)[:150]}")
         resp = None
 
+    # Deterministic guard: cap the score when the draft contains
+    # thinking-out-loud artifacts, independent of the verifier's judgment
+    artifacts = _THINKING_ARTIFACTS.findall(state["draft_answer"] or "")
+    if artifacts and verification.score > 0.5:
+        verification = Verification(
+            score=0.5,
+            passed=False,
+            feedback="Draft contains unresolved reasoning artifacts "
+                     "(thinking out loud / self-corrections). "
+                     + verification.feedback,
+        )
+
     usages = state["usages"]
     if resp is not None:
         usages = usages + [Usage(
@@ -170,9 +217,10 @@ async def verify(state: RouterState) -> RouterState:
 
 async def escalate(state: RouterState) -> RouterState:
     try:
-        resp = await chat(TIER2_MODEL, prompts.ANSWER_SYSTEM, state["query"],
+        resp = await chat(TIER2_MODEL, prompts.FRONTIER_SYSTEM, state["query"],
                           max_tokens=settings.max_frontier_tokens,
-                          reasoning_effort="low")
+                          reasoning_effort=settings.frontier_reasoning_effort,
+                          history=state.get("history"))
         if not resp.content.strip():
             raise LLMError(f"{TIER2_MODEL.openrouter_id}: empty response "
                            "(reasoning consumed the token budget)")
@@ -253,11 +301,7 @@ def get_graph():
     return _graph
 
 
-async def run_query(query: str) -> RunResult:
-    start = time.monotonic()
-    state: RouterState = {"query": query, "usages": [], "escalated": False}
-    final = await get_graph().ainvoke(state)
-
+def _make_run(query: str, final: dict, start: float) -> RunResult:
     usages = final.get("usages", [])
     total_cost = sum(u.cost_usd for u in usages)
     # Baseline: the same in/out volume sent straight to the frontier model
@@ -266,7 +310,7 @@ async def run_query(query: str) -> RunResult:
     baseline_cost = BASELINE_MODEL.estimate_cost(
         max(answer_tokens_in, 100), max(answer_tokens_out, 300))
 
-    run = RunResult(
+    return RunResult(
         id=uuid.uuid4().hex[:12],
         query=query,
         answer=final.get("final_answer", ""),
@@ -283,5 +327,139 @@ async def run_query(query: str) -> RunResult:
         baseline_cost_usd=round(baseline_cost, 6),
         latency_ms=int((time.monotonic() - start) * 1000),
     )
+
+
+def _trim_history(history: list[dict] | None) -> list[dict]:
+    """Answer-level cap: most recent turns, per-turn char truncation."""
+    turns = (history or [])[-settings.history_max_turns_answer:]
+    per_turn = settings.history_max_chars_answer // max(len(turns), 1)
+    return [{"role": t["role"], "content": t["content"][:max(per_turn, 500)]}
+            for t in turns]
+
+
+async def run_query(query: str, history: list[dict] | None = None) -> RunResult:
+    start = time.monotonic()
+    state: RouterState = {"query": query, "history": _trim_history(history),
+                          "usages": [], "escalated": False}
+    final = await get_graph().ainvoke(state)
+    run = _make_run(query, final, start)
     await get_store().save_run(run)
     return run
+
+
+async def run_query_stream(query: str, history: list[dict] | None = None):
+    """Streaming twin of the LangGraph pipeline.
+
+    Reuses the same node functions but drives them imperatively so token
+    deltas and stage transitions can be pushed to the client as they happen.
+    Yields JSON-serializable event dicts; ends with {"type": "done", "run": ...}.
+    """
+    from .llm import chat_stream
+
+    start = time.monotonic()
+    state: dict = {"query": query, "history": _trim_history(history),
+                   "usages": [], "escalated": False}
+
+    yield {"type": "stage", "stage": "bidding"}
+    state.update(await bid_collection(state))
+    state.update(await auction(state))
+    yield {
+        "type": "auction",
+        "bids": [b.model_dump() for b in state["bids"]],
+        "winner": state.get("winner"),
+        "escalated": state.get("escalated", False),
+        "reason": state.get("escalation_reason"),
+    }
+
+    if not state.get("escalated"):
+        spec = TIER1_MODELS[state["winner"]]
+        yield {"type": "stage", "stage": "drafting", "model": spec.display_name}
+        try:
+            # Draft tokens are NOT forwarded to the client: the draft isn't
+            # final until verification passes, and streaming text that later
+            # gets replaced by the frontier answer is a confusing UX.
+            resp = None
+            async for ev in chat_stream(spec, prompts.ANSWER_SYSTEM, query,
+                                        history=state["history"]):
+                if ev["type"] == "final":
+                    resp = ev["response"]
+            if resp is None or not resp.content.strip():
+                state["escalated"] = True
+                state["escalation_reason"] = f"{spec.display_name} returned an empty draft"
+            else:
+                state["draft_answer"] = resp.content
+                state["usages"] = state["usages"] + [Usage(
+                    model_key=spec.key, model_name=spec.display_name, stage="draft",
+                    tokens_in=resp.tokens_in, tokens_out=resp.tokens_out,
+                    cost_usd=spec.estimate_cost(resp.tokens_in, resp.tokens_out,
+                                                resp.served_model),
+                    latency_ms=resp.latency_ms,
+                )]
+        except LLMError as e:
+            state["escalated"] = True
+            state["escalation_reason"] = f"Winner failed to answer: {str(e)[:150]}"
+
+        if state.get("draft_answer"):
+            yield {"type": "stage", "stage": "verifying"}
+            state.update(await verify(state))
+            yield {
+                "type": "verification",
+                **state["verification"].model_dump(),
+                "escalated": state.get("escalated", False),
+                "reason": state.get("escalation_reason"),
+            }
+            if not state.get("escalated"):
+                # Draft is now verified-final: stream it to the client in
+                # chunks (it was generated silently during the draft stage)
+                yield {"type": "stage", "stage": "delivering",
+                       "model": spec.display_name}
+                text = state["draft_answer"]
+                step = 80
+                for i in range(0, len(text), step):
+                    yield {"type": "token", "text": text[i:i + step]}
+                    await asyncio.sleep(0.02)
+
+    if state.get("escalated"):
+        yield {"type": "stage", "stage": "escalating",
+               "model": TIER2_MODEL.display_name,
+               "reason": state.get("escalation_reason")}
+        try:
+            resp = None
+            async for ev in chat_stream(TIER2_MODEL, prompts.FRONTIER_SYSTEM, query,
+                                        max_tokens=settings.max_frontier_tokens,
+                                        reasoning_effort=settings.frontier_reasoning_effort,
+                                        history=state["history"]):
+                if ev["type"] == "delta":
+                    yield {"type": "token", "text": ev["text"]}
+                else:
+                    resp = ev["response"]
+            if resp is None or not resp.content.strip():
+                raise LLMError(f"{TIER2_MODEL.openrouter_id}: empty response")
+            state["final_answer"] = resp.content
+            state["answered_by"] = TIER2_MODEL.display_name
+            state["tier"] = 2
+            state["usages"] = state["usages"] + [Usage(
+                model_key=TIER2_MODEL.key, model_name=TIER2_MODEL.display_name,
+                stage="escalate", tokens_in=resp.tokens_in, tokens_out=resp.tokens_out,
+                cost_usd=TIER2_MODEL.estimate_cost(resp.tokens_in, resp.tokens_out,
+                                                   resp.served_model),
+                latency_ms=resp.latency_ms,
+            )]
+        except LLMError as e:
+            if state.get("draft_answer"):
+                spec = TIER1_MODELS[state["winner"]]
+                state["final_answer"] = state["draft_answer"]
+                state["answered_by"] = f"{spec.display_name} (frontier unavailable)"
+                state["tier"] = 1
+                state["escalation_reason"] = (state.get("escalation_reason") or "") \
+                    + f" | frontier failed: {str(e)[:150]}"
+                yield {"type": "frontier_failed", "reason": str(e)[:200]}
+            else:
+                yield {"type": "error", "message": str(e)[:300]}
+                return
+    else:
+        state.update(await finalize(state))
+
+    run = _make_run(query, state, start)
+    await get_store().save_run(run)
+    yield {"type": "done", "run": run.model_dump(mode="json")}

backend/app/prompts.py

@@ -1,39 +1,113 @@
 """Prompt templates for bidding, answering, and verification."""
 
+
+def format_history(history: list[dict], max_turns: int, max_chars: int) -> str:
+    """Render recent conversation turns as a compact transcript.
+
+    Takes the most recent `max_turns` turns, truncates long turns so the
+    whole transcript fits in `max_chars`, and returns "" for no history.
+    """
+    turns = history[-max_turns:] if history else []
+    if not turns:
+        return ""
+    per_turn = max(200, max_chars // len(turns))
+    lines = []
+    for t in turns:
+        content = t["content"]
+        if len(content) > per_turn:
+            content = content[:per_turn] + " […truncated]"
+        lines.append(f"{t['role'].upper()}: {content}")
+    return "\n".join(lines)[:max_chars]
+
 BID_SYSTEM = """You are a bidding agent for a specific language model competing in an \
 auction to answer a user query. Assess honestly how well YOUR model would handle the \
 query. Overbidding hurts you: your answer will be checked by an independent verifier, \
 and failures lower your historical accuracy in future auctions.
 
+A conversation transcript may precede the query; you are bidding on answering \
+the LATEST user message in that conversation, which may depend on the context.
+
+Calibration rules — you will be given your model's profile:
+- Bid 0.9+ ONLY if the query is squarely within your stated strengths.
+- If the query is outside your profile, bid 0.6 or lower even if you could \
+probably produce a passable answer — another specialist will do it better.
+- If the query is genuinely hard for any model of your size, bid below 0.5.
+
 Respond with ONLY a JSON object:
 {"confidence": <0.0-1.0, probability you produce a correct and complete answer>,
  "estimated_difficulty": <0.0-1.0, how hard this query is for any model>,
  "reason": "<one short sentence explaining your bid>"}"""
 
 
-def bid_user(query: str) -> str:
-    return f"User query to bid on:\n\n{query}"
+def bid_user(query: str, history: list[dict] | None = None,
+             specialty: str = "") -> str:
+    from .config import settings
+    profile = f"YOUR MODEL'S PROFILE: {specialty}\n\n" if specialty else ""
+    transcript = format_history(history or [], settings.history_max_turns_bid,
+                                settings.history_max_chars_bid)
+    if transcript:
+        return (f"{profile}CONVERSATION SO FAR:\n{transcript}\n\n"
+                f"LATEST user message to bid on:\n\n{query}")
+    return f"{profile}User query to bid on:\n\n{query}"
 
 
 ANSWER_SYSTEM = """You are a helpful expert assistant. Answer the user's query \
 accurately and completely. Be concise. If you are unsure about a fact, say so \
 rather than guessing."""
 
+# Escalated queries are the hard ones — the frontier model should show its
+# work rather than compress
+FRONTIER_SYSTEM = """You are an expert assistant handling a question that \
+smaller models could not answer reliably. Give a thorough, well-structured \
+answer: show the key reasoning steps, state the final result clearly, and \
+note any assumptions. Prefer completeness over brevity, but do not pad. If \
+you are unsure about a fact, say so rather than guessing."""
+
 
 VERIFY_SYSTEM = """You are a strict answer verifier. You will receive a user question \
-and a candidate answer produced by another model. Evaluate:
-1. Correctness — are the facts and logic right?
-2. Completeness — does it fully address the question?
-3. Reasoning quality — is the reasoning sound?
-4. Hallucination risk — any invented facts, citations, or details?
+and a candidate answer produced by another model.
+
+A conversation transcript may precede the question; the answer must make sense \
+as a reply to the LATEST question in that context. An answer that ignores the \
+established context fails completeness.
+
+Step 1 — before reading the answer, list what the question actually demands: \
+every specific quantity, proof, or conclusion it asks for. The hardest \
+sub-question is the one that matters most.
+
+Step 2 — grade the answer against that list on four dimensions, each 0.0-1.0:
+- correctness: are the facts and logic right? Verify calculations yourself.
+- completeness: is every demand met? An answer that covers easy parts but \
+never delivers the hard part (e.g. a section titled with the question that \
+never states the result) scores LOW here, no matter how polished it looks.
+- commitment: does it state results plainly and prove them? Hedging \
+("provided we are clever", "this is equivalent, however..."), restating the \
+question instead of answering it, or never landing on a final result is a \
+failure of commitment.
+- presentation: leaked thinking-out-loud ("Wait...", "Hmm...", "let me \
+recalculate", abandoned attempts, self-contradictions, conflicting final \
+values) caps this at 0.5.
+
+The overall score is the MINIMUM of the four — an answer is only as good as \
+its weakest dimension. Calibration: 0.9+ means an expert would sign off on it \
+as complete and correct; 0.7 means right but with real gaps; 0.5 means \
+correct skeleton, core question not actually answered; below 0.3 means wrong \
+or off-topic.
 
-Be skeptical. A confident-sounding wrong answer must fail.
+Be skeptical. Confident-sounding and shallow must fail. Correct-but-evasive \
+must fail.
 
 Respond with ONLY a JSON object:
-{"score": <0.0-1.0 overall quality>,
- "pass": <true if score >= 0.80 and no hallucinations detected>,
- "feedback": "<one or two sentences justifying the score>"}"""
+{"correctness": <0.0-1.0>, "completeness": <0.0-1.0>, "commitment": <0.0-1.0>,
+ "presentation": <0.0-1.0>,
+ "score": <minimum of the four>,
+ "pass": <true if score >= 0.80>,
+ "feedback": "<two or three sentences: what the question demanded, what was missing or wrong>"}"""
 
 
-def verify_user(query: str, answer: str) -> str:
-    return f"QUESTION:\n{query}\n\nCANDIDATE ANSWER:\n{answer}"
+def verify_user(query: str, answer: str, history: list[dict] | None = None) -> str:
+    from .config import settings
+    transcript = format_history(history or [], settings.history_max_turns_verify,
+                                settings.history_max_chars_verify)
+    prefix = f"CONVERSATION SO FAR:\n{transcript}\n\n" if transcript else ""
+    return f"{prefix}QUESTION:\n{query}\n\nCANDIDATE ANSWER:\n{answer}"

backend/app/schemas.py

@@ -6,8 +6,14 @@ from typing import Literal, Optional
 from pydantic import BaseModel, Field
 
 
+class ChatTurn(BaseModel):
+    role: Literal["user", "assistant"]
+    content: str = Field(max_length=8000)
+
+
 class QueryRequest(BaseModel):
     query: str = Field(min_length=1, max_length=8000)
+    history: list[ChatTurn] = Field(default=[], max_length=20)
 
 
 class Bid(BaseModel):

frontend/components/BidArcade.tsx

@@ -8,7 +8,7 @@ import { useEffect, useState } from "react";
 const BOTS = [
   { name: "GEMMA", color: "#fb923c", dark: "#9a3412" },
   { name: "DEEPSEEK", color: "#38bdf8", dark: "#075985" },
-  { name: "QWEN", color: "#a3e635", dark: "#3f6212" },
+  { name: "QWEN-CODER", color: "#a3e635", dark: "#3f6212" },
 ];
 
 // 8x8 robot sprite: 0 empty, 1 body, 2 eye, 3 antenna

frontend/components/Chat.tsx

@@ -1,8 +1,8 @@
 "use client";
 
 import { useRef, useState } from "react";
-import { submitQuery } from "@/lib/api";
-import type { RunResult } from "@/lib/types";
+import { streamQuery } from "@/lib/api";
+import type { ChatTurn, RunResult } from "@/lib/types";
 import { Badge } from "./ui";
 import { Markdown } from "./Markdown";
 
@@ -12,6 +12,35 @@ interface ChatMessage {
   run?: RunResult;
 }
 
+interface LiveState {
+  status: string;
+  text: string;
+  escalating: boolean;
+}
+
+function CopyButton({ text }: { text: string }) {
+  const [copied, setCopied] = useState(false);
+  return (
+    <button
+      onClick={(e) => {
+        e.stopPropagation();
+        navigator.clipboard.writeText(text).then(() => {
+          setCopied(true);
+          setTimeout(() => setCopied(false), 1500);
+        });
+      }}
+      className={`ml-auto border px-1.5 font-[family-name:var(--font-pixel)] text-[7px] uppercase leading-4 ${
+        copied
+          ? "border-green-600 text-green-400"
+          : "border-stone-600 text-stone-500 hover:border-orange-500 hover:text-orange-400"
+      }`}
+      title="copy answer"
+    >
+      {copied ? "✓ copied" : "copy"}
+    </button>
+  );
+}
+
 export function Chat({
   onRun,
   selectedRunId,
@@ -23,32 +52,85 @@ export function Chat({
 }) {
   const [messages, setMessages] = useState<ChatMessage[]>([]);
   const [input, setInput] = useState("");
-  const [busy, setBusy] = useState(false);
+  const [live, setLive] = useState<LiveState | null>(null);
   const bottomRef = useRef<HTMLDivElement>(null);
 
+  const scroll = () =>
+    setTimeout(() => bottomRef.current?.scrollIntoView({ behavior: "smooth" }), 0);
+
   async function send() {
     const query = input.trim();
-    if (!query || busy) return;
+    if (!query || live) return;
+    // Prior turns (excluding errors) give the pipeline conversation context
+    const history: ChatTurn[] = messages
+      .filter((m) => m.role !== "error")
+      .slice(-12)
+      .map((m) => ({
+        role: m.role as "user" | "assistant",
+        content: m.text.slice(0, 8000),
+      }));
     setInput("");
-    setBusy(true);
     setMessages((m) => [...m, { role: "user", text: query }]);
-    setTimeout(() => bottomRef.current?.scrollIntoView({ behavior: "smooth" }), 0);
+    setLive({ status: "⚡ AUCTION IN PROGRESS…", text: "", escalating: false });
+    scroll();
     try {
-      const run = await submitQuery(query);
-      setMessages((m) => [...m, { role: "assistant", text: run.answer, run }]);
-      onRun(run);
+      await streamQuery(query, history, (ev) => {
+        switch (ev.type) {
+          case "stage":
+            if (ev.stage === "bidding")
+              setLive((l) => l && { ...l, status: "⚡ AUCTION IN PROGRESS…" });
+            else if (ev.stage === "drafting")
+              setLive((l) => l && { ...l, status: `✍ ${ev.model} DRAFTING…` });
+            else if (ev.stage === "verifying")
+              setLive((l) => l && { ...l, status: "🔍 VERIFIER JUDGING…" });
+            else if (ev.stage === "delivering")
+              setLive((l) => l && { ...l, status: `✓ VERIFIED — ${ev.model}` });
+            else if (ev.stage === "escalating")
+              // frontier rewrites from scratch: clear the failed draft
+              setLive((l) => l && {
+                status: `⚔ BOSS FIGHT: ${ev.model}…`,
+                text: "",
+                escalating: true,
+              });
+            break;
+          case "token":
+            setLive((l) => l && { ...l, text: l.text + (ev.text ?? "") });
+            break;
+          case "verification":
+            if (!ev.passed)
+              setLive((l) => l && {
+                ...l,
+                status: `✖ VERIFICATION FAILED (${ev.score?.toFixed(2)})`,
+              });
+            break;
+          case "frontier_failed":
+            setLive((l) => l && { ...l, status: "⚠ FRONTIER UNAVAILABLE — USING DRAFT" });
+            break;
+          case "error":
+            throw new Error(ev.message);
+          case "done":
+            if (ev.run) {
+              const run = ev.run;
+              setMessages((m) => [...m, { role: "assistant", text: run.answer, run }]);
+              onRun(run);
+            }
+            setLive(null);
+            scroll();
+            break;
+        }
+      });
     } catch (e) {
       setMessages((m) => [...m, { role: "error", text: String(e) }]);
     } finally {
-      setBusy(false);
-      setTimeout(() => bottomRef.current?.scrollIntoView({ behavior: "smooth" }), 0);
+      setLive(null);
+      scroll();
     }
   }
 
   return (
     <div className="flex h-full flex-col">
       <div className="flex-1 space-y-4 overflow-y-auto pr-2">
-        {messages.length === 0 && (
+        {messages.length === 0 && !live && (
           <div className="flex h-full items-center justify-center">
             <div className="max-w-md border-2 border-dashed border-stone-700 p-5 text-center text-stone-500">
               <div className="mb-2 font-[family-name:var(--font-pixel)] text-[10px] text-orange-500">
@@ -77,9 +159,9 @@ export function Chat({
             </div>
           ) : (
             <div key={i} className="flex justify-start">
-              <button
+              <div
                 onClick={() => msg.run && onSelectRun(msg.run)}
-                className={`max-w-[85%] cursor-pointer border-2 px-3 py-2 text-left shadow-[3px_3px_0_0_#000] ${
+                className={`max-w-[85%] cursor-pointer select-text border-2 px-3 py-2 text-left shadow-[3px_3px_0_0_#000] ${
                   msg.run && msg.run.id === selectedRunId
                     ? "border-orange-500 bg-stone-900"
                     : "border-stone-700 bg-stone-950 hover:border-stone-500"
@@ -94,20 +176,34 @@ export function Chat({
                     ${msg.run?.total_cost_usd.toFixed(5)} ·{" "}
                     {((msg.run?.latency_ms ?? 0) / 1000).toFixed(1)}s
                   </span>
+                  <CopyButton text={msg.text} />
                 </div>
                 <div className="text-stone-200">
                   <Markdown>{msg.text}</Markdown>
                 </div>
-              </button>
+              </div>
             </div>
           ),
         )}
-        {busy && (
-          <div className="flex items-center gap-2 text-orange-500">
-            <span className="blink">▓</span>
-            <span className="text-sm text-stone-400">
-              auction → draft → verify…
-            </span>
+        {live && (
+          <div className="flex justify-start">
+            <div
+              className={`max-w-[85%] border-2 px-3 py-2 shadow-[3px_3px_0_0_#000] ${
+                live.escalating
+                  ? "border-orange-600 bg-orange-950/20"
+                  : "border-stone-700 bg-stone-950"
+              }`}
+            >
+              <div className="mb-1.5 flex items-center gap-2 font-[family-name:var(--font-pixel)] text-[8px] text-orange-400">
+                <span className="blink">▓</span>
+                {live.status}
+              </div>
+              {live.text && (
+                <div className="text-stone-200">
+                  <Markdown>{live.text}</Markdown>
+                </div>
+              )}
+            </div>
           </div>
         )}
         <div ref={bottomRef} />
@@ -129,7 +225,7 @@ export function Chat({
         />
         <button
           onClick={send}
-          disabled={busy || !input.trim()}
+          disabled={!!live || !input.trim()}
           className="pixel-btn bg-orange-950 px-5 font-[family-name:var(--font-pixel)] text-[10px] uppercase text-orange-400 disabled:cursor-not-allowed disabled:text-stone-600"
         >
           send

frontend/components/MetricsDashboard.tsx

@@ -6,9 +6,9 @@ import type { MetricsSummary } from "@/lib/types";
 import { Bar, Card, Stat } from "./ui";
 
 const MODEL_LABELS: Record<string, string> = {
-  gemma: "Gemma 4 26B",
-  deepseek: "DeepSeek",
-  qwen: "Qwen3 Next 80B",
+  gemini: "Gemma 4 (general)",
+  deepseek: "DeepSeek (math)",
+  qwen: "Qwen3 Coder (code)",
 };
 
 export function MetricsDashboard({ refreshKey }: { refreshKey: number }) {

frontend/lib/api.ts

@@ -1,4 +1,4 @@
-import type { MetricsSummary, RunResult } from "./types";
+import type { ChatTurn, MetricsSummary, RunResult } from "./types";
 
 // Same origin in production (FastAPI serves the static export);
 // the local backend during `next dev`.
@@ -15,14 +15,63 @@ async function request<T>(path: string, init?: RequestInit): Promise<T> {
   return res.json();
 }
 
-export function submitQuery(query: string): Promise<RunResult> {
+export function submitQuery(
+  query: string,
+  history: ChatTurn[] = [],
+): Promise<RunResult> {
   return request<RunResult>("/api/query", {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ query }),
+    body: JSON.stringify({ query, history }),
   });
 }
 
+export interface StreamEvent {
+  type: "stage" | "auction" | "token" | "verification" | "frontier_failed" | "error" | "done";
+  stage?: "bidding" | "drafting" | "verifying" | "delivering" | "escalating";
+  model?: string;
+  text?: string;
+  reason?: string | null;
+  message?: string;
+  score?: number;
+  passed?: boolean;
+  feedback?: string;
+  winner?: string | null;
+  escalated?: boolean;
+  run?: RunResult;
+}
+
+// POST + NDJSON reader (EventSource is GET-only)
+export async function streamQuery(
+  query: string,
+  history: ChatTurn[],
+  onEvent: (ev: StreamEvent) => void,
+): Promise<void> {
+  const res = await fetch(`${API_BASE}/api/query/stream`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ query, history }),
+  });
+  if (!res.ok || !res.body) {
+    const body = await res.text().catch(() => "");
+    throw new Error(`${res.status}: ${body.slice(0, 300)}`);
+  }
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+    let nl;
+    while ((nl = buffer.indexOf("\n")) >= 0) {
+      const line = buffer.slice(0, nl).trim();
+      buffer = buffer.slice(nl + 1);
+      if (line) onEvent(JSON.parse(line) as StreamEvent);
+    }
+  }
+}
+
 export function fetchMetrics(): Promise<MetricsSummary> {
   return request<MetricsSummary>("/api/metrics");
 }

frontend/lib/types.ts

@@ -1,3 +1,8 @@
+export interface ChatTurn {
+  role: "user" | "assistant";
+  content: string;
+}
+
 export interface Bid {
   model_key: string;
   model_name: string;