dee/core/llm.py

"""OpenRouter transport — config, one bounded request, and a safe error type.

This is all that survives of the old dee/core/agent.py. That module also held
a session store, a system prompt, a single-shot tool-calling loop
(run_agent_step) and a separate project-consultation turn (run_consult).
Every one of those was superseded by dee/core/orchestrator.py, whose long-
running RUN does the same jobs better — it asks mid-run instead of resolving
in one silent request, and it keeps working across turns. Their last callers
went with the Director and the iframe chat, so they were removed rather than
left as a second way to do the same thing.

What's left is deliberately small: make one chat-completions call, don't leak
anything, and hand the parsed message back. The orchestrator owns the loop,
the prompt, and the state.

SECURITY — the reason this file reads so defensively:
every exception below is logged in full server-side and surfaced to callers
as a fixed, generic message. Never str(exc). A real incident (2026-07-08) is
why: a copy-pasted API key with a trailing newline made the HTTP client raise
an "invalid header value" error whose message embeds the offending header
VERBATIM — i.e. the bearer token — and that string reached the client through
an error response, live, in production. config.api_key must never appear in a
raised AgentError, even indirectly.
"""

from __future__ import annotations

import logging
import os
from dataclasses import dataclass
from typing import Any, Dict, Optional, Tuple

import requests

logger = logging.getLogger("dee.llm")


class AgentError(Exception):
    """A failure worth surfacing to the caller as a clean message.

    kind mirrors the "kind" field the rest of the REST API already sends on
    error responses (e.g. "signin_required", "rate_limited"), so a route can
    pass it straight through instead of collapsing every failure into one
    generic value — letting the frontend render distinct cases (like "the
    model is temporarily unavailable") with their own copy."""

    def __init__(self, message: str, kind: str = "agent_error") -> None:
        super().__init__(message)
        self.kind = kind


# google/gemini-3.5-flash's context window — verified against
# openrouter.ai/google/gemini-3.5-flash (2026-07-08, same check as the model
# slug below). Used only to compute a "% of context used" figure for the UI;
# there's no live per-model lookup here, so if AGENT_MODEL is ever changed to
# a different model, update this constant too.
CONTEXT_WINDOW_TOKENS = 1_000_000


@dataclass
class OpenRouterConfig:
    api_key: str
    model: str
    max_steps: int
    max_cost_usd: float
    # "medium" is google/gemini-3.5-flash's own default (confirmed via
    # GET /api/v1/models — this model's reasoning object reports
    # default_effort: "medium", mandatory: true). Reasoning tokens are
    # billed as OUTPUT tokens, and this model's output is $9.00/M vs
    # $1.50/M input (openrouter.ai/google/gemini-3.5-flash, checked
    # 2026-08-10) — 6x the price, on by default, for a tool-orchestration
    # task (pick from a documented catalog, not open-ended reasoning) that
    # doesn't need medium-depth thinking on every one of up to 8 steps a
    # single run can take. "low" is a fully supported effort level for
    # this model (not experimental) — "none"/"minimal" are NOT used here
    # because mandatory: true means this model can reject effort: "none",
    # and tool-selection accuracy is worth not pushing to the floor.
    reasoning_effort: str = "low"

    # Hard ceiling on a single reply's OUTPUT tokens. Nothing set one before,
    # which left the most expensive token class on the bill uncapped:
    # $9.00/M output against $1.50/M input — 6x — and, unlike input, output
    # can never be served from a prompt cache. Once the input prefix caches,
    # output IS the dominant marginal cost of a run.
    #
    # This is a tail-risk cap, not a routine saving. Normal replies land far
    # under it, so it changes nothing about ordinary quality; what it stops is
    # one run deciding to write an essay, or looping on a verbose tool, and
    # billing the whole way at the priciest rate. max_cost_usd catches that
    # too, but only AFTER the tokens are spent — this catches it per reply.
    #
    # 8,000 is deliberately generous: the longest legitimate thing this agent
    # writes is a methods-style summary, and truncating one of those mid-
    # sentence would be a quality regression to save a fraction of a cent.
    # Raise it if a real reply is ever cut off; do not lower it to save money
    # without checking what got clipped.
    max_output_tokens: int = 8_000

    @classmethod
    def from_env(cls) -> Optional["OpenRouterConfig"]:
        # .strip() — a copy-pasted secret with a trailing newline produced an
        # invalid HTTP header ("Bearer <key>\n\n"), which the underlying HTTP
        # client rejected with an exception whose message embedded the raw
        # header value. See the module docstring: that leak reached production.
        # Stripping here prevents the malformed-header case entirely;
        # _call_openrouter no longer echoes exception text either way.
        api_key = (os.environ.get("OPENROUTER_API_KEY") or "").strip()
        if not api_key:
            return None
        return cls(
            api_key=api_key,
            # Verified against openrouter.ai/google/gemini-3.5-flash directly
            # (2026-07-08) — don't change this without re-checking the slug,
            # Google's Gemini naming moves fast and a stale slug just 400s.
            model=os.environ.get("AGENT_MODEL", "google/gemini-3.5-flash"),
            max_steps=int(os.environ.get("AGENT_MAX_STEPS", "8")),
            max_cost_usd=float(os.environ.get("AGENT_MAX_COST_USD", "0.5")),
            reasoning_effort=os.environ.get("AGENT_REASONING_EFFORT", "low"),
            max_output_tokens=int(os.environ.get("AGENT_MAX_OUTPUT_TOKENS", "8000")),
        )


def call(config: OpenRouterConfig, body: Dict[str, Any]) -> Tuple[Dict[str, Any], float, int, int]:
    """POST one chat-completions request.

    Returns (message_dict, cost_usd, prompt_tokens, cached_tokens).
    prompt_tokens is the size of the WHOLE messages array just sent, so it
    naturally grows as a conversation lengthens — that's what drives the
    context-usage figure. cached_tokens is how much of that the provider
    served from its prompt cache; see the note at the return statement for why
    it is measured at all.

    Raises AgentError, never anything with provider text or the key in it.
    """
    try:
        resp = requests.post(
            "https://openrouter.ai/api/v1/chat/completions",
            headers={
                "Authorization": f"Bearer {config.api_key}",
                "Content-Type": "application/json",
            },
            json=body,
            timeout=30,
        )
    except Exception as exc:  # noqa: BLE001 — deliberately broad, see docstring
        logger.exception("OpenRouter request failed")
        raise AgentError("could not reach OpenRouter") from exc

    # ── Failure copy, by case ────────────────────────────────────────────
    # Two constraints pull against each other here and both are real:
    #
    #   · Never surface billing language. "Insufficient credits" reads as
    #     "this product ran out of money", which is alarming and none of the
    #     user's business.
    #   · Never tell a paying user the product is unfinished. The previous
    #     copy — "Turing is still in active development" — was written when
    #     that was merely true and harmless. To someone evaluating whether to
    #     rely on this for lab work it reads as "this is a prototype, don't
    #     build on it", which is worse than the billing detail it was hiding.
    #
    # What satisfies both: name the SHAPE of the failure (transient, ours,
    # retryable) without its cause, and give a concrete next step. Each case
    # gets its own `kind` so the client can react differently.
    if resp.status_code == 402:
        # OpenRouter's status specifically for "account/key is out of credits"
        # (distinct from 401 auth / 403 permissions / 429 rate limit) —
        # https://openrouter.ai/docs, confirmed 2026-07-10.
        logger.warning("OpenRouter out of credits (402): %s", resp.text[:500])
        raise AgentError(
            "Turing is temporarily at capacity — that's on our side, not "
            "your run. Your workspace is unaffected; try again in a few "
            "minutes.",
            kind="agent_capacity",
        )

    if resp.status_code == 429:
        logger.warning("OpenRouter rate limited (429): %s", resp.text[:500])
        raise AgentError(
            "Turing is handling a lot of requests right now. Give it about a "
            "minute and send that again.",
            kind="agent_busy",
        )

    if resp.status_code >= 500:
        logger.warning("OpenRouter upstream %s: %s", resp.status_code, resp.text[:500])
        raise AgentError(
            "The model provider is having a moment. Nothing was lost — send "
            "that again and it'll pick up where it left off.",
            kind="agent_upstream",
        )

    if resp.status_code != 200:
        # OpenRouter's error body is small and safe to LOG; never echo it
        # verbatim to the client (it can carry the key's own account info).
        # 4xx that isn't 402/429 is a bug in OUR request, not their problem.
        logger.warning("OpenRouter error %s: %s", resp.status_code, resp.text[:500])
        raise AgentError(
            "Turing hit an unexpected error talking to the model. This one is "
            "logged on our side — please retry.",
            kind="agent_error",
        )

    try:
        parsed = resp.json()
        choices = parsed.get("choices") or []
        if not choices:
            raise AgentError("empty response from OpenRouter")
        message = choices[0].get("message") or {}
    except AgentError:
        raise
    except Exception as exc:  # noqa: BLE001 — malformed upstream JSON, etc.
        logger.exception("failed to parse OpenRouter response")
        raise AgentError("could not parse OpenRouter's response") from exc

    usage = parsed.get("usage") or {}
    cost = usage.get("total_cost") or usage.get("cost") or 0.0
    prompt_tokens = int(usage.get("prompt_tokens") or 0)

    # ── Cache visibility ─────────────────────────────────────────────────
    # ~14.5k tokens of tool specs + system prompt are resent on EVERY step of
    # every run, and ~99% of that payload is byte-identical step to step. That
    # repetition, not the model's thinking, is the dominant line item on a run.
    #
    # Gemini caches such a prefix implicitly and bills the hit at a fraction of
    # the input rate, so the single biggest cost lever is simply whether the
    # prefix stays stable enough to hit. Until now nothing here read the field
    # that says whether it did — the cost could have halved or not moved at all
    # and the logs would look identical either way. Measure first: a cache
    # change you cannot observe is a guess, and this session already spent two
    # deploys on confident guesses that were wrong.
    #
    # Providers disagree on where the number lives (OpenRouter normalises to
    # prompt_tokens_details.cached_tokens; some pass through a flat
    # cached_tokens), so read both rather than assume one shape. Absent
    # entirely -> 0, which is honestly indistinguishable from "no hit" and is
    # the safe reading.
    details = usage.get("prompt_tokens_details") or {}
    cached_tokens = int(details.get("cached_tokens")
                        or usage.get("cached_tokens")
                        or 0)

    # Reasoning tokens, logged for the same reason cached_tokens is: they are
    # billed as OUTPUT ($9.00/M, 6x input, never cacheable) and are spent on
    # every step, but nothing here has ever reported how many. Whether
    # reasoning is 5% or 50% of output decides whether dialling effort further
    # is the next real cost lever or a rounding error — and that is a question
    # to answer with a number, not a guess.
    comp = usage.get("completion_tokens_details") or {}
    reasoning_tokens = int(comp.get("reasoning_tokens") or 0)
    completion_tokens = int(usage.get("completion_tokens") or 0)
    if prompt_tokens:
        logger.info(
            "openrouter usage: prompt=%d cached=%d (%.0f%% hit) "
            "completion=%d reasoning=%d cost=%.5f",
            prompt_tokens, cached_tokens,
            100.0 * cached_tokens / prompt_tokens,
            completion_tokens, reasoning_tokens, cost)
    return message, cost, prompt_tokens, cached_tokens