packages/core/src/utils/llm.py

# ============================================================================
# agent verf - LLM Client (Multi-Provider Waterfall)
# Version: 0.2.0
# Last Updated: 2026-01-22
#
# CRITICAL: This is the abstraction layer for all LLM interactions.
#
# Architecture:
#   - Dual-mode: FREE (public) and VENICE (premium)
#   - Waterfall fallback: Try providers in sequence until success
#   - Task-based model selection: TRIAGE (8B fast) vs SYNTHESIS (70B quality)
#
# Capacity (~591 verifications/day):
#   - Free Mode: ~450/day (Groq → Google → Cloudflare for triage,
#                          Cerebras → OpenRouter for synthesis)
#   - Venice Mode: ~141/day (Venice 8B/70B with free fallback)
#
# Usage:
#   client = get_llm_client()
#   result = await client.complete(
#       task=LLMTask.SYNTHESIS,
#       messages=[{"role": "user", "content": "..."}],
#       mode=VerificationMode.FREE,
#   )
# ============================================================================

import os
import json
import time
import asyncio
import structlog
from enum import Enum
from typing import Optional, Type, Any
from pydantic import BaseModel
import httpx

# Provider SDKs
import anthropic
import openai
import groq
import google.generativeai as genai

logger = structlog.get_logger()


# ============================================================================
# Enums
# ============================================================================

class LLMTask(str, Enum):
    """Tasks that require LLM calls."""
    TRIAGE = "triage"          # Fast routing (8B models, ~600 tokens)
    SYNTHESIS = "synthesis"    # Verdict generation (70B models, ~2500 tokens)
    DEEP_DIVE = "deep_dive"    # Complex analysis
    PARAPHRASE = "paraphrase"  # Response variation


class LLMProvider(str, Enum):
    """Supported LLM providers."""
    # Paid providers (fallback)
    ANTHROPIC = "anthropic"
    OPENAI = "openai"
    # Free triage providers
    GROQ = "groq"
    GOOGLE = "google"
    CLOUDFLARE = "cloudflare"
    # Free synthesis providers
    CEREBRAS = "cerebras"
    OPENROUTER = "openrouter"
    # Premium provider
    VENICE = "venice"


class VerificationMode(str, Enum):
    """User verification mode - determines provider priority."""
    FREE = "free"      # Public users: Groq → Google → Cloudflare, Cerebras → OpenRouter
    VENICE = "venice"  # Premium users: Venice first, then free fallback


# ============================================================================
# Provider Configurations
# ============================================================================

PROVIDER_CONFIGS = {
    # === TRIAGE PROVIDERS (8B models, ~600 tokens) ===
    "groq": {
        "base_url": "https://api.groq.com/openai/v1",
        "env_key": "GROQ_API_KEY",
        "task": "triage",
        "models": {
            "triage": "llama-3.1-8b-instant",
        },
        "limits": {
            "rpd": 14_400,   # requests per day (rolling 24h)
            "rpm": 30,       # requests per minute - critical to respect
            "tpd": 500_000,
        },
        "openai_compatible": True,
    },
    "google": {
        "base_url": None,  # Uses SDK directly
        "env_key": "GOOGLE_AI_API_KEY",
        "task": "triage",
        "models": {
            "triage": "gemini-2.0-flash-lite",
        },
        "limits": {
            "rpd": 1_000,
            "rpm": 15,
        },
        "openai_compatible": False,
    },
    "cloudflare": {
        "base_url": "https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1",
        "env_key": "CLOUDFLARE_API_TOKEN",
        "account_id_env": "CLOUDFLARE_ACCOUNT_ID",
        "task": "triage",
        "models": {
            "triage": "@cf/meta/llama-3.1-8b-instruct",
        },
        "limits": {
            "neurons_per_day": 10_000,  # ~200-500 requests
            "rpm": 30,
        },
        "openai_compatible": True,
    },

    # === SYNTHESIS PROVIDERS (70B models, ~2500 tokens) ===
    "cerebras": {
        "base_url": "https://api.cerebras.ai/v1",
        "env_key": "CEREBRAS_API_KEY",
        "task": "synthesis",
        "models": {
            "synthesis": "llama-3.3-70b",
        },
        "limits": {
            "tpd": 1_000_000,  # ~400 synthesis calls at 2500 tokens
            "rpm": 30,
            "tpm": 60_000,
        },
        "openai_compatible": True,
    },
    "openrouter": {
        "base_url": "https://openrouter.ai/api/v1",
        "env_key": "OPENROUTER_API_KEY",
        "task": "synthesis",
        "models": {
            "synthesis": "meta-llama/llama-3.3-70b-instruct:free",
        },
        "limits": {
            "rpd": 50,  # 50/day without $10, 1000/day with $10
            "rpm": 20,
        },
        "openai_compatible": True,
    },
    "venice": {
        "base_url": "https://api.venice.ai/api/v1",
        "env_key": "VENICE_API_KEY",
        "task": "both",  # Venice can do both triage and synthesis
        "models": {
            "triage": "llama-3.1-8b-instruct",
            "synthesis": "llama-3.3-70b",
        },
        "limits": {
            # Stake-dependent, approximately:
            "rpd": 50,   # Conservative estimate for <1% stake
            "rpm": 10,
        },
        "openai_compatible": True,
    },

    # === PAID FALLBACKS ===
    "anthropic": {
        "base_url": None,  # Uses SDK
        "env_key": "ANTHROPIC_API_KEY",
        "task": "both",
        "models": {
            "triage": "claude-3-haiku-20240307",
            "synthesis": "claude-3-5-sonnet-20241022",
        },
        "openai_compatible": False,
    },
    "openai": {
        "base_url": None,  # Uses SDK
        "env_key": "OPENAI_API_KEY",
        "task": "both",
        "models": {
            "triage": "gpt-4o-mini",
            "synthesis": "gpt-4o",
        },
        "openai_compatible": False,
    },
}


# ============================================================================
# Provider Chains (Waterfall Order)
# ============================================================================

PROVIDER_CHAINS = {
    VerificationMode.FREE: {
        LLMTask.TRIAGE: ["groq", "google", "cloudflare"],
        LLMTask.SYNTHESIS: ["cerebras", "openrouter"],
    },
    VerificationMode.VENICE: {
        # Venice first, then fall back to free providers
        LLMTask.TRIAGE: ["venice", "groq", "google", "cloudflare"],
        LLMTask.SYNTHESIS: ["venice", "cerebras", "openrouter"],
    },
}


# ============================================================================
# LLM Response Model
# ============================================================================

class LLMResponse(BaseModel):
    """Standardized response from LLM calls."""
    content: Any  # The actual response (parsed if schema provided)
    raw_content: str  # Raw text response
    model: str  # Model that was used
    provider: str  # Provider that was used
    input_tokens: int  # Tokens in prompt
    output_tokens: int  # Tokens in response
    latency_ms: int  # How long the call took
    cost_usd: float  # Estimated cost (0 for free providers)


# ============================================================================
# Cost Tracking (only for paid providers)
# ============================================================================

MODEL_COSTS = {
    # Anthropic (per 1M tokens)
    "claude-3-haiku-20240307": {"input": 0.25, "output": 1.25},
    "claude-3-sonnet-20240229": {"input": 3.00, "output": 15.00},
    "claude-3-opus-20240229": {"input": 15.00, "output": 75.00},
    "claude-3-5-sonnet-20241022": {"input": 3.00, "output": 15.00},
    # OpenAI (per 1M tokens)
    "gpt-4o": {"input": 2.50, "output": 10.00},
    "gpt-4o-mini": {"input": 0.15, "output": 0.60},
    "gpt-4-turbo": {"input": 10.00, "output": 30.00},
    # Free providers - $0
    "llama-3.1-8b-instant": {"input": 0, "output": 0},
    "llama-3.3-70b": {"input": 0, "output": 0},
    "gemini-2.0-flash-lite": {"input": 0, "output": 0},
}


def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float:
    """Estimate the cost of an LLM call."""
    if model not in MODEL_COSTS:
        return 0.0  # Assume free for unknown models
    costs = MODEL_COSTS[model]
    input_cost = (input_tokens / 1_000_000) * costs["input"]
    output_cost = (output_tokens / 1_000_000) * costs["output"]
    return input_cost + output_cost


# ============================================================================
# Rate Limit Error
# ============================================================================

class RateLimitError(Exception):
    """Raised when a provider returns 429."""
    def __init__(self, provider: str, retry_after: Optional[int] = None):
        self.provider = provider
        self.retry_after = retry_after
        super().__init__(f"{provider} rate limited" + (f", retry after {retry_after}s" if retry_after else ""))


class ProviderUnavailableError(Exception):
    """Raised when a provider is not configured or fails."""
    pass


# ============================================================================
# LLM Client
# ============================================================================

class LLMClient:
    """
    Unified LLM client with multi-provider waterfall fallback.

    Features:
      - Dual-mode operation (FREE vs VENICE)
      - Automatic provider fallback on rate limits
      - OpenAI-compatible API support for most providers
      - Structured output support (JSON mode)
      - Cost and latency tracking
    """

    def __init__(self):
        """Initialize the LLM client with available providers."""
        self._clients: dict[str, Any] = {}
        self._http_client = httpx.AsyncClient(timeout=60.0)

        # Initialize provider clients based on available API keys
        self._init_providers()

        logger.info(
            "LLM client initialized",
            available_providers=list(self._clients.keys()),
        )

    def _init_providers(self):
        """Initialize clients for providers with available API keys."""
        # Anthropic (paid fallback)
        if os.getenv("ANTHROPIC_API_KEY"):
            self._clients["anthropic"] = anthropic.AsyncAnthropic()

        # OpenAI (paid fallback)
        if os.getenv("OPENAI_API_KEY"):
            self._clients["openai"] = openai.AsyncOpenAI()

        # Groq (free triage)
        if os.getenv("GROQ_API_KEY"):
            self._clients["groq"] = groq.AsyncGroq()

        # Google AI Studio (free triage)
        if os.getenv("GOOGLE_AI_API_KEY"):
            genai.configure(api_key=os.getenv("GOOGLE_AI_API_KEY"))
            self._clients["google"] = True  # Flag that it's configured

        # Cloudflare (free triage) - uses httpx directly
        if os.getenv("CLOUDFLARE_API_TOKEN") and os.getenv("CLOUDFLARE_ACCOUNT_ID"):
            self._clients["cloudflare"] = True  # Uses httpx

        # Cerebras (free synthesis) - OpenAI-compatible
        if os.getenv("CEREBRAS_API_KEY"):
            self._clients["cerebras"] = openai.AsyncOpenAI(
                api_key=os.getenv("CEREBRAS_API_KEY"),
                base_url="https://api.cerebras.ai/v1",
            )

        # OpenRouter (free synthesis) - OpenAI-compatible
        if os.getenv("OPENROUTER_API_KEY"):
            self._clients["openrouter"] = openai.AsyncOpenAI(
                api_key=os.getenv("OPENROUTER_API_KEY"),
                base_url="https://openrouter.ai/api/v1",
                default_headers={
                    "HTTP-Referer": os.getenv("APP_URL", "https://agentverf.com"),
                    "X-Title": "agent verf",
                },
            )

        # Venice (premium) - OpenAI-compatible
        if os.getenv("VENICE_API_KEY"):
            self._clients["venice"] = openai.AsyncOpenAI(
                api_key=os.getenv("VENICE_API_KEY"),
                base_url="https://api.venice.ai/api/v1",
            )

    def _get_provider_chain(
        self,
        task: LLMTask,
        mode: VerificationMode,
    ) -> list[str]:
        """Get the provider chain for a task and mode."""
        # Map deep_dive and paraphrase to synthesis/triage
        effective_task = task
        if task == LLMTask.DEEP_DIVE:
            effective_task = LLMTask.SYNTHESIS
        elif task == LLMTask.PARAPHRASE:
            effective_task = LLMTask.TRIAGE

        chain = PROVIDER_CHAINS.get(mode, PROVIDER_CHAINS[VerificationMode.FREE])
        providers = chain.get(effective_task, chain.get(LLMTask.SYNTHESIS, []))

        # Filter to available providers
        return [p for p in providers if p in self._clients]

    def _get_model_for_provider(self, provider: str, task: LLMTask) -> str:
        """Get the model name for a provider and task."""
        config = PROVIDER_CONFIGS.get(provider, {})
        models = config.get("models", {})

        # Map task to model key
        if task in [LLMTask.TRIAGE, LLMTask.PARAPHRASE]:
            return models.get("triage", models.get("synthesis", "unknown"))
        else:
            return models.get("synthesis", models.get("triage", "unknown"))

    async def complete(
        self,
        task: LLMTask,
        messages: list[dict],
        mode: VerificationMode = VerificationMode.FREE,
        output_schema: Optional[Type[BaseModel]] = None,
        system_prompt: Optional[str] = None,
        temperature: float = 0.3,
        max_tokens: int = 4096,
    ) -> LLMResponse:
        """
        Complete an LLM request with automatic provider fallback.

        Args:
            task: The task type (determines model size)
            messages: List of message dicts [{"role": "user", "content": "..."}]
            mode: FREE or VENICE (determines provider priority)
            output_schema: Optional Pydantic model for structured output
            system_prompt: Optional system prompt
            temperature: Sampling temperature
            max_tokens: Maximum tokens in response

        Returns:
            LLMResponse with content and metadata

        Raises:
            Exception: If all providers fail
        """
        providers = self._get_provider_chain(task, mode)

        if not providers:
            raise ProviderUnavailableError(
                f"No providers available for task={task.value}, mode={mode.value}"
            )

        logger.info(
            "LLM request starting",
            task=task.value,
            mode=mode.value,
            provider_chain=providers,
        )

        start_time = time.time()
        last_error = None

        for provider in providers:
            try:
                response = await self._call_provider(
                    provider=provider,
                    task=task,
                    messages=messages,
                    system_prompt=system_prompt,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    output_schema=output_schema,
                )

                response.latency_ms = int((time.time() - start_time) * 1000)

                logger.info(
                    "LLM request completed",
                    task=task.value,
                    provider=provider,
                    model=response.model,
                    latency_ms=response.latency_ms,
                )

                return response

            except RateLimitError as e:
                logger.warning(
                    "Provider rate limited, trying next",
                    provider=provider,
                    retry_after=e.retry_after,
                )
                last_error = e
                continue

            except Exception as e:
                logger.warning(
                    "Provider failed, trying next",
                    provider=provider,
                    error=str(e),
                )
                last_error = e
                continue

        # All providers failed, try paid fallbacks as last resort
        if "anthropic" in self._clients and "anthropic" not in providers:
            try:
                logger.warning("Falling back to paid Anthropic provider")
                response = await self._call_anthropic(
                    model=PROVIDER_CONFIGS["anthropic"]["models"].get(
                        "triage" if task == LLMTask.TRIAGE else "synthesis"
                    ),
                    messages=messages,
                    system_prompt=system_prompt,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    output_schema=output_schema,
                )
                response.latency_ms = int((time.time() - start_time) * 1000)
                return response
            except Exception as e:
                logger.error("Anthropic fallback also failed", error=str(e))

        raise last_error or Exception("All LLM providers failed")

    async def _call_provider(
        self,
        provider: str,
        task: LLMTask,
        messages: list[dict],
        system_prompt: Optional[str],
        temperature: float,
        max_tokens: int,
        output_schema: Optional[Type[BaseModel]],
    ) -> LLMResponse:
        """Route to the appropriate provider handler."""
        model = self._get_model_for_provider(provider, task)

        if provider == "anthropic":
            return await self._call_anthropic(
                model, messages, system_prompt, temperature, max_tokens, output_schema
            )
        elif provider == "google":
            return await self._call_google(
                model, messages, system_prompt, temperature, max_tokens, output_schema
            )
        elif provider == "cloudflare":
            return await self._call_cloudflare(
                model, messages, system_prompt, temperature, max_tokens, output_schema
            )
        elif provider == "groq":
            return await self._call_groq(
                model, messages, system_prompt, temperature, max_tokens, output_schema
            )
        elif provider in ["cerebras", "openrouter", "venice"]:
            return await self._call_openai_compatible(
                provider, model, messages, system_prompt, temperature, max_tokens, output_schema
            )
        elif provider == "openai":
            return await self._call_openai(
                model, messages, system_prompt, temperature, max_tokens, output_schema
            )
        else:
            raise ProviderUnavailableError(f"Unknown provider: {provider}")

    async def _call_groq(
        self,
        model: str,
        messages: list[dict],
        system_prompt: Optional[str],
        temperature: float,
        max_tokens: int,
        output_schema: Optional[Type[BaseModel]],
    ) -> LLMResponse:
        """Call Groq API using their native SDK."""
        client = self._clients.get("groq")
        if not client:
            raise ProviderUnavailableError("Groq client not initialized")

        # Build messages
        api_messages = []
        if system_prompt:
            schema_instruction = ""
            if output_schema:
                schema_json = json.dumps(output_schema.model_json_schema())
                schema_instruction = f"\n\nRespond with valid JSON matching this schema:\n{schema_json}"
            api_messages.append({"role": "system", "content": system_prompt + schema_instruction})
        elif output_schema:
            schema_json = json.dumps(output_schema.model_json_schema())
            api_messages.append({"role": "system", "content": f"Respond with valid JSON matching this schema:\n{schema_json}"})

        api_messages.extend(messages)

        try:
            response = await client.chat.completions.create(
                model=model,
                messages=api_messages,
                temperature=temperature,
                max_tokens=max_tokens,
                response_format={"type": "json_object"} if output_schema else None,
            )
        except groq.RateLimitError as e:
            raise RateLimitError("groq", getattr(e, "retry_after", None))

        raw_content = response.choices[0].message.content
        content = self._parse_output(raw_content, output_schema)

        return LLMResponse(
            content=content,
            raw_content=raw_content,
            model=model,
            provider="groq",
            input_tokens=response.usage.prompt_tokens,
            output_tokens=response.usage.completion_tokens,
            latency_ms=0,
            cost_usd=0.0,
        )

    async def _call_google(
        self,
        model: str,
        messages: list[dict],
        system_prompt: Optional[str],
        temperature: float,
        max_tokens: int,
        output_schema: Optional[Type[BaseModel]],
    ) -> LLMResponse:
        """Call Google AI Studio (Gemini) API."""
        if "google" not in self._clients:
            raise ProviderUnavailableError("Google AI client not initialized")

        # Build the prompt
        prompt_parts = []
        if system_prompt:
            prompt_parts.append(system_prompt)
        if output_schema:
            schema_json = json.dumps(output_schema.model_json_schema())
            prompt_parts.append(f"\nRespond with valid JSON matching this schema:\n{schema_json}")

        # Add messages
        for msg in messages:
            role = msg.get("role", "user")
            content = msg.get("content", "")
            if role == "user":
                prompt_parts.append(f"\nUser: {content}")
            elif role == "assistant":
                prompt_parts.append(f"\nAssistant: {content}")

        prompt = "\n".join(prompt_parts)

        try:
            gemini_model = genai.GenerativeModel(model)
            response = await asyncio.to_thread(
                gemini_model.generate_content,
                prompt,
                generation_config=genai.GenerationConfig(
                    temperature=temperature,
                    max_output_tokens=max_tokens,
                ),
            )
        except Exception as e:
            if "429" in str(e) or "quota" in str(e).lower():
                raise RateLimitError("google")
            raise

        raw_content = response.text
        content = self._parse_output(raw_content, output_schema)

        # Estimate tokens (Google doesn't always return usage)
        input_tokens = len(prompt) // 4  # Rough estimate
        output_tokens = len(raw_content) // 4

        return LLMResponse(
            content=content,
            raw_content=raw_content,
            model=model,
            provider="google",
            input_tokens=input_tokens,
            output_tokens=output_tokens,
            latency_ms=0,
            cost_usd=0.0,
        )

    async def _call_cloudflare(
        self,
        model: str,
        messages: list[dict],
        system_prompt: Optional[str],
        temperature: float,
        max_tokens: int,
        output_schema: Optional[Type[BaseModel]],
    ) -> LLMResponse:
        """Call Cloudflare Workers AI using OpenAI-compatible API."""
        if "cloudflare" not in self._clients:
            raise ProviderUnavailableError("Cloudflare client not initialized")

        account_id = os.getenv("CLOUDFLARE_ACCOUNT_ID")
        api_token = os.getenv("CLOUDFLARE_API_TOKEN")
        base_url = f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1"

        # Build messages
        api_messages = []
        if system_prompt:
            schema_instruction = ""
            if output_schema:
                schema_json = json.dumps(output_schema.model_json_schema())
                schema_instruction = f"\n\nRespond with valid JSON matching this schema:\n{schema_json}"
            api_messages.append({"role": "system", "content": system_prompt + schema_instruction})
        elif output_schema:
            schema_json = json.dumps(output_schema.model_json_schema())
            api_messages.append({"role": "system", "content": f"Respond with valid JSON matching this schema:\n{schema_json}"})

        api_messages.extend(messages)

        try:
            response = await self._http_client.post(
                f"{base_url}/chat/completions",
                headers={
                    "Authorization": f"Bearer {api_token}",
                    "Content-Type": "application/json",
                },
                json={
                    "model": model,
                    "messages": api_messages,
                    "temperature": temperature,
                    "max_tokens": max_tokens,
                },
            )

            if response.status_code == 429:
                raise RateLimitError("cloudflare")

            response.raise_for_status()
            data = response.json()

        except httpx.HTTPStatusError as e:
            if e.response.status_code == 429:
                raise RateLimitError("cloudflare")
            raise

        raw_content = data["result"]["response"]
        content = self._parse_output(raw_content, output_schema)

        return LLMResponse(
            content=content,
            raw_content=raw_content,
            model=model,
            provider="cloudflare",
            input_tokens=data.get("result", {}).get("usage", {}).get("prompt_tokens", 0),
            output_tokens=data.get("result", {}).get("usage", {}).get("completion_tokens", 0),
            latency_ms=0,
            cost_usd=0.0,
        )

    async def _call_openai_compatible(
        self,
        provider: str,
        model: str,
        messages: list[dict],
        system_prompt: Optional[str],
        temperature: float,
        max_tokens: int,
        output_schema: Optional[Type[BaseModel]],
    ) -> LLMResponse:
        """Call OpenAI-compatible APIs (Cerebras, OpenRouter, Venice)."""
        client = self._clients.get(provider)
        if not client:
            raise ProviderUnavailableError(f"{provider} client not initialized")

        # Build messages
        api_messages = []
        if system_prompt:
            schema_instruction = ""
            if output_schema:
                schema_json = json.dumps(output_schema.model_json_schema())
                schema_instruction = f"\n\nRespond with valid JSON matching this schema:\n{schema_json}"
            api_messages.append({"role": "system", "content": system_prompt + schema_instruction})
        elif output_schema:
            schema_json = json.dumps(output_schema.model_json_schema())
            api_messages.append({"role": "system", "content": f"Respond with valid JSON matching this schema:\n{schema_json}"})

        api_messages.extend(messages)

        try:
            response = await client.chat.completions.create(
                model=model,
                messages=api_messages,
                temperature=temperature,
                max_tokens=max_tokens,
            )
        except openai.RateLimitError as e:
            raise RateLimitError(provider, getattr(e, "retry_after", None))

        raw_content = response.choices[0].message.content
        content = self._parse_output(raw_content, output_schema)

        return LLMResponse(
            content=content,
            raw_content=raw_content,
            model=model,
            provider=provider,
            input_tokens=response.usage.prompt_tokens if response.usage else 0,
            output_tokens=response.usage.completion_tokens if response.usage else 0,
            latency_ms=0,
            cost_usd=0.0,
        )

    async def _call_anthropic(
        self,
        model: str,
        messages: list[dict],
        system_prompt: Optional[str],
        temperature: float,
        max_tokens: int,
        output_schema: Optional[Type[BaseModel]],
    ) -> LLMResponse:
        """Call Anthropic's API."""
        client = self._clients.get("anthropic")
        if not client:
            raise ProviderUnavailableError("Anthropic client not initialized")

        request_kwargs = {
            "model": model,
            "messages": messages,
            "temperature": temperature,
            "max_tokens": max_tokens,
        }

        if system_prompt:
            request_kwargs["system"] = system_prompt

        if output_schema:
            schema_json = json.dumps(output_schema.model_json_schema())
            if "system" in request_kwargs:
                request_kwargs["system"] += f"\n\nRespond with valid JSON matching this schema:\n{schema_json}"
            else:
                request_kwargs["system"] = f"Respond with valid JSON matching this schema:\n{schema_json}"

        try:
            response = await client.messages.create(**request_kwargs)
        except anthropic.RateLimitError as e:
            raise RateLimitError("anthropic", getattr(e, "retry_after", None))

        raw_content = response.content[0].text
        content = self._parse_output(raw_content, output_schema)

        return LLMResponse(
            content=content,
            raw_content=raw_content,
            model=model,
            provider="anthropic",
            input_tokens=response.usage.input_tokens,
            output_tokens=response.usage.output_tokens,
            latency_ms=0,
            cost_usd=estimate_cost(model, response.usage.input_tokens, response.usage.output_tokens),
        )

    async def _call_openai(
        self,
        model: str,
        messages: list[dict],
        system_prompt: Optional[str],
        temperature: float,
        max_tokens: int,
        output_schema: Optional[Type[BaseModel]],
    ) -> LLMResponse:
        """Call OpenAI's API."""
        client = self._clients.get("openai")
        if not client:
            raise ProviderUnavailableError("OpenAI client not initialized")

        api_messages = []
        if system_prompt:
            api_messages.append({"role": "system", "content": system_prompt})

        if output_schema:
            schema_json = json.dumps(output_schema.model_json_schema())
            if api_messages:
                api_messages[0]["content"] += f"\n\nRespond with valid JSON matching this schema:\n{schema_json}"
            else:
                api_messages.append({"role": "system", "content": f"Respond with valid JSON matching this schema:\n{schema_json}"})

        api_messages.extend(messages)

        try:
            response = await client.chat.completions.create(
                model=model,
                messages=api_messages,
                temperature=temperature,
                max_tokens=max_tokens,
                response_format={"type": "json_object"} if output_schema else None,
            )
        except openai.RateLimitError as e:
            raise RateLimitError("openai", getattr(e, "retry_after", None))

        raw_content = response.choices[0].message.content
        content = self._parse_output(raw_content, output_schema)

        return LLMResponse(
            content=content,
            raw_content=raw_content,
            model=model,
            provider="openai",
            input_tokens=response.usage.prompt_tokens,
            output_tokens=response.usage.completion_tokens,
            latency_ms=0,
            cost_usd=estimate_cost(model, response.usage.prompt_tokens, response.usage.completion_tokens),
        )

    def _parse_output(
        self,
        raw_content: str,
        output_schema: Optional[Type[BaseModel]],
    ) -> Any:
        """Parse raw content into structured output if schema provided."""
        if not output_schema:
            return raw_content

        try:
            json_str = raw_content
            # Handle markdown code blocks
            if "```json" in json_str:
                json_str = json_str.split("```json")[1].split("```")[0]
            elif "```" in json_str:
                json_str = json_str.split("```")[1].split("```")[0]
            return output_schema.model_validate_json(json_str.strip())
        except Exception as e:
            logger.warning("Failed to parse structured output", error=str(e))
            return raw_content

    async def close(self):
        """Close the HTTP client."""
        await self._http_client.aclose()


# ============================================================================
# Singleton Instance
# ============================================================================

_llm_client: Optional[LLMClient] = None


def get_llm_client() -> LLMClient:
    """Get the singleton LLM client instance."""
    global _llm_client
    if _llm_client is None:
        _llm_client = LLMClient()
    return _llm_client