build/lib/lilith_agent/models.py

from __future__ import annotations

import asyncio
import logging
import time
from collections import deque
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Any, Sequence

from langchain_anthropic import ChatAnthropic
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import BaseMessage, HumanMessage
from langchain_core.runnables import Runnable
from langchain_google_genai import ChatGoogleGenerativeAI
from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
from langchain_ollama import ChatOllama
from langchain_openai import ChatOpenAI

from tenacity import AsyncRetrying, Retrying, retry_if_exception, stop_after_attempt, wait_exponential
from lilith_agent.config import Config

try:
    from google.api_core.exceptions import ResourceExhausted
except ImportError:
    ResourceExhausted = None

try:
    from google.genai.errors import ClientError as GenAIClientError
except ImportError:
    GenAIClientError = None

try:
    from anthropic import RateLimitError as AnthropicRateLimitError
except ImportError:
    AnthropicRateLimitError = None

try:
    from openai import RateLimitError as OpenAIRateLimitError
except ImportError:
    OpenAIRateLimitError = None


def is_retryable_rate_limit(exc: BaseException) -> bool:
    if ResourceExhausted is not None and isinstance(exc, ResourceExhausted):
        return True
    if GenAIClientError is not None and isinstance(exc, GenAIClientError):
        return getattr(exc, "code", None) == 429
    if AnthropicRateLimitError is not None and isinstance(exc, AnthropicRateLimitError):
        return True
    if OpenAIRateLimitError is not None and isinstance(exc, OpenAIRateLimitError):
        return True
    return False


def _tenacity_sleep(seconds: float) -> None:
    time.sleep(seconds)


async def _async_tenacity_sleep(seconds: float) -> None:
    await asyncio.sleep(seconds)


def _base_retry_params() -> dict[str, Any]:
    return dict(
        retry=retry_if_exception(is_retryable_rate_limit),
        wait=wait_exponential(multiplier=2, min=4, max=60),
        stop=stop_after_attempt(5),
        before_sleep=lambda retry_state: log.warning(
            f"LLM Rate Limit (429) hit. Retrying in {retry_state.next_action.sleep}s... "
            f"(Attempt {retry_state.attempt_number}/5)"
        ),
        reraise=True,
    )


def _sync_retry_params() -> dict[str, Any]:
    return {**_base_retry_params(), "sleep": _tenacity_sleep}


def _async_retry_params() -> dict[str, Any]:
    return {**_base_retry_params(), "sleep": _async_tenacity_sleep}

try:
    from langchain_community.cache import SQLiteCache
except ImportError:
    try:
        from langchain.cache import SQLiteCache
    except ImportError:
        SQLiteCache = None

try:
    from langchain_core.globals import set_llm_cache
except ImportError:
    try:
        from langchain.globals import set_llm_cache
    except ImportError:
        set_llm_cache = None

# Enable LLM cache for development efficiency
if SQLiteCache:
    try:
        set_llm_cache(SQLiteCache(database_path=".langchain.db"))
    except Exception:
        # Fail gracefully if sqlite is missing or DB file locked
        pass

log = logging.getLogger(__name__)

LMSTUDIO_DEFAULT_BASE_URL = "http://localhost:1234/v1"
DEEPSEEK_DEFAULT_BASE_URL = "https://api.deepseek.com"
_NO_THINK = "/no_think"
_GEMINI_COOLDOWN_MODELS = {"gemini-3-flash-preview", "gemini-3.1-pro"}
_COOLDOWN_LADDER_SECONDS = (60, 120, 300)
_QUESTION_STREAK_LIMIT = 50
_BATCH_WINDOW_SIZE = 100
_BATCH_WINDOW_RATE_LIMIT_THRESHOLD = 70
_BATCH_PAUSE_LADDER_SECONDS = (300, 600, 1200)
_cooldown_until: dict[tuple[str, str], float] = {}
_rate_limit_exhaustions: dict[tuple[str, str], int] = {}
_question_rate_limit_streak: ContextVar[int | None] = ContextVar("question_rate_limit_streak", default=None)
_batch_rate_limit_window: deque[bool] = deque(maxlen=_BATCH_WINDOW_SIZE)
_batch_pause_count = 0


@dataclass
class RateLimitCooldownError(Exception):
    provider: str
    model: str
    cooldown_seconds: int
    original_error: str

    def __str__(self) -> str:
        return (
            f"rate limited provider={self.provider} model={self.model} "
            f"cooldown={self.cooldown_seconds}s original={self.original_error}"
        )


@dataclass
class BatchAbortRateLimitError(Exception):
    reason: str
    original_error: str

    def __str__(self) -> str:
        return f"batch abort rate limit: {self.reason}; original={self.original_error}"


@dataclass
class QuestionRateLimitStreakError(Exception):
    count: int

    def __str__(self) -> str:
        return f"question hit {self.count} consecutive rate-limit events"


def _reset_rate_limit_state_for_tests() -> None:
    global _batch_pause_count
    _cooldown_until.clear()
    _rate_limit_exhaustions.clear()
    _batch_rate_limit_window.clear()
    _batch_pause_count = 0


@contextmanager
def rate_limit_question_scope():
    token = _question_rate_limit_streak.set(0)
    try:
        yield
    finally:
        _question_rate_limit_streak.reset(token)


def record_rate_limit_observation(exc: BaseException) -> None:
    if not is_retryable_rate_limit(exc):
        return
    _batch_rate_limit_window.append(True)
    current = _question_rate_limit_streak.get()
    if current is None:
        return
    current += 1
    _question_rate_limit_streak.set(current)
    if current >= _QUESTION_STREAK_LIMIT:
        raise QuestionRateLimitStreakError(count=current) from exc


def record_rate_limit_success() -> None:
    _batch_rate_limit_window.append(False)
    if _question_rate_limit_streak.get() is not None:
        _question_rate_limit_streak.set(0)


def batch_rate_limit_pause_seconds() -> int | None:
    global _batch_pause_count
    if len(_batch_rate_limit_window) < _BATCH_WINDOW_SIZE:
        return None
    if sum(_batch_rate_limit_window) < _BATCH_WINDOW_RATE_LIMIT_THRESHOLD:
        return None
    _batch_pause_count += 1
    idx = min(_batch_pause_count - 1, len(_BATCH_PAUSE_LADDER_SECONDS) - 1)
    return _BATCH_PAUSE_LADDER_SECONDS[idx]


def clear_batch_rate_limit_window() -> None:
    _batch_rate_limit_window.clear()


def _gemini_lane(provider: str | None, model_name: str | None) -> tuple[str, str] | None:
    if provider == "google" and model_name in _GEMINI_COOLDOWN_MODELS:
        return (provider, model_name)
    return None


def _cooldown_seconds_for_exhaustion(count: int) -> int:
    idx = min(max(count, 1), len(_COOLDOWN_LADDER_SECONDS)) - 1
    return _COOLDOWN_LADDER_SECONDS[idx]


def _sleep_active_cooldown(lane: tuple[str, str] | None) -> None:
    if lane is None:
        return
    remaining = _cooldown_until.get(lane, 0.0) - time.monotonic()
    if remaining > 0:
        time.sleep(remaining)


async def _sleep_active_cooldown_async(lane: tuple[str, str] | None) -> None:
    if lane is None:
        return
    remaining = _cooldown_until.get(lane, 0.0) - time.monotonic()
    if remaining > 0:
        await asyncio.sleep(remaining)


def _record_success(lane: tuple[str, str] | None) -> None:
    if lane is None:
        return
    _rate_limit_exhaustions[lane] = 0
    _cooldown_until.pop(lane, None)


def _record_exhausted_rate_limit(lane: tuple[str, str], exc: BaseException) -> RateLimitCooldownError:
    count = _rate_limit_exhaustions.get(lane, 0) + 1
    _rate_limit_exhaustions[lane] = count
    cooldown = _cooldown_seconds_for_exhaustion(count)
    _cooldown_until[lane] = time.monotonic() + cooldown
    return RateLimitCooldownError(
        provider=lane[0],
        model=lane[1],
        cooldown_seconds=cooldown,
        original_error=str(exc),
    )


def _iter_error_details(exc: BaseException) -> list[dict[str, Any]]:
    details = getattr(exc, "details", None)
    if isinstance(details, dict):
        nested = details.get("error", {}).get("details", details.get("details", []))
        return nested if isinstance(nested, list) else []
    return details if isinstance(details, list) else []


def _parse_retry_delay_seconds(value: Any) -> float | None:
    if isinstance(value, str) and value.endswith("s"):
        try:
            return float(value[:-1])
        except ValueError:
            return None
    if isinstance(value, (int, float)):
        return float(value)
    return None


def _batch_abort_reason(exc: BaseException) -> str | None:
    for detail in _iter_error_details(exc):
        retry_delay = _parse_retry_delay_seconds(detail.get("retryDelay"))
        if retry_delay is not None and retry_delay > 600:
            return f"retry delay {retry_delay:g}s exceeds batch threshold"
        violations = detail.get("violations", [])
        if isinstance(violations, list):
            for violation in violations:
                if isinstance(violation, dict) and "PerDay" in str(violation.get("quotaId", "")):
                    return f"daily quota exhausted: {violation.get('quotaId')}"
        if "PerDay" in str(detail.get("quotaId", "")):
            return f"daily quota exhausted: {detail.get('quotaId')}"
    return None


class _NoThinkWrapper(BaseChatModel):
    """Wraps a ChatOpenAI model to append /no_think to every HumanMessage.

    Qwen3 Instruct in LM Studio enters thinking (chain-of-thought) mode by
    default, producing a huge reasoning_content blob and empty content.
    Appending /no_think disables it per Qwen3's chat template spec.
    """

    inner: ChatOpenAI
    model_name: str

    @property
    def _llm_type(self) -> str:
        return "lmstudio-no-think"

    def _inject(self, messages: Sequence[BaseMessage]) -> list[BaseMessage]:
        if "qwen" not in self.model_name.lower():
            return list(messages)
            
        out = []
        for msg in messages:
            if isinstance(msg, HumanMessage) and not str(msg.content).endswith(_NO_THINK):
                content = str(msg.content) + f" {_NO_THINK}"
                out.append(HumanMessage(content=content))
            else:
                out.append(msg)
        return out

    def _generate(self, messages, stop=None, run_manager=None, **kwargs):
        return self.inner._generate(self._inject(messages), stop=stop, run_manager=run_manager, **kwargs)

    async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs):
        return await self.inner._agenerate(self._inject(messages), stop=stop, run_manager=run_manager, **kwargs)

    def bind_tools(self, tools: Any, **kwargs: Any):
        # Delegate tool binding to inner model; return a wrapper that injects /no_think
        bound = self.inner.bind_tools(tools, **kwargs)
        wrapper = _BoundNoThinkWrapper(bound=bound, inject=self._inject)
        return wrapper


class _BoundNoThinkWrapper(Runnable):
    """Thin Runnable wrapper around a tool-bound model that injects /no_think."""

    def __init__(self, bound, inject):
        self._bound = bound
        self._inject = inject

    def invoke(self, input, config=None, **kwargs):
        return self._bound.invoke(self._inject(input), config=config, **kwargs)

    async def ainvoke(self, input, config=None, **kwargs):
        return await self._bound.ainvoke(self._inject(input), config=config, **kwargs)

    def __getattr__(self, name):
        return getattr(self._bound, name)


class _RetryWrapper(BaseChatModel):
    """Wraps any BaseChatModel to apply exponential backoff on 429 errors."""

    inner: BaseChatModel
    provider: str | None = None
    model_name: str | None = None

    @property
    def _llm_type(self) -> str:
        return f"retry-{self.inner._llm_type}"

    def _generate(self, *args, **kwargs):
        lane = _gemini_lane(self.provider, self.model_name)
        _sleep_active_cooldown(lane)
        try:
            for attempt in Retrying(**_sync_retry_params()):
                with attempt:
                    try:
                        result = self.inner._generate(*args, **kwargs)
                    except Exception as observed:
                        record_rate_limit_observation(observed)
                        raise
                    record_rate_limit_success()
                    _record_success(lane)
                    return result
        except Exception as exc:
            if lane is not None and is_retryable_rate_limit(exc):
                reason = _batch_abort_reason(exc)
                if reason is not None:
                    raise BatchAbortRateLimitError(reason=reason, original_error=str(exc)) from exc
                raise _record_exhausted_rate_limit(lane, exc) from exc
            raise

    async def _agenerate(self, *args, **kwargs):
        lane = _gemini_lane(self.provider, self.model_name)
        await _sleep_active_cooldown_async(lane)
        try:
            async for attempt in AsyncRetrying(**_async_retry_params()):
                with attempt:
                    try:
                        result = await self.inner._agenerate(*args, **kwargs)
                    except Exception as observed:
                        record_rate_limit_observation(observed)
                        raise
                    record_rate_limit_success()
                    _record_success(lane)
                    return result
        except Exception as exc:
            if lane is not None and is_retryable_rate_limit(exc):
                reason = _batch_abort_reason(exc)
                if reason is not None:
                    raise BatchAbortRateLimitError(reason=reason, original_error=str(exc)) from exc
                raise _record_exhausted_rate_limit(lane, exc) from exc
            raise

    def bind_tools(self, tools: Any, **kwargs: Any):
        bound = self.inner.bind_tools(tools, **kwargs)
        return _BoundRetryWrapper(bound=bound, provider=self.provider, model_name=self.model_name)


class _BoundRetryWrapper(Runnable):
    """Wraps a tool-bound Runnable to apply retry logic to .invoke()."""

    def __init__(self, bound, provider: str | None = None, model_name: str | None = None):
        self._bound = bound
        self._provider = provider
        self._model_name = model_name

    def invoke(self, input, config=None, **kwargs):
        lane = _gemini_lane(self._provider, self._model_name)
        _sleep_active_cooldown(lane)
        try:
            for attempt in Retrying(**_sync_retry_params()):
                with attempt:
                    try:
                        result = self._bound.invoke(input, config=config, **kwargs)
                    except Exception as observed:
                        record_rate_limit_observation(observed)
                        raise
                    record_rate_limit_success()
                    _record_success(lane)
                    return result
        except Exception as exc:
            if lane is not None and is_retryable_rate_limit(exc):
                reason = _batch_abort_reason(exc)
                if reason is not None:
                    raise BatchAbortRateLimitError(reason=reason, original_error=str(exc)) from exc
                raise _record_exhausted_rate_limit(lane, exc) from exc
            raise

    async def ainvoke(self, input, config=None, **kwargs):
        lane = _gemini_lane(self._provider, self._model_name)
        await _sleep_active_cooldown_async(lane)
        try:
            async for attempt in AsyncRetrying(**_async_retry_params()):
                with attempt:
                    try:
                        result = await self._bound.ainvoke(input, config=config, **kwargs)
                    except Exception as observed:
                        record_rate_limit_observation(observed)
                        raise
                    record_rate_limit_success()
                    _record_success(lane)
                    return result
        except Exception as exc:
            if lane is not None and is_retryable_rate_limit(exc):
                reason = _batch_abort_reason(exc)
                if reason is not None:
                    raise BatchAbortRateLimitError(reason=reason, original_error=str(exc)) from exc
                raise _record_exhausted_rate_limit(lane, exc) from exc
            raise

    def __getattr__(self, name):
        return getattr(self._bound, name)


def _resolve_agent_model_choice(cfg: Config) -> tuple[str, str]:
    tier = (cfg.agent_model_tier or "strong").strip().lower()
    tiers = {
        "cheap": (cfg.cheap_provider, cfg.cheap_model),
        "strong": (cfg.strong_provider, cfg.strong_model),
    }
    if tier not in tiers:
        raise ValueError(
            "GAIA_AGENT_MODEL_TIER must be one of: cheap, strong"
        )

    provider, model = tiers[tier]
    return (
        (cfg.agent_provider or provider).strip(),
        (cfg.agent_model or model).strip(),
    )


def _build(provider: str, model: str, cfg: Config, thinking: bool = True) -> BaseChatModel:
    provider = provider.strip().lower()
    log.info("Building model for provider=%r, model=%r", provider, model)
    max_tokens = cfg.max_tokens
    
    # helper to wrap final model
    def _wrap(m):
        return _RetryWrapper(inner=m, provider=provider, model_name=model)

    if provider == "ollama":
        return _wrap(ChatOllama(model=model, num_predict=max_tokens))
    if provider == "anthropic":
        return _wrap(ChatAnthropic(model=model, api_key=cfg.anthropic_api_key, max_tokens=max_tokens))
    if provider == "google":
        from langchain_google_genai import HarmBlockThreshold, HarmCategory
        
        # Suppress all safety filters to avoid silent empty responses on academic tasks
        safety_settings = {
            HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
            HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
            HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
            HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
        }
        return _wrap(ChatGoogleGenerativeAI(
            model=model, 
            google_api_key=cfg.google_api_key, 
            max_output_tokens=max_tokens,
            safety_settings=safety_settings
        ))
    if provider == "huggingface":
        endpoint = HuggingFaceEndpoint(
            repo_id=model, huggingfacehub_api_token=cfg.huggingface_api_key, max_new_tokens=max_tokens
        )
        return _wrap(ChatHuggingFace(llm=endpoint))
    if provider == "deepseek":
        ds_kwargs = dict(
            model=model,
            base_url=cfg.deepseek_base_url or DEEPSEEK_DEFAULT_BASE_URL,
            api_key=cfg.deepseek_api_key,
            max_tokens=max_tokens,
        )
        # DeepSeek v4 defaults to thinking mode, which rejects forced tool_choice
        # (langmem extraction) and leaks raw tool-call markup in free-text calls
        # (finalizer). Opt out explicitly when the caller needs a plain response.
        if not thinking:
            ds_kwargs["extra_body"] = {"thinking": {"type": "disabled"}}
        return _wrap(ChatOpenAI(**ds_kwargs))
    if provider == "lmstudio":
        # LM Studio exposes an OpenAI-compatible API.
        inner = ChatOpenAI(
            model=model,
            base_url=cfg.lmstudio_base_url or LMSTUDIO_DEFAULT_BASE_URL,
            api_key="lm-studio",
            temperature=0,
            max_tokens=max_tokens,
        )
        # Wrap with /no_think injection to disable Qwen3 chain-of-thought mode
        # Then wrap the result with retry logic
        wrapped = _NoThinkWrapper(inner=inner, model_name=model)
        return _wrap(wrapped)
    raise ValueError(f"Unknown provider: {provider}")


def get_cheap_model(cfg: Config, thinking: bool = True) -> BaseChatModel:
    return _build(cfg.cheap_provider, cfg.cheap_model, cfg, thinking=thinking)


def get_strong_model(cfg: Config, thinking: bool = True) -> BaseChatModel:
    return _build(cfg.strong_provider, cfg.strong_model, cfg, thinking=thinking)


def get_extra_strong_model(cfg: Config, thinking: bool = True) -> BaseChatModel:
    provider, model = _resolve_agent_model_choice(cfg)
    return _build(provider, model, cfg, thinking=thinking)