workflows/stateful_react_agent/nodes/main_agent.py

"""Native ReAct agent node - single stateful agent."""

from __future__ import annotations

import logging
import os
import time
from pathlib import Path
from typing import Any

from frontier_agent.components.finalization import (
    ResearchWall,
    check_wall_feasibility,
    nonnegative_seconds,
    positive_seconds,
    resolve_research_wall,
)
from frontier_agent.components.observers.context_size_guard import ContextSizeGuard
from frontier_agent.components.observers.duplicate_query_rollback import (
    DuplicateQueryRollbackObserver,
)
from frontier_agent.components.observers.finalization_reserve import (
    FinalizationReserveObserver,
)
from frontier_agent.components.observers.last_turn_forcer import LastTurnForcer
from frontier_agent.components.observers.leaked_tool_call_retry import (
    LeakedToolCallRetryObserver,
)
from frontier_agent.components.observers.react_step_tracker import ReactStepTracker
from frontier_agent.components.observers.repetition_guard import RepetitionGuard
from frontier_agent.components.observers.sse_observer import SSEObserver
from frontier_agent.components.observers.stuck_target_guard import StuckTargetGuard
from frontier_agent.components.observers.text_repetition_guard import (
    TextRepetitionGuard,
)
from frontier_agent.components.observers.trajectory import TrajectoryFileObserver
from frontier_agent.components.observers.wall_clock_observer import (
    WallClockDeadlineObserver,
)
from frontier_agent.core.loop_types import LoopConfig, LoopPolicy
from frontier_agent.core.messages import text_of, user_msg
from frontier_agent.core.runtime import registry
from frontier_agent.core.runtime.loop.agent_loop import run_agent_loop
from frontier_agent.core.runtime.loop.budget_consistency import (
    check_context_budget,
)
from frontier_agent.core.runtime.loop.compact import KeepLastNToolResultsCompactor
from frontier_agent.core.runtime.loop.llm_client import bind_temperature, extract_model_name
from frontier_agent.core.runtime.loop.model_profile import (
    ModelProfile,
    resolve_history_policy,
)
from frontier_agent.core.runtime.loop.tiered_compact import (
    InputTokenGauge,
    InputTokenThresholdPolicy,
    TieredCompactor,
    compaction_trigger_tokens,
)
from frontier_agent.core.runtime.pause_check import pause_check_from_state
from frontier_agent.core.runtime.resources.manager import ResourceManager
from frontier_agent.core.runtime.session_history import build_session_turn
from frontier_agent.infra.wall_time_lease import WALL_TIME_LEASE_SCOPE_KEY
from frontier_agent.models.node_context import NodeContext
from frontier_agent.state.event_store.sqlite import EventStore
from frontier_agent.utils.history_input import extract_current_query
from frontier_agent.utils.language import (
    detect_language_from_prompt,
    is_language_detect_enabled,
    language_instruction,
    resolve_language,
)
from plugins.tools._bash_policy import reset_policy_mode, set_policy_mode
from plugins.tools._sandbox import (
    BwrapSandbox,
    SandboxUnavailableError,
    bwrap_available,
    clear_task_sandbox,
    make_current_sandbox,
    resolve_mount_dirs,
    resolve_sandbox_mode,
    set_task_sandbox,
)
from plugins.tools.task_board import build_task_board_observer, clear_board
from workflows.stateful_react_agent._runtime import (
    ReactToolResultPostProcessor,
    _minimal_best_effort_answer,
    _strip_leaked_tool_calls,
    _strip_thinking,
    render_system_prompt_notes,
)
from workflows.stateful_react_agent.observers import (
    FinalAnswerSalvageObserver,
    ReporterStreamObserver,
    ReportSynthesisObserver,
    RichConsoleObserver,
)
from workflows.stateful_react_agent.prompts import (
    BOARD_PROMPT_ADDENDUM,
    get_direct_system_prompt,
    get_react_system_prompt,
)

logger = logging.getLogger(__name__)

REACT_MAX_TURNS = 100
REACT_TOOL_TIMEOUT_S = 1800
REACT_LLM_TIMEOUT_S = 1800
REACT_KEEP_LAST_K = 5
REACT_COMPACT_AFTER_TURNS = 0
REACT_CONTEXT_TOKEN_LIMIT = 180_000

_STATEFUL_FINALIZATION_MESSAGE = (
    "Finalization phase has started. Stop new research and implementation "
    "branches. Use the remaining tool-enabled turns to finish the requested "
    "work, copy the best current deliverables to /outputs, and run only the "
    "minimum checks needed to avoid shipping broken files. Then answer the "
    "user in plain text. If full completion is impossible, preserve the best "
    "existing artifacts and give a useful partial answer; never finish with "
    "no deliverable and no answer."
)

_llm_cache: dict[str, tuple[dict[str, Any], Any, ModelProfile | None]] = {}


# Wall-clock arithmetic lives in the shared finalization component; these
# aliases keep the workflow's existing private import surface.
_positive_seconds = positive_seconds
_nonnegative_seconds = nonnegative_seconds


def _resolve_research_wall(
    agent_cfg: dict[str, Any],
    *,
    hard_wall_reserve_s: float | None = None,
) -> ResearchWall:
    """Resolve the research deadline plus the hard ceiling it derives from."""
    reserve_s = (
        nonnegative_seconds(
            agent_cfg.get("wall_deadline_reserve_s"),
            default=180,
            label="stateful wall_deadline_reserve_s",
        )
        if hard_wall_reserve_s is None
        else max(float(hard_wall_reserve_s), 0.0)
    )
    return resolve_research_wall(
        agent_cfg, reserve_s=reserve_s, label_prefix="stateful",
    )


def _resolve_wall_deadline_s(
    agent_cfg: dict[str, Any],
    *,
    hard_wall_reserve_s: float | None = None,
) -> float:
    """Research-only deadline for :class:`WallClockDeadlineObserver`."""
    return _resolve_research_wall(
        agent_cfg, hard_wall_reserve_s=hard_wall_reserve_s,
    ).research_deadline_s


def _resolve_runaway_guardrails(
    agent_cfg: dict[str, Any],
) -> tuple[float | None, int | None, float | None]:
    """Resolve ``(reasoning_only_timeout_s, reasoning_only_max_tokens,
    logical_call_timeout_s)`` from the profile.

    Absent or ``0`` means off for each — and off matters more than it looks:
    the semantic reasoning watchdog only exists on the *streaming* request
    path, and the loop picks that path precisely because one of the
    ``reasoning_only_*`` values is set. With both unset, a reply that spends
    its whole completion budget inside the reasoning channel can only be
    detected after the fact, once the provider has already billed it.
    """
    timeout_raw = agent_cfg.get("reasoning_only_timeout_s")
    tokens_raw = agent_cfg.get("reasoning_only_max_tokens")
    logical_raw = agent_cfg.get("logical_call_timeout_s")
    return (
        float(timeout_raw) if timeout_raw else None,
        int(tokens_raw) if tokens_raw else None,
        float(logical_raw) if logical_raw else None,
    )


def _resolve_finalization_timeout_s(
    agent_cfg: dict[str, Any],
    *,
    llm_timeout_s: float,
) -> float:
    """Resolve the clean-context rescue timeout per fallback leg.

    Absent or ``0`` falls back to ``llm_timeout_s``; ``0`` does NOT mean
    "unlimited" here.
    """
    value = _positive_seconds(
        agent_cfg.get("finalization_timeout_s"),
        label="stateful finalization_timeout_s",
    )
    return value or max(float(llm_timeout_s), 1.0)


def _resolve_reporter_timeout_s(
    agent_cfg: dict[str, Any],
    *,
    llm_timeout_s: float,
) -> float:
    """Resolve a finite per-leg reporter read timeout.

    Absent or ``0`` falls back to ``llm_timeout_s`` — unlike the wall-time keys,
    ``0`` here does NOT mean "unlimited"; an unbounded leg is never wanted.
    """
    value = _positive_seconds(
        agent_cfg.get("reporter_timeout_s"),
        label="stateful reporter_timeout_s",
    )
    return value or max(float(llm_timeout_s), 1.0)


def _resolve_reporter_phase_timeout_s(
    agent_cfg: dict[str, Any],
    *,
    llm_timeout_s: float,
) -> float:
    """Resolve the absolute ceiling around the complete fallback chain.

    Absent or ``0`` falls back to ``llm_timeout_s * 3``; ``0`` does NOT mean
    "unlimited" here. The runtime clamps the result further when a platform
    hard wall leaves less time than this.
    """
    value = _positive_seconds(
        agent_cfg.get("reporter_phase_timeout_s"),
        label="stateful reporter_phase_timeout_s",
    )
    return value or max(float(llm_timeout_s) * 3, 1.0)


def _language_probe(state: dict[str, Any], question: str) -> str:
    """Return only the latest user instruction for answer-language detection.

    Legacy multi-turn requests may fold prior turns into ``question`` with the
    current query appended last. A long history can otherwise dilute the
    heuristic or crowd out an explicit language instruction. Prefer the clean
    ``current_query`` field; ``extract_current_query`` keeps wrapped callers
    safe.
    """
    return (
        str(state.get("current_query") or "").strip()
        or extract_current_query(question).strip()
        or question
    )


def _resolve_answer_language(state: dict[str, Any], question: str) -> str:
    """Resolve the query language used by the agent and optional reporter.

    The SDK currently seeds every run with legacy ``language="en"`` even when
    the caller supplied no preference.  Treat that default like ``"auto"`` so
    non-English queries are detected, matching the agent_team reporter's
    query-language behavior.  Other explicit language values remain
    authoritative.

    Detection uses the cleaned task text (``question`` has already had the
    protocol ``# Task`` wrapper removed).
    """
    requested = str(state.get("language", "auto") or "auto").strip()
    if requested.lower() == "en":
        requested = "auto"
    language_state = {
        "language": requested,
        "original_question": question,
    }
    return resolve_language(language_state) or "English"


def _flag(value: Any, *, default: bool) -> bool:
    """Coerce a profile/metadata boolean that may have come from env substitution.

    ``_resolve_env_vars`` yields strings, so a profile writing
    ``reporter: ${REPORTER:-false}`` hands this the string ``"false"`` — and
    ``bool("false")`` is True, which silently inverts the operator's intent.
    Vocabulary matches ``workflows/agent_team/nodes/main_agent.py``.
    """
    if value is None:
        return default
    if isinstance(value, str):
        return value.strip().lower() not in ("", "0", "false", "no", "off")
    return bool(value)


async def _resolve_answer_language_with_llm(
    state: dict[str, Any],
    question: str,
    *,
    answer_language: str,
    reporter_enabled: bool,
    llm: Any,
    llm_timeout: float,
    profile: dict[str, Any] | None,
    metadata: dict[str, Any],
) -> str:
    """Upgrade auto language detection using the already-resolved profile LLM."""
    requested = str(state.get("language", "auto") or "auto").strip().lower()
    if (
        not reporter_enabled
        or requested not in ("", "auto", "en")
        or not is_language_detect_enabled()
    ):
        return answer_language

    async def _detect_ask(prompt: str) -> str:
        resp = await llm.chat([user_msg(prompt)], timeout=llm_timeout)
        usage = dict(getattr(resp, "usage", None) or {})
        provider = str(
            (getattr(resp, "response_metadata", None) or {}).get(
                "provider_actually_used",
            )
            or "",
        )
        model = (
            getattr(resp, "model", "")
            or extract_model_name(llm, profile)
            or ""
        )
        from workflows._shared.sdk_shim import (
            record_language_detect_usage,
        )

        record_language_detect_usage(
            metadata.get("sdk_protocol_usage_aggregator"),
            usage=usage,
            provider=provider,
            model=model,
        )
        return _strip_thinking(text_of(resp.content))

    return (
        await detect_language_from_prompt(question, _detect_ask)
        or answer_language
    )


def _resolve_llm_and_profile(
    profile_name: str | None,
    *,
    profile_overrides: dict[str, Any] | None = None,
    profile_inline: dict[str, Any] | None = None,
) -> tuple[Any, dict[str, Any] | None, ModelProfile | None]:
    """Return ``(llm, profile_dict, model_profile)`` for this run."""
    if profile_name or profile_inline:
        from workflows.stateful_react_agent.profile import (
            build_react_model_profile,
            create_react_llm,
            load_react_profile,
        )

        bypass_cache = bool(profile_overrides) or profile_inline is not None
        cache_key = profile_name or "__inline__"
        if bypass_cache or cache_key not in _llm_cache:
            profile = load_react_profile(
                profile_name or "",
                overrides=profile_overrides,
                inline=profile_inline,
            )
            entry = (
                profile,
                create_react_llm(profile),
                build_react_model_profile(profile),
            )
            if not bypass_cache:
                _llm_cache[cache_key] = entry
        else:
            entry = _llm_cache[cache_key]
        profile, llm, model_profile = entry
        return llm, profile, model_profile

    llm = registry.get(ResourceManager).get_llm("stateful_react")
    return bind_temperature(llm, 0.0), None, None


def _resolve_trajectory_dir(state: dict[str, Any], task_id: str) -> Path:
    trial_dir = (state.get("metadata") or {}).get("_trial_dir")
    if trial_dir:
        return Path(trial_dir) / "agent" / "trajectories"
    if run_dir := os.environ.get("APODEX_RUN_DIR", "").strip():
        return Path(run_dir) / "trajectories" / task_id
    return Path("logs") / "stateful_react" / task_id / "trajectories"


def _resolve_worktree_root(state: dict[str, Any], task_id: str) -> Path:
    md = state.get("metadata") or {}
    trial_dir = md.get("_trial_dir")
    if trial_dir:
        return Path(trial_dir) / "sandbox" / "worktree"
    experiment = md.get("experiment")
    bench_task_id = md.get("bench_task_id")
    if experiment and bench_task_id:
        return (
            Path("experiments")
            / str(experiment)
            / "questions"
            / str(bench_task_id)
            / "worktree"
        )
    coding_root = md.get("coding_workspace_root")
    if coding_root:
        return Path(coding_root)
    return Path("logs") / "stateful_react" / task_id / "worktree"


def _resolve_sandbox_binds(
    state: dict[str, Any],
    worktree_root: Path,
) -> tuple[tuple[tuple[str, str, bool], ...], Path]:
    """Resolve benchmark-provided ``/inputs`` and shared ``/outputs`` mounts."""
    metadata = state.get("metadata") or {}
    outputs_dir = worktree_root.parent / "outputs"

    binds: list[tuple[str, str, bool]] = []
    dataset_root = str(metadata.get("_dataset_root") or "")
    for mount in metadata.get("_sandbox_mounts") or []:
        src = str(mount.get("src", "")).strip()
        dst = str(mount.get("dst", "")).strip()
        if not src or not dst:
            continue
        if not dst.startswith("/inputs"):
            logger.warning("sandbox mount dst not under /inputs, skipped: %s", dst)
            continue
        src_path = Path(src)
        if not src_path.is_absolute() and dataset_root:
            src_path = Path(dataset_root) / src_path
        read_only = str(mount.get("mode", "ro")).lower() != "rw"
        binds.append((str(src_path.expanduser().resolve()), dst, read_only))

    binds.append((str(outputs_dir), "/outputs", False))
    return tuple(binds), outputs_dir


def _shallow_entries(root: str, *, limit: int = 40) -> list[str]:
    """Depth-1 listing of *root* (dirs suffixed ``/``), for fallback probing."""
    try:
        p = Path(root)
        if not p.is_dir():
            return []
        names: list[str] = []
        for e in sorted(p.iterdir()):
            try:
                names.append(e.name + ("/" if e.is_dir() else ""))
            except OSError:
                names.append(e.name)
            if len(names) >= limit:
                names.append("… (truncated)")
                break
        return names
    except OSError:
        return []


def _log_inputs_dir_contents(
    roots: list[tuple[str, str]],
    *,
    fallback_roots: list[tuple[str, str]] | None = None,
    max_files: int = 200,
) -> None:
    """Diagnostic: log what actually lives under each ``/inputs`` root at runtime.

    ``roots`` is ``[(label, host_path), ...]`` where ``host_path`` is the real
    directory the model's file tools (``read_file`` / ``glob_search`` /
    ``grep_search``) will see as ``/inputs`` — in container mode the mounted
    ``/inputs`` itself, in bwrap mode the host ``src`` of each ``/inputs`` bind.
    For each root it logs the resolved absolute path, whether it exists, and
    every file beneath it (name + absolute path + size) so a mount/path mismatch
    (S3 files landing at a path the tools don't read) is visible in the worker
    log stream.

    When no input file surfaces at any expected root, ``fallback_roots`` are
    probed shallowly (depth-1) so a misplaced mount (e.g. files under
    ``/workspace`` or a nested UUID subdir) shows up in the same log burst.
    Never raises — a diagnostic must not break the run.
    """
    found_any = False
    for label, root in roots:
        try:
            p = Path(root)
            exists = p.exists()
            is_dir = exists and p.is_dir()
            logger.info(
                "[stateful_react inputs] %s path=%s exists=%s is_dir=%s",
                label, p, exists, is_dir,
            )
            if not is_dir:
                continue
            files: list[Path] = []
            for f in sorted(p.rglob("*")):
                try:
                    if f.is_file():
                        files.append(f)
                except OSError:
                    continue
                if len(files) >= max_files:
                    break
            if not files:
                logger.warning(
                    "[stateful_react inputs] %s path=%s is EMPTY — read_file / "
                    "glob_search will find nothing here", label, p,
                )
                continue
            found_any = True
            logger.info(
                "[stateful_react inputs] %s path=%s has %d file(s):",
                label, p, len(files),
            )
            for f in files:
                try:
                    size = f.stat().st_size
                except OSError:
                    size = -1
                logger.info(
                    "[stateful_react inputs]   name=%r abs=%s size=%s", f.name, f, size,
                )
        except Exception as exc:
            logger.warning(
                "[stateful_react inputs] failed to scan %s path=%s: %s",
                label, root, exc,
            )

    if not found_any and fallback_roots:
        logger.warning(
            "[stateful_react inputs] no files at expected input path(s); probing "
            "fallback locations to find where the mounted files landed",
        )
        for label, root in fallback_roots:
            p = Path(root)
            logger.warning(
                "[stateful_react inputs] fallback %s path=%s exists=%s entries=%s",
                label, p, p.exists(), _shallow_entries(root) or "(none/not-a-dir)",
            )


def _loop_policy() -> LoopPolicy:
    return LoopPolicy(terminal_tool_names=(), no_tool_behavior="stop")


def _tools_for_stateful_react(
    resource_mgr: ResourceManager,
    agent_cfg: dict[str, Any],
) -> list[Any]:
    override = agent_cfg.get("agent_tools")
    if not override:
        return resource_mgr.get_tools_for_role("stateful_react")

    names: list[str] = []
    for raw in override:
        name = str(raw).strip()
        if name and name not in names:
            names.append(name)
    # Closed-book has to be enforced *here* too, not only on the role's tool
    # pool. A profile's ``agent_tools`` list wins over the pool, and every
    # shipped profile lists the web tools explicitly — so honouring
    # REACT_NO_WEB only in the AgentDefinition made it silently inert for any
    # real run. This is the list that actually gets bound to the model.
    import os as _os

    from workflows.stateful_react_agent import WEB_TOOL_NAMES
    no_web = _os.environ.get("REACT_NO_WEB", "").strip().lower() in (
        "1", "true", "yes", "on",
    )
    if no_web:
        dropped = [n for n in names if n in WEB_TOOL_NAMES]
        if dropped:
            names = [n for n in names if n not in WEB_TOOL_NAMES]
            logger.info(
                "closed-book (REACT_NO_WEB): dropped profile web tools %s", dropped,
            )
    # Profiles created before controlled downloads existed commonly list
    # web_fetch explicitly. Preserve that narrowing while adding its new
    # binary-file companion without requiring every deployed profile YAML to
    # change in lockstep.
    if "web_fetch" in names and "download_file" not in names:
        names.insert(names.index("web_fetch") + 1, "download_file")

    policy = resource_mgr.global_tool_policy
    all_tools = resource_mgr.all_tools
    tools: list[Any] = []
    skipped: list[str] = []
    for name in names:
        if policy is not None and not policy.allows(name):
            skipped.append(name)
            continue
        tool = all_tools.get(name)
        if tool is None:
            skipped.append(name)
            continue
        tools.append(tool)
    if skipped:
        logger.warning("stateful_react profile tools skipped: %s", skipped)
    logger.info("stateful_react tools selected by profile: %s", [t.name for t in tools])
    return tools


def _replace_tool_impls(tools: list[Any], agent_cfg: dict[str, Any]) -> list[Any]:
    out = list(tools)
    if (agent_cfg.get("web_search_impl") or "original") == "aligned":
        from plugins.tools.web_search_aligned import web_search_aligned

        out = [web_search_aligned if getattr(t, "name", "") == "web_search" else t for t in out]
    if (agent_cfg.get("web_fetch_impl") or "original") == "aligned":
        from plugins.tools.web_fetch_aligned import web_fetch_aligned

        out = [web_fetch_aligned if getattr(t, "name", "") == "web_fetch" else t for t in out]
    return out


async def react_agent_node(state: dict[str, Any], ctx: NodeContext) -> dict[str, Any]:
    """Run the single stateful ReAct agent."""
    question = state.get("original_question", "")
    if not question:
        raise ValueError("react_agent_node requires 'original_question' in state")
    if question.startswith("# Task\n\n"):
        question = question[len("# Task\n\n") :]
    question = question.rstrip()

    metadata = state.get("metadata") or {}
    profile_name = (
        metadata.get("profile")
        or metadata.get("stateful_profile")
        or metadata.get("react_profile")
        or metadata.get("swarm_profile")
    )
    llm, profile, model_profile = _resolve_llm_and_profile(
        str(profile_name) if profile_name else None,
        profile_overrides=metadata.get("profile_overrides"),
        profile_inline=metadata.get("profile_inline"),
    )
    agent_cfg = (profile or {}).get("agent", {})

    max_turns = int(agent_cfg.get("main_max_turns", agent_cfg.get("max_turns", REACT_MAX_TURNS)))
    tool_timeout = float(agent_cfg.get("tool_timeout_s", REACT_TOOL_TIMEOUT_S))
    llm_timeout = float(agent_cfg.get("llm_timeout_s", REACT_LLM_TIMEOUT_S))
    first_chunk_s = agent_cfg.get("first_chunk_s")
    (
        reasoning_only_timeout_s,
        reasoning_only_max_tokens,
        logical_call_timeout_s,
    ) = _resolve_runaway_guardrails(agent_cfg)
    history_policy = resolve_history_policy(agent_cfg)
    keep_last_k = int(agent_cfg.get("keep_last_k", REACT_KEEP_LAST_K))
    compact_after_turns = int(agent_cfg.get("compact_after_turns", REACT_COMPACT_AFTER_TURNS))
    context_token_limit = int(agent_cfg.get("context_token_limit", REACT_CONTEXT_TOKEN_LIMIT))
    max_input_tokens = int(agent_cfg.get("max_input_tokens", 0) or 0)
    # Tiered context compaction (opt-in via profile ``context_compaction``):
    #   "off" (default) → legacy keep_last_k path.
    #   "tiered"        → compact ONLY when the REAL input tokens cross
    #     ``max_len`` * 0.8; Tier1 keeps the last ``tier1_keep_tool_result`` tool
    #     results (drops older), Tier2 LLM-summarises the middle only if Tier1
    #     left the estimate above max_len*0.6. ``max_len`` = model context window.
    context_compaction = str(agent_cfg.get("context_compaction", "off")).lower()
    compaction_spill = _flag(agent_cfg.get("compaction_spill"), default=False)
    max_len = int(agent_cfg.get("max_len", 0) or 0)
    # See the sibling call in agent_team: the sglang doctor covers the compose
    # path only, and nothing checked the values the loop is actually built from.
    check_context_budget(
        max_len=max_len,
        max_input_tokens=max_input_tokens,
        max_tokens=int((profile or {}).get("llm", {}).get("max_tokens", 0) or 0) or None,
        reasoning_only_max_tokens=reasoning_only_max_tokens,
        label="stateful_react",
    )
    tier1_keep_tool_result = int(agent_cfg.get("tier1_keep_tool_result", 5))
    keep_recent_turns = int(agent_cfg.get("keep_recent_turns", 5))
    fs_mode = bool(agent_cfg.get("fs_mode", False)) or bool(metadata.get("fs_mode", False))
    # Opt-in task-board mode: append the board addendum so the agent uses the
    # add_task / update_task board tools as a status checklist (open →
    # in_progress → resolved / cancelled). Off by default; the board tools must
    # also be in the profile's ``agent_tools``.
    task_board = bool(agent_cfg.get("task_board", False))
    # Opt-in direct-inference mode: NO tools are bound, so the model
    # answers from its own knowledge in a single turn (the first tool-free reply
    # is the final answer). Uses the tool-free direct system prompt and skips the
    # task-board / sandbox-FS prompt scaffolding, which are meaningless without
    # tools. The sandbox is still constructed below (harmless — unused when no
    # tool can invoke it).
    direct = bool(agent_cfg.get("direct", False))
    # Salvage philosophy for abnormal/infra exits (llm_error / wall_deadline /
    # budget_exhausted / max_attempts): True (default) = make a clean-context
    # LLM rescue call; False = skip that extra call. Both paths still return a
    # deterministic non-empty best-effort status when no answer was generated.
    salvage_infra_errors = bool(agent_cfg.get("salvage_infra_errors", True))
    # Stop-loss (both opt-in, 0 = off):
    #   ``research_wall_time_s`` (legacy alias ``wall_deadline_s``) — research
    #     budget only. The loop exits cleanly with ``wall_deadline`` and the
    #     reporter takes over outside that budget.
    #   ``stuck_target_hint_after`` — confirmed failures for one network host
    #     within the last ``stuck_target_window`` network turns. A fetch success
    #     resets that host. The first threshold asks for a route change; the
    #     escalation threshold (HARD failures only) quarantines the host for the
    #     rest of the loop.
    finalization_reserve_turns = int(
        agent_cfg.get("finalization_reserve_turns", 8) or 8,
    )
    finalization_timeout_s = _resolve_finalization_timeout_s(
        agent_cfg,
        llm_timeout_s=llm_timeout,
    )
    stuck_hint_after = int(agent_cfg.get("stuck_target_hint_after", 0) or 0)
    stuck_escalate_after = int(
        agent_cfg.get("stuck_target_escalate_after", stuck_hint_after * 2) or 0,
    )
    stuck_window = int(agent_cfg.get("stuck_target_window", 20) or 20)
    # Opt-in lightweight reporter: when on, a single tool-free
    # streaming LLM call synthesises a structured, cited report over the whole
    # conversation at loop end (standard-mode reporter parity), and it
    # REPLACES the raw-answer ReporterStreamObserver. Off by default → direct
    # answer, current behaviour. Profile ``agent.reporter`` sets it; a metadata
    # ``reporter`` key overrides (per-request opt-in without a profile edit).
    reporter_enabled = _flag(agent_cfg.get("reporter"), default=False)
    if metadata.get("reporter") is not None:
        reporter_enabled = _flag(metadata.get("reporter"), default=False)
    reporter_timeout_s = _resolve_reporter_timeout_s(
        agent_cfg,
        llm_timeout_s=llm_timeout,
    )
    reporter_phase_timeout_s = _resolve_reporter_phase_timeout_s(
        agent_cfg,
        llm_timeout_s=llm_timeout,
    )
    configured_wall_reserve_s = _nonnegative_seconds(
        agent_cfg.get("wall_deadline_reserve_s"),
        default=180,
        label="stateful wall_deadline_reserve_s",
    )
    landing_budget_s = (
        reporter_phase_timeout_s
        if reporter_enabled
        else finalization_timeout_s
    )
    # A tool may start just before the turn-end observer checks the research
    # deadline. Reserve enough for that complete overrun plus the bounded
    # reporter/finalization phase. The overrun is the tool's OUTER wait, not
    # the configured timeout — budget-aware tools get a grace on top so they
    # can report their own timeout — so ask the loop rather than assuming.
    from frontier_agent.core.runtime.loop.tool_exec import max_tool_wall_time_s
    worst_case_tool_s = max_tool_wall_time_s(tool_timeout)
    wall_deadline_reserve_s = max(
        configured_wall_reserve_s,
        worst_case_tool_s + landing_budget_s,
    )
    research_wall = _resolve_research_wall(
        agent_cfg,
        hard_wall_reserve_s=wall_deadline_reserve_s,
    )
    wall_deadline_s = research_wall.research_deadline_s
    # ``soft_wall_deadline_s`` floors research at half the wall, so on a short
    # wall the reserve above is NOT what actually survives. Hand the
    # finalization stage an absolute instant instead of a static budget: it
    # clamps itself to the time really left and fails open with a real answer,
    # rather than being killed mid-call by the platform ceiling (the live serve
    # ``RenewableWallTimeLease`` gives only a few seconds of grace).
    node_started_monotonic = time.monotonic()
    hard_deadline_monotonic = (
        node_started_monotonic + research_wall.hard_total_s
        if research_wall.hard_total_s > 0
        else None
    )
    check_wall_feasibility(
        hard_total_s=research_wall.hard_total_s,
        research_deadline_s=wall_deadline_s,
        tool_timeout_s=worst_case_tool_s,
        landing_budget_s=landing_budget_s,
        label_prefix="stateful",
    )
    reporter_context_default = (
        max(1_024, max_input_tokens - 4_096)
        if max_input_tokens > 0
        else 220_000
    )
    reporter_context_max_tokens = int(
        agent_cfg.get(
            "reporter_context_max_tokens",
            reporter_context_default,
        )
        or reporter_context_default,
    )
    language_probe_text = _language_probe(state, question)
    answer_language = _resolve_answer_language(state, language_probe_text)
    # When the caller left the language unpinned, upgrade the character
    # heuristic to a single-call LLM detector (arbitrary-language coverage —
    # Spanish/Vietnamese/… that the CJK heuristic collapses to English).
    # The shared detector falls back to that same heuristic on any LLM error,
    # so this never does worse than the line above. The helper also gates on
    # reporter mode and ``LANGUAGE_DETECT_ENABLED``.
    answer_language = await _resolve_answer_language_with_llm(
        state,
        language_probe_text,
        answer_language=answer_language,
        reporter_enabled=reporter_enabled,
        llm=llm,
        llm_timeout=llm_timeout,
        profile=profile,
        metadata=metadata,
    )

    resource_mgr = registry.get(ResourceManager)
    tools = _replace_tool_impls(
        _tools_for_stateful_react(resource_mgr, agent_cfg),
        agent_cfg,
    )
    # Direct-inference mode: drop all tools so the model answers in one turn.
    if direct:
        tools, tool_names = [], []
    else:
        tool_names = [getattr(t, "name", "") for t in tools if getattr(t, "name", "")]

    if direct:
        system_prompt = get_direct_system_prompt()
    else:
        system_prompt = get_react_system_prompt(fs_mode=fs_mode)
    addendum = str(metadata.get("_sys_prompt_addendum") or "").strip()
    if addendum:
        system_prompt = f"{system_prompt}\n\n{addendum}"
    # task-board + sandbox-FS notes are tool-dependent — skip them in direct mode.
    if task_board and not direct:
        system_prompt = f"{system_prompt}{BOARD_PROMPT_ADDENDUM}"
    if reporter_enabled:
        # Keep the research agent's draft/salvage answer in the same language
        # as the reporter.  This also makes the fail-open path language-stable
        # if report synthesis fails.
        system_prompt = f"{system_prompt}{language_instruction(answer_language)}"

    # Sandbox mode: trusted deployment config must explicitly select container;
    # a profile may only tighten auto to bwrap, never attest container isolation.
    #   container — the surrounding docker container IS the isolation; tools
    #     operate directly on the mounted /workspace, /outputs, /inputs via a
    #     CurrentSandbox. No bwrap needed. This is the production model.
    #   bwrap / auto — local benchmark path: bwrap namespaces (unchanged).
    sandbox_mode = resolve_sandbox_mode(agent_cfg)
    sandbox_binds: tuple[tuple[str, str, bool], ...] = ()
    workspace_root_str = ""
    # Bound only on the container/native branch below; the one reader is guarded
    # by the same sandbox_mode test, so "" never reaches it.
    inputs_dir = ""
    if sandbox_mode in ("container", "native"):
        workspace_dir, outputs_dir_str, inputs_dir = resolve_mount_dirs()
        worktree_root = Path(workspace_dir)
        outputs_dir = Path(outputs_dir_str)
        workspace_root_str = str(worktree_root)
    else:
        worktree_root = _resolve_worktree_root(state, ctx.task_id)
        sandbox_binds, outputs_dir = _resolve_sandbox_binds(state, worktree_root)
    worktree_root.mkdir(parents=True, exist_ok=True)
    outputs_dir.mkdir(parents=True, exist_ok=True)

    # /inputs is an external bind-mount (Worker Shell syncs it from S3); the
    # harness never populates it. Log what actually landed there so a
    # missing/misplaced input file is diagnosable from the worker log.
    if sandbox_mode in ("container", "native"):
        _log_inputs_dir_contents(
            [("container /inputs", inputs_dir)],
            fallback_roots=[
                ("workspace", str(worktree_root)),
                ("root", "/"),
                ("cwd", str(Path.cwd())),
            ],
        )
    else:
        _log_inputs_dir_contents(
            [
                (f"bwrap bind {dst}", src)
                for (src, dst, _ro) in sandbox_binds
                if dst.startswith("/inputs")
            ]
        )

    # Filesystem tools are usable in container mode OR when bwrap is present;
    # add the /workspace, /outputs, /inputs convention note accordingly.
    fs_enabled = sandbox_mode in ("container", "native") or bwrap_available()
    if fs_enabled and not direct:
        # Charts are written through the same filesystem tools, so the
        # clipping rule rides along with the /workspace-vs-/outputs note.
        runtime_notes = render_system_prompt_notes(
            sandbox_mode=sandbox_mode,
            tool_names=tool_names,
        )
        system_prompt = f"{system_prompt}{runtime_notes}"
    elif not direct:
        # auto/bwrap mode without bwrap must fail closed. CurrentSandbox only
        # changes cwd; it does not isolate the host filesystem or network.
        # Container mode is safe only when trusted deployment configuration
        # selected it explicitly and the surrounding container is the boundary.
        raise SandboxUnavailableError(
            "stateful-react-agent requires bubblewrap for tool execution "
            "(or SANDBOX_BACKEND=container inside an isolated task container); "
            "refusing unisolated host fallback"
        )

    event_store = registry.get_optional(EventStore)
    model_name = extract_model_name(llm)
    observers: list[Any] = [
        LeakedToolCallRetryObserver(tool_names=tool_names),
        RichConsoleObserver(),
        TrajectoryFileObserver(
            _resolve_trajectory_dir(state, ctx.task_id),
            filename="react_agent",
            tools=tools,
            model_name=model_name,
            system_prompt=system_prompt,
            user_message=question,
        ),
        ReactStepTracker(),
    ]
    if not direct:
        # Repetition stop-loss. Both of these stay hint-only: this agent IS
        # the run, so a false positive must cost one message, never the answer.
        observers.append(RepetitionGuard())
        observers.append(TextRepetitionGuard())
        if DuplicateQueryRollbackObserver.DEFAULT_TOOL_NAMES.intersection(tool_names):
            # Pops the turn before the duplicate search runs. Matters most
            # here: main_max_turns reaches 600 in the TUI profile, and tiered
            # compaction discards the older search results that would
            # otherwise remind the model it already ran this query.
            observers.append(DuplicateQueryRollbackObserver())
    # Task-board mode: re-inject the board on a cooldown so it survives
    # KeepLastN compaction (parity with agent_team). No-op if the agent never
    # writes a board; meaningless in direct mode (no tools).
    if task_board and not direct:
        observers.append(build_task_board_observer())
    if not direct:
        observers.append(FinalizationReserveObserver(
            reserve_turns=finalization_reserve_turns,
            message=_STATEFUL_FINALIZATION_MESSAGE,
        ))
        # The reserved turns above are tool-enabled so artifacts can still be
        # completed.  Strip tools only for the actual landing turn.
        observers.append(LastTurnForcer(terminal_tool=""))
    if max_input_tokens > 0:
        observers.append(ContextSizeGuard(
            max_input_tokens=max_input_tokens,
            force_compaction_first=(context_compaction == "tiered" and max_len > 0),
        ))
    # Stop-loss, both no-ops unless the profile opts in (see above).
    if stuck_hint_after > 0 and not direct:
        observers.append(StuckTargetGuard(
            hint_after=stuck_hint_after,
            escalate_after=stuck_escalate_after or stuck_hint_after * 2,
            window=stuck_window,
        ))
    if wall_deadline_s > 0:
        observers.append(WallClockDeadlineObserver(
            deadline_s=wall_deadline_s,
            # ``_resolve_wall_deadline_s`` already converted any hard
            # operational wall into this research-only soft deadline.
            reserve_s=0,
        ))
    if event_store is not None:
        observers.append(
            SSEObserver(
                event_store=event_store,
                task_id=ctx.task_id,
                run_id=str(metadata.get("run_id") or ""),
                run_type=str(metadata.get("run_type") or ""),
            ),
        )
    # Reporter-disabled runs need a clean-context rescue on bounded/infra exits.
    # When the reporter is enabled it is already the authoritative clean-context
    # LLM synthesis chain, so a coordinator rescue here would duplicate the
    # largest call immediately before it.
    if not reporter_enabled:
        observers.append(FinalAnswerSalvageObserver(
            llm=llm,
            timeout=finalization_timeout_s,
            phase_deadline_monotonic=hard_deadline_monotonic,
            language=answer_language,
            task_description=question,
            thinking_format=(
                model_profile.thinking_format if model_profile is not None else "tag"
            ),
            salvage_infra_errors=salvage_infra_errors,
        ))

    # Reporter output stream (serve stdout): re-emit the final answer as
    # ``response.swarm.llm_delta`` frames with ``agent_id="reporter"`` at
    # on_loop_end — immediately before the protocol stream observer's
    # terminal ``final``. MUST be appended to the node's own observers before the serve
    # chain in ``sdk_extra_observers`` so its critical on_loop_end runs before
    # the terminal.
    #
    # Two mutually exclusive modes:
    #   reporter_enabled → ReportSynthesisObserver: ONE streaming LLM call
    #     synthesises a cited report over the whole conversation and rewrites
    #     ``final_answer`` (runs regardless of an emitter — it improves the
    #     returned/traced answer; the emitter only adds live streaming).
    #   otherwise        → ReporterStreamObserver: re-streams the raw resolved
    #     answer (no-op without an emitter, so wire it only when one is present).
    sdk_emitter = metadata.get("sdk_protocol_emitter")
    if reporter_enabled:
        thinking_fmt = (
            model_profile.thinking_format if model_profile is not None else "tag"
        )
        # Inline-thinking risk: tag-mode + ``enable_thinking`` means the endpoint
        # MAY return ``<think>…</think>`` inside ``content`` — and SGLang/Qwen
        # often drop the opening tag while always emitting the closing one. The
        # reporter's stream filter needs to know so it never ships reasoning as
        # report text (it self-releases the hold on a native reasoning delta).
        chat_template_kwargs = (
            ((profile or {}).get("llm") or {}).get("extra_body") or {}
        ).get("chat_template_kwargs") or {}
        observers.append(ReportSynthesisObserver(
            llm=llm,
            # Finite timeout per fallback leg; deliberately no whole-reporter
            # phase wall, so the chain can advance after research has stopped.
            timeout=reporter_timeout_s,
            task_description=question,
            emitter=sdk_emitter,
            usage_aggregator=metadata.get("sdk_protocol_usage_aggregator"),
            language=answer_language,
            thinking_format=thinking_fmt,
            inline_thinking=(
                thinking_fmt == "tag"
                and bool(chat_template_kwargs.get("enable_thinking"))
            ),
            extra_observers=metadata.get("sdk_extra_observers"),
            context_max_tokens=reporter_context_max_tokens,
            phase_timeout=reporter_phase_timeout_s,
            phase_deadline_monotonic=hard_deadline_monotonic,
        ))
    elif sdk_emitter is not None:
        observers.append(ReporterStreamObserver(sdk_emitter))

    extra_observers = metadata.get("sdk_extra_observers") or []
    if extra_observers:
        observers.extend(list(extra_observers))

    sandbox = None
    sb_token = None
    if not direct:
        if sandbox_mode in ("container", "native"):
            sandbox = make_current_sandbox(worktree_root)
        else:
            sandbox = BwrapSandbox(workspace=worktree_root, binds=sandbox_binds)
        sb_token = set_task_sandbox(sandbox)
    # Default this eval workflow to the bash command allowlist (deny anything
    # off the read/analyse/python allowlist). This is defense-in-depth on top
    # of the required bwrap isolation.
    # Overridable via BASH_ALLOWLIST_MODE (env wins in resolve_mode()).
    policy_token = set_policy_mode("enforce")
    # Assemble the compactor + trigger. Tiered reuses KeepLastN (Tier1) +
    # LLMSummaryCompactor (Tier2) behind a real-input-token threshold; else the
    # legacy keep-last-k path (unchanged when ``context_compaction`` is off).
    keep_recent_msgs = max(6, keep_recent_turns * 3)  # ~3 msgs/turn (AI + tool(s))
    compaction_policy: Any = None
    if context_compaction == "tiered" and max_len > 0:
        gauge = InputTokenGauge()
        observers.append(gauge)
        from plugins.tools._overflow import spill_compacted_body

        compactor: Any = TieredCompactor(
            keep_tool_result=tier1_keep_tool_result,
            summary_llm=llm,
            relief_target=int(max_len * 0.6),
            gauge=gauge,  # calibrate relief to real tokens (unit-match trigger)
            spill=spill_compacted_body if compaction_spill else None,
            summary_retry_timeout_s=llm_timeout,
        )
        compaction_policy = InputTokenThresholdPolicy(
            gauge, compaction_trigger_tokens(max_len),
        )
    else:
        compactor = KeepLastNToolResultsCompactor(keep_tool_result=keep_last_k)

    try:
        import sys
        loop_fn = getattr(sys.modules.get("apodex.session"), "run_agent_loop", run_agent_loop)
        result = await loop_fn(
            system_prompt=system_prompt,
            user_message=question,
            llm=llm,
            tools=tools,
            config=LoopConfig(
                max_turns=max_turns,
                task_id=ctx.task_id,
                llm_session_id=str(metadata.get("session_id") or ctx.task_id),
                role_id="stateful_react",
                tool_timeout=int(tool_timeout),
                llm_timeout=int(llm_timeout),
                first_chunk_timeout=(
                    float(first_chunk_s) if first_chunk_s is not None else None
                ),
                reasoning_only_timeout_s=reasoning_only_timeout_s,
                reasoning_only_max_tokens=reasoning_only_max_tokens,
                logical_call_timeout_s=logical_call_timeout_s,
                context_token_limit=context_token_limit,
                compact_after_turns=compact_after_turns,
                keep_recent=keep_recent_msgs,
                loop_policy=_loop_policy(),
                compactor=compactor,
                compaction_policy=compaction_policy,
                tool_result_post_processor=ReactToolResultPostProcessor(),
            ),
            model_profile=model_profile,
            history_policy=history_policy,
            observers=observers,
            pause_check=pause_check_from_state(state),
            scope_metadata={
                "root_task_id": ctx.task_id,
                "run_id": metadata.get("run_id"),
                "run_type": metadata.get("run_type"),
                "pipeline_id": "stateful-react-agent",
                # Live ``add`` intervention renews this shared lease before its
                # queued ack. WallClockDeadlineObserver binds to it so the
                # in-process soft deadline follows the same sliding window.
                # Binding creates a per-loop deadline view; never forward the
                # view itself to sibling/concurrent loops.
                WALL_TIME_LEASE_SCOPE_KEY: metadata.get(
                    WALL_TIME_LEASE_SCOPE_KEY
                ),
                # Container mode: authorize file tools' direct-local access to
                # the mounted /workspace (consumed by plugins.tools._path_auth).
                # Empty string in bwrap mode → no host-workspace authorization.
                "workspace_root": workspace_root_str,
            },
        )
    finally:
        reset_policy_mode(policy_token)
        if sb_token is not None:
            clear_task_sandbox(sb_token)
        # Drop this run's task board (no-op when task_board is off) so boards
        # don't leak across trials in a long-lived worker process.
        clear_board(ctx.task_id)
        kill = getattr(sandbox, "kill", None)
        if callable(kill):
            kill()

    # ``force_final_answer`` for salvage stops ran at loop end via
    # Finalization is complete here: reporter-enabled runs were synthesised by
    # ``ReportSynthesisObserver``; reporter-disabled bounded exits were handled
    # by ``FinalAnswerSalvageObserver``.
    final_text = result.metadata.get("final_answer") or result.final_content
    final_text = _strip_leaked_tool_calls(str(final_text or ""))
    if not final_text:
        final_text = _minimal_best_effort_answer(
            question, result.stopped_by, language=answer_language,
        )
        result.metadata["final_answer_source"] = "deterministic_fallback"
    answer_source = str(result.metadata.get("final_answer_source") or "agent")
    answer_status = (
        "not_found"
        if answer_source == "deterministic_fallback"
        else "best_effort"
        if answer_source in {
            "clean_context_llm",
            "existing_partial",
            "collected_reports",
        }
        else "complete"
    )
    return {
        "final_answer": final_text,
        "final_content": final_text,
        "session_turn": build_session_turn(
            _language_probe(state, question),
            result.messages,
            final_text,
            steps=result.metadata.get("react_steps", []),
        ),
        "react_steps": result.metadata.get("react_steps", []),
        "language": answer_language,
        # Keep the user-facing answer non-empty while preserving the old
        # machine-readable infra/eval failure signal out of band.
        "answer_status": answer_status,
        "answer_sentinel": (
            "<ANSWER_NOT_FOUND>" if answer_status == "not_found" else ""
        ),
        "final_answer_rescued": bool(
            result.metadata.get("final_answer_rescued", False),
        ),
        "final_answer_rescue_mode": str(
            result.metadata.get("final_answer_rescue_mode") or "",
        ),
        "final_answer_source": answer_source,
        "stopped_by": result.stopped_by,
        # Preserve the provider failure separately from deterministic fallback
        # prose so terminal clients can render an actionable error rather than
        # mislabeling the fallback as a completed report.
        "llm_error": str(result.metadata.get("llm_error") or ""),
        "llm_error_reason": str(result.metadata.get("llm_error_reason") or ""),
    }