Use evaluator API_KEY for LLM proxy and strengthen env

Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.11-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
 
 WORKDIR /app
 
@@ -14,6 +15,14 @@ RUN python -m pip install --upgrade pip \
     && python -m pip install --no-cache-dir -r requirements.txt \
     && python -m pip install --no-cache-dir .
 
+RUN useradd --create-home --uid 10001 appuser \
+    && chown -R appuser:appuser /app
+
 EXPOSE 7860
 
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/health', timeout=3)"
+
+USER appuser
+
 CMD ["uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -383,9 +383,10 @@ TASK_ID=3 python inference.py
 
 Set these environment variables first:
 
-- `API_BASE_URL`
-- `MODEL_NAME`
-- `HF_TOKEN`
+  - `API_BASE_URL`
+  - `MODEL_NAME`
+  - `API_KEY`
+  - `HF_TOKEN`
 
 Then run:
 

ROADMAP.md

@@ -130,7 +130,7 @@ These come directly from `required.md` and `KNOWLEDGE.md`:
 - 3 tasks exist and remain meaningfully different
 - grader scores stay in `[0.0, 1.0]`
 - `inference.py` runs reproducibly without crashing
-- `inference.py` uses the OpenAI client with `API_BASE_URL`, `MODEL_NAME`, and `HF_TOKEN`
+- `inference.py` uses the OpenAI client with `API_BASE_URL`, `MODEL_NAME`, and the evaluator-injected `API_KEY` (`HF_TOKEN` remains a local fallback)
 - structured stdout logs follow the official `[START]`, `[STEP]`, and `[END]` format
 - `openenv validate` passes
 - Docker builds and starts cleanly

data/dataset.json

@@ -574,6 +574,126 @@
         "resolution_action":  "fulfill",
         "ambiguity_note":  "Contractor onboarding blocked by access issue, routed to service desk",
         "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-046",
+        "title":  "Privileged admin login blocked during security review",
+        "requester":  "security-ops@atlasbank.io",
+        "description":  "Our privileged admin account was locked during an internal security review. We need access restored, but the security team must verify the incident trail before the account is reopened.",
+        "issue_type":  "identity_access",
+        "priority":  "critical",
+        "assignment_group":  "security_team",
+        "resolution_action":  "escalate",
+        "ambiguity_note":  "Looks like a login problem, but security owns the privileged-access review and release decision.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-047",
+        "title":  "Temporary sandbox extension for signed pilot",
+        "requester":  "solutions@bluequarry.io",
+        "description":  "The commercial pilot is already approved. We only need the existing sandbox kept alive for two more weeks so the customer can finish testing.",
+        "issue_type":  "service_request",
+        "priority":  "medium",
+        "assignment_group":  "service_desk",
+        "resolution_action":  "fulfill",
+        "ambiguity_note":  "Commercial context is present, but the actual action is an operational extension the service desk can fulfill directly.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-048",
+        "title":  "Who approves seat-transfer terms in the vendor questionnaire?",
+        "requester":  "vendorops@aurorahealth.org",
+        "description":  "Our procurement team is filling out your vendor questionnaire and needs clarification on who approves seat-transfer language before we continue the review.",
+        "issue_type":  "general_inquiry",
+        "priority":  "medium",
+        "assignment_group":  "procurement",
+        "resolution_action":  "assign",
+        "ambiguity_note":  "The request is a question, but it belongs with the commercial owner rather than the generic service desk.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-049",
+        "title":  "Credential-defense rollout is causing auth API failures",
+        "requester":  "platform@nightferry.dev",
+        "description":  "The authentication API is returning intermittent 403 errors after a credential-stuffing defense rule was enabled. Product behavior is broken, but security needs to triage the mitigation first.",
+        "issue_type":  "application_support",
+        "priority":  "high",
+        "assignment_group":  "security_team",
+        "resolution_action":  "escalate",
+        "ambiguity_note":  "The symptom looks like application support, but the active security control owns the first response path.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-050",
+        "title":  "Acquired-team onboarding needs cross-functional coordination",
+        "requester":  "integration@mergerco.com",
+        "description":  "Thirty acquired employees start next week and need onboarding, access setup, hardware coordination, and shared mailbox provisioning across multiple internal teams.",
+        "issue_type":  "onboarding",
+        "priority":  "high",
+        "assignment_group":  "service_desk",
+        "resolution_action":  "assign",
+        "ambiguity_note":  "The workflow is onboarding, but it requires central service-desk coordination instead of a single onboarding-ops fulfillment step.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-051",
+        "title":  "Renewal credit memo requires contract amendment approval",
+        "requester":  "procurement@crownlogistics.com",
+        "description":  "Finance approved the renewal credit memo, but the contract amendment still needs commercial approval before the invoice can be corrected.",
+        "issue_type":  "billing_license",
+        "priority":  "medium",
+        "assignment_group":  "procurement",
+        "resolution_action":  "assign",
+        "ambiguity_note":  "This sounds billing-related, but the remaining work is a commercial contract amendment owned by procurement.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-052",
+        "title":  "Need remediation evidence package for product vulnerability",
+        "requester":  "assurance@clientgrid.com",
+        "description":  "Our assurance team needs the remediation evidence package for a previously confirmed application vulnerability before we close the compliance review.",
+        "issue_type":  "security_compliance",
+        "priority":  "high",
+        "assignment_group":  "application_team",
+        "resolution_action":  "fulfill",
+        "ambiguity_note":  "The request is compliance-driven, but the application team must provide the concrete remediation evidence.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-053",
+        "title":  "Customer requests penetration-test window and allowlist",
+        "requester":  "engsec@vectorlabs.io",
+        "description":  "We want to schedule a penetration test and need the approved window plus the process for allowlisting our source IPs.",
+        "issue_type":  "service_request",
+        "priority":  "medium",
+        "assignment_group":  "security_team",
+        "resolution_action":  "assign",
+        "ambiguity_note":  "This is a request, but the security team owns approval and coordination instead of procurement.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-054",
+        "title":  "Need archived invoice copies for board audit binder",
+        "requester":  "boardops@silverpine.com",
+        "description":  "The board audit binder needs PDF copies of invoices from the last four quarters. No billing change is required, just document retrieval.",
+        "issue_type":  "general_inquiry",
+        "priority":  "low",
+        "assignment_group":  "license_ops",
+        "resolution_action":  "fulfill",
+        "ambiguity_note":  "The request is informational, but license operations owns the archived invoice records and can fulfill it directly.",
+        "related_ticket_id":  null
+    },
+    {
+        "ticket_id":  "ticket-055",
+        "title":  "Re: Renewal credit memo requires contract amendment approval",
+        "requester":  "procurement@crownlogistics.com",
+        "description":  "Following up on ticket-051. Quarter close is tomorrow and the contract amendment is still pending, so the corrected invoice cannot be issued yet.",
+        "issue_type":  "billing_license",
+        "priority":  "high",
+        "assignment_group":  "procurement",
+        "resolution_action":  "escalate",
+        "ambiguity_note":  null,
+        "related_ticket_id":  "ticket-051"
     }
 ]
 

inference.py

@@ -16,8 +16,12 @@ MODEL_NAME
     Model identifier to use for LLM inference.
     Default: ``<your-active-model>``
 
+API_KEY
+    Proxy/API authentication token injected by the evaluator.
+    No default is set.
+
 HF_TOKEN
-    HuggingFace authentication token for the LLM provider.
+    Backward-compatible local fallback alias for API_KEY.
     No default is set.
 
 TASK_ID
@@ -33,8 +37,9 @@ LOCAL_IMAGE_NAME
     Optional compatibility variable from the sample inference pattern.
     This script does not use ``from_docker_image()``, so the value is unused here.
 
-When both MODEL_NAME and HF_TOKEN are set explicitly, the script calls the LLM via the
-OpenAI-compatible API at API_BASE_URL. Otherwise it falls back to the deterministic
+When MODEL_NAME and API_KEY are set explicitly, the script calls the LLM via the
+OpenAI-compatible API at API_BASE_URL. For local compatibility, HF_TOKEN is accepted
+as a fallback alias for API_KEY. Otherwise it falls back to the deterministic
 heuristic baseline automatically.
 
 All stdout logs use the required structured tags: ``[START]``, ``[STEP]``, and ``[END]``.
@@ -83,6 +88,7 @@ def _get_int_env(name: str, default: int) -> int:
 API_BASE_URL = os.getenv("API_BASE_URL", DEFAULT_API_BASE_URL)
 MODEL_NAME = os.getenv("MODEL_NAME", DEFAULT_MODEL_NAME)
 HF_TOKEN = os.getenv("HF_TOKEN")
+API_KEY = os.getenv("API_KEY") or HF_TOKEN
 LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME")
 ENV_URL = os.getenv("ENV_URL", "http://localhost:7860")
 
@@ -100,12 +106,12 @@ RUN_ALL_TASKS_ENV = os.getenv("RUN_ALL_TASKS", "").strip().lower() in {
 
 
 def llm_mode_enabled() -> bool:
-    return bool(HF_TOKEN) and MODEL_NAME != DEFAULT_MODEL_NAME
+    return bool(API_KEY) and MODEL_NAME != DEFAULT_MODEL_NAME
 
 
 llm_client: OpenAI | None = None
 if llm_mode_enabled():
-    llm_client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
+    llm_client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
 
 
 RECENT_HISTORY_LIMIT = 2
@@ -698,21 +704,108 @@ def should_investigate(ticket: dict, history: list[dict[str, Any]]) -> tuple[boo
     if not ticket:
         return False, None
     context_status = ticket.get("context_status") or {}
-    remaining_tools = context_status.get("remaining_tools") or []
-    if remaining_tools:
-        return True, str(remaining_tools[0])
     current_ticket_id = ticket.get("ticket_id")
+    prior_ticket_history = [
+        entry
+        for entry in history
+        if entry.get("ticket_id") == current_ticket_id
+    ]
     already_investigated = any(
         entry.get("ticket_id") == current_ticket_id
         and entry.get("predicted", {}).get("action_type") == "investigate"
         for entry in history
     )
-    if already_investigated:
+    investigations_used = sum(
+        1
+        for entry in prior_ticket_history
+        if entry.get("predicted", {}).get("action_type") == "investigate"
+    )
+    hidden_context_remaining = bool(context_status.get("hidden_context_remaining"))
+    if investigations_used >= 3:
+        return False, None
+
+    used_tools = {
+        entry.get("predicted", {}).get("tool_name")
+        for entry in prior_ticket_history
+        if entry.get("predicted", {}).get("action_type") == "investigate"
+    }
+    routing_text = build_routing_text(ticket)
+    last_tool_result = ticket.get("last_tool_result") or {}
+    last_tool_name = str(last_tool_result.get("tool_name", "") or "")
+
+    follow_up_signal = any(
+        phrase in routing_text
+        for phrase in (
+            "re:",
+            "follow-up",
+            "following up",
+            "regression",
+            "reference ticket",
+            "third update",
+            "still",
+            "unresolved",
+        )
+    )
+    routing_ambiguity_signal = any(
+        phrase in routing_text
+        for phrase in (
+            "billing-style",
+            "prorating",
+            "seat expansion",
+            "vendor offer",
+            "pricing",
+            "compliance scan",
+            "vulnerability",
+            "onboarding workflow",
+            "blocked by an account problem",
+            "permissions error",
+            "mixed workflow",
+        )
+    )
+    requester_history_signal = any(
+        phrase in routing_text
+        for phrase in (
+            "still haven't",
+            "third update",
+            "again",
+            "follow-up",
+            "priority",
+            "legal",
+            "overdue",
+            "escalating",
+        )
+    )
+
+    preferred_tools: list[str] = []
+    if last_tool_name == "lookup_related_ticket":
+        preferred_tools.append("lookup_requester_history")
+    if last_tool_name == "lookup_requester_history":
+        preferred_tools.append("lookup_internal_routing_note")
+    if follow_up_signal or ticket.get("related_ticket_id"):
+        preferred_tools.append("lookup_related_ticket")
+    if routing_ambiguity_signal or hidden_context_remaining:
+        preferred_tools.append("lookup_internal_routing_note")
+    if requester_history_signal:
+        preferred_tools.append("lookup_requester_history")
+    if hidden_context_remaining:
+        preferred_tools.extend(
+            [
+                "lookup_related_ticket",
+                "lookup_internal_routing_note",
+                "lookup_requester_history",
+            ]
+        )
+
+    for tool_name in preferred_tools:
+        if tool_name not in used_tools:
+            return True, tool_name
+
+    if already_investigated and not hidden_context_remaining:
         return False, None
-    if ticket.get("related_ticket_id"):
+    if ticket.get("ambiguity_note") and "lookup_internal_routing_note" not in used_tools:
+        return True, "lookup_internal_routing_note"
+    if ticket.get("related_ticket_id") and "lookup_related_ticket" not in used_tools:
         return True, "lookup_related_ticket"
-    if ticket.get("ambiguity_note"):
-        return True, "lookup_requester_history"
     return False, None
 
 

openenv.yaml

@@ -32,7 +32,11 @@ api:
     - /reset
     - /step
     - /state
+    - /ws
     - /tasks
+    - /web
+    - /baseline
+    - /grader
     - /docs
 
 evaluation:
@@ -51,9 +55,13 @@ inference:
   env_vars:
     - API_BASE_URL
     - MODEL_NAME
+    - API_KEY
     - HF_TOKEN
     - ENV_URL
     - TASK_ID
+    - SEED
+    - RUN_ALL_TASKS
+    - LOCAL_IMAGE_NAME
 
 requirements:
   python: ">=3.11"

policy_learning.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import argparse
 import importlib
 import json
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 from statistics import mean
 from typing import Any, Callable, Iterable
@@ -18,13 +18,13 @@ from vocabulary import TASK_IDS
 DEFAULT_COMPARE_POLICIES = (
     "no_investigation",
     "investigate_when_context_hidden",
+    "adaptive_cue_bandit",
 )
 DEFAULT_SEARCH_POLICIES = (
     "no_investigation",
     "legacy_single_probe",
     "investigate_when_context_hidden",
-    "context_chain",
-    "hybrid_context",
+    "adaptive_cue_bandit",
 )
 DEFAULT_OUTPUT_DIR = "analysis/policy_learning_runs"
 
@@ -40,11 +40,13 @@ class PolicyConfig:
     investigate_ambiguity_history: bool
     max_investigations_per_ticket: int
     description: str
+    strategy: str = "static"
 
 
 POLICY_LIBRARY: dict[str, PolicyConfig] = {
     "no_investigation": PolicyConfig(
         name="no_investigation",
+        strategy="static",
         investigate_hidden_context=False,
         investigate_related_ticket_hint=False,
         investigate_ambiguity_history=False,
@@ -53,6 +55,7 @@ POLICY_LIBRARY: dict[str, PolicyConfig] = {
     ),
     "legacy_single_probe": PolicyConfig(
         name="legacy_single_probe",
+        strategy="static",
         investigate_hidden_context=False,
         investigate_related_ticket_hint=True,
         investigate_ambiguity_history=True,
@@ -61,30 +64,105 @@ POLICY_LIBRARY: dict[str, PolicyConfig] = {
     ),
     "investigate_when_context_hidden": PolicyConfig(
         name="investigate_when_context_hidden",
+        strategy="static",
         investigate_hidden_context=True,
         investigate_related_ticket_hint=False,
         investigate_ambiguity_history=False,
         max_investigations_per_ticket=1,
-        description="Investigate once when the environment says context is hidden.",
+        description="Investigate once when the environment shows hidden-context pressure.",
     ),
-    "context_chain": PolicyConfig(
-        name="context_chain",
-        investigate_hidden_context=True,
-        investigate_related_ticket_hint=False,
-        investigate_ambiguity_history=False,
-        max_investigations_per_ticket=3,
-        description="Follow the environment's required-tool chain until context is revealed.",
-    ),
-    "hybrid_context": PolicyConfig(
-        name="hybrid_context",
+    "adaptive_cue_bandit": PolicyConfig(
+        name="adaptive_cue_bandit",
+        strategy="adaptive",
         investigate_hidden_context=True,
         investigate_related_ticket_hint=True,
         investigate_ambiguity_history=True,
         max_investigations_per_ticket=3,
-        description="Use hidden-context signals first, then legacy ambiguity hints.",
+        description=(
+            "Learn cue-conditioned tool preferences from investigation rewards on train seeds."
+        ),
     ),
 }
 
+AVAILABLE_TOOLS = (
+    "lookup_related_ticket",
+    "lookup_requester_history",
+    "lookup_internal_routing_note",
+)
+
+
+@dataclass
+class AdaptiveToolBandit:
+    exploration_rounds: int = 1
+    cue_tool_totals: dict[str, dict[str, float]] = field(default_factory=dict)
+    cue_tool_counts: dict[str, dict[str, int]] = field(default_factory=dict)
+    global_tool_totals: dict[str, float] = field(default_factory=dict)
+    global_tool_counts: dict[str, int] = field(default_factory=dict)
+
+    def choose_tool(self, cue: str, candidate_tools: list[str]) -> str:
+        for tool_name in candidate_tools:
+            if self.cue_tool_counts.get(cue, {}).get(tool_name, 0) < self.exploration_rounds:
+                return tool_name
+        return max(
+            candidate_tools,
+            key=lambda tool_name: (
+                self._cue_average(cue, tool_name),
+                self._global_average(tool_name),
+                -candidate_tools.index(tool_name),
+            ),
+        )
+
+    def record_reward(self, cue: str, tool_name: str, reward: float) -> None:
+        cue_totals = self.cue_tool_totals.setdefault(cue, {})
+        cue_counts = self.cue_tool_counts.setdefault(cue, {})
+        cue_totals[tool_name] = cue_totals.get(tool_name, 0.0) + reward
+        cue_counts[tool_name] = cue_counts.get(tool_name, 0) + 1
+        self.global_tool_totals[tool_name] = self.global_tool_totals.get(tool_name, 0.0) + reward
+        self.global_tool_counts[tool_name] = self.global_tool_counts.get(tool_name, 0) + 1
+
+    def export(self) -> dict[str, Any]:
+        return {
+            "exploration_rounds": self.exploration_rounds,
+            "cue_tool_averages": {
+                cue: {
+                    tool_name: round(self._cue_average(cue, tool_name), 6)
+                    for tool_name in sorted(tool_totals)
+                }
+                for cue, tool_totals in sorted(self.cue_tool_totals.items())
+            },
+            "global_tool_averages": {
+                tool_name: round(self._global_average(tool_name), 6)
+                for tool_name in sorted(self.global_tool_totals)
+            },
+        }
+
+    def frozen_copy(self) -> "AdaptiveToolBandit":
+        return AdaptiveToolBandit(
+            exploration_rounds=self.exploration_rounds,
+            cue_tool_totals={
+                cue: dict(tool_totals) for cue, tool_totals in self.cue_tool_totals.items()
+            },
+            cue_tool_counts={
+                cue: dict(tool_counts) for cue, tool_counts in self.cue_tool_counts.items()
+            },
+            global_tool_totals=dict(self.global_tool_totals),
+            global_tool_counts=dict(self.global_tool_counts),
+        )
+
+    def _cue_average(self, cue: str, tool_name: str) -> float:
+        total = self.cue_tool_totals.get(cue, {}).get(tool_name, 0.0)
+        count = self.cue_tool_counts.get(cue, {}).get(tool_name, 0)
+        if count == 0:
+            return self._global_average(tool_name)
+        return total / count
+
+    def _global_average(self, tool_name: str) -> float:
+        total = self.global_tool_totals.get(tool_name, 0.0)
+        count = self.global_tool_counts.get(tool_name, 0)
+        if count == 0:
+            return 0.0
+        return total / count
+
 
 def _dedupe_preserving_order(values: Iterable[int]) -> list[int]:
     seen: set[int] = set()
@@ -154,29 +232,199 @@ def default_submit_builder(
     return HelpdeskTicketAction(**candidate)
 
 
+def _routing_text(ticket: dict[str, Any]) -> str:
+    parts = [
+        str(ticket.get("title", "")),
+        str(ticket.get("description", "")),
+        str(ticket.get("ambiguity_note", "")),
+        json.dumps(ticket.get("last_tool_result") or {}, sort_keys=True),
+    ]
+    related_preview = ticket.get("related_ticket_preview") or {}
+    parts.extend(
+        [
+            str(related_preview.get("title", "")),
+            str(related_preview.get("description", "")),
+        ]
+    )
+    return " ".join(parts).lower()
+
+
+def infer_ticket_cue(ticket: dict[str, Any]) -> str:
+    text = _routing_text(ticket)
+    if any(
+        phrase in text
+        for phrase in ("re:", "follow-up", "following up", "regression", "reference ticket", "third update")
+    ):
+        return "follow_up"
+    if any(
+        phrase in text
+        for phrase in (
+            "pricing",
+            "quote",
+            "vendor offer",
+            "prorating",
+            "seat expansion",
+            "commercial",
+        )
+    ):
+        return "commercial_ambiguity"
+    if any(
+        phrase in text
+        for phrase in (
+            "onboarding",
+            "contractor",
+            "permissions error",
+            "blocked by an account problem",
+        )
+    ):
+        return "workflow_blocker"
+    if any(
+        phrase in text
+        for phrase in ("compliance scan", "vulnerability", "policy issue", "routing note")
+    ):
+        return "routing_note"
+    if any(
+        phrase in text
+        for phrase in ("still", "again", "overdue", "legal", "priority")
+    ):
+        return "history_pressure"
+    return "generic_hidden_context"
+
+
+def preferred_tool_order(
+    ticket: dict[str, Any],
+    *,
+    hidden_context_remaining: bool,
+) -> list[str]:
+    text = _routing_text(ticket)
+    last_tool_result = ticket.get("last_tool_result") or {}
+    last_tool_name = str(last_tool_result.get("tool_name", "") or "")
+
+    preferred_tools: list[str] = []
+    if last_tool_name == "lookup_related_ticket":
+        preferred_tools.append("lookup_requester_history")
+    if last_tool_name == "lookup_requester_history":
+        preferred_tools.append("lookup_internal_routing_note")
+
+    if any(
+        phrase in text
+        for phrase in ("re:", "follow-up", "following up", "regression", "reference ticket")
+    ) or ticket.get("related_ticket_id"):
+        preferred_tools.append("lookup_related_ticket")
+
+    if any(
+        phrase in text
+        for phrase in (
+            "pricing",
+            "quote",
+            "vendor offer",
+            "prorating",
+            "seat expansion",
+            "billing-style",
+            "compliance scan",
+            "vulnerability",
+            "onboarding workflow",
+            "permissions error",
+            "blocked by an account problem",
+        )
+    ):
+        preferred_tools.append("lookup_internal_routing_note")
+
+    if any(
+        phrase in text
+        for phrase in ("still", "again", "overdue", "legal", "third update", "priority")
+    ):
+        preferred_tools.append("lookup_requester_history")
+
+    if hidden_context_remaining:
+        preferred_tools.extend(
+            [
+                "lookup_internal_routing_note",
+                "lookup_related_ticket",
+                "lookup_requester_history",
+            ]
+        )
+
+    deduped_tools: list[str] = []
+    for tool_name in preferred_tools:
+        if tool_name not in deduped_tools:
+            deduped_tools.append(tool_name)
+    return deduped_tools
+
+
+def select_cue_based_tool(
+    ticket: dict[str, Any],
+    *,
+    hidden_context_remaining: bool,
+    used_tools: set[str],
+) -> str | None:
+    preferred_tools = preferred_tool_order(
+        ticket,
+        hidden_context_remaining=hidden_context_remaining,
+    )
+    for tool_name in preferred_tools:
+        if tool_name not in used_tools:
+            return tool_name
+    return None
+
+
 def choose_policy_action(
     policy: PolicyConfig,
     observation: HelpdeskTicketObservation,
     investigations_by_ticket: dict[str, int],
     submit_builder: SubmitBuilder,
-) -> tuple[HelpdeskTicketAction, str]:
+    *,
+    used_tools_by_ticket: dict[str, set[str]] | None = None,
+    adaptive_bandit: AdaptiveToolBandit | None = None,
+) -> tuple[HelpdeskTicketAction, str, str | None]:
     ticket = observation.current_ticket or {}
     ticket_id = str(ticket.get("ticket_id", ""))
     ticket_investigations = investigations_by_ticket.get(ticket_id, 0)
-    revealed_tools = set(((ticket.get("context_status") or {}).get("revealed_tools") or []))
-    remaining_tools = list(((ticket.get("context_status") or {}).get("remaining_tools") or []))
+    used_tools = set()
+    if used_tools_by_ticket is not None:
+        used_tools = set(used_tools_by_ticket.get(ticket_id, set()))
+    context_status = ticket.get("context_status") or {}
+    hidden_context_remaining = bool(context_status.get("hidden_context_remaining"))
 
     if ticket_investigations < policy.max_investigations_per_ticket:
-        if policy.investigate_hidden_context and remaining_tools:
-            tool_name = str(remaining_tools[0])
-            return (
-                HelpdeskTicketAction(action_type="investigate", tool_name=tool_name),
-                "investigate_hidden_context",
+        if policy.strategy == "adaptive" and adaptive_bandit is not None and hidden_context_remaining:
+            candidate_tools = [
+                tool_name
+                for tool_name in preferred_tool_order(
+                    ticket,
+                    hidden_context_remaining=hidden_context_remaining,
+                )
+                if tool_name not in used_tools
+            ]
+            if not candidate_tools:
+                candidate_tools = [
+                    tool_name for tool_name in AVAILABLE_TOOLS if tool_name not in used_tools
+                ]
+            if candidate_tools:
+                cue = infer_ticket_cue(ticket)
+                tool_name = adaptive_bandit.choose_tool(cue, candidate_tools)
+                return (
+                    HelpdeskTicketAction(action_type="investigate", tool_name=tool_name),
+                    "adaptive_bandit_investigate",
+                    cue,
+                )
+
+        if policy.investigate_hidden_context and hidden_context_remaining:
+            tool_name = select_cue_based_tool(
+                ticket,
+                hidden_context_remaining=hidden_context_remaining,
+                used_tools=used_tools,
             )
+            if tool_name is not None:
+                return (
+                    HelpdeskTicketAction(action_type="investigate", tool_name=tool_name),
+                    "investigate_hidden_context",
+                    infer_ticket_cue(ticket),
+                )
         if (
             policy.investigate_related_ticket_hint
             and ticket.get("related_ticket_id")
-            and "lookup_related_ticket" not in revealed_tools
+            and "lookup_related_ticket" not in used_tools
         ):
             return (
                 HelpdeskTicketAction(
@@ -184,11 +432,16 @@ def choose_policy_action(
                     tool_name="lookup_related_ticket",
                 ),
                 "investigate_related_ticket_hint",
+                infer_ticket_cue(ticket),
             )
         if (
             policy.investigate_ambiguity_history
-            and ticket.get("ambiguity_note")
-            and "lookup_requester_history" not in revealed_tools
+            and (
+                ticket.get("ambiguity_note")
+                or ticket.get("feedback_summary")
+                or hidden_context_remaining
+            )
+            and "lookup_requester_history" not in used_tools
         ):
             return (
                 HelpdeskTicketAction(
@@ -196,9 +449,10 @@ def choose_policy_action(
                     tool_name="lookup_requester_history",
                 ),
                 "investigate_ambiguity_history",
+                infer_ticket_cue(ticket),
             )
 
-    return submit_builder(ticket, list(observation.allowed_fields)), "submit"
+    return submit_builder(ticket, list(observation.allowed_fields)), "submit", None
 
 
 def rollout_episode(
@@ -208,27 +462,39 @@ def rollout_episode(
     seed: int,
     task_id: int,
     submit_builder: SubmitBuilder,
+    adaptive_bandit: AdaptiveToolBandit | None = None,
+    update_adaptive: bool = False,
 ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
     task = get_task_definition(task_id)
     observation = env.reset(seed=seed, task_id=task_id)
     investigations_by_ticket: dict[str, int] = {}
+    used_tools_by_ticket: dict[str, set[str]] = {}
     episode_return = 0.0
     trajectories: list[dict[str, Any]] = []
 
     while not observation.done:
         ticket = observation.current_ticket or {}
         ticket_id = str(ticket.get("ticket_id", ""))
-        action, action_source = choose_policy_action(
+        action, action_source, action_cue = choose_policy_action(
             policy,
             observation,
             investigations_by_ticket,
             submit_builder,
+            used_tools_by_ticket=used_tools_by_ticket,
+            adaptive_bandit=adaptive_bandit,
         )
         next_observation = env.step(action)
         reward_value = float(next_observation.reward or 0.0)
         episode_return += reward_value
         if action.action_type == "investigate" and ticket_id:
             investigations_by_ticket[ticket_id] = investigations_by_ticket.get(ticket_id, 0) + 1
+            used_tools_by_ticket.setdefault(ticket_id, set()).add(str(action.tool_name))
+            if policy.strategy == "adaptive" and adaptive_bandit is not None and update_adaptive:
+                adaptive_bandit.record_reward(
+                    action_cue or infer_ticket_cue(ticket),
+                    str(action.tool_name),
+                    reward_value,
+                )
 
         history_entry = env.state.history_entries[-1] if env.state.history_entries else {}
         trajectories.append(
@@ -241,6 +507,7 @@ def rollout_episode(
                 "step_index": len(trajectories) + 1,
                 "ticket_id": history_entry.get("ticket_id", ticket_id),
                 "action_source": action_source,
+                "action_cue": action_cue,
                 "action": action.model_dump(exclude_none=True),
                 "step_reward": reward_value,
                 "rubric_reward": next_observation.rubric_reward,
@@ -280,6 +547,8 @@ def rollout_episode(
         "average_ticket_score": env.state.average_score_so_far,
         "per_ticket_scores": list(env.state.per_ticket_scores),
     }
+    if adaptive_bandit is not None and policy.strategy == "adaptive":
+        summary["learned_tool_values"] = adaptive_bandit.export()
     return summary, trajectories
 
 
@@ -352,6 +621,8 @@ def evaluate_policy(
     *,
     env_factory: EnvFactory = HelpdeskTicketRoutingEnvironment,
     submit_builder: SubmitBuilder = default_submit_builder,
+    adaptive_bandit: AdaptiveToolBandit | None = None,
+    update_adaptive: bool = False,
 ) -> dict[str, Any]:
     episode_summaries: list[dict[str, Any]] = []
     trajectories: list[dict[str, Any]] = []
@@ -365,16 +636,21 @@ def evaluate_policy(
                 seed=seed,
                 task_id=task_id,
                 submit_builder=submit_builder,
+                adaptive_bandit=adaptive_bandit,
+                update_adaptive=update_adaptive,
             )
             episode_summaries.append(summary)
             trajectories.extend(episode_trajectories)
 
-    return {
+    result = {
         "policy": policy.name,
         "summary": summarize_policy_episodes(policy, episode_summaries),
         "episodes": episode_summaries,
         "trajectories": trajectories,
     }
+    if adaptive_bandit is not None and policy.strategy == "adaptive":
+        result["adaptive_bandit"] = adaptive_bandit.export()
+    return result
 
 
 def _selection_tuple(summary: dict[str, Any]) -> tuple[float, float, float, float]:
@@ -416,16 +692,20 @@ def compare_policies(
     submit_builder: SubmitBuilder = default_submit_builder,
 ) -> dict[str, Any]:
     output_dir = Path(output_dir)
-    policy_runs = [
-        evaluate_policy(
-            policy,
-            seeds,
-            task_ids,
-            env_factory=env_factory,
-            submit_builder=submit_builder,
+    policy_runs = []
+    for policy in policies:
+        adaptive_bandit = AdaptiveToolBandit() if policy.strategy == "adaptive" else None
+        policy_runs.append(
+            evaluate_policy(
+                policy,
+                seeds,
+                task_ids,
+                env_factory=env_factory,
+                submit_builder=submit_builder,
+                adaptive_bandit=adaptive_bandit,
+                update_adaptive=policy.strategy == "adaptive",
+            )
         )
-        for policy in policies
-    ]
     best_run = select_best_policy(policy_runs)
     baseline_run = policy_runs[0]
 
@@ -461,6 +741,11 @@ def compare_policies(
                 reverse=True,
             )
         ],
+        "adaptive_bandits": {
+            run["policy"]: run["adaptive_bandit"]
+            for run in policy_runs
+            if "adaptive_bandit" in run
+        },
         "artifacts": {
             "summary": str(output_dir / "compare_summary.json"),
             "episodes": str(output_dir / "compare_episodes.jsonl"),
@@ -496,16 +781,22 @@ def search_policies(
     baseline_policy_name: str = "no_investigation",
 ) -> dict[str, Any]:
     output_dir = Path(output_dir)
-    train_runs = [
-        evaluate_policy(
+    train_runs = []
+    trained_bandits: dict[str, AdaptiveToolBandit] = {}
+    for policy in candidate_policies:
+        adaptive_bandit = AdaptiveToolBandit() if policy.strategy == "adaptive" else None
+        train_run = evaluate_policy(
             policy,
             train_seeds,
             task_ids,
             env_factory=env_factory,
             submit_builder=submit_builder,
+            adaptive_bandit=adaptive_bandit,
+            update_adaptive=policy.strategy == "adaptive",
         )
-        for policy in candidate_policies
-    ]
+        train_runs.append(train_run)
+        if adaptive_bandit is not None:
+            trained_bandits[policy.name] = adaptive_bandit.frozen_copy()
     selected_run = select_best_policy(train_runs)
     selected_policy = POLICY_LIBRARY[selected_run["policy"]]
     eval_selected = evaluate_policy(
@@ -514,6 +805,8 @@ def search_policies(
         task_ids,
         env_factory=env_factory,
         submit_builder=submit_builder,
+        adaptive_bandit=trained_bandits.get(selected_policy.name),
+        update_adaptive=False,
     )
 
     baseline_policy = POLICY_LIBRARY.get(baseline_policy_name, candidate_policies[0])
@@ -523,6 +816,8 @@ def search_policies(
         task_ids,
         env_factory=env_factory,
         submit_builder=submit_builder,
+        adaptive_bandit=trained_bandits.get(baseline_policy.name),
+        update_adaptive=False,
     )
 
     report = {
@@ -535,6 +830,9 @@ def search_policies(
         "selected_policy": selected_policy.name,
         "baseline_policy": baseline_policy.name,
         "train_policy_summaries": [run["summary"] for run in train_runs],
+        "trained_adaptive_bandits": {
+            name: bandit.export() for name, bandit in trained_bandits.items()
+        },
         "eval_selected_summary": eval_selected["summary"],
         "eval_baseline_summary": eval_baseline["summary"],
         "eval_improvement_vs_baseline": {

required.md

@@ -154,11 +154,12 @@ All of these must pass:
 
 ### Required inference environment variables
 
-- `API_BASE_URL`
-- `MODEL_NAME`
-- `HF_TOKEN`
+  - `API_BASE_URL`
+  - `MODEL_NAME`
+  - `API_KEY`
+  - `HF_TOKEN`
 
-The official text also mentions `OPENAI_API_KEY` in one place, but the more specific submission instructions above consistently emphasize `API_BASE_URL`, `MODEL_NAME`, and `HF_TOKEN`. We should follow the later, more specific instruction while continuing to use the OpenAI client.
+Use `API_KEY` as the primary evaluator-injected credential for the OpenAI client. `HF_TOKEN` can remain as a backward-compatible local fallback, but submission-time LLM traffic should flow through the injected proxy key.
 
 ### Inference script constraints
 
@@ -302,7 +303,7 @@ The project keeps three tasks:
 ### Inference
 
 - heuristic mode works without model credentials
-- LLM mode reads `API_BASE_URL`, `MODEL_NAME`, and `HF_TOKEN`
+- LLM mode reads `API_BASE_URL`, `MODEL_NAME`, and `API_KEY` (`HF_TOKEN` remains a local fallback)
 - uses the OpenAI client
 - stdout follows `[START]`, `[STEP]`, and `[END]`
 - output is reproducible when the seed is fixed

server/Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.11-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
 
 WORKDIR /app
 
@@ -14,6 +15,14 @@ RUN python -m pip install --upgrade pip \
     && python -m pip install --no-cache-dir -r requirements.txt \
     && python -m pip install --no-cache-dir .
 
+RUN useradd --create-home --uid 10001 appuser \
+    && chown -R appuser:appuser /app
+
 EXPOSE 7860
 
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/health', timeout=3)"
+
+USER appuser
+
 CMD ["uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860"]

server/app.py

@@ -1,17 +1,21 @@
 import sys
 from pathlib import Path
+from typing import Any
 
 # Ensure repo root is on sys.path so `models` and `server` are importable
 _repo_root = str(Path(__file__).resolve().parent.parent)
 if _repo_root not in sys.path:
     sys.path.insert(0, _repo_root)
 
-from fastapi.responses import HTMLResponse
+from fastapi import HTTPException
+from pydantic import BaseModel
+from fastapi.responses import HTMLResponse, RedirectResponse
 from openenv.core.env_server import create_app
 
 from models import HelpdeskTicketAction, HelpdeskTicketObservation
 from server.environment import HelpdeskTicketRoutingEnvironment
-from server.tasks import TASKS
+from server.grader import grade_action
+from server.tasks import TASKS, load_dataset
 from vocabulary import APP_ENV_NAME
 
 app = create_app(
@@ -22,6 +26,17 @@ app = create_app(
 )
 
 
+class GraderRequest(BaseModel):
+    task_id: int
+    ticket_id: str
+    action: dict[str, Any]
+
+
+@app.get("/", include_in_schema=False)
+def root_redirect():
+    return RedirectResponse(url="/web", status_code=307)
+
+
 @app.get("/tasks")
 def list_tasks():
     return {
@@ -57,6 +72,109 @@ def web_ui():
     return HTMLResponse(content=html)
 
 
+def _build_baseline_submit_action(
+    ticket: dict[str, Any], allowed_fields: list[str]
+) -> HelpdeskTicketAction:
+    import inference
+
+    candidate = inference.heuristic_action(ticket, allowed_fields)
+    candidate, _ = inference.apply_domain_overrides(ticket, candidate, allowed_fields)
+    return HelpdeskTicketAction(**candidate)
+
+
+@app.get("/baseline")
+def baseline_rollout(task_id: int = 1, seed: int = 42):
+    import inference
+
+    env = HelpdeskTicketRoutingEnvironment()
+    observation = env.reset(seed=seed, task_id=task_id)
+    steps: list[dict[str, Any]] = []
+
+    while not observation.done:
+        ticket = observation.current_ticket
+        if ticket is None:
+            break
+
+        investigate, tool_name = inference.should_investigate(ticket, observation.history)
+        if (
+            investigate
+            and tool_name is not None
+            and observation.investigation_budget_remaining > 0
+        ):
+            investigate_action = HelpdeskTicketAction(
+                action_type="investigate",
+                tool_name=tool_name,
+                tool_target_ticket_id=ticket.get("related_ticket_id"),
+            )
+            observation = env.step(investigate_action)
+            steps.append(
+                {
+                    "action": investigate_action.model_dump(exclude_none=True),
+                    "reward": observation.reward,
+                    "done": observation.done,
+                    "action_source": "baseline_investigate",
+                }
+            )
+            if observation.done:
+                break
+            ticket = observation.current_ticket
+            if ticket is None:
+                break
+
+        action = _build_baseline_submit_action(
+            inference.merge_ticket_context(ticket, observation),
+            list(observation.allowed_fields),
+        )
+        observation = env.step(action)
+        steps.append(
+            {
+                "action": action.model_dump(exclude_none=True),
+                "reward": observation.reward,
+                "done": observation.done,
+                "action_source": "baseline_submit",
+            }
+        )
+
+    return {
+        "task_id": task_id,
+        "seed": seed,
+        "step_count": len(steps),
+        "final_reward": observation.reward,
+        "rubric_reward": observation.rubric_reward,
+        "steps": steps,
+    }
+
+
+@app.post("/grader")
+def grader_preview(request: GraderRequest):
+    ticket = next(
+        (record for record in load_dataset() if record.ticket_id == request.ticket_id),
+        None,
+    )
+    if ticket is None:
+        raise HTTPException(status_code=404, detail=f"Unknown ticket_id: {request.ticket_id}")
+
+    try:
+        action = HelpdeskTicketAction.model_validate(request.action)
+    except Exception as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
+
+    score, breakdown = grade_action(action, ticket, request.task_id)
+    return {
+        "task_id": request.task_id,
+        "ticket_id": request.ticket_id,
+        "score": score,
+        "breakdown": breakdown,
+        "expected": {
+            "issue_type": ticket.issue_type,
+            "priority": ticket.priority,
+            "assignment_group": ticket.assignment_group,
+            "resolution_action": ticket.resolution_action,
+        },
+        "submitted": action.model_dump(exclude_none=True),
+    }
+
+
 def main() -> None:
     import uvicorn
 

server/environment.py

@@ -13,8 +13,15 @@ from models import (
     HelpdeskTicketState,
 )
 from server.grader import grade_action
-from server.reward import compute_step_reward, compute_trajectory_reward
+from server.reward import (
+    compute_step_adjustments,
+    compute_trajectory_adjustments,
+)
 from server.tasks import get_task_definition, load_dataset
+from vocabulary import (
+    ISSUE_TYPE_TO_ASSIGNMENT_GROUP,
+    ISSUE_TYPE_TO_RESOLUTION_ACTION,
+)
 
 
 QUEUE_SIZE_RANGE = (3, 5)
@@ -29,6 +36,12 @@ EXTRA_INVESTIGATION_COST = 0.02
 MAX_EXTRA_INVESTIGATION_PENALTY = 0.15
 USEFUL_INVESTIGATION_REWARD = 0.08
 PREMATURE_SUBMIT_PENALTY = 0.10
+CONTEXT_COMPLETION_BONUS = 0.04
+TRAJECTORY_CONTEXT_COMPLETION_BONUS = 0.03
+PRIORITY_UNDERSHOOT_PENALTY = 0.03
+SEVERE_PRIORITY_UNDERSHOOT_PENALTY = 0.07
+DANGEROUS_RESOLUTION_PENALTY = 0.05
+NONDEFAULT_ROUTING_FOLLOWTHROUGH_BONUS = 0.02
 
 TASK3_INVESTIGATION_TOOL_PLAN: dict[str, tuple[str, ...]] = {
     "ticket-021": ("lookup_related_ticket", "lookup_requester_history"),
@@ -190,11 +203,16 @@ class HelpdeskTicketRoutingEnvironment(
             is_done = self._state.current_ticket_index >= len(self._queue)
             self._state.done = is_done
             trajectory_reward = None
+            trajectory_components = None
             investigation_penalty = self._compute_episode_penalty() if is_done else 0.0
             if is_done:
-                trajectory_reward = compute_trajectory_reward(
-                    self._state.per_ticket_scores, len(self._queue), self._state.step_count
+                trajectory_components = compute_trajectory_adjustments(
+                    self._state.per_ticket_scores,
+                    len(self._queue),
+                    self._state.step_count,
+                    completion_bonus=self._trajectory_consistency_bonus(),
                 )
+                trajectory_reward = trajectory_components["final_reward"]
                 final_reward = self._apply_episode_economics(trajectory_reward)
                 self._state.total_reward = final_reward
             else:
@@ -208,6 +226,23 @@ class HelpdeskTicketRoutingEnvironment(
                 trajectory_reward=trajectory_reward,
                 investigation_penalty=investigation_penalty,
                 penalty_reason=f"extra_fields: {sorted(extra_fields)}",
+                extra_details={
+                    "trajectory_average_reward": (
+                        trajectory_components["average_reward"]
+                        if trajectory_components is not None
+                        else None
+                    ),
+                    "trajectory_completion_bonus": (
+                        trajectory_components["completion_bonus"]
+                        if trajectory_components is not None
+                        else None
+                    ),
+                    "trajectory_consistency_bonus": (
+                        trajectory_components["consistency_bonus"]
+                        if trajectory_components is not None
+                        else None
+                    ),
+                },
             )
             self._state.history_entries.append(
                 self._build_history_entry(
@@ -235,13 +270,30 @@ class HelpdeskTicketRoutingEnvironment(
                 rubric_reward=final_reward if is_done else None,
             )
 
+        previous_average = self._current_average_score()
         score, breakdown = grade_action(action, current_ticket, task_id)
-        step_reward = compute_step_reward(score)
-        context_penalty, missing_required_tools = self._submit_context_penalty(current_ticket)
-        milestone_adjustment = step_reward - score
+        context_penalty, missing_required_count = self._submit_context_penalty(current_ticket)
+        process_bonus = self._context_completion_bonus(
+            current_ticket,
+            missing_required_count=missing_required_count,
+            score=score,
+        )
+        risk_penalty = self._operational_risk_penalty(
+            current_ticket,
+            action,
+            task_id=task_id,
+        )
+        step_adjustments = compute_step_adjustments(
+            score,
+            previous_average=previous_average,
+            process_bonus=process_bonus,
+            risk_penalty=risk_penalty,
+        )
+        step_reward = step_adjustments["final_reward"]
 
         is_done = (self._state.current_ticket_index + 1) >= len(self._queue)
         trajectory_reward = None
+        trajectory_components = None
         investigation_penalty = 0.0
         rubric_reward = None
 
@@ -250,11 +302,13 @@ class HelpdeskTicketRoutingEnvironment(
             self._state.average_score_so_far = self._current_average_score()
             self._state.step_count += 1
             self._state.current_ticket_index += 1
-            trajectory_reward = compute_trajectory_reward(
+            trajectory_components = compute_trajectory_adjustments(
                 self._state.per_ticket_scores,
                 len(self._queue),
                 self._state.step_count,
+                completion_bonus=self._trajectory_consistency_bonus(),
             )
+            trajectory_reward = trajectory_components["final_reward"]
             rubric_reward = self._apply_episode_economics(trajectory_reward)
             final_reward = max(0.0, min(1.0, rubric_reward - context_penalty))
             self._state.total_reward = rubric_reward
@@ -272,14 +326,35 @@ class HelpdeskTicketRoutingEnvironment(
             shaped_step_reward=step_reward,
             reward_kind="trajectory" if is_done else "step",
             final_reward=final_reward,
-            milestone_adjustment=milestone_adjustment,
+            milestone_adjustment=step_adjustments["milestone_adjustment"],
             trajectory_reward=trajectory_reward,
             investigation_penalty=investigation_penalty,
             extra_details={
                 "context_gap_penalty": context_penalty,
-                "required_tools": self._required_tools_for_ticket(current_ticket),
-                "remaining_required_tools": missing_required_tools,
+                "context_completion_bonus": process_bonus,
+                "risk_penalty": risk_penalty,
+                "delta_adjustment": step_adjustments["delta_adjustment"],
+                "required_investigation_count": len(self._required_tools_for_ticket(current_ticket)),
+                "hidden_context_remaining_count": missing_required_count,
+                "hidden_context_revealed_count": len(
+                    self._used_tools_for_ticket(current_ticket.ticket_id)
+                ),
                 "rubric_reward": rubric_reward,
+                "trajectory_average_reward": (
+                    trajectory_components["average_reward"]
+                    if trajectory_components is not None
+                    else None
+                ),
+                "trajectory_completion_bonus": (
+                    trajectory_components["completion_bonus"]
+                    if trajectory_components is not None
+                    else None
+                ),
+                "trajectory_consistency_bonus": (
+                    trajectory_components["consistency_bonus"]
+                    if trajectory_components is not None
+                    else None
+                ),
             },
         )
 
@@ -335,6 +410,35 @@ class HelpdeskTicketRoutingEnvironment(
             return 0.0
         return sum(self._state.per_ticket_scores) / len(self._state.per_ticket_scores)
 
+    def _ticket_has_nondefault_routing(self, ticket: HelpdeskTicketRecord) -> bool:
+        return (
+            ticket.assignment_group
+            != ISSUE_TYPE_TO_ASSIGNMENT_GROUP.get(ticket.issue_type, ticket.assignment_group)
+            or ticket.resolution_action
+            != ISSUE_TYPE_TO_RESOLUTION_ACTION.get(
+                ticket.issue_type, ticket.resolution_action
+            )
+        )
+
+    def _ticket_mentions_follow_up(self, ticket: HelpdeskTicketRecord) -> bool:
+        text = f"{ticket.title} {ticket.description}".lower()
+        return any(
+            phrase in text
+            for phrase in (
+                "re:",
+                "follow-up",
+                "following up",
+                "still",
+                "third update",
+                "reference ticket",
+                "regression",
+                "unresolved",
+            )
+        )
+
+    def _ticket_repeated_requester_count(self, ticket: HelpdeskTicketRecord) -> int:
+        return sum(1 for candidate in self._dataset if candidate.requester == ticket.requester)
+
     def _required_tools_for_ticket(
         self,
         ticket: HelpdeskTicketRecord,
@@ -343,7 +447,25 @@ class HelpdeskTicketRoutingEnvironment(
         resolved_task_id = self._state.current_task_id if task_id is None else task_id
         if resolved_task_id != 3:
             return []
-        return list(TASK3_INVESTIGATION_TOOL_PLAN.get(ticket.ticket_id, ()))
+        required_tools: list[str] = list(TASK3_INVESTIGATION_TOOL_PLAN.get(ticket.ticket_id, ()))
+        if ticket.related_ticket_id is not None and "lookup_related_ticket" not in required_tools:
+            required_tools.append("lookup_related_ticket")
+        if (
+            ticket.ambiguity_note is not None or self._ticket_has_nondefault_routing(ticket)
+        ) and "lookup_internal_routing_note" not in required_tools:
+            required_tools.append("lookup_internal_routing_note")
+        if (
+            self._ticket_repeated_requester_count(ticket) >= 2
+            and (
+                ticket.related_ticket_id is not None
+                or self._ticket_mentions_follow_up(ticket)
+                or self._ticket_has_nondefault_routing(ticket)
+                or ticket.priority in {"high", "critical"}
+            )
+            and "lookup_requester_history" not in required_tools
+        ):
+            required_tools.append("lookup_requester_history")
+        return required_tools
 
     def _used_tools_for_ticket(self, ticket_id: str) -> list[str]:
         return list(self._state.ticket_tool_usage.get(ticket_id, []))
@@ -362,35 +484,122 @@ class HelpdeskTicketRoutingEnvironment(
         if tool_name not in used:
             used.append(tool_name)
 
-    def _investigation_hints_for_ticket(self, ticket: HelpdeskTicketRecord) -> list[str]:
-        hints: list[str] = []
+    def _tool_progress_for_ticket(self, ticket: HelpdeskTicketRecord) -> dict[str, Any]:
+        required_tools = self._required_tools_for_ticket(ticket)
+        revealed_tools = self._used_tools_for_ticket(ticket.ticket_id)
         remaining_tools = self._remaining_tools_for_ticket(ticket)
-        if "lookup_internal_routing_note" in remaining_tools:
-            hints.append("An internal routing note may disambiguate the correct workflow.")
-        if "lookup_related_ticket" in remaining_tools:
-            hints.append("A linked prior ticket can reveal important follow-up context.")
-        if "lookup_requester_history" in remaining_tools:
-            hints.append("Requester history may clarify severity or routing intent.")
-        return hints
+        total_required = max(1, len(required_tools))
+        return {
+            "required_tools": required_tools,
+            "revealed_tools": revealed_tools,
+            "remaining_tools": remaining_tools,
+            "revealed_count": len(revealed_tools),
+            "remaining_count": len(remaining_tools),
+            "completeness": round(len(revealed_tools) / total_required, 2),
+        }
+
+    def _default_redacted_description(self, ticket: HelpdeskTicketRecord) -> str:
+        if ticket.related_ticket_id is not None:
+            return (
+                "This is a follow-up operational issue that references prior work. "
+                "Additional routing context is available via investigation."
+            )
+        if ticket.ambiguity_note is not None:
+            return (
+                "This ticket mixes multiple plausible workflows. "
+                "Additional routing context is available via investigation."
+            )
+        if self._ticket_has_nondefault_routing(ticket):
+            return (
+                "The visible request looks straightforward, but the decisive routing "
+                "detail is hidden until investigation."
+            )
+        return (
+            "Additional routing context is available via investigation before final submission."
+        )
 
     def _visible_description(self, ticket: HelpdeskTicketRecord) -> str:
-        if (
-            self._state.current_task_id == 3
-            and self._remaining_tools_for_ticket(ticket)
-            and ticket.ticket_id in HARD_TASK_DESCRIPTION_REDACTIONS
-        ):
-            return HARD_TASK_DESCRIPTION_REDACTIONS[ticket.ticket_id]
+        if self._state.current_task_id == 3 and self._remaining_tools_for_ticket(ticket):
+            return HARD_TASK_DESCRIPTION_REDACTIONS.get(
+                ticket.ticket_id,
+                self._default_redacted_description(ticket),
+            )
         return ticket.description
 
-    def _submit_context_penalty(self, ticket: HelpdeskTicketRecord) -> tuple[float, list[str]]:
-        required_tools = self._required_tools_for_ticket(ticket)
-        if not required_tools:
-            return 0.0, []
-        remaining_tools = self._remaining_tools_for_ticket(ticket)
-        if not remaining_tools:
-            return 0.0, []
-        penalty = PREMATURE_SUBMIT_PENALTY * (len(remaining_tools) / len(required_tools))
-        return penalty, remaining_tools
+    def _submit_context_penalty(self, ticket: HelpdeskTicketRecord) -> tuple[float, int]:
+        progress = self._tool_progress_for_ticket(ticket)
+        required_tools = progress["required_tools"]
+        remaining_tools = progress["remaining_tools"]
+        if not required_tools or not remaining_tools:
+            return 0.0, 0
+        penalty = PREMATURE_SUBMIT_PENALTY * (
+            len(remaining_tools) / max(1, len(required_tools))
+        )
+        return penalty, len(remaining_tools)
+
+    def _context_completion_bonus(
+        self,
+        ticket: HelpdeskTicketRecord,
+        *,
+        missing_required_count: int,
+        score: float,
+    ) -> float:
+        if not self._required_tools_for_ticket(ticket):
+            return 0.0
+        if missing_required_count != 0 or score < 0.75:
+            return 0.0
+        bonus = CONTEXT_COMPLETION_BONUS
+        if self._ticket_has_nondefault_routing(ticket):
+            bonus += NONDEFAULT_ROUTING_FOLLOWTHROUGH_BONUS
+        return bonus
+
+    def _trajectory_consistency_bonus(self) -> float:
+        if not self._queue:
+            return 0.0
+        hidden_context_tickets = [
+            ticket for ticket in self._queue if self._required_tools_for_ticket(ticket)
+        ]
+        if not hidden_context_tickets:
+            return 0.0
+        resolved = sum(
+            1 for ticket in hidden_context_tickets if not self._remaining_tools_for_ticket(ticket)
+        )
+        resolution_rate = resolved / len(hidden_context_tickets)
+        return round(TRAJECTORY_CONTEXT_COMPLETION_BONUS * resolution_rate, 4)
+
+    def _operational_risk_penalty(
+        self,
+        ticket: HelpdeskTicketRecord,
+        action: HelpdeskTicketAction,
+        *,
+        task_id: int,
+    ) -> float:
+        if task_id < 2 or action.priority is None:
+            priority_penalty = 0.0
+        else:
+            priority_rank = {"critical": 3, "high": 2, "medium": 1, "low": 0}
+            expected_rank = priority_rank.get(ticket.priority, 0)
+            predicted_rank = priority_rank.get(action.priority, 0)
+            gap = expected_rank - predicted_rank
+            if gap >= 2:
+                priority_penalty = SEVERE_PRIORITY_UNDERSHOOT_PENALTY
+            elif gap == 1 and ticket.priority in {"high", "critical"}:
+                priority_penalty = PRIORITY_UNDERSHOOT_PENALTY
+            else:
+                priority_penalty = 0.0
+
+        resolution_penalty = 0.0
+        if task_id == 3 and action.resolution_action is not None:
+            if (
+                ticket.issue_type in {"identity_access", "application_support", "security_compliance"}
+                and ticket.priority in {"high", "critical"}
+                and action.resolution_action == "acknowledge"
+            ):
+                resolution_penalty += DANGEROUS_RESOLUTION_PENALTY
+            if ticket.issue_type == "spam_phishing" and action.resolution_action == "fulfill":
+                resolution_penalty += PRIORITY_UNDERSHOOT_PENALTY
+
+        return round(priority_penalty + resolution_penalty, 4)
 
     def _build_reward_components(
         self,
@@ -547,6 +756,7 @@ class HelpdeskTicketRoutingEnvironment(
         self._state.reward = investigation_reward
         self._state.done = False
         self._state.investigation_penalty_applied = self._compute_episode_penalty()
+        progress = self._tool_progress_for_ticket(current_ticket)
         reward_components = self._build_reward_components(
             ticket_score=0.0,
             field_breakdown={},
@@ -556,8 +766,10 @@ class HelpdeskTicketRoutingEnvironment(
             investigation_penalty=self._state.investigation_penalty_applied,
             extra_details={
                 "new_context_revealed": useful_investigation,
-                "required_tools": required_tools,
-                "remaining_required_tools": self._remaining_tools_for_ticket(current_ticket),
+                "required_investigation_count": len(required_tools),
+                "hidden_context_remaining_count": progress["remaining_count"],
+                "hidden_context_revealed_count": progress["revealed_count"],
+                "context_completeness": progress["completeness"],
                 "tool_name": action.tool_name,
             },
         )
@@ -578,21 +790,22 @@ class HelpdeskTicketRoutingEnvironment(
         return self._build_observation(task, done=False, reward=investigation_reward)
 
     def _build_ticket_view(self, ticket: HelpdeskTicketRecord) -> dict[str, Any]:
-        required_tools = self._required_tools_for_ticket(ticket)
-        revealed_tools = self._used_tools_for_ticket(ticket.ticket_id)
-        remaining_tools = self._remaining_tools_for_ticket(ticket)
+        progress = self._tool_progress_for_ticket(ticket)
+        remaining_tools = progress["remaining_tools"]
         ticket_view: dict[str, Any] = {
             "ticket_id": ticket.ticket_id,
             "title": ticket.title,
             "requester": ticket.requester,
             "description": self._visible_description(ticket),
         }
-        if required_tools:
+        if progress["required_tools"]:
             ticket_view["context_status"] = {
                 "investigation_required": True,
-                "revealed_tools": revealed_tools,
-                "remaining_tools": remaining_tools,
-                "hints": self._investigation_hints_for_ticket(ticket),
+                "hidden_context_remaining": bool(progress["remaining_count"]),
+                "context_gap_count": progress["remaining_count"],
+                "revealed_context_count": progress["revealed_count"],
+                "context_completeness": progress["completeness"],
+                "investigations_used_for_ticket": progress["revealed_count"],
             }
         if ticket.ambiguity_note is not None and "lookup_internal_routing_note" not in remaining_tools:
             ticket_view["ambiguity_note"] = ticket.ambiguity_note
@@ -646,9 +859,19 @@ class HelpdeskTicketRoutingEnvironment(
             context_gap_penalty = reward_components.get("context_gap_penalty")
             if context_gap_penalty:
                 parts.append(f"context_gap_penalty={context_gap_penalty:.2f}")
-            remaining_required_tools = reward_components.get("remaining_required_tools") or []
-            if remaining_required_tools:
-                parts.append(f"missing_context={remaining_required_tools}")
+            hidden_context_remaining_count = reward_components.get(
+                "hidden_context_remaining_count"
+            )
+            if hidden_context_remaining_count:
+                parts.append(
+                    f"hidden_context_remaining={hidden_context_remaining_count}"
+                )
+            context_completion_bonus = reward_components.get("context_completion_bonus")
+            if context_completion_bonus:
+                parts.append(f"context_bonus={context_completion_bonus:.2f}")
+            risk_penalty = reward_components.get("risk_penalty")
+            if risk_penalty:
+                parts.append(f"risk_penalty={risk_penalty:.2f}")
 
         return "; ".join(parts)
 
@@ -667,8 +890,8 @@ class HelpdeskTicketRoutingEnvironment(
         tool_result: dict[str, Any] | None = None,
         reward_components: dict[str, Any] | None = None,
     ) -> dict[str, Any]:
-        remaining_tools = self._remaining_tools_for_ticket(ticket)
-        revealed_tools = self._used_tools_for_ticket(ticket.ticket_id)
+        progress = self._tool_progress_for_ticket(ticket)
+        remaining_tools = progress["remaining_tools"]
         history_entry: dict[str, Any] = {
             "ticket_id": ticket.ticket_id,
             "title": ticket.title,
@@ -702,8 +925,13 @@ class HelpdeskTicketRoutingEnvironment(
             history_entry["tool_result"] = tool_result
         if reward_components is not None:
             history_entry["reward_components"] = reward_components
-        if revealed_tools:
-            history_entry["revealed_tools"] = revealed_tools
+        if progress["required_tools"]:
+            history_entry["context_progress"] = {
+                "hidden_context_remaining": bool(progress["remaining_count"]),
+                "context_gap_count": progress["remaining_count"],
+                "revealed_context_count": progress["revealed_count"],
+                "context_completeness": progress["completeness"],
+            }
         history_entry["feedback_summary"] = self._build_feedback_summary(
             predicted=predicted,
             score=score,
@@ -751,6 +979,10 @@ class HelpdeskTicketRoutingEnvironment(
             "has_related_ticket_context": bool(
                 ticket_view and ticket_view.get("related_ticket_preview")
             ),
+            "has_hidden_context": bool(
+                ticket_view
+                and (ticket_view.get("context_status") or {}).get("hidden_context_remaining")
+            ),
             "action_mode": "investigate_or_submit",
             "available_action_types": list(AVAILABLE_ACTION_TYPES),
             "average_score_so_far": self._state.average_score_so_far,

server/reward.py

@@ -4,21 +4,113 @@ MILESTONE_HIGH_THRESHOLD = 0.8
 MILESTONE_LOW_THRESHOLD = 0.2
 MILESTONE_BONUS = 0.05
 MILESTONE_PENALTY = 0.05
+DELTA_REWARD_WEIGHT = 0.08
+DELTA_REWARD_CAP = 0.04
+PROCESS_BONUS_CAP = 0.08
+RISK_PENALTY_CAP = 0.12
 
 
-def compute_step_reward(score: float) -> float:
-    base = max(0.0, min(1.0, score))
+def _clamp_unit_interval(value: float) -> float:
+    return max(0.0, min(1.0, value))
+
+
+def compute_step_adjustments(
+    score: float,
+    *,
+    previous_average: float = 0.0,
+    process_bonus: float = 0.0,
+    risk_penalty: float = 0.0,
+) -> dict[str, float]:
+    base = _clamp_unit_interval(score)
+
     if score >= MILESTONE_HIGH_THRESHOLD:
-        return min(1.0, base + MILESTONE_BONUS)
-    if score < MILESTONE_LOW_THRESHOLD:
-        return max(0.0, base - MILESTONE_PENALTY)
-    return base
+        milestone_adjustment = MILESTONE_BONUS
+    elif score < MILESTONE_LOW_THRESHOLD:
+        milestone_adjustment = -MILESTONE_PENALTY
+    else:
+        milestone_adjustment = 0.0
 
+    delta_adjustment = _clamp_delta((base - previous_average) * DELTA_REWARD_WEIGHT)
+    bounded_process_bonus = max(0.0, min(PROCESS_BONUS_CAP, process_bonus))
+    bounded_risk_penalty = max(0.0, min(RISK_PENALTY_CAP, risk_penalty))
+    final_reward = _clamp_unit_interval(
+        base
+        + milestone_adjustment
+        + delta_adjustment
+        + bounded_process_bonus
+        - bounded_risk_penalty
+    )
 
-def compute_trajectory_reward(
-    per_ticket_scores: list[float], queue_size: int, steps_taken: int
+    return {
+        "base_reward": base,
+        "milestone_adjustment": milestone_adjustment,
+        "delta_adjustment": delta_adjustment,
+        "process_bonus": bounded_process_bonus,
+        "risk_penalty": bounded_risk_penalty,
+        "final_reward": final_reward,
+    }
+
+
+def _clamp_delta(value: float) -> float:
+    return max(-DELTA_REWARD_CAP, min(DELTA_REWARD_CAP, value))
+
+
+def compute_step_reward(
+    score: float,
+    *,
+    previous_average: float = 0.0,
+    process_bonus: float = 0.0,
+    risk_penalty: float = 0.0,
 ) -> float:
+    return compute_step_adjustments(
+        score,
+        previous_average=previous_average,
+        process_bonus=process_bonus,
+        risk_penalty=risk_penalty,
+    )["final_reward"]
+
+
+def compute_trajectory_adjustments(
+    per_ticket_scores: list[float],
+    queue_size: int,
+    steps_taken: int,
+    *,
+    completion_bonus: float = 0.0,
+    consistency_bonus: float = 0.0,
+) -> dict[str, float]:
     if not per_ticket_scores:
-        return 0.0
+        return {
+            "average_reward": 0.0,
+            "completion_bonus": 0.0,
+            "consistency_bonus": 0.0,
+            "final_reward": 0.0,
+        }
     avg = sum(per_ticket_scores) / len(per_ticket_scores)
-    return max(0.0, min(1.0, avg))
+    bounded_completion_bonus = max(0.0, min(0.08, completion_bonus))
+    bounded_consistency_bonus = max(0.0, min(0.05, consistency_bonus))
+    final_reward = _clamp_unit_interval(
+        avg + bounded_completion_bonus + bounded_consistency_bonus
+    )
+    return {
+        "average_reward": avg,
+        "completion_bonus": bounded_completion_bonus,
+        "consistency_bonus": bounded_consistency_bonus,
+        "final_reward": final_reward,
+    }
+
+
+def compute_trajectory_reward(
+    per_ticket_scores: list[float],
+    queue_size: int,
+    steps_taken: int,
+    *,
+    completion_bonus: float = 0.0,
+    consistency_bonus: float = 0.0,
+) -> float:
+    return compute_trajectory_adjustments(
+        per_ticket_scores,
+        queue_size,
+        steps_taken,
+        completion_bonus=completion_bonus,
+        consistency_bonus=consistency_bonus,
+    )["final_reward"]

tests/test_competitive_upgrade.py

@@ -245,27 +245,27 @@ class TestMilestoneRewardShaping(unittest.TestCase):
 
     def test_high_score_gets_bonus(self) -> None:
         # score=0.9 >= 0.8 threshold → base=0.9, bonus=0.05 → 0.95
-        result = compute_step_reward(0.9)
+        result = compute_step_reward(0.9, previous_average=0.9)
         self.assertAlmostEqual(result, 0.95, places=9)
 
     def test_low_score_gets_penalty(self) -> None:
         # score=0.1 < 0.2 threshold → base=0.1, penalty=0.05 → 0.05
-        result = compute_step_reward(0.1)
+        result = compute_step_reward(0.1, previous_average=0.1)
         self.assertAlmostEqual(result, 0.05, places=9)
 
     def test_mid_score_is_neutral(self) -> None:
         # score=0.5 is in [0.2, 0.8) → no shaping → 0.5
-        result = compute_step_reward(0.5)
+        result = compute_step_reward(0.5, previous_average=0.5)
         self.assertAlmostEqual(result, 0.5, places=9)
 
     def test_boundary_high_threshold_gets_bonus(self) -> None:
         # score=0.8 exactly → bonus applies → 0.85
-        result = compute_step_reward(0.8)
+        result = compute_step_reward(0.8, previous_average=0.8)
         self.assertAlmostEqual(result, 0.85, places=9)
 
     def test_boundary_low_threshold_is_neutral(self) -> None:
         # score=0.2 exactly → not < 0.2, so neutral → 0.2
-        result = compute_step_reward(0.2)
+        result = compute_step_reward(0.2, previous_average=0.2)
         self.assertAlmostEqual(result, 0.2, places=9)
 
     def test_reward_clamped_to_unit_interval(self) -> None:
@@ -274,6 +274,11 @@ class TestMilestoneRewardShaping(unittest.TestCase):
         self.assertLessEqual(result, 1.0)
         self.assertGreaterEqual(result, 0.0)
 
+    def test_improvement_delta_adds_small_bonus(self) -> None:
+        improved = compute_step_reward(0.7, previous_average=0.2)
+        flat = compute_step_reward(0.7, previous_average=0.7)
+        self.assertGreater(improved, flat)
+
     def test_zero_score_clamped_to_zero(self) -> None:
         # score=0.0 < 0.2 → base=0.0, penalty → max(0.0, -0.05) = 0.0
         result = compute_step_reward(0.0)
@@ -348,10 +353,8 @@ class TestAmbiguityNoteInObservation(unittest.TestCase):
         self.assertIsNotNone(obs.current_ticket)
         self.assertNotIn("ambiguity_note", obs.current_ticket)
         self.assertIn("context_status", obs.current_ticket)
-        self.assertIn(
-            "lookup_internal_routing_note",
-            obs.current_ticket["context_status"]["remaining_tools"],
-        )
+        self.assertTrue(obs.current_ticket["context_status"]["hidden_context_remaining"])
+        self.assertGreater(obs.current_ticket["context_status"]["context_gap_count"], 0)
 
         obs = env.step(
             HelpdeskTicketAction(
@@ -436,10 +439,8 @@ class TestRelatedTicketPreviewInObservation(unittest.TestCase):
         self.assertIsNotNone(obs.current_ticket)
         self.assertNotIn("related_ticket_preview", obs.current_ticket)
         self.assertIn("context_status", obs.current_ticket)
-        self.assertIn(
-            "lookup_related_ticket",
-            obs.current_ticket["context_status"]["remaining_tools"],
-        )
+        self.assertTrue(obs.current_ticket["context_status"]["hidden_context_remaining"])
+        self.assertGreater(obs.current_ticket["context_status"]["context_gap_count"], 0)
 
         obs = env.step(
             HelpdeskTicketAction(
@@ -766,8 +767,8 @@ class TestDatasetNonDefaultRouting(unittest.TestCase):
             if t.assignment_group != ISSUE_TYPE_TO_ASSIGNMENT_GROUP.get(t.issue_type)
         ]
         self.assertGreaterEqual(
-            len(non_default), 3,
-            f"Expected >= 3 non-default routing tickets, found {len(non_default)}: "
+            len(non_default), 10,
+            f"Expected >= 10 non-default routing tickets, found {len(non_default)}: "
             + str([(t.ticket_id, t.issue_type, t.assignment_group) for t in non_default])
         )
 

tests/test_inference_unit.py

@@ -129,7 +129,7 @@ class FakeEnvClient:
 
 
 class InferenceUnitTests(unittest.TestCase):
-    def test_hf_token_has_no_default_and_model_name_keeps_allowed_default(self) -> None:
+    def test_api_credentials_have_no_defaults_and_model_name_keeps_allowed_default(self) -> None:
         inference = _load_inference_module()
 
         self.assertEqual(
@@ -137,9 +137,22 @@ class InferenceUnitTests(unittest.TestCase):
             "https://router.huggingface.co/v1",
         )
         self.assertEqual(inference.MODEL_NAME, "<your-active-model>")
+        self.assertIsNone(inference.API_KEY)
         self.assertIsNone(inference.HF_TOKEN)
         self.assertFalse(inference.llm_mode_enabled())
 
+    def test_api_key_enables_llm_mode_without_hf_token(self) -> None:
+        inference = _load_inference_module(
+            {
+                "API_KEY": "validator-proxy-key",
+                "MODEL_NAME": "meta/test-model",
+            }
+        )
+
+        self.assertEqual(inference.API_KEY, "validator-proxy-key")
+        self.assertIsNone(inference.HF_TOKEN)
+        self.assertTrue(inference.llm_mode_enabled())
+
     def test_seed_env_override_is_respected(self) -> None:
         inference = _load_inference_module({"SEED": "7"})
 
@@ -199,9 +212,11 @@ class InferenceUnitTests(unittest.TestCase):
             "description": "Access permissions are blocking contractor setup.",
             "context_status": {
                 "investigation_required": True,
-                "revealed_tools": [],
-                "remaining_tools": ["lookup_internal_routing_note"],
-                "hints": ["An internal routing note may disambiguate the correct workflow."],
+                "hidden_context_remaining": True,
+                "context_gap_count": 1,
+                "revealed_context_count": 0,
+                "context_completeness": 0.0,
+                "investigations_used_for_ticket": 0,
             },
             "last_tool_result": {"tool_name": "lookup_requester_history", "found": False},
             "feedback_summary": "Ticket score=0.40; field_scores[issue_type=0.40]; reward=0.40",
@@ -475,24 +490,24 @@ class InferenceUnitTests(unittest.TestCase):
         self.assertEqual(merged["tickets_remaining"], 4)
         self.assertEqual(merged["last_tool_result"]["tool_name"], "lookup_requester_history")
 
-    def test_should_investigate_uses_remaining_tools_from_context_status(self) -> None:
+    def test_should_investigate_uses_hidden_context_and_ticket_cues(self) -> None:
         inference = _load_inference_module()
 
         investigate, tool_name = inference.should_investigate(
             {
-                "ticket_id": "ticket-021",
+                "ticket_id": "TKT-NONDEFAULT-003",
+                "title": "Contractor onboarding blocked by access issue",
+                "description": "Additional routing context is available via investigation.",
                 "context_status": {
-                    "remaining_tools": [
-                        "lookup_related_ticket",
-                        "lookup_requester_history",
-                    ]
-                },
+                    "hidden_context_remaining": True,
+                    "context_gap_count": 1,
+                }
             },
             [],
         )
 
         self.assertTrue(investigate)
-        self.assertEqual(tool_name, "lookup_related_ticket")
+        self.assertEqual(tool_name, "lookup_internal_routing_note")
 
 
 if __name__ == "__main__":

tests/test_policy_learning.py

@@ -32,6 +32,7 @@ from policy_learning import (
     POLICY_LIBRARY,
     choose_policy_action,
     compare_policies,
+    infer_ticket_cue,
     parse_int_spec,
     rollout_episode,
     search_policies,
@@ -99,35 +100,55 @@ class PolicyLearningTests(unittest.TestCase):
         observation = HelpdeskTicketObservation(
             current_ticket={
                 "ticket_id": "ticket-021",
+                "title": "Re: Production checkout throwing null reference exception",
+                "description": "Additional routing context is available via investigation.",
                 "context_status": {
-                    "remaining_tools": ["lookup_related_ticket", "lookup_requester_history"],
-                    "revealed_tools": [],
+                    "hidden_context_remaining": True,
+                    "context_gap_count": 2,
+                    "revealed_context_count": 0,
+                    "context_completeness": 0.0,
                 }
             },
             allowed_fields=["issue_type"],
         )
 
-        action, source = choose_policy_action(policy, observation, {}, _context_sensitive_submit_builder)
+        action, source, cue = choose_policy_action(
+            policy,
+            observation,
+            {},
+            _context_sensitive_submit_builder,
+            used_tools_by_ticket={},
+        )
 
         self.assertEqual(action.action_type, "investigate")
         self.assertEqual(action.tool_name, "lookup_related_ticket")
         self.assertEqual(source, "investigate_hidden_context")
+        self.assertEqual(cue, "follow_up")
 
     def test_choose_policy_action_submits_when_investigation_disabled(self) -> None:
         policy = POLICY_LIBRARY["no_investigation"]
         observation = HelpdeskTicketObservation(
             current_ticket={
                 "ticket_id": "ticket-021",
-                "context_status": {"remaining_tools": ["lookup_related_ticket"]},
+                "title": "Re: Production checkout throwing null reference exception",
+                "description": "Additional routing context is available via investigation.",
+                "context_status": {"hidden_context_remaining": True, "context_gap_count": 1},
             },
             allowed_fields=["issue_type", "priority"],
         )
 
-        action, source = choose_policy_action(policy, observation, {}, _context_sensitive_submit_builder)
+        action, source, cue = choose_policy_action(
+            policy,
+            observation,
+            {},
+            _context_sensitive_submit_builder,
+            used_tools_by_ticket={},
+        )
 
         self.assertEqual(action.action_type, "submit")
         self.assertEqual(action.issue_type, "identity_access")
         self.assertEqual(source, "submit")
+        self.assertIsNone(cue)
 
     def test_rollout_episode_rewards_context_aware_policy(self) -> None:
         no_investigation = POLICY_LIBRARY["no_investigation"]
@@ -152,11 +173,11 @@ class PolicyLearningTests(unittest.TestCase):
         self.assertLess(no_summary["normalized_return"], context_summary["normalized_return"])
         self.assertEqual(context_summary["investigation_steps"], 1)
 
-    def test_search_policies_selects_better_policy(self) -> None:
+    def test_search_policies_selects_adaptive_policy(self) -> None:
         report = search_policies(
             [
                 POLICY_LIBRARY["no_investigation"],
-                POLICY_LIBRARY["investigate_when_context_hidden"],
+                POLICY_LIBRARY["adaptive_cue_bandit"],
             ],
             train_seeds=[41, 42],
             eval_seeds=[43],
@@ -166,17 +187,18 @@ class PolicyLearningTests(unittest.TestCase):
             submit_builder=_context_sensitive_submit_builder,
         )
 
-        self.assertEqual(report["selected_policy"], "investigate_when_context_hidden")
+        self.assertEqual(report["selected_policy"], "adaptive_cue_bandit")
         self.assertGreater(
             report["eval_improvement_vs_baseline"]["avg_normalized_return"],
             0.0,
         )
+        self.assertIn("adaptive_cue_bandit", report["trained_adaptive_bandits"])
 
     def test_compare_policies_reports_improvement(self) -> None:
         report = compare_policies(
             [
                 POLICY_LIBRARY["no_investigation"],
-                POLICY_LIBRARY["investigate_when_context_hidden"],
+                POLICY_LIBRARY["adaptive_cue_bandit"],
             ],
             seeds=[42],
             task_ids=[3],
@@ -185,9 +207,18 @@ class PolicyLearningTests(unittest.TestCase):
             submit_builder=_context_sensitive_submit_builder,
         )
 
-        self.assertEqual(report["best_policy"], "investigate_when_context_hidden")
+        self.assertEqual(report["best_policy"], "adaptive_cue_bandit")
         self.assertGreater(report["improvement_vs_baseline"]["avg_terminal_reward"], 0.0)
 
+    def test_infer_ticket_cue_distinguishes_workflow_blocker(self) -> None:
+        cue = infer_ticket_cue(
+            {
+                "title": "Contractor onboarding blocked by access issue",
+                "description": "A contractor onboarding workflow is blocked by a permissions error.",
+            }
+        )
+        self.assertEqual(cue, "workflow_blocker")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_real_openenv_integration.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+SITE_PACKAGES = os.path.join(REPO_ROOT, ".venv", "Lib", "site-packages")
+if SITE_PACKAGES not in sys.path:
+    sys.path.insert(0, SITE_PACKAGES)
+
+for module_name in list(sys.modules):
+    if module_name == "openenv" or module_name.startswith("openenv."):
+        del sys.modules[module_name]
+for module_name in list(sys.modules):
+    if module_name in {"models", "server.app", "server.environment", "client"}:
+        del sys.modules[module_name]
+
+try:
+    from starlette.testclient import TestClient
+    from server.app import app
+
+    REAL_OPENENV_AVAILABLE = True
+    IMPORT_ERROR: Exception | None = None
+except Exception as exc:  # pragma: no cover - only used for skip messaging
+    REAL_OPENENV_AVAILABLE = False
+    IMPORT_ERROR = exc
+
+
+@unittest.skipUnless(
+    REAL_OPENENV_AVAILABLE,
+    f"real OpenEnv stack unavailable: {IMPORT_ERROR}",
+)
+class RealOpenEnvIntegrationTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.client = TestClient(app)
+
+    def test_root_redirects_to_web(self) -> None:
+        response = self.client.get("/", follow_redirects=False)
+        self.assertEqual(response.status_code, 307)
+        self.assertEqual(response.headers["location"], "/web")
+
+    def test_grader_endpoint_scores_known_action(self) -> None:
+        response = self.client.post(
+            "/grader",
+            json={
+                "task_id": 3,
+                "ticket_id": "ticket-002",
+                "action": {
+                    "issue_type": "identity_access",
+                    "priority": "high",
+                    "assignment_group": "service_desk",
+                    "resolution_action": "fulfill",
+                },
+            },
+        )
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        self.assertEqual(payload["score"], 1.0)
+        self.assertEqual(payload["breakdown"]["issue_type"], 1.0)
+
+    def test_baseline_endpoint_runs_episode(self) -> None:
+        response = self.client.get("/baseline", params={"task_id": 3, "seed": 42})
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        self.assertEqual(payload["task_id"], 3)
+        self.assertGreater(payload["step_count"], 0)
+        self.assertIn("steps", payload)
+        self.assertIsInstance(payload["steps"], list)
+
+    def test_websocket_round_trip_reset_state_step(self) -> None:
+        with self.client.websocket_connect("/ws") as websocket:
+            websocket.send_json({"type": "reset", "data": {"task_id": 1, "seed": 42}})
+            reset_message = websocket.receive_json()
+            self.assertEqual(reset_message["type"], "observation")
+            reset_payload = reset_message["data"]
+            reset_obs = reset_payload.get("observation", reset_payload)
+            self.assertEqual(reset_obs["task_id"], 1)
+            self.assertFalse(reset_payload.get("done", reset_obs.get("done", False)))
+
+            websocket.send_json({"type": "state"})
+            state_message = websocket.receive_json()
+            self.assertEqual(state_message["type"], "state")
+            self.assertEqual(state_message["data"]["current_task_id"], 1)
+
+            websocket.send_json(
+                {
+                    "type": "step",
+                    "data": {
+                        "issue_type": "billing_license",
+                    },
+                }
+            )
+            step_message = websocket.receive_json()
+            self.assertEqual(step_message["type"], "observation")
+            step_payload = step_message["data"]
+            step_obs = step_payload.get("observation", step_payload)
+            reward = step_payload.get("reward", step_obs.get("reward"))
+            self.assertGreaterEqual(reward, 0.0)
+            self.assertLessEqual(reward, 1.0)
+
+
+if __name__ == "__main__":
+    unittest.main()