tests/test_chat_library_cap.py

"""The chat library caps at 20, and the app has to say so where it matters.

The incident: a reviewer typed 30 into "Variants to generate", asked Turing
for a library, and got 10 rows. Their words — "did not explicitly or clearly
state anywhere that only 10 variants would be scored… you have to search for
this info in small text in a banner in a different section."

The cap is real and correct: ``_tool_design_variant_library`` runs inside one
reply rather than as a background job, so it clamps ``k`` to 1–20 and defaults
to 10. The bug was purely that nothing said so at the input, and the one thing
that did say so ("a chat run just caps the library size") named no number.

The disclosure now names 20 and 10 in two places, and the browser has no way
to ask the server what they are. So these tests read the numbers out of the
Python and fail if the copy has drifted from the behaviour it describes — a
disclosure with a stale number is worse than none, because the user then has a
specific wrong number to plan around.
"""
import re

import pytest

from dee.core import agent_tools as t

_APP_JS = "dee/static/app.js"
_COCKPIT = "dee/static/cockpit.js"
_INDEX = "dee/static/index.html"


def _read(p):
    with open(p, encoding="utf-8") as fh:
        return fh.read()


def _python_bounds():
    """(default_k, cap) as the chat tool actually enforces them."""
    src = _read("dee/core/agent_tools.py")
    body = re.search(
        r"def _tool_design_variant_library\(.*?\n(?=\ndef |\n# )", src, re.S).group(0)
    default = int(re.search(r'int\(args\.get\("k",\s*(\d+)\)\)', body).group(1))
    cap = int(re.search(r"k\s*=\s*max\(1,\s*min\((\d+),\s*k\)\)", body).group(1))
    return default, cap


# --------------------------------------------------------------------------- #
# the numbers themselves
# --------------------------------------------------------------------------- #
def test_the_cap_is_what_the_tool_actually_enforces():
    """Executed, not read: a 30-variant request has to come back clamped."""
    default, cap = _python_bounds()
    assert (default, cap) == (10, 20)
    spec = next(s for s in __import__("dee.core.orchestrator", fromlist=["x"]).TOOL_SPECS
                if s["function"]["name"] == "design_variant_library")
    desc = spec["function"]["parameters"]["properties"]["k"]["description"]
    assert str(cap) in desc and str(default) in desc, desc


def test_the_client_constants_match_the_server_clamp():
    app = _read(_APP_JS)
    default, cap = _python_bounds()
    assert f"const CHAT_LIBRARY_CAP = {cap};" in app
    assert f"const CHAT_LIBRARY_DEFAULT_K = {default};" in app


# --------------------------------------------------------------------------- #
# WHERE the number is entered — the whole point of the complaint
# --------------------------------------------------------------------------- #
def test_the_cap_is_disclosed_at_the_input_not_only_in_a_banner():
    html = _read(_INDEX)
    default, cap = _python_bounds()
    # the note has to sit with the K field, not somewhere else on the page
    block = re.search(
        r'<input type="number" id="settingK".*?</label>', html, re.S)
    assert block, "settingK field not found — did the settings card move?"
    note = block.group(0)
    assert 'class="setting-note"' in note, note
    assert str(cap) in note and str(default) in note, note
    # and it must distinguish the two paths, or it just looks like the sidebar
    # run is capped at 20 too
    assert "Directed Evolution" in note and "Turing" in note


def test_the_note_has_styles_to_render_with():
    """A class with no rule is invisible, and this one is built into static
    HTML — nothing else in the app would fail if the stylesheet lost it."""
    css = _read("dee/static/trace.css")
    assert ".setting-note" in css
    assert ".count-note" in css


# --------------------------------------------------------------------------- #
# the delivered result has to state requested vs scored
# --------------------------------------------------------------------------- #
def test_the_painted_library_states_requested_versus_scored():
    app = _read(_APP_JS)
    fn = re.search(r"function paintAgentDesignRun\(.*?\n\}", app, re.S).group(0)
    assert "callArgs" in fn, "the requested k is only available on the tool_call args"
    assert "You asked for" in fn
    assert "CHAT_LIBRARY_CAP" in fn
    # the summary line above the table carries it too, for the sidebar path
    assert 'class="count-note"' in app
    assert "Requested" in app


def test_the_requested_count_actually_reaches_the_painter():
    """The tool RESULT carries what came back and nothing about what was
    asked for. Without the tool_call args being stashed and handed over, the
    painter has no requested number and the sentence cannot be written —
    which is a silent no-op, not an error."""
    cp = _read(_COCKPIT)
    assert "state.toolArgs[ev.id] = ev.args" in cp
    assert "toolArgs: {}" in cp
    # handed to the painter, and the design painter accepts it
    assert re.search(r"PAINTERS\[ev\.name\]\(ui\.panel,\s*\(ev\.id && state\.toolArgs\[ev\.id\]\)",
                     cp), "painter called without the call args"
    assert re.search(r"design_variant_library: function \(panel, args\)", cp)
    assert "TDDesign.paintAgentRun(panel, args)" in cp
    # ...and cleared with the run, so run 2 can't inherit run 1's arguments
    assert "state.toolArgs = {}" in cp


# --------------------------------------------------------------------------- #
# round 2 in chat has its own, smaller, hardcoded ceiling
# --------------------------------------------------------------------------- #
def test_chat_round_two_returns_at_most_ten_and_says_so_in_its_own_payload():
    """propose_round2 hardcodes k=10 and then slices [:10] again. Nothing in
    the UI reads that path's count, so this only guards the number quoted in
    the tool description the model reads."""
    src = _read("dee/core/agent_tools.py")
    body = re.search(r"def _tool_propose_round2\(.*?\n(?=\ndef |\n# )", src, re.S).group(0)
    assert re.search(r'"k":\s*10', body)
    assert re.search(r"\[:10\]", body)


@pytest.mark.parametrize("asked, delivered", [(30, 10), (20, 20), (5, 5)])
def test_the_sentence_only_claims_a_shortfall_when_there_is_one(asked, delivered):
    """Three cases and they say different things; collapsing them into one
    line ("a chat run just caps the library size") is what left a user
    staring at 10 rows after typing 30."""
    app = _read(_APP_JS)
    fn = re.search(r"function paintAgentDesignRun\(.*?\n\}", app, re.S).group(0)
    assert "asked > delivered" in fn
    assert "the ${asked} you asked for" in fn