"""The chat library caps at 20, and the app has to say so where it matters. The incident: a reviewer typed 30 into "Variants to generate", asked Turing for a library, and got 10 rows. Their words — "did not explicitly or clearly state anywhere that only 10 variants would be scored… you have to search for this info in small text in a banner in a different section." The cap is real and correct: ``_tool_design_variant_library`` runs inside one reply rather than as a background job, so it clamps ``k`` to 1–20 and defaults to 10. The bug was purely that nothing said so at the input, and the one thing that did say so ("a chat run just caps the library size") named no number. The disclosure now names 20 and 10 in two places, and the browser has no way to ask the server what they are. So these tests read the numbers out of the Python and fail if the copy has drifted from the behaviour it describes — a disclosure with a stale number is worse than none, because the user then has a specific wrong number to plan around. """ import re import pytest from dee.core import agent_tools as t _APP_JS = "dee/static/app.js" _COCKPIT = "dee/static/cockpit.js" _INDEX = "dee/static/index.html" def _read(p): with open(p, encoding="utf-8") as fh: return fh.read() def _python_bounds(): """(default_k, cap) as the chat tool actually enforces them.""" src = _read("dee/core/agent_tools.py") body = re.search( r"def _tool_design_variant_library\(.*?\n(?=\ndef |\n# )", src, re.S).group(0) default = int(re.search(r'int\(args\.get\("k",\s*(\d+)\)\)', body).group(1)) cap = int(re.search(r"k\s*=\s*max\(1,\s*min\((\d+),\s*k\)\)", body).group(1)) return default, cap # --------------------------------------------------------------------------- # # the numbers themselves # --------------------------------------------------------------------------- # def test_the_cap_is_what_the_tool_actually_enforces(): """Executed, not read: a 30-variant request has to come back clamped.""" default, cap = _python_bounds() assert (default, cap) == (10, 20) spec = next(s for s in __import__("dee.core.orchestrator", fromlist=["x"]).TOOL_SPECS if s["function"]["name"] == "design_variant_library") desc = spec["function"]["parameters"]["properties"]["k"]["description"] assert str(cap) in desc and str(default) in desc, desc def test_the_client_constants_match_the_server_clamp(): app = _read(_APP_JS) default, cap = _python_bounds() assert f"const CHAT_LIBRARY_CAP = {cap};" in app assert f"const CHAT_LIBRARY_DEFAULT_K = {default};" in app # --------------------------------------------------------------------------- # # WHERE the number is entered — the whole point of the complaint # --------------------------------------------------------------------------- # def test_the_cap_is_disclosed_at_the_input_not_only_in_a_banner(): html = _read(_INDEX) default, cap = _python_bounds() # the note has to sit with the K field, not somewhere else on the page block = re.search( r'', html, re.S) assert block, "settingK field not found — did the settings card move?" note = block.group(0) assert 'class="setting-note"' in note, note assert str(cap) in note and str(default) in note, note # and it must distinguish the two paths, or it just looks like the sidebar # run is capped at 20 too assert "Directed Evolution" in note and "Turing" in note def test_the_note_has_styles_to_render_with(): """A class with no rule is invisible, and this one is built into static HTML — nothing else in the app would fail if the stylesheet lost it.""" css = _read("dee/static/trace.css") assert ".setting-note" in css assert ".count-note" in css # --------------------------------------------------------------------------- # # the delivered result has to state requested vs scored # --------------------------------------------------------------------------- # def test_the_painted_library_states_requested_versus_scored(): app = _read(_APP_JS) fn = re.search(r"function paintAgentDesignRun\(.*?\n\}", app, re.S).group(0) assert "callArgs" in fn, "the requested k is only available on the tool_call args" assert "You asked for" in fn assert "CHAT_LIBRARY_CAP" in fn # the summary line above the table carries it too, for the sidebar path assert 'class="count-note"' in app assert "Requested" in app def test_the_requested_count_actually_reaches_the_painter(): """The tool RESULT carries what came back and nothing about what was asked for. Without the tool_call args being stashed and handed over, the painter has no requested number and the sentence cannot be written — which is a silent no-op, not an error.""" cp = _read(_COCKPIT) assert "state.toolArgs[ev.id] = ev.args" in cp assert "toolArgs: {}" in cp # handed to the painter, and the design painter accepts it assert re.search(r"PAINTERS\[ev\.name\]\(ui\.panel,\s*\(ev\.id && state\.toolArgs\[ev\.id\]\)", cp), "painter called without the call args" assert re.search(r"design_variant_library: function \(panel, args\)", cp) assert "TDDesign.paintAgentRun(panel, args)" in cp # ...and cleared with the run, so run 2 can't inherit run 1's arguments assert "state.toolArgs = {}" in cp # --------------------------------------------------------------------------- # # round 2 in chat has its own, smaller, hardcoded ceiling # --------------------------------------------------------------------------- # def test_chat_round_two_returns_at_most_ten_and_says_so_in_its_own_payload(): """propose_round2 hardcodes k=10 and then slices [:10] again. Nothing in the UI reads that path's count, so this only guards the number quoted in the tool description the model reads.""" src = _read("dee/core/agent_tools.py") body = re.search(r"def _tool_propose_round2\(.*?\n(?=\ndef |\n# )", src, re.S).group(0) assert re.search(r'"k":\s*10', body) assert re.search(r"\[:10\]", body) @pytest.mark.parametrize("asked, delivered", [(30, 10), (20, 20), (5, 5)]) def test_the_sentence_only_claims_a_shortfall_when_there_is_one(asked, delivered): """Three cases and they say different things; collapsing them into one line ("a chat run just caps the library size") is what left a user staring at 10 rows after typing 30.""" app = _read(_APP_JS) fn = re.search(r"function paintAgentDesignRun\(.*?\n\}", app, re.S).group(0) assert "asked > delivered" in fn assert "the ${asked} you asked for" in fn