File size: 19,091 Bytes
a46f204
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
"""Tests for the xv01.1 spec. Run from the repository root or spec/:

    python -m pytest spec/test_spec.py -q

Needs only pytest and jsonschema.
"""
from __future__ import annotations

import json
import sys
from pathlib import Path

import pytest
from jsonschema import Draft202012Validator

SPEC = Path(__file__).resolve().parent
sys.path.insert(0, str(SPEC))

import render
import validate

KNOWN_TOOL_NAMES = {
    "fetch_page", "web_search", "web_rag", "kb_retrieve", "drug_lookup", "dsld_lookup", "pharmgkb_lookup",
    "open_fda", "check_drug_interactions", "open_food_facts", "usda_food_search", "semantic_scholar_search",
    "google_trends_search", "openaq_air_quality", "calculate_bmi", "calculate_dosage", "calculate",
    "memory_write_bound", "code_agent", "generate_image", "google_calendar_list_events",
    "google_calendar_create_event", "google_calendar_delete_event", "google_calendar_list_calendars",
    "google_health_read", "google_health_list_data_sources",
}

BASE_TOOLS = ["fetch_page", "web_search", "web_rag", "drug_lookup", "code_agent", "calculate",
              "usda_food_search", "open_food_facts", "generate_image", "memory_write_bound"]


def _load(name):
    return json.loads((SPEC / name).read_text(encoding="utf-8"))


def ex(message="hello", history=(), **kw):
    base = {"message": message, "history": [{"role": r, "content": c} for r, c in history],
            "attached_image": False, "remembered_image": None, "document_attached": False,
            "memory_enabled": True, "signed_in": True, "api_client": False,
            "offered_tools": list(BASE_TOOLS), "locale": None}
    base.update(kw)
    return base


V = {"schema_version": "xv01.1"}


# ── the files agree with each other and with the backend ─────────────────────

def test_schemas_are_valid_draft_2020_12():
    for name in ("turn_plan.schema.json", "example_input.schema.json"):
        Draft202012Validator.check_schema(_load(name))
    for tool in _load("tools.json")["tools"]:
        Draft202012Validator.check_schema(tool["parameters"])


def test_every_object_in_the_plan_schema_is_closed():
    def walk(node, path):
        if isinstance(node, dict):
            if node.get("type") == "object" or (isinstance(node.get("type"), list) and "object" in node["type"]):
                assert "additionalProperties" in node, f"{path} has no additionalProperties"
            for k, v in node.items():
                walk(v, f"{path}/{k}")
        elif isinstance(node, list):
            for i, v in enumerate(node):
                walk(v, f"{path}/{i}")
    walk(_load("turn_plan.schema.json"), "")
    walk(_load("example_input.schema.json"), "")


def test_catalogue_covers_every_bound_tool():
    names = {t["name"] for t in _load("tools.json")["tools"]}
    assert KNOWN_TOOL_NAMES <= names
    assert names - KNOWN_TOOL_NAMES == {"handoff_to"}


def test_schema_enums_match_the_catalogues():
    schema = _load("turn_plan.schema.json")
    tool_enum = set(schema["properties"]["tools"]["items"]["properties"]["name"]["enum"])
    plannable = {t["name"] for t in _load("tools.json")["tools"] if t["plannable"]}
    assert tool_enum == plannable
    directive_enum = schema["properties"]["directives"]["items"]["enum"]
    assert directive_enum == render.planner_directive_ids()


def test_plan_keys_follow_the_schema_order():
    assert list(_load("turn_plan.schema.json")["properties"]) == list(render.PLAN_KEYS)


def test_directive_ids_are_unique_and_planner_entries_complete():
    entries = _load("directives.json")["directives"]
    ids = [d["id"] for d in entries]
    assert len(ids) == len(set(ids))
    for d in entries:
        assert d["kind"] in ("planner", "derived")
        if d["kind"] == "planner":
            assert d["combine"] in ("union", "planner_gated") and d["paths"]


def test_side_effect_tools_needing_confirmation():
    confirm = {t["name"] for t in _load("tools.json")["tools"] if t["requires_confirmation"]}
    assert confirm == {"google_calendar_create_event", "google_calendar_delete_event", "memory_write_bound"}


def test_worked_examples_are_valid_and_canonical():
    rows = [json.loads(line) for line in (SPEC / "examples.jsonl").read_text(encoding="utf-8").splitlines() if line]
    assert len(rows) >= 15
    for row in rows:
        assert validate.validate_example(row["input"]) == [], row["note"]
        assert validate.validate_target(row["target"], row["input"]) == [], row["note"]


# ── render ────────────────────────────────────────────────────────────────────

def test_render_messages_shape_and_tool_catalogue():
    msgs = render.render_messages(ex("what is 12 * 7?"))
    assert [m["role"] for m in msgs] == ["system", "user"]
    system, user = msgs[0]["content"], msgs[1]["content"]
    assert "- calculate(expression: str)" in system
    assert "- web_search" not in system and "- generate_image" not in system   # driven by plan fields
    assert "{tool_catalogue}" not in system and "{directive_list}" not in system
    assert "- style.casual:" in system and "frame.realtime_results" not in system
    assert user.rstrip().endswith("MESSAGE\nwhat is 12 * 7?")


def test_render_clips_history_and_long_messages():
    history = [("user" if i % 2 == 0 else "assistant", f"turn {i} " + "x" * 2000) for i in range(20)]
    msg = "A" * 5000 + " what does this mean?"
    user = render.render_context(ex(msg, history))
    chat = user.split("RECENT CHAT\n", 1)[1].split("\nTURN\n", 1)[0].splitlines()
    assert len(chat) == render.MAX_HISTORY_MESSAGES and chat[0].startswith("user: turn 12")
    assert all(len(line) <= len("assistant: ") + render.ASSISTANT_TURN_CHARS for line in chat)
    assert user.endswith("what does this mean?") and "characters left out" in user


def test_remembered_upload_marker_is_rendered_as_an_upload():
    user = render.render_context(ex(remembered_image="the image you uploaded"))
    assert "remembered image: yes, an earlier upload (no description)" in user


def test_render_target_drops_defaults_and_orders_keys():
    plan = {"directives": ["style.casual"], "needs_tools": True, "domain": "medical",
            "image": {"action": "none"}, "safety_concern": "none", "schema_version": "xv01.1",
            "tools": [], "drug": None, "clarify": None, "memory": {"action": "none"}}
    assert render.render_target(plan) == \
        '{"schema_version":"xv01.1","domain":"medical","needs_tools":true,"directives":["style.casual"]}'


def test_canonicalize_is_idempotent_and_normalises_numbers_and_tool_defaults():
    plan = {**V, "data_tool": "barcode", "needs_tools": True,
            "tools": [{"arguments": {"search_by": "barcode", "query": "8901234567890"}, "name": "open_food_facts"}],
            "directives": ["lang.devanagari", "safety.medication"], "domain": "medical"}
    once = render.canonicalize(plan)
    assert render.canonicalize(once) == once
    assert list(once["tools"][0]) == ["name", "arguments"]
    assert list(once["tools"][0]["arguments"]) == ["query", "search_by"]
    assert once["directives"] == ["safety.medication", "lang.devanagari"]
    bmi = render.canonicalize({**V, "tools": [{"name": "open_fda", "arguments": {"query": "x", "category": "drug", "limit": 3.0}}]})
    assert bmi["tools"][0]["arguments"] == {"query": "x"}   # defaults dropped


def test_fill_defaults_round_trip():
    full = render.fill_defaults(V)
    assert full["image"] == {"action": "none"} and full["tools"] == [] and full["depth"] == "normal"
    assert render.canonicalize(full) == V


# ── validate: the rules ───────────────────────────────────────────────────────

def errors(plan, example=None):
    return validate.validate_plan(plan, example if example is not None else ex())


def test_minimal_plan_is_valid():
    assert errors(V) == []


def test_schema_version_is_required_and_fixed():
    assert errors({"domain": "general"})
    assert errors({"schema_version": "xv01.0"})


def test_unknown_field_is_rejected():
    assert any("Additional properties" in e for e in errors({**V, "route": "T3"}))


def test_tool_must_be_offered():
    plan = {**V, "needs_tools": True, "tools": [{"name": "dsld_lookup", "arguments": {"query": "vitamin d3"}}]}
    assert any("not offered" in e for e in errors(plan))
    assert errors(plan, ex(offered_tools=BASE_TOOLS + ["dsld_lookup"])) == []


def test_tools_driven_by_plan_fields_are_never_listed():
    plan = {**V, "needs_tools": True, "tools": [{"name": "web_search", "arguments": {"query": "x"}}]}
    assert errors(plan)   # rejected by the schema enum and by the semantic rule


def test_tool_arguments_follow_the_tool_schema():
    bad = {**V, "needs_tools": True, "data_tool": "calc",
           "tools": [{"name": "calculate", "arguments": {"expr": "1+1"}}]}
    assert any("expression" in e for e in errors(bad))
    unsafe = {**V, "needs_tools": True, "data_tool": "calc",
              "tools": [{"name": "calculate", "arguments": {"expression": "__import__('os')"}}]}
    assert errors(unsafe)
    ok = {**V, "needs_tools": True, "data_tool": "calc",
          "tools": [{"name": "calculate", "arguments": {"expression": "round(316 / 1605 * 100, 2)"}}]}
    assert errors(ok) == []


def test_dose_per_kg_needs_its_basis():
    example = ex(offered_tools=BASE_TOOLS + ["calculate_dosage"])
    plan = {**V, "data_tool": "dose", "domain": "medical", "needs_tools": True,
            "tools": [{"name": "calculate_dosage", "arguments": {"drug_name": "amoxicillin", "weight_kg": 20, "dose_per_kg": 45}}]}
    assert any("dose_basis" in e for e in errors(plan, example))
    plan["tools"][0]["arguments"]["dose_basis"] = "per_day"
    assert errors(plan, example) == []


def test_message_reference_only_for_string_arguments():
    ok = {**V, "domain": "coding", "needs_tools": True, "tools": [{"name": "code_agent", "arguments": {"query": "$MESSAGE"}}]}
    assert errors(ok) == []
    plan = {**V, "needs_tools": True, "tools": [{"name": "kb_retrieve", "arguments": {"query": "x", "top_k": "$MESSAGE"}}]}
    assert errors(plan, ex(offered_tools=["kb_retrieve"]))


def test_edit_needs_an_image_to_edit():
    plan = {**V, "image": {"action": "edit", "instruction": "make the sky purple"}}
    assert any("edit needs an attached or remembered image" in e for e in errors(plan))
    assert errors(plan, ex(remembered_image="a beach at noon")) == []
    assert errors(plan, ex(attached_image=True)) == []


def test_attached_image_is_analyzed_or_edited():
    assert any("analyze or edit" in e for e in errors(V, ex(attached_image=True)))
    assert errors({**V, "image": {"action": "analyze"}}, ex(attached_image=True)) == []
    assert any("analyze needs an attached image" in e for e in errors({**V, "image": {"action": "analyze"}}))


def test_generate_needs_prompt_and_visible_text_without_placeholders():
    plan = {**V, "image": {"action": "generate", "prompt": "a red kite over green hills"}}
    assert any("visible_text" in e for e in errors(plan))
    plan["image"]["visible_text"] = "Spring Fair [Date]"
    assert any("placeholder" in e for e in errors(plan))
    plan["image"]["visible_text"] = ""
    assert errors(plan) == []


def test_picture_turn_carries_no_other_work():
    plan = {**V, "image": {"action": "generate", "prompt": "a red kite", "visible_text": ""},
            "needs_search": True, "search_queries": ["kites"], "needs_tools": True}
    assert any("picture turn" in e for e in errors(plan))


def test_document_needs_its_format():
    assert any("doc_format" in e for e in errors({**V, "request_kind": "document"}))
    assert any("doc_format" in e for e in errors({**V, "doc_format": "pdf"}))
    assert errors({**V, "request_kind": "document", "doc_format": "pptx"}) == []


def test_search_queries_go_with_needs_search_and_avoid_banned_tokens():
    assert errors({**V, "needs_search": True, "needs_tools": True})
    assert errors({**V, "search_queries": ["x y z"]})
    bad = {**V, "needs_search": True, "needs_tools": True, "search_queries": ["site:reddit.com best running shoes"]}
    assert any("query uses" in e for e in errors(bad))
    good = {**V, "needs_search": True, "needs_tools": True, "search_queries": ["gold price India today"]}
    assert errors(good) == []


def test_personal_and_data_tool_turns_are_not_searched():
    plan = {**V, "domain": "personal_memory", "needs_tools": True, "needs_search": True, "search_queries": ["where do i live"]}
    assert any("personal and coding" in e for e in errors(plan))
    plan = {**V, "data_tool": "calc", "needs_tools": True, "needs_search": True, "search_queries": ["12 x 7"],
            "tools": [{"name": "calculate", "arguments": {"expression": "12 * 7"}}]}
    assert any("not a web question" in e for e in errors(plan))


def test_data_tool_calls_its_offered_tool():
    plan = {**V, "data_tool": "calc", "needs_tools": True}
    assert any("must call it" in e for e in errors(plan))
    assert errors(plan, ex(offered_tools=["web_search"])) == []   # not offered: the decision stands alone


def test_route_rules():
    assert any("quick" in e for e in errors({**V, "depth": "quick", "needs_tools": True}))
    assert any("deep" in e for e in errors({**V, "depth": "deep"}))
    assert any("needs_tools must be true" in e for e in errors({**V, "domain": "coding"}))
    assert errors({**V, "depth": "research", "needs_tools": True, "domain": "medical"}) == []


def test_clarify_carries_no_other_decision():
    plan = {**V, "clarify": "Which country do you mean?", "needs_search": True, "needs_tools": True,
            "search_queries": ["president"]}
    assert any("clarifying turn" in e for e in errors(plan))
    assert errors({**V, "clarify": "Which country do you mean?"}) == []


def test_standalone_question_needs_history_and_must_differ():
    plan = {**V, "standalone_question": "How old is Lionel Messi?"}
    assert any("no history" in e for e in errors(plan, ex("how old is he?")))
    history = [("user", "who is the best footballer?"), ("assistant", "Many would say Lionel Messi.")]
    assert errors(plan, ex("how old is he?", history)) == []
    same = {**V, "standalone_question": "How old is he"}
    assert any("same as the message" in e for e in errors(same, ex("how old is he?", history)))


def test_memory_shapes():
    assert errors({**V, "memory": {"action": "remember", "content": "Vegetarian"}, "needs_tools": True,
                   "domain": "personal_memory"})
    assert errors({**V, "memory": {"action": "forget", "content": "*"}, "needs_tools": True,
                   "domain": "personal_memory"}) == []
    assert any("recall" in e for e in errors({**V, "memory": {"action": "recall"}, "needs_tools": True}))


def test_safety_only_escalates_and_blocks_pictures_and_casual_style():
    plan = {**V, "safety_concern": "possible_emergency", "image": {"action": "generate", "prompt": "a diagram", "visible_text": ""}}
    assert any("no picture" in e for e in errors(plan))
    plan = {**V, "safety_concern": "possible_crisis", "domain": "medical", "needs_tools": True, "directives": ["style.casual"]}
    assert any("stands aside" in e for e in errors(plan))
    assert any("medical turn" in e for e in errors({**V, "safety_concern": "medication_risk"}))


def test_derived_directives_are_never_emitted():
    assert errors({**V, "needs_tools": True, "directives": ["frame.realtime_results"]})


def test_directive_preconditions_on_the_input():
    casual = {**V, "needs_tools": True, "directives": ["style.casual"]}
    assert any("document_attached" in e for e in errors(casual, ex(document_attached=True)))
    unseen = {**V, "needs_tools": True, "directives": ["context.unseen_reference"]}
    assert any("history_empty" in e for e in errors(unseen, ex(history=[("user", "hi")])))
    self_fact = {**V, "needs_tools": True, "directives": ["memory.self_fact"]}
    assert any("memory_enabled" in e for e in errors(self_fact, ex(memory_enabled=False)))
    assert any("agent path" in e for e in errors({**V, "directives": ["style.casual"]}))
    assert errors({**V, "directives": ["lang.devanagari"]}) == []   # language notes ride the fast path too


def test_directive_combinations():
    plan = {**V, "needs_tools": True}
    two_langs = {**plan, "directives": ["lang.hinglish", "lang.devanagari"]}
    assert any("one language note" in e for e in errors(two_langs))
    both_styles = {**plan, "directives": ["style.casual", "style.followup"]}
    assert any("stands aside" in e for e in errors(both_styles, ex(history=[("user", "hi")])))


def test_validate_target_requires_canonical_text():
    assert validate.validate_target('{"schema_version":"xv01.1"}') == []
    assert any("not canonical" in e for e in validate.validate_target('{"schema_version": "xv01.1"}'))
    assert any("not canonical" in e for e in validate.validate_target('{"schema_version":"xv01.1","domain":"general"}'))
    assert any("not JSON" in e for e in validate.validate_target("Sure! Here is the plan"))


def test_example_input_validation():
    assert validate.validate_example(ex()) == []
    bad = ex()
    bad["user_id"] = "u-1"
    assert validate.validate_example(bad)
    assert any("not in tools.json" in e for e in validate.validate_example(ex(offered_tools=["send_email"])))


# ── guided decoding schema ────────────────────────────────────────────────────

def test_guided_schema_narrows_tools_to_the_offered_ones():
    schema = validate.guided_schema(["calculate", "web_search", "code_agent"])
    Draft202012Validator.check_schema(schema)
    v = Draft202012Validator(schema)
    ok = {**V, "needs_tools": True, "tools": [{"name": "calculate", "arguments": {"expression": "2+2"}}]}
    assert not list(v.iter_errors(ok))
    not_offered = {**V, "needs_tools": True, "tools": [{"name": "dsld_lookup", "arguments": {"query": "zinc"}}]}
    assert list(v.iter_errors(not_offered))
    wrong_args = {**V, "needs_tools": True, "tools": [{"name": "calculate", "arguments": {"query": "2+2"}}]}
    assert list(v.iter_errors(wrong_args))


def test_guided_schema_without_plannable_tools_allows_no_tools():
    v = Draft202012Validator(validate.guided_schema(["web_search", "drug_lookup"]))
    assert not list(v.iter_errors(V))
    assert list(v.iter_errors({**V, "tools": [{"name": "calculate", "arguments": {"expression": "1"}}]}))


@pytest.mark.parametrize("name", sorted(KNOWN_TOOL_NAMES | {"handoff_to"}))
def test_every_tool_has_purpose_and_flags(name):
    tool = render.tool_catalogue()[name]
    for key in ("purpose", "group", "plannable", "planned_via", "side_effect",
                "requires_confirmation", "parameters"):
        assert key in tool, f"{name} lacks {key}"
    assert tool["parameters"]["additionalProperties"] is False