"""Tests for the xv01.1 spec. Run from the repository root or spec/: python -m pytest spec/test_spec.py -q Needs only pytest and jsonschema. """ from __future__ import annotations import json import sys from pathlib import Path import pytest from jsonschema import Draft202012Validator SPEC = Path(__file__).resolve().parent sys.path.insert(0, str(SPEC)) import render import validate KNOWN_TOOL_NAMES = { "fetch_page", "web_search", "web_rag", "kb_retrieve", "drug_lookup", "dsld_lookup", "pharmgkb_lookup", "open_fda", "check_drug_interactions", "open_food_facts", "usda_food_search", "semantic_scholar_search", "google_trends_search", "openaq_air_quality", "calculate_bmi", "calculate_dosage", "calculate", "memory_write_bound", "code_agent", "generate_image", "google_calendar_list_events", "google_calendar_create_event", "google_calendar_delete_event", "google_calendar_list_calendars", "google_health_read", "google_health_list_data_sources", } BASE_TOOLS = ["fetch_page", "web_search", "web_rag", "drug_lookup", "code_agent", "calculate", "usda_food_search", "open_food_facts", "generate_image", "memory_write_bound"] def _load(name): return json.loads((SPEC / name).read_text(encoding="utf-8")) def ex(message="hello", history=(), **kw): base = {"message": message, "history": [{"role": r, "content": c} for r, c in history], "attached_image": False, "remembered_image": None, "document_attached": False, "memory_enabled": True, "signed_in": True, "api_client": False, "offered_tools": list(BASE_TOOLS), "locale": None} base.update(kw) return base V = {"schema_version": "xv01.1"} # ── the files agree with each other and with the backend ───────────────────── def test_schemas_are_valid_draft_2020_12(): for name in ("turn_plan.schema.json", "example_input.schema.json"): Draft202012Validator.check_schema(_load(name)) for tool in _load("tools.json")["tools"]: Draft202012Validator.check_schema(tool["parameters"]) def test_every_object_in_the_plan_schema_is_closed(): def walk(node, path): if isinstance(node, dict): if node.get("type") == "object" or (isinstance(node.get("type"), list) and "object" in node["type"]): assert "additionalProperties" in node, f"{path} has no additionalProperties" for k, v in node.items(): walk(v, f"{path}/{k}") elif isinstance(node, list): for i, v in enumerate(node): walk(v, f"{path}/{i}") walk(_load("turn_plan.schema.json"), "") walk(_load("example_input.schema.json"), "") def test_catalogue_covers_every_bound_tool(): names = {t["name"] for t in _load("tools.json")["tools"]} assert KNOWN_TOOL_NAMES <= names assert names - KNOWN_TOOL_NAMES == {"handoff_to"} def test_schema_enums_match_the_catalogues(): schema = _load("turn_plan.schema.json") tool_enum = set(schema["properties"]["tools"]["items"]["properties"]["name"]["enum"]) plannable = {t["name"] for t in _load("tools.json")["tools"] if t["plannable"]} assert tool_enum == plannable directive_enum = schema["properties"]["directives"]["items"]["enum"] assert directive_enum == render.planner_directive_ids() def test_plan_keys_follow_the_schema_order(): assert list(_load("turn_plan.schema.json")["properties"]) == list(render.PLAN_KEYS) def test_directive_ids_are_unique_and_planner_entries_complete(): entries = _load("directives.json")["directives"] ids = [d["id"] for d in entries] assert len(ids) == len(set(ids)) for d in entries: assert d["kind"] in ("planner", "derived") if d["kind"] == "planner": assert d["combine"] in ("union", "planner_gated") and d["paths"] def test_side_effect_tools_needing_confirmation(): confirm = {t["name"] for t in _load("tools.json")["tools"] if t["requires_confirmation"]} assert confirm == {"google_calendar_create_event", "google_calendar_delete_event", "memory_write_bound"} def test_worked_examples_are_valid_and_canonical(): rows = [json.loads(line) for line in (SPEC / "examples.jsonl").read_text(encoding="utf-8").splitlines() if line] assert len(rows) >= 15 for row in rows: assert validate.validate_example(row["input"]) == [], row["note"] assert validate.validate_target(row["target"], row["input"]) == [], row["note"] # ── render ──────────────────────────────────────────────────────────────────── def test_render_messages_shape_and_tool_catalogue(): msgs = render.render_messages(ex("what is 12 * 7?")) assert [m["role"] for m in msgs] == ["system", "user"] system, user = msgs[0]["content"], msgs[1]["content"] assert "- calculate(expression: str)" in system assert "- web_search" not in system and "- generate_image" not in system # driven by plan fields assert "{tool_catalogue}" not in system and "{directive_list}" not in system assert "- style.casual:" in system and "frame.realtime_results" not in system assert user.rstrip().endswith("MESSAGE\nwhat is 12 * 7?") def test_render_clips_history_and_long_messages(): history = [("user" if i % 2 == 0 else "assistant", f"turn {i} " + "x" * 2000) for i in range(20)] msg = "A" * 5000 + " what does this mean?" user = render.render_context(ex(msg, history)) chat = user.split("RECENT CHAT\n", 1)[1].split("\nTURN\n", 1)[0].splitlines() assert len(chat) == render.MAX_HISTORY_MESSAGES and chat[0].startswith("user: turn 12") assert all(len(line) <= len("assistant: ") + render.ASSISTANT_TURN_CHARS for line in chat) assert user.endswith("what does this mean?") and "characters left out" in user def test_remembered_upload_marker_is_rendered_as_an_upload(): user = render.render_context(ex(remembered_image="the image you uploaded")) assert "remembered image: yes, an earlier upload (no description)" in user def test_render_target_drops_defaults_and_orders_keys(): plan = {"directives": ["style.casual"], "needs_tools": True, "domain": "medical", "image": {"action": "none"}, "safety_concern": "none", "schema_version": "xv01.1", "tools": [], "drug": None, "clarify": None, "memory": {"action": "none"}} assert render.render_target(plan) == \ '{"schema_version":"xv01.1","domain":"medical","needs_tools":true,"directives":["style.casual"]}' def test_canonicalize_is_idempotent_and_normalises_numbers_and_tool_defaults(): plan = {**V, "data_tool": "barcode", "needs_tools": True, "tools": [{"arguments": {"search_by": "barcode", "query": "8901234567890"}, "name": "open_food_facts"}], "directives": ["lang.devanagari", "safety.medication"], "domain": "medical"} once = render.canonicalize(plan) assert render.canonicalize(once) == once assert list(once["tools"][0]) == ["name", "arguments"] assert list(once["tools"][0]["arguments"]) == ["query", "search_by"] assert once["directives"] == ["safety.medication", "lang.devanagari"] bmi = render.canonicalize({**V, "tools": [{"name": "open_fda", "arguments": {"query": "x", "category": "drug", "limit": 3.0}}]}) assert bmi["tools"][0]["arguments"] == {"query": "x"} # defaults dropped def test_fill_defaults_round_trip(): full = render.fill_defaults(V) assert full["image"] == {"action": "none"} and full["tools"] == [] and full["depth"] == "normal" assert render.canonicalize(full) == V # ── validate: the rules ─────────────────────────────────────────────────────── def errors(plan, example=None): return validate.validate_plan(plan, example if example is not None else ex()) def test_minimal_plan_is_valid(): assert errors(V) == [] def test_schema_version_is_required_and_fixed(): assert errors({"domain": "general"}) assert errors({"schema_version": "xv01.0"}) def test_unknown_field_is_rejected(): assert any("Additional properties" in e for e in errors({**V, "route": "T3"})) def test_tool_must_be_offered(): plan = {**V, "needs_tools": True, "tools": [{"name": "dsld_lookup", "arguments": {"query": "vitamin d3"}}]} assert any("not offered" in e for e in errors(plan)) assert errors(plan, ex(offered_tools=BASE_TOOLS + ["dsld_lookup"])) == [] def test_tools_driven_by_plan_fields_are_never_listed(): plan = {**V, "needs_tools": True, "tools": [{"name": "web_search", "arguments": {"query": "x"}}]} assert errors(plan) # rejected by the schema enum and by the semantic rule def test_tool_arguments_follow_the_tool_schema(): bad = {**V, "needs_tools": True, "data_tool": "calc", "tools": [{"name": "calculate", "arguments": {"expr": "1+1"}}]} assert any("expression" in e for e in errors(bad)) unsafe = {**V, "needs_tools": True, "data_tool": "calc", "tools": [{"name": "calculate", "arguments": {"expression": "__import__('os')"}}]} assert errors(unsafe) ok = {**V, "needs_tools": True, "data_tool": "calc", "tools": [{"name": "calculate", "arguments": {"expression": "round(316 / 1605 * 100, 2)"}}]} assert errors(ok) == [] def test_dose_per_kg_needs_its_basis(): example = ex(offered_tools=BASE_TOOLS + ["calculate_dosage"]) plan = {**V, "data_tool": "dose", "domain": "medical", "needs_tools": True, "tools": [{"name": "calculate_dosage", "arguments": {"drug_name": "amoxicillin", "weight_kg": 20, "dose_per_kg": 45}}]} assert any("dose_basis" in e for e in errors(plan, example)) plan["tools"][0]["arguments"]["dose_basis"] = "per_day" assert errors(plan, example) == [] def test_message_reference_only_for_string_arguments(): ok = {**V, "domain": "coding", "needs_tools": True, "tools": [{"name": "code_agent", "arguments": {"query": "$MESSAGE"}}]} assert errors(ok) == [] plan = {**V, "needs_tools": True, "tools": [{"name": "kb_retrieve", "arguments": {"query": "x", "top_k": "$MESSAGE"}}]} assert errors(plan, ex(offered_tools=["kb_retrieve"])) def test_edit_needs_an_image_to_edit(): plan = {**V, "image": {"action": "edit", "instruction": "make the sky purple"}} assert any("edit needs an attached or remembered image" in e for e in errors(plan)) assert errors(plan, ex(remembered_image="a beach at noon")) == [] assert errors(plan, ex(attached_image=True)) == [] def test_attached_image_is_analyzed_or_edited(): assert any("analyze or edit" in e for e in errors(V, ex(attached_image=True))) assert errors({**V, "image": {"action": "analyze"}}, ex(attached_image=True)) == [] assert any("analyze needs an attached image" in e for e in errors({**V, "image": {"action": "analyze"}})) def test_generate_needs_prompt_and_visible_text_without_placeholders(): plan = {**V, "image": {"action": "generate", "prompt": "a red kite over green hills"}} assert any("visible_text" in e for e in errors(plan)) plan["image"]["visible_text"] = "Spring Fair [Date]" assert any("placeholder" in e for e in errors(plan)) plan["image"]["visible_text"] = "" assert errors(plan) == [] def test_picture_turn_carries_no_other_work(): plan = {**V, "image": {"action": "generate", "prompt": "a red kite", "visible_text": ""}, "needs_search": True, "search_queries": ["kites"], "needs_tools": True} assert any("picture turn" in e for e in errors(plan)) def test_document_needs_its_format(): assert any("doc_format" in e for e in errors({**V, "request_kind": "document"})) assert any("doc_format" in e for e in errors({**V, "doc_format": "pdf"})) assert errors({**V, "request_kind": "document", "doc_format": "pptx"}) == [] def test_search_queries_go_with_needs_search_and_avoid_banned_tokens(): assert errors({**V, "needs_search": True, "needs_tools": True}) assert errors({**V, "search_queries": ["x y z"]}) bad = {**V, "needs_search": True, "needs_tools": True, "search_queries": ["site:reddit.com best running shoes"]} assert any("query uses" in e for e in errors(bad)) good = {**V, "needs_search": True, "needs_tools": True, "search_queries": ["gold price India today"]} assert errors(good) == [] def test_personal_and_data_tool_turns_are_not_searched(): plan = {**V, "domain": "personal_memory", "needs_tools": True, "needs_search": True, "search_queries": ["where do i live"]} assert any("personal and coding" in e for e in errors(plan)) plan = {**V, "data_tool": "calc", "needs_tools": True, "needs_search": True, "search_queries": ["12 x 7"], "tools": [{"name": "calculate", "arguments": {"expression": "12 * 7"}}]} assert any("not a web question" in e for e in errors(plan)) def test_data_tool_calls_its_offered_tool(): plan = {**V, "data_tool": "calc", "needs_tools": True} assert any("must call it" in e for e in errors(plan)) assert errors(plan, ex(offered_tools=["web_search"])) == [] # not offered: the decision stands alone def test_route_rules(): assert any("quick" in e for e in errors({**V, "depth": "quick", "needs_tools": True})) assert any("deep" in e for e in errors({**V, "depth": "deep"})) assert any("needs_tools must be true" in e for e in errors({**V, "domain": "coding"})) assert errors({**V, "depth": "research", "needs_tools": True, "domain": "medical"}) == [] def test_clarify_carries_no_other_decision(): plan = {**V, "clarify": "Which country do you mean?", "needs_search": True, "needs_tools": True, "search_queries": ["president"]} assert any("clarifying turn" in e for e in errors(plan)) assert errors({**V, "clarify": "Which country do you mean?"}) == [] def test_standalone_question_needs_history_and_must_differ(): plan = {**V, "standalone_question": "How old is Lionel Messi?"} assert any("no history" in e for e in errors(plan, ex("how old is he?"))) history = [("user", "who is the best footballer?"), ("assistant", "Many would say Lionel Messi.")] assert errors(plan, ex("how old is he?", history)) == [] same = {**V, "standalone_question": "How old is he"} assert any("same as the message" in e for e in errors(same, ex("how old is he?", history))) def test_memory_shapes(): assert errors({**V, "memory": {"action": "remember", "content": "Vegetarian"}, "needs_tools": True, "domain": "personal_memory"}) assert errors({**V, "memory": {"action": "forget", "content": "*"}, "needs_tools": True, "domain": "personal_memory"}) == [] assert any("recall" in e for e in errors({**V, "memory": {"action": "recall"}, "needs_tools": True})) def test_safety_only_escalates_and_blocks_pictures_and_casual_style(): plan = {**V, "safety_concern": "possible_emergency", "image": {"action": "generate", "prompt": "a diagram", "visible_text": ""}} assert any("no picture" in e for e in errors(plan)) plan = {**V, "safety_concern": "possible_crisis", "domain": "medical", "needs_tools": True, "directives": ["style.casual"]} assert any("stands aside" in e for e in errors(plan)) assert any("medical turn" in e for e in errors({**V, "safety_concern": "medication_risk"})) def test_derived_directives_are_never_emitted(): assert errors({**V, "needs_tools": True, "directives": ["frame.realtime_results"]}) def test_directive_preconditions_on_the_input(): casual = {**V, "needs_tools": True, "directives": ["style.casual"]} assert any("document_attached" in e for e in errors(casual, ex(document_attached=True))) unseen = {**V, "needs_tools": True, "directives": ["context.unseen_reference"]} assert any("history_empty" in e for e in errors(unseen, ex(history=[("user", "hi")]))) self_fact = {**V, "needs_tools": True, "directives": ["memory.self_fact"]} assert any("memory_enabled" in e for e in errors(self_fact, ex(memory_enabled=False))) assert any("agent path" in e for e in errors({**V, "directives": ["style.casual"]})) assert errors({**V, "directives": ["lang.devanagari"]}) == [] # language notes ride the fast path too def test_directive_combinations(): plan = {**V, "needs_tools": True} two_langs = {**plan, "directives": ["lang.hinglish", "lang.devanagari"]} assert any("one language note" in e for e in errors(two_langs)) both_styles = {**plan, "directives": ["style.casual", "style.followup"]} assert any("stands aside" in e for e in errors(both_styles, ex(history=[("user", "hi")]))) def test_validate_target_requires_canonical_text(): assert validate.validate_target('{"schema_version":"xv01.1"}') == [] assert any("not canonical" in e for e in validate.validate_target('{"schema_version": "xv01.1"}')) assert any("not canonical" in e for e in validate.validate_target('{"schema_version":"xv01.1","domain":"general"}')) assert any("not JSON" in e for e in validate.validate_target("Sure! Here is the plan")) def test_example_input_validation(): assert validate.validate_example(ex()) == [] bad = ex() bad["user_id"] = "u-1" assert validate.validate_example(bad) assert any("not in tools.json" in e for e in validate.validate_example(ex(offered_tools=["send_email"]))) # ── guided decoding schema ──────────────────────────────────────────────────── def test_guided_schema_narrows_tools_to_the_offered_ones(): schema = validate.guided_schema(["calculate", "web_search", "code_agent"]) Draft202012Validator.check_schema(schema) v = Draft202012Validator(schema) ok = {**V, "needs_tools": True, "tools": [{"name": "calculate", "arguments": {"expression": "2+2"}}]} assert not list(v.iter_errors(ok)) not_offered = {**V, "needs_tools": True, "tools": [{"name": "dsld_lookup", "arguments": {"query": "zinc"}}]} assert list(v.iter_errors(not_offered)) wrong_args = {**V, "needs_tools": True, "tools": [{"name": "calculate", "arguments": {"query": "2+2"}}]} assert list(v.iter_errors(wrong_args)) def test_guided_schema_without_plannable_tools_allows_no_tools(): v = Draft202012Validator(validate.guided_schema(["web_search", "drug_lookup"])) assert not list(v.iter_errors(V)) assert list(v.iter_errors({**V, "tools": [{"name": "calculate", "arguments": {"expression": "1"}}]})) @pytest.mark.parametrize("name", sorted(KNOWN_TOOL_NAMES | {"handoff_to"})) def test_every_tool_has_purpose_and_flags(name): tool = render.tool_catalogue()[name] for key in ("purpose", "group", "plannable", "planned_via", "side_effect", "requires_confirmation", "parameters"): assert key in tool, f"{name} lacks {key}" assert tool["parameters"]["additionalProperties"] is False