xevyo-01 / code /spec /test_spec.py
meet4150's picture
xevyo-01 round 4: merged weights, adapter, model card, code
a46f204 verified
Raw History Blame Contribute Delete
19.1 kB
"""Tests for the xv01.1 spec. Run from the repository root or spec/:
python -m pytest spec/test_spec.py -q
Needs only pytest and jsonschema.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
from jsonschema import Draft202012Validator
SPEC = Path(__file__).resolve().parent
sys.path.insert(0, str(SPEC))
import render
import validate
KNOWN_TOOL_NAMES = {
"fetch_page", "web_search", "web_rag", "kb_retrieve", "drug_lookup", "dsld_lookup", "pharmgkb_lookup",
"open_fda", "check_drug_interactions", "open_food_facts", "usda_food_search", "semantic_scholar_search",
"google_trends_search", "openaq_air_quality", "calculate_bmi", "calculate_dosage", "calculate",
"memory_write_bound", "code_agent", "generate_image", "google_calendar_list_events",
"google_calendar_create_event", "google_calendar_delete_event", "google_calendar_list_calendars",
"google_health_read", "google_health_list_data_sources",
}
BASE_TOOLS = ["fetch_page", "web_search", "web_rag", "drug_lookup", "code_agent", "calculate",
"usda_food_search", "open_food_facts", "generate_image", "memory_write_bound"]
def _load(name):
return json.loads((SPEC / name).read_text(encoding="utf-8"))
def ex(message="hello", history=(), **kw):
base = {"message": message, "history": [{"role": r, "content": c} for r, c in history],
"attached_image": False, "remembered_image": None, "document_attached": False,
"memory_enabled": True, "signed_in": True, "api_client": False,
"offered_tools": list(BASE_TOOLS), "locale": None}
base.update(kw)
return base
V = {"schema_version": "xv01.1"}
# ── the files agree with each other and with the backend ─────────────────────
def test_schemas_are_valid_draft_2020_12():
for name in ("turn_plan.schema.json", "example_input.schema.json"):
Draft202012Validator.check_schema(_load(name))
for tool in _load("tools.json")["tools"]:
Draft202012Validator.check_schema(tool["parameters"])
def test_every_object_in_the_plan_schema_is_closed():
def walk(node, path):
if isinstance(node, dict):
if node.get("type") == "object" or (isinstance(node.get("type"), list) and "object" in node["type"]):
assert "additionalProperties" in node, f"{path} has no additionalProperties"
for k, v in node.items():
walk(v, f"{path}/{k}")
elif isinstance(node, list):
for i, v in enumerate(node):
walk(v, f"{path}/{i}")
walk(_load("turn_plan.schema.json"), "")
walk(_load("example_input.schema.json"), "")
def test_catalogue_covers_every_bound_tool():
names = {t["name"] for t in _load("tools.json")["tools"]}
assert KNOWN_TOOL_NAMES <= names
assert names - KNOWN_TOOL_NAMES == {"handoff_to"}
def test_schema_enums_match_the_catalogues():
schema = _load("turn_plan.schema.json")
tool_enum = set(schema["properties"]["tools"]["items"]["properties"]["name"]["enum"])
plannable = {t["name"] for t in _load("tools.json")["tools"] if t["plannable"]}
assert tool_enum == plannable
directive_enum = schema["properties"]["directives"]["items"]["enum"]
assert directive_enum == render.planner_directive_ids()
def test_plan_keys_follow_the_schema_order():
assert list(_load("turn_plan.schema.json")["properties"]) == list(render.PLAN_KEYS)
def test_directive_ids_are_unique_and_planner_entries_complete():
entries = _load("directives.json")["directives"]
ids = [d["id"] for d in entries]
assert len(ids) == len(set(ids))
for d in entries:
assert d["kind"] in ("planner", "derived")
if d["kind"] == "planner":
assert d["combine"] in ("union", "planner_gated") and d["paths"]
def test_side_effect_tools_needing_confirmation():
confirm = {t["name"] for t in _load("tools.json")["tools"] if t["requires_confirmation"]}
assert confirm == {"google_calendar_create_event", "google_calendar_delete_event", "memory_write_bound"}
def test_worked_examples_are_valid_and_canonical():
rows = [json.loads(line) for line in (SPEC / "examples.jsonl").read_text(encoding="utf-8").splitlines() if line]
assert len(rows) >= 15
for row in rows:
assert validate.validate_example(row["input"]) == [], row["note"]
assert validate.validate_target(row["target"], row["input"]) == [], row["note"]
# ── render ────────────────────────────────────────────────────────────────────
def test_render_messages_shape_and_tool_catalogue():
msgs = render.render_messages(ex("what is 12 * 7?"))
assert [m["role"] for m in msgs] == ["system", "user"]
system, user = msgs[0]["content"], msgs[1]["content"]
assert "- calculate(expression: str)" in system
assert "- web_search" not in system and "- generate_image" not in system # driven by plan fields
assert "{tool_catalogue}" not in system and "{directive_list}" not in system
assert "- style.casual:" in system and "frame.realtime_results" not in system
assert user.rstrip().endswith("MESSAGE\nwhat is 12 * 7?")
def test_render_clips_history_and_long_messages():
history = [("user" if i % 2 == 0 else "assistant", f"turn {i} " + "x" * 2000) for i in range(20)]
msg = "A" * 5000 + " what does this mean?"
user = render.render_context(ex(msg, history))
chat = user.split("RECENT CHAT\n", 1)[1].split("\nTURN\n", 1)[0].splitlines()
assert len(chat) == render.MAX_HISTORY_MESSAGES and chat[0].startswith("user: turn 12")
assert all(len(line) <= len("assistant: ") + render.ASSISTANT_TURN_CHARS for line in chat)
assert user.endswith("what does this mean?") and "characters left out" in user
def test_remembered_upload_marker_is_rendered_as_an_upload():
user = render.render_context(ex(remembered_image="the image you uploaded"))
assert "remembered image: yes, an earlier upload (no description)" in user
def test_render_target_drops_defaults_and_orders_keys():
plan = {"directives": ["style.casual"], "needs_tools": True, "domain": "medical",
"image": {"action": "none"}, "safety_concern": "none", "schema_version": "xv01.1",
"tools": [], "drug": None, "clarify": None, "memory": {"action": "none"}}
assert render.render_target(plan) == \
'{"schema_version":"xv01.1","domain":"medical","needs_tools":true,"directives":["style.casual"]}'
def test_canonicalize_is_idempotent_and_normalises_numbers_and_tool_defaults():
plan = {**V, "data_tool": "barcode", "needs_tools": True,
"tools": [{"arguments": {"search_by": "barcode", "query": "8901234567890"}, "name": "open_food_facts"}],
"directives": ["lang.devanagari", "safety.medication"], "domain": "medical"}
once = render.canonicalize(plan)
assert render.canonicalize(once) == once
assert list(once["tools"][0]) == ["name", "arguments"]
assert list(once["tools"][0]["arguments"]) == ["query", "search_by"]
assert once["directives"] == ["safety.medication", "lang.devanagari"]
bmi = render.canonicalize({**V, "tools": [{"name": "open_fda", "arguments": {"query": "x", "category": "drug", "limit": 3.0}}]})
assert bmi["tools"][0]["arguments"] == {"query": "x"} # defaults dropped
def test_fill_defaults_round_trip():
full = render.fill_defaults(V)
assert full["image"] == {"action": "none"} and full["tools"] == [] and full["depth"] == "normal"
assert render.canonicalize(full) == V
# ── validate: the rules ───────────────────────────────────────────────────────
def errors(plan, example=None):
return validate.validate_plan(plan, example if example is not None else ex())
def test_minimal_plan_is_valid():
assert errors(V) == []
def test_schema_version_is_required_and_fixed():
assert errors({"domain": "general"})
assert errors({"schema_version": "xv01.0"})
def test_unknown_field_is_rejected():
assert any("Additional properties" in e for e in errors({**V, "route": "T3"}))
def test_tool_must_be_offered():
plan = {**V, "needs_tools": True, "tools": [{"name": "dsld_lookup", "arguments": {"query": "vitamin d3"}}]}
assert any("not offered" in e for e in errors(plan))
assert errors(plan, ex(offered_tools=BASE_TOOLS + ["dsld_lookup"])) == []
def test_tools_driven_by_plan_fields_are_never_listed():
plan = {**V, "needs_tools": True, "tools": [{"name": "web_search", "arguments": {"query": "x"}}]}
assert errors(plan) # rejected by the schema enum and by the semantic rule
def test_tool_arguments_follow_the_tool_schema():
bad = {**V, "needs_tools": True, "data_tool": "calc",
"tools": [{"name": "calculate", "arguments": {"expr": "1+1"}}]}
assert any("expression" in e for e in errors(bad))
unsafe = {**V, "needs_tools": True, "data_tool": "calc",
"tools": [{"name": "calculate", "arguments": {"expression": "__import__('os')"}}]}
assert errors(unsafe)
ok = {**V, "needs_tools": True, "data_tool": "calc",
"tools": [{"name": "calculate", "arguments": {"expression": "round(316 / 1605 * 100, 2)"}}]}
assert errors(ok) == []
def test_dose_per_kg_needs_its_basis():
example = ex(offered_tools=BASE_TOOLS + ["calculate_dosage"])
plan = {**V, "data_tool": "dose", "domain": "medical", "needs_tools": True,
"tools": [{"name": "calculate_dosage", "arguments": {"drug_name": "amoxicillin", "weight_kg": 20, "dose_per_kg": 45}}]}
assert any("dose_basis" in e for e in errors(plan, example))
plan["tools"][0]["arguments"]["dose_basis"] = "per_day"
assert errors(plan, example) == []
def test_message_reference_only_for_string_arguments():
ok = {**V, "domain": "coding", "needs_tools": True, "tools": [{"name": "code_agent", "arguments": {"query": "$MESSAGE"}}]}
assert errors(ok) == []
plan = {**V, "needs_tools": True, "tools": [{"name": "kb_retrieve", "arguments": {"query": "x", "top_k": "$MESSAGE"}}]}
assert errors(plan, ex(offered_tools=["kb_retrieve"]))
def test_edit_needs_an_image_to_edit():
plan = {**V, "image": {"action": "edit", "instruction": "make the sky purple"}}
assert any("edit needs an attached or remembered image" in e for e in errors(plan))
assert errors(plan, ex(remembered_image="a beach at noon")) == []
assert errors(plan, ex(attached_image=True)) == []
def test_attached_image_is_analyzed_or_edited():
assert any("analyze or edit" in e for e in errors(V, ex(attached_image=True)))
assert errors({**V, "image": {"action": "analyze"}}, ex(attached_image=True)) == []
assert any("analyze needs an attached image" in e for e in errors({**V, "image": {"action": "analyze"}}))
def test_generate_needs_prompt_and_visible_text_without_placeholders():
plan = {**V, "image": {"action": "generate", "prompt": "a red kite over green hills"}}
assert any("visible_text" in e for e in errors(plan))
plan["image"]["visible_text"] = "Spring Fair [Date]"
assert any("placeholder" in e for e in errors(plan))
plan["image"]["visible_text"] = ""
assert errors(plan) == []
def test_picture_turn_carries_no_other_work():
plan = {**V, "image": {"action": "generate", "prompt": "a red kite", "visible_text": ""},
"needs_search": True, "search_queries": ["kites"], "needs_tools": True}
assert any("picture turn" in e for e in errors(plan))
def test_document_needs_its_format():
assert any("doc_format" in e for e in errors({**V, "request_kind": "document"}))
assert any("doc_format" in e for e in errors({**V, "doc_format": "pdf"}))
assert errors({**V, "request_kind": "document", "doc_format": "pptx"}) == []
def test_search_queries_go_with_needs_search_and_avoid_banned_tokens():
assert errors({**V, "needs_search": True, "needs_tools": True})
assert errors({**V, "search_queries": ["x y z"]})
bad = {**V, "needs_search": True, "needs_tools": True, "search_queries": ["site:reddit.com best running shoes"]}
assert any("query uses" in e for e in errors(bad))
good = {**V, "needs_search": True, "needs_tools": True, "search_queries": ["gold price India today"]}
assert errors(good) == []
def test_personal_and_data_tool_turns_are_not_searched():
plan = {**V, "domain": "personal_memory", "needs_tools": True, "needs_search": True, "search_queries": ["where do i live"]}
assert any("personal and coding" in e for e in errors(plan))
plan = {**V, "data_tool": "calc", "needs_tools": True, "needs_search": True, "search_queries": ["12 x 7"],
"tools": [{"name": "calculate", "arguments": {"expression": "12 * 7"}}]}
assert any("not a web question" in e for e in errors(plan))
def test_data_tool_calls_its_offered_tool():
plan = {**V, "data_tool": "calc", "needs_tools": True}
assert any("must call it" in e for e in errors(plan))
assert errors(plan, ex(offered_tools=["web_search"])) == [] # not offered: the decision stands alone
def test_route_rules():
assert any("quick" in e for e in errors({**V, "depth": "quick", "needs_tools": True}))
assert any("deep" in e for e in errors({**V, "depth": "deep"}))
assert any("needs_tools must be true" in e for e in errors({**V, "domain": "coding"}))
assert errors({**V, "depth": "research", "needs_tools": True, "domain": "medical"}) == []
def test_clarify_carries_no_other_decision():
plan = {**V, "clarify": "Which country do you mean?", "needs_search": True, "needs_tools": True,
"search_queries": ["president"]}
assert any("clarifying turn" in e for e in errors(plan))
assert errors({**V, "clarify": "Which country do you mean?"}) == []
def test_standalone_question_needs_history_and_must_differ():
plan = {**V, "standalone_question": "How old is Lionel Messi?"}
assert any("no history" in e for e in errors(plan, ex("how old is he?")))
history = [("user", "who is the best footballer?"), ("assistant", "Many would say Lionel Messi.")]
assert errors(plan, ex("how old is he?", history)) == []
same = {**V, "standalone_question": "How old is he"}
assert any("same as the message" in e for e in errors(same, ex("how old is he?", history)))
def test_memory_shapes():
assert errors({**V, "memory": {"action": "remember", "content": "Vegetarian"}, "needs_tools": True,
"domain": "personal_memory"})
assert errors({**V, "memory": {"action": "forget", "content": "*"}, "needs_tools": True,
"domain": "personal_memory"}) == []
assert any("recall" in e for e in errors({**V, "memory": {"action": "recall"}, "needs_tools": True}))
def test_safety_only_escalates_and_blocks_pictures_and_casual_style():
plan = {**V, "safety_concern": "possible_emergency", "image": {"action": "generate", "prompt": "a diagram", "visible_text": ""}}
assert any("no picture" in e for e in errors(plan))
plan = {**V, "safety_concern": "possible_crisis", "domain": "medical", "needs_tools": True, "directives": ["style.casual"]}
assert any("stands aside" in e for e in errors(plan))
assert any("medical turn" in e for e in errors({**V, "safety_concern": "medication_risk"}))
def test_derived_directives_are_never_emitted():
assert errors({**V, "needs_tools": True, "directives": ["frame.realtime_results"]})
def test_directive_preconditions_on_the_input():
casual = {**V, "needs_tools": True, "directives": ["style.casual"]}
assert any("document_attached" in e for e in errors(casual, ex(document_attached=True)))
unseen = {**V, "needs_tools": True, "directives": ["context.unseen_reference"]}
assert any("history_empty" in e for e in errors(unseen, ex(history=[("user", "hi")])))
self_fact = {**V, "needs_tools": True, "directives": ["memory.self_fact"]}
assert any("memory_enabled" in e for e in errors(self_fact, ex(memory_enabled=False)))
assert any("agent path" in e for e in errors({**V, "directives": ["style.casual"]}))
assert errors({**V, "directives": ["lang.devanagari"]}) == [] # language notes ride the fast path too
def test_directive_combinations():
plan = {**V, "needs_tools": True}
two_langs = {**plan, "directives": ["lang.hinglish", "lang.devanagari"]}
assert any("one language note" in e for e in errors(two_langs))
both_styles = {**plan, "directives": ["style.casual", "style.followup"]}
assert any("stands aside" in e for e in errors(both_styles, ex(history=[("user", "hi")])))
def test_validate_target_requires_canonical_text():
assert validate.validate_target('{"schema_version":"xv01.1"}') == []
assert any("not canonical" in e for e in validate.validate_target('{"schema_version": "xv01.1"}'))
assert any("not canonical" in e for e in validate.validate_target('{"schema_version":"xv01.1","domain":"general"}'))
assert any("not JSON" in e for e in validate.validate_target("Sure! Here is the plan"))
def test_example_input_validation():
assert validate.validate_example(ex()) == []
bad = ex()
bad["user_id"] = "u-1"
assert validate.validate_example(bad)
assert any("not in tools.json" in e for e in validate.validate_example(ex(offered_tools=["send_email"])))
# ── guided decoding schema ────────────────────────────────────────────────────
def test_guided_schema_narrows_tools_to_the_offered_ones():
schema = validate.guided_schema(["calculate", "web_search", "code_agent"])
Draft202012Validator.check_schema(schema)
v = Draft202012Validator(schema)
ok = {**V, "needs_tools": True, "tools": [{"name": "calculate", "arguments": {"expression": "2+2"}}]}
assert not list(v.iter_errors(ok))
not_offered = {**V, "needs_tools": True, "tools": [{"name": "dsld_lookup", "arguments": {"query": "zinc"}}]}
assert list(v.iter_errors(not_offered))
wrong_args = {**V, "needs_tools": True, "tools": [{"name": "calculate", "arguments": {"query": "2+2"}}]}
assert list(v.iter_errors(wrong_args))
def test_guided_schema_without_plannable_tools_allows_no_tools():
v = Draft202012Validator(validate.guided_schema(["web_search", "drug_lookup"]))
assert not list(v.iter_errors(V))
assert list(v.iter_errors({**V, "tools": [{"name": "calculate", "arguments": {"expression": "1"}}]}))
@pytest.mark.parametrize("name", sorted(KNOWN_TOOL_NAMES | {"handoff_to"}))
def test_every_tool_has_purpose_and_flags(name):
tool = render.tool_catalogue()[name]
for key in ("purpose", "group", "plannable", "planned_via", "side_effect",
"requires_confirmation", "parameters"):
assert key in tool, f"{name} lacks {key}"
assert tool["parameters"]["additionalProperties"] is False