"""Validate xevyo-01 turn plans (spec xv01.1). validate_plan(plan, example=None) -> list[str] # [] means valid validate_target(text, example=None) -> list[str] # also checks JSON + canonical form validate_example(example) -> list[str] # an input against example_input.schema.json guided_schema(offered_tools) -> dict # per-turn schema for vLLM guided_json Two layers. The JSON Schema (turn_plan.schema.json) checks shape and closed value sets. The semantic rules below check what a schema cannot: tools offered this turn, arguments that fit each tool, conditions between fields, and directive preconditions on the input. Every rule mirrors a behaviour of the serving application. Needs only the `jsonschema` package. Command line, one JSON object per line with keys "input" (example_input) and "plan" (object) or "target" (string): python spec/validate.py examples.jsonl """ from __future__ import annotations import ast import copy import json import sys from functools import lru_cache from pathlib import Path from typing import Any, Iterable, Optional from jsonschema import Draft202012Validator SPEC_DIR = Path(__file__).resolve().parent if str(SPEC_DIR) not in sys.path: sys.path.insert(0, str(SPEC_DIR)) import render MESSAGE_REF = "$MESSAGE" UPLOADED_IMAGE_MARKER = render.UPLOADED_IMAGE_MARKER #: : a query naming one of these returns nothing. BANNED_QUERY_TOKENS = ( "site:", "inurl:", "intitle:", "filetype:", "twitter.com", "x.com", "facebook.com", "instagram.com", "linkedin.com", "tiktok.com", "reddit.com/r/", ) #: data_tool_intent._TOOL_FOR, with the call a plan makes. DATA_TOOL_FOR = { "bmi": ("calculate_bmi",), "dose": ("calculate_dosage",), "barcode": ("open_food_facts",), "usda": ("usda_food_search",), "food": ("usda_food_search", "open_food_facts"), "calc": ("calculate",), } #: Tool -> the data_tool values that may list it. DATA_KIND_OF_TOOL = { "calculate_bmi": {"bmi"}, "calculate_dosage": {"dose"}, "calculate": {"calc"}, "open_food_facts": {"barcode", "food"}, "usda_food_search": {"usda", "food"}, } #: Domains that take the agent path (complexity.py: coding 811-819, personal 897-919, #: prompt_authoring 734-742, medical 801-808 and ). AGENT_DOMAINS = {"medical", "coding", "personal_memory", "prompt_authoring"} SAFETY_RANK = {"none": 0, "medication_risk": 1, "possible_emergency": 2, "possible_crisis": 3} # ── spec files ──────────────────────────────────────────────────────────────── @lru_cache(maxsize=None) def _json(name: str) -> Any: return json.loads((SPEC_DIR / name).read_text(encoding="utf-8")) @lru_cache(maxsize=None) def _plan_validator() -> Draft202012Validator: return Draft202012Validator(_json("turn_plan.schema.json")) @lru_cache(maxsize=None) def _example_validator() -> Draft202012Validator: return Draft202012Validator(_json("example_input.schema.json")) def _tools() -> dict[str, dict]: return render.tool_catalogue() def _directives() -> dict[str, dict]: return {d["id"]: d for d in render.directive_catalogue()} def _schema_errors(validator: Draft202012Validator, instance: Any, label: str) -> list[str]: out = [] for err in sorted(validator.iter_errors(instance), key=lambda e: list(e.absolute_path)): where = "/".join(str(p) for p in err.absolute_path) or "(root)" out.append(f"{label} schema: {where}: {err.message}") return out # ── helpers ─────────────────────────────────────────────────────────────────── def _norm(text: str) -> str: """followup_rewrite.is_restated's comparison form.""" return " ".join((text or "").lower().split()).rstrip("?.! ") def _has_image_source(example: Optional[dict]) -> bool: if example is None: return True # unknown: do not fail a plan on what we cannot see return bool(example.get("attached_image")) or bool((example.get("remembered_image") or "").strip()) def _calc_expression_ok(expr: str) -> Optional[str]: """None if `expr` is what evaluates, else why not.""" if not isinstance(expr, str) or not expr.strip(): return "empty expression" if expr.strip() == MESSAGE_REF: return "calculate needs an expression, not $MESSAGE" try: tree = ast.parse(expr.strip(), mode="eval") except SyntaxError: return "not a valid expression" allowed_bin = (ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow) def ok(node) -> bool: if isinstance(node, ast.Expression): return ok(node.body) if isinstance(node, ast.Constant): return isinstance(node.value, (int, float)) and not isinstance(node.value, bool) if isinstance(node, ast.BinOp): return isinstance(node.op, allowed_bin) and ok(node.left) and ok(node.right) if isinstance(node, ast.UnaryOp): return isinstance(node.op, (ast.UAdd, ast.USub)) and ok(node.operand) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and not node.keywords: if node.func.id == "abs": return len(node.args) == 1 and all(ok(a) for a in node.args) if node.func.id == "round": return len(node.args) in (1, 2) and all(ok(a) for a in node.args) return False return None if ok(tree) else "only numbers, + - * / // % **, parentheses, round() and abs() are allowed" def _check_tool_call(call: dict, idx: int, offered: Optional[set]) -> list[str]: errs: list[str] = [] name = call.get("name") args = call.get("arguments") or {} tool = _tools().get(name) where = f"tools[{idx}] {name}" if tool is None: return [f"{where}: not in tools.json"] if not tool["plannable"]: errs.append(f"{where}: never listed in tools; it is driven by plan field '{tool['planned_via']}'") if offered is not None and name not in offered: errs.append(f"{where}: not offered this turn") # $MESSAGE stands for the user's text: accepted for any string parameter. props = tool["parameters"].get("properties", {}) substituted = {} for key, value in args.items(): if value == MESSAGE_REF: spec = props.get(key, {}) kinds = spec.get("type", []) kinds = kinds if isinstance(kinds, list) else [kinds] if "string" not in kinds: errs.append(f"{where}: $MESSAGE used for non-string parameter '{key}'") substituted[key] = "message text" else: substituted[key] = value for err in Draft202012Validator(tool["parameters"]).iter_errors(substituted): path = "/".join(str(p) for p in err.absolute_path) or "arguments" errs.append(f"{where}: {path}: {err.message}") # Rules the parameter schemas cannot state. if name == "calculate" and "expression" in args: why = _calc_expression_ok(args["expression"]) if why: errs.append(f"{where}: expression: {why}") if name == "calculate_dosage" and args.get("dose_per_kg") and not args.get("dose_basis"): errs.append(f"{where}: dose_basis is required whenever dose_per_kg is given (calculator.py _DOSAGE_DOC)") if name == "calculate_bmi": if not any(k in args for k in ("weight_kg", "weight_lb", "weight_st")) or \ not any(k in args for k in ("height_cm", "height_m", "height_ft", "height_in")): errs.append(f"{where}: needs a weight and a height") return errs # ── the semantic rules ──────────────────────────────────────────────────────── def _semantic_errors(plan: dict, example: Optional[dict]) -> list[str]: p = render.fill_defaults(plan) errs: list[str] = [] image = p["image"] if isinstance(p["image"], dict) else {"action": "none"} action = image.get("action", "none") memory = p["memory"] if isinstance(p["memory"], dict) else {"action": "none"} mem_action = memory.get("action", "none") drug = p["drug"] if isinstance(p["drug"], dict) else None tools = [t for t in p["tools"] if isinstance(t, dict)] tool_names = [t.get("name") for t in tools] directives = [d for d in p["directives"] if isinstance(d, str)] safety = p["safety_concern"] offered = set(example.get("offered_tools") or []) if example is not None else None history = [m for m in (example or {}).get("history") or [] if m.get("role") in ("user", "assistant")] # Image. extra = {k for k in ("prompt", "visible_text", "instruction") if k in image} if action == "none" and extra: errs.append(f"image: action none carries {sorted(extra)}") if action == "generate": if "prompt" not in image: errs.append("image: generate needs a prompt") if "visible_text" not in image: errs.append("image: generate needs visible_text ('' for no words)") if "instruction" in image: errs.append("image: instruction belongs to edit, not generate") if action == "edit": if "instruction" not in image: errs.append("image: edit needs an instruction") if extra - {"instruction"}: errs.append("image: edit carries only an instruction") if not _has_image_source(example): errs.append("image: edit needs an attached or remembered image") if action == "analyze": if extra: errs.append("image: analyze carries no prompt, visible_text or instruction") if example is not None and not example.get("attached_image"): errs.append("image: analyze needs an attached image") if example is not None and example.get("attached_image") and action not in ("analyze", "edit"): errs.append("image: with an image attached the action is analyze or edit") vt = image.get("visible_text") if isinstance(vt, str): if "\n" in vt: errs.append("image: visible_text is one line") if any(b in vt for b in ("[", "]", "{", "}")): errs.append("image: visible_text holds no bracket placeholders") if action in ("generate", "edit"): # The image turn ends at the image service. if p["request_kind"] != "chat": errs.append("image: a picture turn is not a document turn") if p["needs_search"] or p["data_tool"] != "none" or drug or tools or directives or p["clarify"]: errs.append("image: a picture turn has no search, data tool, drug, tools, directives or clarify") # Request kind. if (p["request_kind"] == "document") != (p["doc_format"] is not None): errs.append("request: doc_format is set exactly when request_kind is document") if p["request_kind"] == "document" and (tools or p["needs_search"] or directives): errs.append("request: a document turn goes to doc-gen: no tools, search or directives") # Clarify. if p["clarify"]: if action != "none" or p["request_kind"] != "chat" or p["needs_search"] or p["data_tool"] != "none" \ or drug or mem_action != "none" or tools or directives: errs.append("clarify: a clarifying turn carries no other decision") # Standalone question. sq = p["standalone_question"] if sq is not None: if example is not None and not history and mem_action not in ("remember", "forget"): errs.append("standalone_question: no history to resolve from (followup_rewrite.should_rewrite)") if example is not None and _norm(sq) == _norm(example.get("message", "")): errs.append("standalone_question: same as the message; omit it (followup_rewrite.is_restated)") # Search. queries = p["search_queries"] or [] if p["needs_search"] != bool(queries): errs.append("search: search_queries are given exactly when needs_search is true") lowered = [q.lower().strip() for q in queries if isinstance(q, str)] if len(set(lowered)) != len(lowered): errs.append("search: duplicate queries") for q in lowered: bad = [t for t in BANNED_QUERY_TOKENS if t in q] if bad: errs.append(f"search: query uses {bad}") if p["needs_search"] and p["domain"] in ("personal_memory", "coding"): errs.append("search: personal and coding turns are never searched") if p["needs_search"] and p["data_tool"] != "none": errs.append("search: a calculator/food-data question is not a web question (data_tool_intent.answered_without_web_search)") # Data tool <-> tools. for tool_name, kinds in DATA_KIND_OF_TOOL.items(): if tool_name in tool_names and p["data_tool"] not in kinds: errs.append(f"data_tool: {tool_name} in tools needs data_tool in {sorted(kinds)}") if p["data_tool"] != "none" and offered is not None: wanted = DATA_TOOL_FOR[p["data_tool"]] if any(t in offered for t in wanted) and not any(t in tool_names for t in wanted): errs.append(f"data_tool: {p['data_tool']} with {list(wanted)} offered must call it in tools") if p["data_tool"] == "barcode": for t in tools: if t.get("name") == "open_food_facts" and (t.get("arguments") or {}).get("search_by") != "barcode": errs.append("data_tool: a barcode lookup calls open_food_facts with search_by barcode") # Drug. if drug: names = drug.get("names") or [] if any(isinstance(n, str) and n != n.lower() for n in names): errs.append("drug: names are lowercased") if len({str(n) for n in names}) != len(names): errs.append("drug: duplicate names") if "semantic_scholar_search" in tool_names and drug and not drug.get("wants_research"): errs.append("drug: research papers for a medication question only when wants_research") # Memory. if mem_action == "remember" and not ("memory_type" in memory and memory.get("content")): errs.append("memory: remember needs memory_type and content") if mem_action == "forget" and ("memory_type" in memory or not memory.get("content")): errs.append("memory: forget needs content ('*' for everything) and no memory_type") if mem_action in ("recall", "none") and ({"memory_type", "content"} & set(memory)): errs.append(f"memory: {mem_action} carries no memory_type or content") if mem_action == "recall" and p["domain"] != "personal_memory": errs.append("memory: recall is a personal_memory turn") # Route. depth, needs_tools, domain = p["depth"], p["needs_tools"], p["domain"] if depth == "quick" and needs_tools: errs.append("route: quick (T1) never needs tools") if depth in ("deep", "research") and not needs_tools: errs.append(f"route: {depth} is an agent tier and needs tools") if not p["clarify"] and action not in ("generate", "edit") and p["request_kind"] == "chat": why = [] if domain in AGENT_DOMAINS: why.append(f"domain {domain}") if p["needs_search"]: why.append("needs_search") if p["data_tool"] != "none": why.append("data_tool") if drug: why.append("drug") if tools: why.append("tools") if mem_action != "none": why.append("memory") if why and not needs_tools: errs.append(f"route: needs_tools must be true with {', '.join(why)}") # Safety. if safety in ("possible_crisis", "possible_emergency"): if action in ("generate", "edit"): errs.append("safety: no picture on a crisis or emergency turn") if p["request_kind"] != "chat": errs.append("safety: no file on a crisis or emergency turn") if safety in ("medication_risk", "possible_emergency") and domain != "medical": errs.append(f"safety: {safety} is a medical turn") # Tools. for i, call in enumerate(tools): errs.extend(_check_tool_call(call, i, offered)) if "handoff_to" in tool_names and len(tools) > 1: errs.append("tools: handoff_to ends the turn, so it is the only tool") seen = set() for call in tools: key = json.dumps(call, sort_keys=True) if key in seen: errs.append(f"tools: duplicate call {call.get('name')}") seen.add(key) if "code_agent" in tool_names and domain != "coding": errs.append("tools: code_agent is for coding turns") # Directives. catalogue = _directives() for d in directives: entry = catalogue.get(d) if entry is None: errs.append(f"directives: unknown id {d}") continue if entry["kind"] != "planner": errs.append(f"directives: {d} is derived by code; the plan never names it") continue if domain in entry.get("excluded_domains", []): errs.append(f"directives: {d} is not used on a {domain} turn") if action in entry.get("excluded_image_actions", []): errs.append(f"directives: {d} is not used on a picture turn") if safety in entry.get("excluded_safety", []): errs.append(f"directives: {d} stands aside on a {safety} turn") if entry.get("paths") == ["agent"] and not needs_tools: errs.append(f"directives: {d} is applied on the agent path only (needs_tools true)") if example is not None: req = entry.get("requires", {}) if "history_empty" in req and req["history_empty"] != (not history): errs.append(f"directives: {d} requires history_empty={req['history_empty']}") for flag in ("document_attached", "signed_in", "memory_enabled"): if flag in req and bool(example.get(flag)) != req[flag]: errs.append(f"directives: {d} requires {flag}={req[flag]}") if "style.casual" in directives and "style.followup" in directives: errs.append("directives: style.followup stands aside for style.casual") if "style.statement_only" in directives and ({"style.casual", "style.followup"} & set(directives)): errs.append("directives: style.statement_only replaces the other style notes") langs = [d for d in directives if d.startswith("lang.")] if len(langs) > 1: errs.append(f"directives: one language note at most, got {langs}") if "memory.self_fact" in directives and mem_action in ("remember", "forget"): errs.append("directives: memory.self_fact is not used on a turn with a memory command") if "style.statement_only" in directives and (p["needs_search"] or tools): errs.append("directives: a statement-only turn has nothing to look up") return errs # ── entry points ────────────────────────────────────────────────────────────── def validate_example(example: dict) -> list[str]: errs = _schema_errors(_example_validator(), example, "input") unknown = [t for t in (example.get("offered_tools") or []) if t not in _tools()] if unknown: errs.append(f"input: offered_tools not in tools.json: {unknown}") return errs def validate_plan(plan: Any, example: Optional[dict] = None) -> list[str]: """All errors for `plan` (and, given the input, the rules that depend on it). An empty list means the plan is valid.""" if not isinstance(plan, dict): return ["plan: not a JSON object"] errs = _schema_errors(_plan_validator(), plan, "plan") if errs: # Semantic rules assume the shape; report shape errors alone. return errs return _semantic_errors(plan, example) def validate_target(text: str, example: Optional[dict] = None) -> list[str]: """A model output or training target: JSON, valid, and in canonical form.""" try: plan = json.loads(text) except (TypeError, ValueError) as exc: return [f"target: not JSON ({exc})"] errs = validate_plan(plan, example) if not errs and render.render_target(plan) != text: errs.append("target: valid but not canonical (render.render_target)") return errs def guided_schema(offered_tools: Iterable[str]) -> dict: """turn_plan.schema.json narrowed to this turn: each tools item is one of the plannable offered tools with that tool's own argument schema. Pass it to vLLM as guided_json. No plannable tool offered -> tools must be empty.""" schema = copy.deepcopy(_json("turn_plan.schema.json")) offered = [t for t in _tools().values() if t["plannable"] and t["name"] in set(offered_tools or ())] tools_prop = schema["properties"]["tools"] if not offered: tools_prop["maxItems"] = 0 tools_prop["items"] = {"type": "object", "additionalProperties": False, "properties": {}} return schema variants = [] for tool in offered: args = copy.deepcopy(tool["parameters"]) args.pop("minProperties", None) variants.append({ "type": "object", "additionalProperties": False, "required": ["name", "arguments"], "properties": {"name": {"const": tool["name"]}, "arguments": args}, }) tools_prop["items"] = variants[0] if len(variants) == 1 else {"anyOf": variants} return schema def _main(argv: list[str]) -> int: if len(argv) != 2: print(__doc__) return 2 total = bad = 0 with open(argv[1], encoding="utf-8") as fh: for lineno, line in enumerate(fh, 1): if not line.strip(): continue total += 1 row = json.loads(line) example = row.get("input") errs = validate_example(example) if example is not None else [] if "target" in row: errs += validate_target(row["target"], example) else: errs += validate_plan(row.get("plan"), example) if errs: bad += 1 for e in errs: print(f"line {lineno}: {e}") print(f"{total - bad}/{total} valid") return 1 if bad else 0 if __name__ == "__main__": sys.stdout.reconfigure(encoding="utf-8") raise SystemExit(_main(sys.argv))