xevyo-01 / code /spec /validate.py
meet4150's picture
xevyo-01 round 4: merged weights, adapter, model card, code
a46f204 verified
Raw History Blame Contribute Delete
22.6 kB
"""Validate xevyo-01 turn plans (spec xv01.1).
validate_plan(plan, example=None) -> list[str] # [] means valid
validate_target(text, example=None) -> list[str] # also checks JSON + canonical form
validate_example(example) -> list[str] # an input against example_input.schema.json
guided_schema(offered_tools) -> dict # per-turn schema for vLLM guided_json
Two layers. The JSON Schema (turn_plan.schema.json) checks shape and closed value
sets. The semantic rules below check what a schema cannot: tools offered this turn,
arguments that fit each tool, conditions between fields, and directive
preconditions on the input. Every rule mirrors a behaviour of the serving application.
Needs only the `jsonschema` package. Command line, one JSON object per line with
keys "input" (example_input) and "plan" (object) or "target" (string):
python spec/validate.py examples.jsonl
"""
from __future__ import annotations
import ast
import copy
import json
import sys
from functools import lru_cache
from pathlib import Path
from typing import Any, Iterable, Optional
from jsonschema import Draft202012Validator
SPEC_DIR = Path(__file__).resolve().parent
if str(SPEC_DIR) not in sys.path:
sys.path.insert(0, str(SPEC_DIR))
import render
MESSAGE_REF = "$MESSAGE"
UPLOADED_IMAGE_MARKER = render.UPLOADED_IMAGE_MARKER
#: : a query naming one of these returns nothing.
BANNED_QUERY_TOKENS = (
"site:", "inurl:", "intitle:", "filetype:",
"twitter.com", "x.com", "facebook.com", "instagram.com", "linkedin.com", "tiktok.com", "reddit.com/r/",
)
#: data_tool_intent._TOOL_FOR, with the call a plan makes.
DATA_TOOL_FOR = {
"bmi": ("calculate_bmi",),
"dose": ("calculate_dosage",),
"barcode": ("open_food_facts",),
"usda": ("usda_food_search",),
"food": ("usda_food_search", "open_food_facts"),
"calc": ("calculate",),
}
#: Tool -> the data_tool values that may list it.
DATA_KIND_OF_TOOL = {
"calculate_bmi": {"bmi"},
"calculate_dosage": {"dose"},
"calculate": {"calc"},
"open_food_facts": {"barcode", "food"},
"usda_food_search": {"usda", "food"},
}
#: Domains that take the agent path (complexity.py: coding 811-819, personal 897-919,
#: prompt_authoring 734-742, medical 801-808 and ).
AGENT_DOMAINS = {"medical", "coding", "personal_memory", "prompt_authoring"}
SAFETY_RANK = {"none": 0, "medication_risk": 1, "possible_emergency": 2, "possible_crisis": 3}
# ── spec files ────────────────────────────────────────────────────────────────
@lru_cache(maxsize=None)
def _json(name: str) -> Any:
return json.loads((SPEC_DIR / name).read_text(encoding="utf-8"))
@lru_cache(maxsize=None)
def _plan_validator() -> Draft202012Validator:
return Draft202012Validator(_json("turn_plan.schema.json"))
@lru_cache(maxsize=None)
def _example_validator() -> Draft202012Validator:
return Draft202012Validator(_json("example_input.schema.json"))
def _tools() -> dict[str, dict]:
return render.tool_catalogue()
def _directives() -> dict[str, dict]:
return {d["id"]: d for d in render.directive_catalogue()}
def _schema_errors(validator: Draft202012Validator, instance: Any, label: str) -> list[str]:
out = []
for err in sorted(validator.iter_errors(instance), key=lambda e: list(e.absolute_path)):
where = "/".join(str(p) for p in err.absolute_path) or "(root)"
out.append(f"{label} schema: {where}: {err.message}")
return out
# ── helpers ───────────────────────────────────────────────────────────────────
def _norm(text: str) -> str:
"""followup_rewrite.is_restated's comparison form."""
return " ".join((text or "").lower().split()).rstrip("?.! ")
def _has_image_source(example: Optional[dict]) -> bool:
if example is None:
return True # unknown: do not fail a plan on what we cannot see
return bool(example.get("attached_image")) or bool((example.get("remembered_image") or "").strip())
def _calc_expression_ok(expr: str) -> Optional[str]:
"""None if `expr` is what evaluates, else why not."""
if not isinstance(expr, str) or not expr.strip():
return "empty expression"
if expr.strip() == MESSAGE_REF:
return "calculate needs an expression, not $MESSAGE"
try:
tree = ast.parse(expr.strip(), mode="eval")
except SyntaxError:
return "not a valid expression"
allowed_bin = (ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow)
def ok(node) -> bool:
if isinstance(node, ast.Expression):
return ok(node.body)
if isinstance(node, ast.Constant):
return isinstance(node.value, (int, float)) and not isinstance(node.value, bool)
if isinstance(node, ast.BinOp):
return isinstance(node.op, allowed_bin) and ok(node.left) and ok(node.right)
if isinstance(node, ast.UnaryOp):
return isinstance(node.op, (ast.UAdd, ast.USub)) and ok(node.operand)
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and not node.keywords:
if node.func.id == "abs":
return len(node.args) == 1 and all(ok(a) for a in node.args)
if node.func.id == "round":
return len(node.args) in (1, 2) and all(ok(a) for a in node.args)
return False
return None if ok(tree) else "only numbers, + - * / // % **, parentheses, round() and abs() are allowed"
def _check_tool_call(call: dict, idx: int, offered: Optional[set]) -> list[str]:
errs: list[str] = []
name = call.get("name")
args = call.get("arguments") or {}
tool = _tools().get(name)
where = f"tools[{idx}] {name}"
if tool is None:
return [f"{where}: not in tools.json"]
if not tool["plannable"]:
errs.append(f"{where}: never listed in tools; it is driven by plan field '{tool['planned_via']}'")
if offered is not None and name not in offered:
errs.append(f"{where}: not offered this turn")
# $MESSAGE stands for the user's text: accepted for any string parameter.
props = tool["parameters"].get("properties", {})
substituted = {}
for key, value in args.items():
if value == MESSAGE_REF:
spec = props.get(key, {})
kinds = spec.get("type", [])
kinds = kinds if isinstance(kinds, list) else [kinds]
if "string" not in kinds:
errs.append(f"{where}: $MESSAGE used for non-string parameter '{key}'")
substituted[key] = "message text"
else:
substituted[key] = value
for err in Draft202012Validator(tool["parameters"]).iter_errors(substituted):
path = "/".join(str(p) for p in err.absolute_path) or "arguments"
errs.append(f"{where}: {path}: {err.message}")
# Rules the parameter schemas cannot state.
if name == "calculate" and "expression" in args:
why = _calc_expression_ok(args["expression"])
if why:
errs.append(f"{where}: expression: {why}")
if name == "calculate_dosage" and args.get("dose_per_kg") and not args.get("dose_basis"):
errs.append(f"{where}: dose_basis is required whenever dose_per_kg is given (calculator.py _DOSAGE_DOC)")
if name == "calculate_bmi":
if not any(k in args for k in ("weight_kg", "weight_lb", "weight_st")) or \
not any(k in args for k in ("height_cm", "height_m", "height_ft", "height_in")):
errs.append(f"{where}: needs a weight and a height")
return errs
# ── the semantic rules ────────────────────────────────────────────────────────
def _semantic_errors(plan: dict, example: Optional[dict]) -> list[str]:
p = render.fill_defaults(plan)
errs: list[str] = []
image = p["image"] if isinstance(p["image"], dict) else {"action": "none"}
action = image.get("action", "none")
memory = p["memory"] if isinstance(p["memory"], dict) else {"action": "none"}
mem_action = memory.get("action", "none")
drug = p["drug"] if isinstance(p["drug"], dict) else None
tools = [t for t in p["tools"] if isinstance(t, dict)]
tool_names = [t.get("name") for t in tools]
directives = [d for d in p["directives"] if isinstance(d, str)]
safety = p["safety_concern"]
offered = set(example.get("offered_tools") or []) if example is not None else None
history = [m for m in (example or {}).get("history") or [] if m.get("role") in ("user", "assistant")]
# Image.
extra = {k for k in ("prompt", "visible_text", "instruction") if k in image}
if action == "none" and extra:
errs.append(f"image: action none carries {sorted(extra)}")
if action == "generate":
if "prompt" not in image:
errs.append("image: generate needs a prompt")
if "visible_text" not in image:
errs.append("image: generate needs visible_text ('' for no words)")
if "instruction" in image:
errs.append("image: instruction belongs to edit, not generate")
if action == "edit":
if "instruction" not in image:
errs.append("image: edit needs an instruction")
if extra - {"instruction"}:
errs.append("image: edit carries only an instruction")
if not _has_image_source(example):
errs.append("image: edit needs an attached or remembered image")
if action == "analyze":
if extra:
errs.append("image: analyze carries no prompt, visible_text or instruction")
if example is not None and not example.get("attached_image"):
errs.append("image: analyze needs an attached image")
if example is not None and example.get("attached_image") and action not in ("analyze", "edit"):
errs.append("image: with an image attached the action is analyze or edit")
vt = image.get("visible_text")
if isinstance(vt, str):
if "\n" in vt:
errs.append("image: visible_text is one line")
if any(b in vt for b in ("[", "]", "{", "}")):
errs.append("image: visible_text holds no bracket placeholders")
if action in ("generate", "edit"):
# The image turn ends at the image service.
if p["request_kind"] != "chat":
errs.append("image: a picture turn is not a document turn")
if p["needs_search"] or p["data_tool"] != "none" or drug or tools or directives or p["clarify"]:
errs.append("image: a picture turn has no search, data tool, drug, tools, directives or clarify")
# Request kind.
if (p["request_kind"] == "document") != (p["doc_format"] is not None):
errs.append("request: doc_format is set exactly when request_kind is document")
if p["request_kind"] == "document" and (tools or p["needs_search"] or directives):
errs.append("request: a document turn goes to doc-gen: no tools, search or directives")
# Clarify.
if p["clarify"]:
if action != "none" or p["request_kind"] != "chat" or p["needs_search"] or p["data_tool"] != "none" \
or drug or mem_action != "none" or tools or directives:
errs.append("clarify: a clarifying turn carries no other decision")
# Standalone question.
sq = p["standalone_question"]
if sq is not None:
if example is not None and not history and mem_action not in ("remember", "forget"):
errs.append("standalone_question: no history to resolve from (followup_rewrite.should_rewrite)")
if example is not None and _norm(sq) == _norm(example.get("message", "")):
errs.append("standalone_question: same as the message; omit it (followup_rewrite.is_restated)")
# Search.
queries = p["search_queries"] or []
if p["needs_search"] != bool(queries):
errs.append("search: search_queries are given exactly when needs_search is true")
lowered = [q.lower().strip() for q in queries if isinstance(q, str)]
if len(set(lowered)) != len(lowered):
errs.append("search: duplicate queries")
for q in lowered:
bad = [t for t in BANNED_QUERY_TOKENS if t in q]
if bad:
errs.append(f"search: query uses {bad}")
if p["needs_search"] and p["domain"] in ("personal_memory", "coding"):
errs.append("search: personal and coding turns are never searched")
if p["needs_search"] and p["data_tool"] != "none":
errs.append("search: a calculator/food-data question is not a web question (data_tool_intent.answered_without_web_search)")
# Data tool <-> tools.
for tool_name, kinds in DATA_KIND_OF_TOOL.items():
if tool_name in tool_names and p["data_tool"] not in kinds:
errs.append(f"data_tool: {tool_name} in tools needs data_tool in {sorted(kinds)}")
if p["data_tool"] != "none" and offered is not None:
wanted = DATA_TOOL_FOR[p["data_tool"]]
if any(t in offered for t in wanted) and not any(t in tool_names for t in wanted):
errs.append(f"data_tool: {p['data_tool']} with {list(wanted)} offered must call it in tools")
if p["data_tool"] == "barcode":
for t in tools:
if t.get("name") == "open_food_facts" and (t.get("arguments") or {}).get("search_by") != "barcode":
errs.append("data_tool: a barcode lookup calls open_food_facts with search_by barcode")
# Drug.
if drug:
names = drug.get("names") or []
if any(isinstance(n, str) and n != n.lower() for n in names):
errs.append("drug: names are lowercased")
if len({str(n) for n in names}) != len(names):
errs.append("drug: duplicate names")
if "semantic_scholar_search" in tool_names and drug and not drug.get("wants_research"):
errs.append("drug: research papers for a medication question only when wants_research")
# Memory.
if mem_action == "remember" and not ("memory_type" in memory and memory.get("content")):
errs.append("memory: remember needs memory_type and content")
if mem_action == "forget" and ("memory_type" in memory or not memory.get("content")):
errs.append("memory: forget needs content ('*' for everything) and no memory_type")
if mem_action in ("recall", "none") and ({"memory_type", "content"} & set(memory)):
errs.append(f"memory: {mem_action} carries no memory_type or content")
if mem_action == "recall" and p["domain"] != "personal_memory":
errs.append("memory: recall is a personal_memory turn")
# Route.
depth, needs_tools, domain = p["depth"], p["needs_tools"], p["domain"]
if depth == "quick" and needs_tools:
errs.append("route: quick (T1) never needs tools")
if depth in ("deep", "research") and not needs_tools:
errs.append(f"route: {depth} is an agent tier and needs tools")
if not p["clarify"] and action not in ("generate", "edit") and p["request_kind"] == "chat":
why = []
if domain in AGENT_DOMAINS:
why.append(f"domain {domain}")
if p["needs_search"]:
why.append("needs_search")
if p["data_tool"] != "none":
why.append("data_tool")
if drug:
why.append("drug")
if tools:
why.append("tools")
if mem_action != "none":
why.append("memory")
if why and not needs_tools:
errs.append(f"route: needs_tools must be true with {', '.join(why)}")
# Safety.
if safety in ("possible_crisis", "possible_emergency"):
if action in ("generate", "edit"):
errs.append("safety: no picture on a crisis or emergency turn")
if p["request_kind"] != "chat":
errs.append("safety: no file on a crisis or emergency turn")
if safety in ("medication_risk", "possible_emergency") and domain != "medical":
errs.append(f"safety: {safety} is a medical turn")
# Tools.
for i, call in enumerate(tools):
errs.extend(_check_tool_call(call, i, offered))
if "handoff_to" in tool_names and len(tools) > 1:
errs.append("tools: handoff_to ends the turn, so it is the only tool")
seen = set()
for call in tools:
key = json.dumps(call, sort_keys=True)
if key in seen:
errs.append(f"tools: duplicate call {call.get('name')}")
seen.add(key)
if "code_agent" in tool_names and domain != "coding":
errs.append("tools: code_agent is for coding turns")
# Directives.
catalogue = _directives()
for d in directives:
entry = catalogue.get(d)
if entry is None:
errs.append(f"directives: unknown id {d}")
continue
if entry["kind"] != "planner":
errs.append(f"directives: {d} is derived by code; the plan never names it")
continue
if domain in entry.get("excluded_domains", []):
errs.append(f"directives: {d} is not used on a {domain} turn")
if action in entry.get("excluded_image_actions", []):
errs.append(f"directives: {d} is not used on a picture turn")
if safety in entry.get("excluded_safety", []):
errs.append(f"directives: {d} stands aside on a {safety} turn")
if entry.get("paths") == ["agent"] and not needs_tools:
errs.append(f"directives: {d} is applied on the agent path only (needs_tools true)")
if example is not None:
req = entry.get("requires", {})
if "history_empty" in req and req["history_empty"] != (not history):
errs.append(f"directives: {d} requires history_empty={req['history_empty']}")
for flag in ("document_attached", "signed_in", "memory_enabled"):
if flag in req and bool(example.get(flag)) != req[flag]:
errs.append(f"directives: {d} requires {flag}={req[flag]}")
if "style.casual" in directives and "style.followup" in directives:
errs.append("directives: style.followup stands aside for style.casual")
if "style.statement_only" in directives and ({"style.casual", "style.followup"} & set(directives)):
errs.append("directives: style.statement_only replaces the other style notes")
langs = [d for d in directives if d.startswith("lang.")]
if len(langs) > 1:
errs.append(f"directives: one language note at most, got {langs}")
if "memory.self_fact" in directives and mem_action in ("remember", "forget"):
errs.append("directives: memory.self_fact is not used on a turn with a memory command")
if "style.statement_only" in directives and (p["needs_search"] or tools):
errs.append("directives: a statement-only turn has nothing to look up")
return errs
# ── entry points ──────────────────────────────────────────────────────────────
def validate_example(example: dict) -> list[str]:
errs = _schema_errors(_example_validator(), example, "input")
unknown = [t for t in (example.get("offered_tools") or []) if t not in _tools()]
if unknown:
errs.append(f"input: offered_tools not in tools.json: {unknown}")
return errs
def validate_plan(plan: Any, example: Optional[dict] = None) -> list[str]:
"""All errors for `plan` (and, given the input, the rules that depend on it).
An empty list means the plan is valid."""
if not isinstance(plan, dict):
return ["plan: not a JSON object"]
errs = _schema_errors(_plan_validator(), plan, "plan")
if errs:
# Semantic rules assume the shape; report shape errors alone.
return errs
return _semantic_errors(plan, example)
def validate_target(text: str, example: Optional[dict] = None) -> list[str]:
"""A model output or training target: JSON, valid, and in canonical form."""
try:
plan = json.loads(text)
except (TypeError, ValueError) as exc:
return [f"target: not JSON ({exc})"]
errs = validate_plan(plan, example)
if not errs and render.render_target(plan) != text:
errs.append("target: valid but not canonical (render.render_target)")
return errs
def guided_schema(offered_tools: Iterable[str]) -> dict:
"""turn_plan.schema.json narrowed to this turn: each tools item is one of the
plannable offered tools with that tool's own argument schema. Pass it to vLLM
as guided_json. No plannable tool offered -> tools must be empty."""
schema = copy.deepcopy(_json("turn_plan.schema.json"))
offered = [t for t in _tools().values() if t["plannable"] and t["name"] in set(offered_tools or ())]
tools_prop = schema["properties"]["tools"]
if not offered:
tools_prop["maxItems"] = 0
tools_prop["items"] = {"type": "object", "additionalProperties": False, "properties": {}}
return schema
variants = []
for tool in offered:
args = copy.deepcopy(tool["parameters"])
args.pop("minProperties", None)
variants.append({
"type": "object",
"additionalProperties": False,
"required": ["name", "arguments"],
"properties": {"name": {"const": tool["name"]}, "arguments": args},
})
tools_prop["items"] = variants[0] if len(variants) == 1 else {"anyOf": variants}
return schema
def _main(argv: list[str]) -> int:
if len(argv) != 2:
print(__doc__)
return 2
total = bad = 0
with open(argv[1], encoding="utf-8") as fh:
for lineno, line in enumerate(fh, 1):
if not line.strip():
continue
total += 1
row = json.loads(line)
example = row.get("input")
errs = validate_example(example) if example is not None else []
if "target" in row:
errs += validate_target(row["target"], example)
else:
errs += validate_plan(row.get("plan"), example)
if errs:
bad += 1
for e in errs:
print(f"line {lineno}: {e}")
print(f"{total - bad}/{total} valid")
return 1 if bad else 0
if __name__ == "__main__":
sys.stdout.reconfigure(encoding="utf-8")
raise SystemExit(_main(sys.argv))