"""Render xevyo-01 inputs and targets (spec xv01.1). render_messages(example) -> [{"role": "system", ...}, {"role": "user", ...}] render_target(plan) -> canonical compact JSON string (the training target) canonicalize(plan) -> the same plan as a dict: defaults dropped, keys ordered `example` follows example_input.schema.json; `plan` follows turn_plan.schema.json. Standard library only, so the data, training, eval and serving code can all import it. Chat template: the caller applies Qwen's template with thinking OFF (tokenizer.apply_chat_template(messages, add_generation_prompt=True, enable_thinking=False)). Clipping follows the deciders this replaces: the image decider reads 4 turns of 200 chars, the follow-up rewrite 6 messages of 1500 chars, request_intent 6-8 turns of 400 chars. We keep the last 8 messages (4 exchanges, enough for a two-hop follow-up) and clip each; a long current message keeps its head and its tail, where a question after a pasted document sits (data_tool_intent.scan_window). """ from __future__ import annotations import json import math from functools import lru_cache from pathlib import Path from typing import Any SPEC_DIR = Path(__file__).resolve().parent SCHEMA_VERSION = "xv01.1" MAX_HISTORY_MESSAGES = 8 USER_TURN_CHARS = 500 ASSISTANT_TURN_CHARS = 600 MESSAGE_HEAD_CHARS = 1500 MESSAGE_TAIL_CHARS = 500 REMEMBERED_IMAGE_CHARS = 120 UPLOADED_IMAGE_MARKER = "the image you uploaded" #: Context windows (settings.XEVYO01_CONTEXT). "trained" is the window above, the one #: xevyo-01 was trained on and the golden fixtures pin; it stays the default and its #: output is unchanged byte for byte. "agent" shows the planner the earlier messages #: an answering agent is shown (newest first, up to 24 #: messages within 12,000 characters, user turns clipped at 2,000, the latest answer at #: 6,000, older answers at 1,200) and more of a long new message. Same format, longer: #: the model never saw inputs this long in training, so measure it before relying on it. WINDOWS = ("trained", "agent") AGENT_HISTORY_MAX_MESSAGES = 24 AGENT_HISTORY_CHAR_BUDGET = 12_000 AGENT_USER_TURN_CHARS = 2_000 AGENT_LATEST_ANSWER_CHARS = 6_000 AGENT_OLDER_ANSWER_CHARS = 1_200 AGENT_MESSAGE_HEAD_CHARS = 6_000 AGENT_MESSAGE_TAIL_CHARS = 2_000 #: Canonical key order of a plan (most consequential decisions first). PLAN_KEYS = ( "schema_version", "safety_concern", "image", "request_kind", "doc_format", "clarify", "standalone_question", "needs_search", "search_queries", "data_tool", "drug", "memory", "domain", "depth", "needs_tools", "tools", "directives", ) IMAGE_KEYS = ("action", "prompt", "visible_text", "instruction") DRUG_KEYS = ("names", "wants_research") MEMORY_KEYS = ("action", "memory_type", "content") TOOL_CALL_KEYS = ("name", "arguments") #: Field defaults. A field holding its default is left out of the canonical plan. DEFAULTS: dict[str, Any] = { "safety_concern": "none", "image": {"action": "none"}, "request_kind": "chat", "doc_format": None, "clarify": None, "standalone_question": None, "needs_search": False, "search_queries": [], "data_tool": "none", "drug": None, "memory": {"action": "none"}, "domain": "general", "depth": "normal", "needs_tools": False, "tools": [], "directives": [], } _PROMPT_START = "" _PROMPT_END = "" # ── spec files ──────────────────────────────────────────────────────────────── @lru_cache(maxsize=None) def _load(name: str) -> Any: return json.loads((SPEC_DIR / name).read_text(encoding="utf-8")) def tool_catalogue() -> dict[str, dict]: """tools.json entries by name.""" return {t["name"]: t for t in _load("tools.json")["tools"]} def directive_catalogue() -> list[dict]: """directives.json entries in catalogue order.""" return list(_load("directives.json")["directives"]) def planner_directive_ids() -> list[str]: """The ids the planner may emit, in canonical order.""" return [d["id"] for d in directive_catalogue() if d["kind"] == "planner"] @lru_cache(maxsize=None) def _prompt_template() -> str: text = (SPEC_DIR / "system_prompt.md").read_text(encoding="utf-8") start, end = text.index(_PROMPT_START) + len(_PROMPT_START), text.index(_PROMPT_END) return text[start:end].strip() # ── input rendering ─────────────────────────────────────────────────────────── def _one_line(text: str, limit: int) -> str: """Client text placed inside our own frame stays on its line and within size.""" flat = " ".join((text or "").split()) return flat if len(flat) <= limit else flat[: limit - 1].rstrip() + "…" def _clip_message(text: str) -> str: text = (text or "").strip() if len(text) <= MESSAGE_HEAD_CHARS + MESSAGE_TAIL_CHARS: return text return f"{text[:MESSAGE_HEAD_CHARS].rstrip()}\n[… {len(text) - MESSAGE_HEAD_CHARS - MESSAGE_TAIL_CHARS} characters left out …]\n{text[-MESSAGE_TAIL_CHARS:].lstrip()}" def _clip_message_agent(text: str) -> str: """_clip_message with the "agent" window's longer head and tail (same marker).""" text = (text or "").strip() head, tail = AGENT_MESSAGE_HEAD_CHARS, AGENT_MESSAGE_TAIL_CHARS if len(text) <= head + tail: return text return f"{text[:head].rstrip()}\n[… {len(text) - head - tail} characters left out …]\n{text[-tail:].lstrip()}" def _agent_history_lines(history: list) -> list[str]: """The "agent" window: newest backwards, each turn clipped like the answering agent's (the latest answer longer than older ones), until 24 messages or the 12,000-character budget runs out (the newest message always fits); oldest first.""" picked: list[str] = [] used = 0 seen_answer = False for m in reversed(history): if m["role"] == "assistant": limit = AGENT_OLDER_ANSWER_CHARS if seen_answer else AGENT_LATEST_ANSWER_CHARS seen_answer = True else: limit = AGENT_USER_TURN_CHARS content = _one_line(m.get("content", ""), limit) if picked and used + len(content) > AGENT_HISTORY_CHAR_BUDGET: break picked.append(f"{m['role']}: {content}") used += len(content) if len(picked) >= AGENT_HISTORY_MAX_MESSAGES: break picked.reverse() return picked def _type_name(param: dict) -> str: if "enum" in param: return "|".join(json.dumps(v) for v in param["enum"]) kind = param.get("type", "any") if isinstance(kind, list): kind = "|".join(k for k in kind if k != "null") return {"string": "str", "number": "num", "integer": "int", "boolean": "bool"}.get(kind, kind) def render_tool_line(tool: dict) -> str: """One catalogue line: name(arg: type, opt?: type=default): purpose.""" params = tool["parameters"] required = set(params.get("required", [])) parts = [] for pname, spec in params.get("properties", {}).items(): piece = f"{pname}{'' if pname in required else '?'}: {_type_name(spec)}" if "default" in spec and spec["default"] not in ("", None): piece += f"={json.dumps(spec['default'])}" parts.append(piece) return f"- {tool['name']}({', '.join(parts)}): {tool['purpose']}" def render_tool_catalogue(offered_tools) -> str: """The plannable offered tools, in tools.json order. Tools driven by another plan field (web search, drug_lookup, generate_image, memory writes) are not listed: the plan never names them. Unknown names are ignored.""" offered = set(offered_tools or ()) lines = [render_tool_line(t) for t in tool_catalogue().values() if t["plannable"] and t["name"] in offered] return "\n".join(lines) if lines else "(none)" def render_directive_list() -> str: return "\n".join(f"- {d['id']}: {d['meaning']}" for d in directive_catalogue() if d["kind"] == "planner") def render_system_prompt(offered_tools) -> str: return (_prompt_template() .replace("{directive_list}", render_directive_list()) .replace("{tool_catalogue}", render_tool_catalogue(offered_tools))) def _yes(flag: bool) -> str: return "yes" if flag else "no" def render_context(example: dict, window: str = "trained") -> str: """The user message: the context block, then the message last (nothing after it, so nothing in it can pass for our own frame). `window`: see WINDOWS.""" history = [m for m in example.get("history") or [] if m.get("role") in ("user", "assistant")] if window == "agent": lines = _agent_history_lines(history) chat = "\n".join(lines) if lines else "none" message = _clip_message_agent(example.get("message", "")) else: history = history[-MAX_HISTORY_MESSAGES:] if history: chat = "\n".join( f"{m['role']}: {_one_line(m.get('content', ''), USER_TURN_CHARS if m['role'] == 'user' else ASSISTANT_TURN_CHARS)}" for m in history ) else: chat = "none" message = _clip_message(example.get("message", "")) remembered = example.get("remembered_image") if remembered is None or not str(remembered).strip(): remembered_line = "none" elif remembered.strip().lower() == UPLOADED_IMAGE_MARKER: remembered_line = "yes, an earlier upload (no description)" else: remembered_line = '"' + _one_line(remembered, REMEMBERED_IMAGE_CHARS).replace('"', "'") + '"' return ( "RECENT CHAT\n" f"{chat}\n" "TURN\n" f"image attached: {_yes(example.get('attached_image', False))}\n" f"remembered image: {remembered_line}\n" f"document attached: {_yes(example.get('document_attached', False))}\n" f"memory: {'on' if example.get('memory_enabled', False) else 'off'}\n" f"signed in: {_yes(example.get('signed_in', False))}\n" f"api client: {_yes(example.get('api_client', False))}\n" f"locale: {example.get('locale') or 'unknown'}\n" "MESSAGE\n" f"{message}" ) def render_messages(example: dict, window: str = "trained") -> list[dict]: """The chat messages xevyo-01 reads for `example` (example_input.schema.json).""" return [ {"role": "system", "content": render_system_prompt(example.get("offered_tools") or [])}, {"role": "user", "content": render_context(example, window)}, ] # ── target rendering ────────────────────────────────────────────────────────── def _number(value: Any) -> Any: """70.0 -> 70, so the same plan always prints the same way.""" if isinstance(value, float) and math.isfinite(value) and value.is_integer(): return int(value) return value def _ordered(obj: dict, keys) -> dict: out = {k: obj[k] for k in keys if k in obj} # Unknown keys are kept at the end so validation can still report them. out.update({k: v for k, v in obj.items() if k not in out}) return out def _canonical_arguments(name: str, arguments: dict) -> dict: tool = tool_catalogue().get(name) props = (tool or {}).get("parameters", {}).get("properties", {}) order = list(props) + [k for k in arguments if k not in props] out = {} for key in order: if key not in arguments: continue value = _number(arguments[key]) if key in props and "default" in props[key] and value == props[key]["default"] \ and key not in (tool or {}).get("parameters", {}).get("required", []): continue out[key] = value return out def canonicalize(plan: dict) -> dict: """`plan` in canonical form: schema_version forced first, keys in PLAN_KEYS order, nested keys ordered, defaults (and nulls) dropped, directives in catalogue order, integral floats as ints, tool arguments equal to their defaults dropped. Content is never rewritten (no case changes, no trimming of text): a plan that is wrong stays wrong here and validate.py says why.""" out: dict[str, Any] = {"schema_version": plan.get("schema_version", SCHEMA_VERSION)} for key in PLAN_KEYS[1:]: if key not in plan: continue value = plan[key] if value is None: continue if key == "image" and isinstance(value, dict): value = {k: v for k, v in _ordered(value, IMAGE_KEYS).items() if v is not None} elif key == "drug" and isinstance(value, dict): value = _ordered(value, DRUG_KEYS) if value.get("wants_research") is False: value.pop("wants_research") elif key == "memory" and isinstance(value, dict): value = {k: v for k, v in _ordered(value, MEMORY_KEYS).items() if v not in (None, "")} elif key == "tools" and isinstance(value, list): value = [ _ordered({**call, "arguments": _canonical_arguments(call.get("name", ""), call.get("arguments") or {})}, TOOL_CALL_KEYS) if isinstance(call, dict) else call for call in value ] elif key == "directives" and isinstance(value, list): rank = {d: i for i, d in enumerate(d["id"] for d in directive_catalogue())} value = sorted(dict.fromkeys(value), key=lambda d: rank.get(d, len(rank))) if value == DEFAULTS.get(key, object()): continue out[key] = value out.update({k: v for k, v in plan.items() if k not in out and k not in PLAN_KEYS}) return out def fill_defaults(plan: dict) -> dict: """`plan` with every omitted field set to its default, for per-field scoring.""" full = {"schema_version": plan.get("schema_version", SCHEMA_VERSION)} for key in PLAN_KEYS[1:]: value = plan.get(key) full[key] = json.loads(json.dumps(DEFAULTS[key])) if value is None else value if isinstance(full["drug"], dict): full["drug"] = {"wants_research": False, **full["drug"]} return full def render_target(plan: dict) -> str: """The training target: canonical plan as compact JSON, stable key order, UTF-8 kept (Hindi stays Hindi).""" return json.dumps(canonicalize(plan), ensure_ascii=False, separators=(",", ":")) __all__ = [ "SCHEMA_VERSION", "PLAN_KEYS", "DEFAULTS", "render_messages", "render_context", "render_system_prompt", "render_tool_catalogue", "render_directive_list", "render_target", "canonicalize", "fill_defaults", "tool_catalogue", "directive_catalogue", "planner_directive_ids", ]