xevyo-01 / code /spec /render.py
meet4150's picture
xevyo-01 round 4: merged weights, adapter, model card, code
a46f204 verified
Raw History Blame Contribute Delete
14.9 kB
"""Render xevyo-01 inputs and targets (spec xv01.1).
render_messages(example) -> [{"role": "system", ...}, {"role": "user", ...}]
render_target(plan) -> canonical compact JSON string (the training target)
canonicalize(plan) -> the same plan as a dict: defaults dropped, keys ordered
`example` follows example_input.schema.json; `plan` follows turn_plan.schema.json.
Standard library only, so the data, training, eval and serving code can all import it.
Chat template: the caller applies Qwen's template with thinking OFF
(tokenizer.apply_chat_template(messages, add_generation_prompt=True, enable_thinking=False)).
Clipping follows the deciders this replaces: the image decider reads 4 turns of 200
chars, the follow-up rewrite 6 messages of 1500 chars, request_intent 6-8 turns of 400 chars. We keep the last 8 messages (4 exchanges, enough for a
two-hop follow-up) and clip each; a long current message keeps its head and its tail,
where a question after a pasted document sits (data_tool_intent.scan_window).
"""
from __future__ import annotations
import json
import math
from functools import lru_cache
from pathlib import Path
from typing import Any
SPEC_DIR = Path(__file__).resolve().parent
SCHEMA_VERSION = "xv01.1"
MAX_HISTORY_MESSAGES = 8
USER_TURN_CHARS = 500
ASSISTANT_TURN_CHARS = 600
MESSAGE_HEAD_CHARS = 1500
MESSAGE_TAIL_CHARS = 500
REMEMBERED_IMAGE_CHARS = 120
UPLOADED_IMAGE_MARKER = "the image you uploaded"
#: Context windows (settings.XEVYO01_CONTEXT). "trained" is the window above, the one
#: xevyo-01 was trained on and the golden fixtures pin; it stays the default and its
#: output is unchanged byte for byte. "agent" shows the planner the earlier messages
#: an answering agent is shown (newest first, up to 24
#: messages within 12,000 characters, user turns clipped at 2,000, the latest answer at
#: 6,000, older answers at 1,200) and more of a long new message. Same format, longer:
#: the model never saw inputs this long in training, so measure it before relying on it.
WINDOWS = ("trained", "agent")
AGENT_HISTORY_MAX_MESSAGES = 24
AGENT_HISTORY_CHAR_BUDGET = 12_000
AGENT_USER_TURN_CHARS = 2_000
AGENT_LATEST_ANSWER_CHARS = 6_000
AGENT_OLDER_ANSWER_CHARS = 1_200
AGENT_MESSAGE_HEAD_CHARS = 6_000
AGENT_MESSAGE_TAIL_CHARS = 2_000
#: Canonical key order of a plan (most consequential decisions first).
PLAN_KEYS = (
"schema_version", "safety_concern", "image", "request_kind", "doc_format", "clarify",
"standalone_question", "needs_search", "search_queries", "data_tool", "drug", "memory",
"domain", "depth", "needs_tools", "tools", "directives",
)
IMAGE_KEYS = ("action", "prompt", "visible_text", "instruction")
DRUG_KEYS = ("names", "wants_research")
MEMORY_KEYS = ("action", "memory_type", "content")
TOOL_CALL_KEYS = ("name", "arguments")
#: Field defaults. A field holding its default is left out of the canonical plan.
DEFAULTS: dict[str, Any] = {
"safety_concern": "none",
"image": {"action": "none"},
"request_kind": "chat",
"doc_format": None,
"clarify": None,
"standalone_question": None,
"needs_search": False,
"search_queries": [],
"data_tool": "none",
"drug": None,
"memory": {"action": "none"},
"domain": "general",
"depth": "normal",
"needs_tools": False,
"tools": [],
"directives": [],
}
_PROMPT_START = "<!-- PROMPT START -->"
_PROMPT_END = "<!-- PROMPT END -->"
# ── spec files ────────────────────────────────────────────────────────────────
@lru_cache(maxsize=None)
def _load(name: str) -> Any:
return json.loads((SPEC_DIR / name).read_text(encoding="utf-8"))
def tool_catalogue() -> dict[str, dict]:
"""tools.json entries by name."""
return {t["name"]: t for t in _load("tools.json")["tools"]}
def directive_catalogue() -> list[dict]:
"""directives.json entries in catalogue order."""
return list(_load("directives.json")["directives"])
def planner_directive_ids() -> list[str]:
"""The ids the planner may emit, in canonical order."""
return [d["id"] for d in directive_catalogue() if d["kind"] == "planner"]
@lru_cache(maxsize=None)
def _prompt_template() -> str:
text = (SPEC_DIR / "system_prompt.md").read_text(encoding="utf-8")
start, end = text.index(_PROMPT_START) + len(_PROMPT_START), text.index(_PROMPT_END)
return text[start:end].strip()
# ── input rendering ───────────────────────────────────────────────────────────
def _one_line(text: str, limit: int) -> str:
"""Client text placed inside our own frame stays on its line and within size."""
flat = " ".join((text or "").split())
return flat if len(flat) <= limit else flat[: limit - 1].rstrip() + "…"
def _clip_message(text: str) -> str:
text = (text or "").strip()
if len(text) <= MESSAGE_HEAD_CHARS + MESSAGE_TAIL_CHARS:
return text
return f"{text[:MESSAGE_HEAD_CHARS].rstrip()}\n[… {len(text) - MESSAGE_HEAD_CHARS - MESSAGE_TAIL_CHARS} characters left out …]\n{text[-MESSAGE_TAIL_CHARS:].lstrip()}"
def _clip_message_agent(text: str) -> str:
"""_clip_message with the "agent" window's longer head and tail (same marker)."""
text = (text or "").strip()
head, tail = AGENT_MESSAGE_HEAD_CHARS, AGENT_MESSAGE_TAIL_CHARS
if len(text) <= head + tail:
return text
return f"{text[:head].rstrip()}\n[… {len(text) - head - tail} characters left out …]\n{text[-tail:].lstrip()}"
def _agent_history_lines(history: list) -> list[str]:
"""The "agent" window: newest backwards, each turn clipped like the answering
agent's (the latest answer longer than older ones), until 24 messages or the
12,000-character budget runs out (the newest message always fits); oldest first."""
picked: list[str] = []
used = 0
seen_answer = False
for m in reversed(history):
if m["role"] == "assistant":
limit = AGENT_OLDER_ANSWER_CHARS if seen_answer else AGENT_LATEST_ANSWER_CHARS
seen_answer = True
else:
limit = AGENT_USER_TURN_CHARS
content = _one_line(m.get("content", ""), limit)
if picked and used + len(content) > AGENT_HISTORY_CHAR_BUDGET:
break
picked.append(f"{m['role']}: {content}")
used += len(content)
if len(picked) >= AGENT_HISTORY_MAX_MESSAGES:
break
picked.reverse()
return picked
def _type_name(param: dict) -> str:
if "enum" in param:
return "|".join(json.dumps(v) for v in param["enum"])
kind = param.get("type", "any")
if isinstance(kind, list):
kind = "|".join(k for k in kind if k != "null")
return {"string": "str", "number": "num", "integer": "int", "boolean": "bool"}.get(kind, kind)
def render_tool_line(tool: dict) -> str:
"""One catalogue line: name(arg: type, opt?: type=default): purpose."""
params = tool["parameters"]
required = set(params.get("required", []))
parts = []
for pname, spec in params.get("properties", {}).items():
piece = f"{pname}{'' if pname in required else '?'}: {_type_name(spec)}"
if "default" in spec and spec["default"] not in ("", None):
piece += f"={json.dumps(spec['default'])}"
parts.append(piece)
return f"- {tool['name']}({', '.join(parts)}): {tool['purpose']}"
def render_tool_catalogue(offered_tools) -> str:
"""The plannable offered tools, in tools.json order. Tools driven by another
plan field (web search, drug_lookup, generate_image, memory writes) are not
listed: the plan never names them. Unknown names are ignored."""
offered = set(offered_tools or ())
lines = [render_tool_line(t) for t in tool_catalogue().values() if t["plannable"] and t["name"] in offered]
return "\n".join(lines) if lines else "(none)"
def render_directive_list() -> str:
return "\n".join(f"- {d['id']}: {d['meaning']}" for d in directive_catalogue() if d["kind"] == "planner")
def render_system_prompt(offered_tools) -> str:
return (_prompt_template()
.replace("{directive_list}", render_directive_list())
.replace("{tool_catalogue}", render_tool_catalogue(offered_tools)))
def _yes(flag: bool) -> str:
return "yes" if flag else "no"
def render_context(example: dict, window: str = "trained") -> str:
"""The user message: the context block, then the message last (nothing after
it, so nothing in it can pass for our own frame). `window`: see WINDOWS."""
history = [m for m in example.get("history") or [] if m.get("role") in ("user", "assistant")]
if window == "agent":
lines = _agent_history_lines(history)
chat = "\n".join(lines) if lines else "none"
message = _clip_message_agent(example.get("message", ""))
else:
history = history[-MAX_HISTORY_MESSAGES:]
if history:
chat = "\n".join(
f"{m['role']}: {_one_line(m.get('content', ''), USER_TURN_CHARS if m['role'] == 'user' else ASSISTANT_TURN_CHARS)}"
for m in history
)
else:
chat = "none"
message = _clip_message(example.get("message", ""))
remembered = example.get("remembered_image")
if remembered is None or not str(remembered).strip():
remembered_line = "none"
elif remembered.strip().lower() == UPLOADED_IMAGE_MARKER:
remembered_line = "yes, an earlier upload (no description)"
else:
remembered_line = '"' + _one_line(remembered, REMEMBERED_IMAGE_CHARS).replace('"', "'") + '"'
return (
"RECENT CHAT\n"
f"{chat}\n"
"TURN\n"
f"image attached: {_yes(example.get('attached_image', False))}\n"
f"remembered image: {remembered_line}\n"
f"document attached: {_yes(example.get('document_attached', False))}\n"
f"memory: {'on' if example.get('memory_enabled', False) else 'off'}\n"
f"signed in: {_yes(example.get('signed_in', False))}\n"
f"api client: {_yes(example.get('api_client', False))}\n"
f"locale: {example.get('locale') or 'unknown'}\n"
"MESSAGE\n"
f"{message}"
)
def render_messages(example: dict, window: str = "trained") -> list[dict]:
"""The chat messages xevyo-01 reads for `example` (example_input.schema.json)."""
return [
{"role": "system", "content": render_system_prompt(example.get("offered_tools") or [])},
{"role": "user", "content": render_context(example, window)},
]
# ── target rendering ──────────────────────────────────────────────────────────
def _number(value: Any) -> Any:
"""70.0 -> 70, so the same plan always prints the same way."""
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return int(value)
return value
def _ordered(obj: dict, keys) -> dict:
out = {k: obj[k] for k in keys if k in obj}
# Unknown keys are kept at the end so validation can still report them.
out.update({k: v for k, v in obj.items() if k not in out})
return out
def _canonical_arguments(name: str, arguments: dict) -> dict:
tool = tool_catalogue().get(name)
props = (tool or {}).get("parameters", {}).get("properties", {})
order = list(props) + [k for k in arguments if k not in props]
out = {}
for key in order:
if key not in arguments:
continue
value = _number(arguments[key])
if key in props and "default" in props[key] and value == props[key]["default"] \
and key not in (tool or {}).get("parameters", {}).get("required", []):
continue
out[key] = value
return out
def canonicalize(plan: dict) -> dict:
"""`plan` in canonical form: schema_version forced first, keys in PLAN_KEYS order,
nested keys ordered, defaults (and nulls) dropped, directives in catalogue order,
integral floats as ints, tool arguments equal to their defaults dropped.
Content is never rewritten (no case changes, no trimming of text): a plan that
is wrong stays wrong here and validate.py says why."""
out: dict[str, Any] = {"schema_version": plan.get("schema_version", SCHEMA_VERSION)}
for key in PLAN_KEYS[1:]:
if key not in plan:
continue
value = plan[key]
if value is None:
continue
if key == "image" and isinstance(value, dict):
value = {k: v for k, v in _ordered(value, IMAGE_KEYS).items() if v is not None}
elif key == "drug" and isinstance(value, dict):
value = _ordered(value, DRUG_KEYS)
if value.get("wants_research") is False:
value.pop("wants_research")
elif key == "memory" and isinstance(value, dict):
value = {k: v for k, v in _ordered(value, MEMORY_KEYS).items() if v not in (None, "")}
elif key == "tools" and isinstance(value, list):
value = [
_ordered({**call, "arguments": _canonical_arguments(call.get("name", ""), call.get("arguments") or {})},
TOOL_CALL_KEYS) if isinstance(call, dict) else call
for call in value
]
elif key == "directives" and isinstance(value, list):
rank = {d: i for i, d in enumerate(d["id"] for d in directive_catalogue())}
value = sorted(dict.fromkeys(value), key=lambda d: rank.get(d, len(rank)))
if value == DEFAULTS.get(key, object()):
continue
out[key] = value
out.update({k: v for k, v in plan.items() if k not in out and k not in PLAN_KEYS})
return out
def fill_defaults(plan: dict) -> dict:
"""`plan` with every omitted field set to its default, for per-field scoring."""
full = {"schema_version": plan.get("schema_version", SCHEMA_VERSION)}
for key in PLAN_KEYS[1:]:
value = plan.get(key)
full[key] = json.loads(json.dumps(DEFAULTS[key])) if value is None else value
if isinstance(full["drug"], dict):
full["drug"] = {"wants_research": False, **full["drug"]}
return full
def render_target(plan: dict) -> str:
"""The training target: canonical plan as compact JSON, stable key order,
UTF-8 kept (Hindi stays Hindi)."""
return json.dumps(canonicalize(plan), ensure_ascii=False, separators=(",", ":"))
__all__ = [
"SCHEMA_VERSION", "PLAN_KEYS", "DEFAULTS",
"render_messages", "render_context", "render_system_prompt", "render_tool_catalogue",
"render_directive_list", "render_target", "canonicalize", "fill_defaults",
"tool_catalogue", "directive_catalogue", "planner_directive_ids",
]