File size: 14,930 Bytes
a46f204
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
"""Render xevyo-01 inputs and targets (spec xv01.1).

    render_messages(example) -> [{"role": "system", ...}, {"role": "user", ...}]
    render_target(plan)      -> canonical compact JSON string (the training target)
    canonicalize(plan)       -> the same plan as a dict: defaults dropped, keys ordered

`example` follows example_input.schema.json; `plan` follows turn_plan.schema.json.
Standard library only, so the data, training, eval and serving code can all import it.
Chat template: the caller applies Qwen's template with thinking OFF
(tokenizer.apply_chat_template(messages, add_generation_prompt=True, enable_thinking=False)).

Clipping follows the deciders this replaces: the image decider reads 4 turns of 200
chars, the follow-up rewrite 6 messages of 1500 chars, request_intent 6-8 turns of 400 chars. We keep the last 8 messages (4 exchanges, enough for a
two-hop follow-up) and clip each; a long current message keeps its head and its tail,
where a question after a pasted document sits (data_tool_intent.scan_window).
"""
from __future__ import annotations

import json
import math
from functools import lru_cache
from pathlib import Path
from typing import Any

SPEC_DIR = Path(__file__).resolve().parent
SCHEMA_VERSION = "xv01.1"

MAX_HISTORY_MESSAGES = 8
USER_TURN_CHARS = 500
ASSISTANT_TURN_CHARS = 600
MESSAGE_HEAD_CHARS = 1500
MESSAGE_TAIL_CHARS = 500
REMEMBERED_IMAGE_CHARS = 120
UPLOADED_IMAGE_MARKER = "the image you uploaded"

#: Context windows (settings.XEVYO01_CONTEXT). "trained" is the window above, the one
#: xevyo-01 was trained on and the golden fixtures pin; it stays the default and its
#: output is unchanged byte for byte. "agent" shows the planner the earlier messages
#: an answering agent is shown (newest first, up to 24
#: messages within 12,000 characters, user turns clipped at 2,000, the latest answer at
#: 6,000, older answers at 1,200) and more of a long new message. Same format, longer:
#: the model never saw inputs this long in training, so measure it before relying on it.
WINDOWS = ("trained", "agent")
AGENT_HISTORY_MAX_MESSAGES = 24
AGENT_HISTORY_CHAR_BUDGET = 12_000
AGENT_USER_TURN_CHARS = 2_000
AGENT_LATEST_ANSWER_CHARS = 6_000
AGENT_OLDER_ANSWER_CHARS = 1_200
AGENT_MESSAGE_HEAD_CHARS = 6_000
AGENT_MESSAGE_TAIL_CHARS = 2_000

#: Canonical key order of a plan (most consequential decisions first).
PLAN_KEYS = (
    "schema_version", "safety_concern", "image", "request_kind", "doc_format", "clarify",
    "standalone_question", "needs_search", "search_queries", "data_tool", "drug", "memory",
    "domain", "depth", "needs_tools", "tools", "directives",
)
IMAGE_KEYS = ("action", "prompt", "visible_text", "instruction")
DRUG_KEYS = ("names", "wants_research")
MEMORY_KEYS = ("action", "memory_type", "content")
TOOL_CALL_KEYS = ("name", "arguments")

#: Field defaults. A field holding its default is left out of the canonical plan.
DEFAULTS: dict[str, Any] = {
    "safety_concern": "none",
    "image": {"action": "none"},
    "request_kind": "chat",
    "doc_format": None,
    "clarify": None,
    "standalone_question": None,
    "needs_search": False,
    "search_queries": [],
    "data_tool": "none",
    "drug": None,
    "memory": {"action": "none"},
    "domain": "general",
    "depth": "normal",
    "needs_tools": False,
    "tools": [],
    "directives": [],
}

_PROMPT_START = "<!-- PROMPT START -->"
_PROMPT_END = "<!-- PROMPT END -->"


# ── spec files ────────────────────────────────────────────────────────────────

@lru_cache(maxsize=None)
def _load(name: str) -> Any:
    return json.loads((SPEC_DIR / name).read_text(encoding="utf-8"))


def tool_catalogue() -> dict[str, dict]:
    """tools.json entries by name."""
    return {t["name"]: t for t in _load("tools.json")["tools"]}


def directive_catalogue() -> list[dict]:
    """directives.json entries in catalogue order."""
    return list(_load("directives.json")["directives"])


def planner_directive_ids() -> list[str]:
    """The ids the planner may emit, in canonical order."""
    return [d["id"] for d in directive_catalogue() if d["kind"] == "planner"]


@lru_cache(maxsize=None)
def _prompt_template() -> str:
    text = (SPEC_DIR / "system_prompt.md").read_text(encoding="utf-8")
    start, end = text.index(_PROMPT_START) + len(_PROMPT_START), text.index(_PROMPT_END)
    return text[start:end].strip()


# ── input rendering ───────────────────────────────────────────────────────────

def _one_line(text: str, limit: int) -> str:
    """Client text placed inside our own frame stays on its line and within size."""
    flat = " ".join((text or "").split())
    return flat if len(flat) <= limit else flat[: limit - 1].rstrip() + "…"


def _clip_message(text: str) -> str:
    text = (text or "").strip()
    if len(text) <= MESSAGE_HEAD_CHARS + MESSAGE_TAIL_CHARS:
        return text
    return f"{text[:MESSAGE_HEAD_CHARS].rstrip()}\n[… {len(text) - MESSAGE_HEAD_CHARS - MESSAGE_TAIL_CHARS} characters left out …]\n{text[-MESSAGE_TAIL_CHARS:].lstrip()}"


def _clip_message_agent(text: str) -> str:
    """_clip_message with the "agent" window's longer head and tail (same marker)."""
    text = (text or "").strip()
    head, tail = AGENT_MESSAGE_HEAD_CHARS, AGENT_MESSAGE_TAIL_CHARS
    if len(text) <= head + tail:
        return text
    return f"{text[:head].rstrip()}\n[… {len(text) - head - tail} characters left out …]\n{text[-tail:].lstrip()}"


def _agent_history_lines(history: list) -> list[str]:
    """The "agent" window: newest backwards, each turn clipped like the answering
    agent's (the latest answer longer than older ones), until 24 messages or the
    12,000-character budget runs out (the newest message always fits); oldest first."""
    picked: list[str] = []
    used = 0
    seen_answer = False
    for m in reversed(history):
        if m["role"] == "assistant":
            limit = AGENT_OLDER_ANSWER_CHARS if seen_answer else AGENT_LATEST_ANSWER_CHARS
            seen_answer = True
        else:
            limit = AGENT_USER_TURN_CHARS
        content = _one_line(m.get("content", ""), limit)
        if picked and used + len(content) > AGENT_HISTORY_CHAR_BUDGET:
            break
        picked.append(f"{m['role']}: {content}")
        used += len(content)
        if len(picked) >= AGENT_HISTORY_MAX_MESSAGES:
            break
    picked.reverse()
    return picked


def _type_name(param: dict) -> str:
    if "enum" in param:
        return "|".join(json.dumps(v) for v in param["enum"])
    kind = param.get("type", "any")
    if isinstance(kind, list):
        kind = "|".join(k for k in kind if k != "null")
    return {"string": "str", "number": "num", "integer": "int", "boolean": "bool"}.get(kind, kind)


def render_tool_line(tool: dict) -> str:
    """One catalogue line: name(arg: type, opt?: type=default): purpose."""
    params = tool["parameters"]
    required = set(params.get("required", []))
    parts = []
    for pname, spec in params.get("properties", {}).items():
        piece = f"{pname}{'' if pname in required else '?'}: {_type_name(spec)}"
        if "default" in spec and spec["default"] not in ("", None):
            piece += f"={json.dumps(spec['default'])}"
        parts.append(piece)
    return f"- {tool['name']}({', '.join(parts)}): {tool['purpose']}"


def render_tool_catalogue(offered_tools) -> str:
    """The plannable offered tools, in tools.json order. Tools driven by another
    plan field (web search, drug_lookup, generate_image, memory writes) are not
    listed: the plan never names them. Unknown names are ignored."""
    offered = set(offered_tools or ())
    lines = [render_tool_line(t) for t in tool_catalogue().values() if t["plannable"] and t["name"] in offered]
    return "\n".join(lines) if lines else "(none)"


def render_directive_list() -> str:
    return "\n".join(f"- {d['id']}: {d['meaning']}" for d in directive_catalogue() if d["kind"] == "planner")


def render_system_prompt(offered_tools) -> str:
    return (_prompt_template()
            .replace("{directive_list}", render_directive_list())
            .replace("{tool_catalogue}", render_tool_catalogue(offered_tools)))


def _yes(flag: bool) -> str:
    return "yes" if flag else "no"


def render_context(example: dict, window: str = "trained") -> str:
    """The user message: the context block, then the message last (nothing after
    it, so nothing in it can pass for our own frame). `window`: see WINDOWS."""
    history = [m for m in example.get("history") or [] if m.get("role") in ("user", "assistant")]
    if window == "agent":
        lines = _agent_history_lines(history)
        chat = "\n".join(lines) if lines else "none"
        message = _clip_message_agent(example.get("message", ""))
    else:
        history = history[-MAX_HISTORY_MESSAGES:]
        if history:
            chat = "\n".join(
                f"{m['role']}: {_one_line(m.get('content', ''), USER_TURN_CHARS if m['role'] == 'user' else ASSISTANT_TURN_CHARS)}"
                for m in history
            )
        else:
            chat = "none"
        message = _clip_message(example.get("message", ""))
    remembered = example.get("remembered_image")
    if remembered is None or not str(remembered).strip():
        remembered_line = "none"
    elif remembered.strip().lower() == UPLOADED_IMAGE_MARKER:
        remembered_line = "yes, an earlier upload (no description)"
    else:
        remembered_line = '"' + _one_line(remembered, REMEMBERED_IMAGE_CHARS).replace('"', "'") + '"'
    return (
        "RECENT CHAT\n"
        f"{chat}\n"
        "TURN\n"
        f"image attached: {_yes(example.get('attached_image', False))}\n"
        f"remembered image: {remembered_line}\n"
        f"document attached: {_yes(example.get('document_attached', False))}\n"
        f"memory: {'on' if example.get('memory_enabled', False) else 'off'}\n"
        f"signed in: {_yes(example.get('signed_in', False))}\n"
        f"api client: {_yes(example.get('api_client', False))}\n"
        f"locale: {example.get('locale') or 'unknown'}\n"
        "MESSAGE\n"
        f"{message}"
    )


def render_messages(example: dict, window: str = "trained") -> list[dict]:
    """The chat messages xevyo-01 reads for `example` (example_input.schema.json)."""
    return [
        {"role": "system", "content": render_system_prompt(example.get("offered_tools") or [])},
        {"role": "user", "content": render_context(example, window)},
    ]


# ── target rendering ──────────────────────────────────────────────────────────

def _number(value: Any) -> Any:
    """70.0 -> 70, so the same plan always prints the same way."""
    if isinstance(value, float) and math.isfinite(value) and value.is_integer():
        return int(value)
    return value


def _ordered(obj: dict, keys) -> dict:
    out = {k: obj[k] for k in keys if k in obj}
    # Unknown keys are kept at the end so validation can still report them.
    out.update({k: v for k, v in obj.items() if k not in out})
    return out


def _canonical_arguments(name: str, arguments: dict) -> dict:
    tool = tool_catalogue().get(name)
    props = (tool or {}).get("parameters", {}).get("properties", {})
    order = list(props) + [k for k in arguments if k not in props]
    out = {}
    for key in order:
        if key not in arguments:
            continue
        value = _number(arguments[key])
        if key in props and "default" in props[key] and value == props[key]["default"] \
                and key not in (tool or {}).get("parameters", {}).get("required", []):
            continue
        out[key] = value
    return out


def canonicalize(plan: dict) -> dict:
    """`plan` in canonical form: schema_version forced first, keys in PLAN_KEYS order,
    nested keys ordered, defaults (and nulls) dropped, directives in catalogue order,
    integral floats as ints, tool arguments equal to their defaults dropped.

    Content is never rewritten (no case changes, no trimming of text): a plan that
    is wrong stays wrong here and validate.py says why."""
    out: dict[str, Any] = {"schema_version": plan.get("schema_version", SCHEMA_VERSION)}
    for key in PLAN_KEYS[1:]:
        if key not in plan:
            continue
        value = plan[key]
        if value is None:
            continue
        if key == "image" and isinstance(value, dict):
            value = {k: v for k, v in _ordered(value, IMAGE_KEYS).items() if v is not None}
        elif key == "drug" and isinstance(value, dict):
            value = _ordered(value, DRUG_KEYS)
            if value.get("wants_research") is False:
                value.pop("wants_research")
        elif key == "memory" and isinstance(value, dict):
            value = {k: v for k, v in _ordered(value, MEMORY_KEYS).items() if v not in (None, "")}
        elif key == "tools" and isinstance(value, list):
            value = [
                _ordered({**call, "arguments": _canonical_arguments(call.get("name", ""), call.get("arguments") or {})},
                         TOOL_CALL_KEYS) if isinstance(call, dict) else call
                for call in value
            ]
        elif key == "directives" and isinstance(value, list):
            rank = {d: i for i, d in enumerate(d["id"] for d in directive_catalogue())}
            value = sorted(dict.fromkeys(value), key=lambda d: rank.get(d, len(rank)))
        if value == DEFAULTS.get(key, object()):
            continue
        out[key] = value
    out.update({k: v for k, v in plan.items() if k not in out and k not in PLAN_KEYS})
    return out


def fill_defaults(plan: dict) -> dict:
    """`plan` with every omitted field set to its default, for per-field scoring."""
    full = {"schema_version": plan.get("schema_version", SCHEMA_VERSION)}
    for key in PLAN_KEYS[1:]:
        value = plan.get(key)
        full[key] = json.loads(json.dumps(DEFAULTS[key])) if value is None else value
    if isinstance(full["drug"], dict):
        full["drug"] = {"wants_research": False, **full["drug"]}
    return full


def render_target(plan: dict) -> str:
    """The training target: canonical plan as compact JSON, stable key order,
    UTF-8 kept (Hindi stays Hindi)."""
    return json.dumps(canonicalize(plan), ensure_ascii=False, separators=(",", ":"))


__all__ = [
    "SCHEMA_VERSION", "PLAN_KEYS", "DEFAULTS",
    "render_messages", "render_context", "render_system_prompt", "render_tool_catalogue",
    "render_directive_list", "render_target", "canonicalize", "fill_defaults",
    "tool_catalogue", "directive_catalogue", "planner_directive_ids",
]