File size: 5,510 Bytes
a2784ae 435b6b8 a2784ae 435b6b8 a2784ae 435b6b8 a2784ae 435b6b8 e18e763 435b6b8 e18e763 435b6b8 e18e763 a2784ae 435b6b8 e18e763 a2784ae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | #!/usr/bin/env python3
"""Verify the Sharp template: minimal diff from upstream, correct rendering, minify round-trip.
python3 scripts/verify_template.py
Run this before publishing, and after any edit to chat_template.jinja. It fetches current
upstream (v22.1) fresh, so it also catches the case where froggeric ships a new version and our
splice silently sits on top of a template we haven't looked at.
The checks exist because each of them has a real failure mode:
* diff-vs-upstream -- guards against accidentally shipping edits we didn't intend to make
* terseness-once -- a double-splice would send the instruction twice and waste context
* system-preserved -- the else-branch is the whole reason a user's own prompt survives
* think-retained -- froggeric's retention fix is the thing we must NOT have broken
* minify round-trip -- the minifier collapses newlines; our {% set %} body must survive it
"""
from __future__ import annotations
import difflib
import pathlib
import sys
import urllib.request
from jinja2 import Environment
HERE = pathlib.Path(__file__).resolve().parent.parent
UPSTREAM = ("https://huggingface.co/froggeric/Qwen-Fixed-Chat-Templates/"
"resolve/main/chat_template.jinja")
MARKER = "Never: open with preamble"
fails: list[str] = []
def check(ok: bool, label: str) -> None:
print(f" {'PASS' if ok else 'FAIL'} {label}")
if not ok:
fails.append(label)
def render(src: str, msgs: list[dict], **kw) -> str:
return Environment().from_string(src).render(
messages=msgs, add_generation_prompt=True, **kw)
def main() -> int:
full = (HERE / "chat_template.jinja").read_text()
mini = (HERE / "chat_template_oneline.txt").read_text()
print("=== diff vs upstream v22.1")
with urllib.request.urlopen(UPSTREAM, timeout=60) as r:
up = r.read().decode()
diff = [l for l in difflib.unified_diff(up.splitlines(), full.splitlines(), lineterm="")
if l.startswith(("+", "-")) and not l.startswith(("+++", "---"))]
check(all(l.startswith("+") for l in diff), "only insertions, no upstream lines changed")
check(len(diff) == 11, f"exactly 11 inserted lines (got {len(diff)})")
check('template_version = "qwen3.8-froggeric-v22.1"' in full, "upstream version is v22.1")
check("Nail" not in full and "Dagger" not in full, "no model-specific identity in template")
print("\n=== rendering")
cases = {
"no system prompt": [{"role": "user", "content": "hi"}],
"with system prompt": [{"role": "system", "content": "Be a pirate."},
{"role": "user", "content": "hi"}],
"multi-turn w/ think": [{"role": "user", "content": "Q1"},
{"role": "assistant", "content": "<think>t</think>A1"},
{"role": "user", "content": "Q2"}],
}
for name, msgs in cases.items():
out = render(full, msgs)
check(out.count(MARKER) == 1, f"{name}: terseness appears exactly once")
check("Be a pirate." in render(full, cases["with system prompt"]),
"user's own system prompt is preserved")
check("<think>t</think>" in render(full, cases["multi-turn w/ think"]),
"prior thinking is retained across turns")
# v22.1 defaults reasoning_effort to MEDIUM, which injects no steering line -- so out-of-the-box
# behavior is terseness-only with no forced effort, matching tuned v1, WITHOUT us suppressing
# anything (upstream fixed the old forced-xhigh default). An EXPLICIT effort still injects steering,
# so froggeric's feature is honored as opt-in.
STEER = "Reasoning effort is set to"
check(STEER not in render(full, cases["no system prompt"]),
"default (no reasoning_effort): no steering line (upstream medium default)")
check(STEER not in render(full, cases["with system prompt"]),
"default with system prompt: still no steering line")
check("Reasoning effort is set to low" in render(full, cases["no system prompt"],
reasoning_effort="low"), "explicit reasoning_effort=low is still honored (opt-in)")
check(MARKER in render(full, cases["no system prompt"], reasoning_effort="low"),
"terseness still present when an explicit effort is requested")
tools = [{"type": "function", "function": {"name": "get_weather", "description": "w",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}]
out = render(full, [{"role": "user", "content": "weather?"}], tools=tools)
check("get_weather" in out and out.count(MARKER) == 1, "tool definitions still render")
print("\n=== minified round-trip")
for name, msgs in cases.items():
check(render(full, msgs) == render(mini, msgs), f"{name}: oneline == full")
sysblk = render(mini, cases["no system prompt"]).split("<|im_start|>system")[1] \
.split("<|im_end|>")[0].strip()
# the terseness tail starts with "Answer directly," (at medium default no steering precedes it);
# isolate it and confirm ITS newlines survived minification as 4 lines.
terse = "Answer directly," + sysblk.split("Answer directly,", 1)[1]
check(len(terse.splitlines()) == 4, "terseness survives minification as 4 lines")
print(f"\n{'ALL CHECKS PASSED' if not fails else str(len(fails)) + ' FAILED: ' + '; '.join(fails)}")
return 1 if fails else 0
if __name__ == "__main__":
sys.exit(main())
|