scripts/verify_template.py

#!/usr/bin/env python3
"""Verify the Sharp template: minimal diff from upstream, correct rendering, minify round-trip.

    python3 scripts/verify_template.py

Run this before publishing, and after any edit to chat_template.jinja. It fetches current
upstream (v22.1) fresh, so it also catches the case where froggeric ships a new version and our
splice silently sits on top of a template we haven't looked at.

The checks exist because each of them has a real failure mode:
  * diff-vs-upstream  -- guards against accidentally shipping edits we didn't intend to make
  * terseness-once    -- a double-splice would send the instruction twice and waste context
  * system-preserved  -- the else-branch is the whole reason a user's own prompt survives
  * think-retained    -- froggeric's retention fix is the thing we must NOT have broken
  * minify round-trip -- the minifier collapses newlines; our {% set %} body must survive it
"""
from __future__ import annotations

import difflib
import pathlib
import sys
import urllib.request

from jinja2 import Environment

HERE = pathlib.Path(__file__).resolve().parent.parent
UPSTREAM = ("https://huggingface.co/froggeric/Qwen-Fixed-Chat-Templates/"
            "resolve/main/chat_template.jinja")
MARKER = "Never: open with preamble"

fails: list[str] = []


def check(ok: bool, label: str) -> None:
    print(f"  {'PASS' if ok else 'FAIL'}  {label}")
    if not ok:
        fails.append(label)


def render(src: str, msgs: list[dict], **kw) -> str:
    return Environment().from_string(src).render(
        messages=msgs, add_generation_prompt=True, **kw)


def main() -> int:
    full = (HERE / "chat_template.jinja").read_text()
    mini = (HERE / "chat_template_oneline.txt").read_text()

    print("=== diff vs upstream v22.1")
    with urllib.request.urlopen(UPSTREAM, timeout=60) as r:
        up = r.read().decode()
    diff = [l for l in difflib.unified_diff(up.splitlines(), full.splitlines(), lineterm="")
            if l.startswith(("+", "-")) and not l.startswith(("+++", "---"))]
    check(all(l.startswith("+") for l in diff), "only insertions, no upstream lines changed")
    check(len(diff) == 11, f"exactly 11 inserted lines (got {len(diff)})")
    check('template_version = "qwen3.8-froggeric-v22.1"' in full, "upstream version is v22.1")
    check("Nail" not in full and "Dagger" not in full, "no model-specific identity in template")

    print("\n=== rendering")
    cases = {
        "no system prompt": [{"role": "user", "content": "hi"}],
        "with system prompt": [{"role": "system", "content": "Be a pirate."},
                               {"role": "user", "content": "hi"}],
        "multi-turn w/ think": [{"role": "user", "content": "Q1"},
                                {"role": "assistant", "content": "<think>t</think>A1"},
                                {"role": "user", "content": "Q2"}],
    }
    for name, msgs in cases.items():
        out = render(full, msgs)
        check(out.count(MARKER) == 1, f"{name}: terseness appears exactly once")
    check("Be a pirate." in render(full, cases["with system prompt"]),
          "user's own system prompt is preserved")
    check("<think>t</think>" in render(full, cases["multi-turn w/ think"]),
          "prior thinking is retained across turns")

    # v22.1 defaults reasoning_effort to MEDIUM, which injects no steering line -- so out-of-the-box
    # behavior is terseness-only with no forced effort, matching tuned v1, WITHOUT us suppressing
    # anything (upstream fixed the old forced-xhigh default). An EXPLICIT effort still injects steering,
    # so froggeric's feature is honored as opt-in.
    STEER = "Reasoning effort is set to"
    check(STEER not in render(full, cases["no system prompt"]),
          "default (no reasoning_effort): no steering line (upstream medium default)")
    check(STEER not in render(full, cases["with system prompt"]),
          "default with system prompt: still no steering line")
    check("Reasoning effort is set to low" in render(full, cases["no system prompt"],
          reasoning_effort="low"), "explicit reasoning_effort=low is still honored (opt-in)")
    check(MARKER in render(full, cases["no system prompt"], reasoning_effort="low"),
          "terseness still present when an explicit effort is requested")

    tools = [{"type": "function", "function": {"name": "get_weather", "description": "w",
              "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}]
    out = render(full, [{"role": "user", "content": "weather?"}], tools=tools)
    check("get_weather" in out and out.count(MARKER) == 1, "tool definitions still render")

    print("\n=== minified round-trip")
    for name, msgs in cases.items():
        check(render(full, msgs) == render(mini, msgs), f"{name}: oneline == full")
    sysblk = render(mini, cases["no system prompt"]).split("<|im_start|>system")[1] \
                                                    .split("<|im_end|>")[0].strip()
    # the terseness tail starts with "Answer directly," (at medium default no steering precedes it);
    # isolate it and confirm ITS newlines survived minification as 4 lines.
    terse = "Answer directly," + sysblk.split("Answer directly,", 1)[1]
    check(len(terse.splitlines()) == 4, "terseness survives minification as 4 lines")

    print(f"\n{'ALL CHECKS PASSED' if not fails else str(len(fails)) + ' FAILED: ' + '; '.join(fails)}")
    return 1 if fails else 0


if __name__ == "__main__":
    sys.exit(main())