Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Advanced Usage and Customizing Your Chat Templates","local":"advanced-usage-and-customizing-your-chat-templates","sections":[{"title":"How do chat templates work?","local":"how-do-chat-templates-work","sections":[],"depth":2},{"title":"How do I create a chat template?","local":"how-do-i-create-a-chat-template","sections":[],"depth":2},{"title":"Why do some models have multiple templates?","local":"why-do-some-models-have-multiple-templates","sections":[],"depth":2},{"title":"What template should I use?","local":"what-template-should-i-use","sections":[],"depth":2},{"title":"I want to add some chat templates! How should I get started?","local":"i-want-to-add-some-chat-templates-how-should-i-get-started","sections":[{"title":"Trimming whitespace","local":"trimming-whitespace","sections":[],"depth":3},{"title":"Special variables","local":"special-variables","sections":[],"depth":3},{"title":"Callable functions","local":"callable-functions","sections":[],"depth":3},{"title":"Compatibility with non-Python Jinja","local":"compatibility-with-non-python-jinja","sections":[],"depth":3},{"title":"Writing generation prompts","local":"writing-generation-prompts","sections":[],"depth":3},{"title":"Writing and debugging larger templates","local":"writing-and-debugging-larger-templates","sections":[],"depth":3}],"depth":2},{"title":"Writing templates for tools","local":"writing-templates-for-tools","sections":[{"title":"Tool definitions","local":"tool-definitions","sections":[],"depth":3},{"title":"Tool calls","local":"tool-calls","sections":[],"depth":3},{"title":"Tool responses","local":"tool-responses","sections":[],"depth":3}],"depth":2}],"depth":1}"> | |
| <link href="/docs/transformers/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/entry/start.8cba93b8.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/scheduler.bdbef820.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/singletons.6587ca2b.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/index.8a885b74.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/paths.2c06013a.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/entry/app.00a1188c.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/index.33f81d56.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/nodes/0.876f3f60.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/nodes/11.aa0b8f13.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/Tip.34194030.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/CodeBlock.362b34a4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/en/_app/immutable/chunks/EditOnGithub.a9246e21.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Advanced Usage and Customizing Your Chat Templates","local":"advanced-usage-and-customizing-your-chat-templates","sections":[{"title":"How do chat templates work?","local":"how-do-chat-templates-work","sections":[],"depth":2},{"title":"How do I create a chat template?","local":"how-do-i-create-a-chat-template","sections":[],"depth":2},{"title":"Why do some models have multiple templates?","local":"why-do-some-models-have-multiple-templates","sections":[],"depth":2},{"title":"What template should I use?","local":"what-template-should-i-use","sections":[],"depth":2},{"title":"I want to add some chat templates! How should I get started?","local":"i-want-to-add-some-chat-templates-how-should-i-get-started","sections":[{"title":"Trimming whitespace","local":"trimming-whitespace","sections":[],"depth":3},{"title":"Special variables","local":"special-variables","sections":[],"depth":3},{"title":"Callable functions","local":"callable-functions","sections":[],"depth":3},{"title":"Compatibility with non-Python Jinja","local":"compatibility-with-non-python-jinja","sections":[],"depth":3},{"title":"Writing generation prompts","local":"writing-generation-prompts","sections":[],"depth":3},{"title":"Writing and debugging larger templates","local":"writing-and-debugging-larger-templates","sections":[],"depth":3}],"depth":2},{"title":"Writing templates for tools","local":"writing-templates-for-tools","sections":[{"title":"Tool definitions","local":"tool-definitions","sections":[],"depth":3},{"title":"Tool calls","local":"tool-calls","sections":[],"depth":3},{"title":"Tool responses","local":"tool-responses","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="advanced-usage-and-customizing-your-chat-templates" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#advanced-usage-and-customizing-your-chat-templates"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Advanced Usage and Customizing Your Chat Templates</span></h1> <p data-svelte-h="svelte-mj86xb">In this page, we’ll explore more advanced techniques for working with chat templates in Transformers. Whether you’re looking to write your own templates, create custom components, or optimize your templates for efficiency, we’ll cover everything you need to take your templates to the next level. Let’s dive into the tools and strategies that will help you get the most out of your chat models.</p> <h2 class="relative group"><a id="how-do-chat-templates-work" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-do-chat-templates-work"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How do chat templates work?</span></h2> <p data-svelte-h="svelte-1czh8av">The chat template for a model is stored on the <code>tokenizer.chat_template</code> attribute. Let’s take a look at a <code>Zephyr</code> chat template, though note this | |
| one is a little simplified from the actual one!</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="language-xml">{%- for message in messages %} | |
| </span><span class="hljs-template-variable">{{<span class="hljs-name">-</span> <span class="hljs-string">'<|'</span> + message['role'] + <span class="hljs-string">'|>\n'</span> }}</span><span class="language-xml"> | |
| </span><span class="hljs-template-variable">{{<span class="hljs-name">-</span> message['content'] + eos_token }}</span><span class="language-xml"> | |
| {%- endfor %} | |
| {%- if add_generation_prompt %} | |
| </span><span class="hljs-template-variable">{{<span class="hljs-name">-</span> <span class="hljs-string">'<|assistant|>\n'</span> }}</span><span class="language-xml"> | |
| {%- endif %}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-zet1qo">If you’ve never seen one of these before, this is a <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/" rel="nofollow">Jinja template</a>. | |
| Jinja is a templating language that allows you to write simple code that generates text. In many ways, the code and | |
| syntax resembles Python. In pure Python, this template would look something like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">for</span> message <span class="hljs-keyword">in</span> messages: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f'<|<span class="hljs-subst">{message[<span class="hljs-string">"role"</span>]}</span>|>'</span>) | |
| <span class="hljs-built_in">print</span>(message[<span class="hljs-string">'content'</span>] + eos_token) | |
| <span class="hljs-keyword">if</span> add_generation_prompt: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">'<|assistant|>'</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-9bdwn1">Effectively, the template does three things:</p> <ol data-svelte-h="svelte-1ax9ke1"><li>For each message, print the role enclosed in <code><|</code> and <code>|></code>, like <code><|user|></code> or <code><|assistant|></code>.</li> <li>Next, print the content of the message, followed by the end-of-sequence token.</li> <li>Finally, if <code>add_generation_prompt</code> is set, print the assistant token, so that the model knows to start generating | |
| an assistant response.</li></ol> <p data-svelte-h="svelte-4zn2hx">This is a pretty simple template but Jinja gives you a lot of flexibility to do more complex things! Let’s see a Jinja | |
| template that can format inputs similarly to the way LLaMA formats them (note that the real LLaMA template includes | |
| handling for default system messages and slightly different system message handling in general - don’t use this one | |
| in your actual code!)</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{%- <span class="hljs-keyword">for</span> message <span class="hljs-keyword">in</span> messages %} | |
| {%- <span class="hljs-keyword">if</span> message[<span class="hljs-string">'role'</span>] == <span class="hljs-string">'user'</span> %} | |
| {{- bos_token + <span class="hljs-string">'[INST] '</span> + message[<span class="hljs-string">'content'</span>] + <span class="hljs-string">' [/INST]'</span> }} | |
| {%- elif message[<span class="hljs-string">'role'</span>] == <span class="hljs-string">'system'</span> %} | |
| {{- <span class="hljs-string">'<<SYS>>\\n'</span> + message[<span class="hljs-string">'content'</span>] + <span class="hljs-string">'\\n<</SYS>>\\n\\n'</span> }} | |
| {%- elif message[<span class="hljs-string">'role'</span>] == <span class="hljs-string">'assistant'</span> %} | |
| {{- <span class="hljs-string">' '</span> + message[<span class="hljs-string">'content'</span>] + <span class="hljs-string">' '</span> + eos_token }} | |
| {%- endif %} | |
| {%- endfor %}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-prz4we">Hopefully if you stare at this for a little bit you can see what this template is doing - it adds specific tokens like | |
| <code>[INST]</code> and <code>[/INST]</code> based on the role of each message. User, assistant and system messages are clearly | |
| distinguishable to the model because of the tokens they’re wrapped in.</p> <h2 class="relative group"><a id="how-do-i-create-a-chat-template" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-do-i-create-a-chat-template"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How do I create a chat template?</span></h2> <p data-svelte-h="svelte-1ubxgh9">Simple, just write a jinja template and set <code>tokenizer.chat_template</code>. You may find it easier to start with an | |
| existing template from another model and simply edit it for your needs! For example, we could take the LLaMA template | |
| above and add ”[ASST]” and ”[/ASST]” to assistant messages:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{%- <span class="hljs-keyword">for</span> message <span class="hljs-keyword">in</span> messages %} | |
| {%- <span class="hljs-keyword">if</span> message[<span class="hljs-string">'role'</span>] == <span class="hljs-string">'user'</span> %} | |
| {{- bos_token + <span class="hljs-string">'[INST] '</span> + message[<span class="hljs-string">'content'</span>].<span class="hljs-keyword">strip</span>() + <span class="hljs-string">' [/INST]'</span> }} | |
| {%- elif message[<span class="hljs-string">'role'</span>] == <span class="hljs-string">'system'</span> %} | |
| {{- <span class="hljs-string">'<<SYS>>\\n'</span> + message[<span class="hljs-string">'content'</span>].<span class="hljs-keyword">strip</span>() + <span class="hljs-string">'\\n<</SYS>>\\n\\n'</span> }} | |
| {%- elif message[<span class="hljs-string">'role'</span>] == <span class="hljs-string">'assistant'</span> %} | |
| {{- <span class="hljs-string">'[ASST] '</span> + message[<span class="hljs-string">'content'</span>] + <span class="hljs-string">' [/ASST]'</span> + eos_token }} | |
| {%- endif %} | |
| {%- endfor %}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-t74z56">Now, simply set the <code>tokenizer.chat_template</code> attribute. Next time you use <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template">apply_chat_template()</a>, it will | |
| use your new template! This attribute will be saved in the <code>tokenizer_config.json</code> file, so you can use | |
| <a href="/docs/transformers/main/en/main_classes/model#transformers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> to upload your new template to the Hub and make sure everyone’s using the right | |
| template for your model!</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->template = tokenizer.chat_template | |
| template = template.replace(<span class="hljs-string">"SYS"</span>, <span class="hljs-string">"SYSTEM"</span>) <span class="hljs-comment"># Change the system token</span> | |
| tokenizer.chat_template = template <span class="hljs-comment"># Set the new template</span> | |
| tokenizer.push_to_hub(<span class="hljs-string">"model_name"</span>) <span class="hljs-comment"># Upload your new template to the Hub!</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-pl6d88">The method <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template">apply_chat_template()</a> which uses your chat template is called by the <a href="/docs/transformers/main/en/main_classes/pipelines#transformers.TextGenerationPipeline">TextGenerationPipeline</a> class, so | |
| once you set the correct chat template, your model will automatically become compatible with <a href="/docs/transformers/main/en/main_classes/pipelines#transformers.TextGenerationPipeline">TextGenerationPipeline</a>.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">If you're fine-tuning a model for chat, in addition to setting a chat template, you should probably add any new chat | |
| control tokens as special tokens in the tokenizer. Special tokens are never split, | |
| ensuring that your control tokens are always handled as single tokens rather than being tokenized in pieces. You | |
| should also set the tokenizer's `eos_token` attribute to the token that marks the end of assistant generations in your | |
| template. This will ensure that text generation tools can correctly figure out when to stop generating text.</div> <h2 class="relative group"><a id="why-do-some-models-have-multiple-templates" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-do-some-models-have-multiple-templates"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Why do some models have multiple templates?</span></h2> <p data-svelte-h="svelte-1d7cql4">Some models use different templates for different use cases. For example, they might use one template for normal chat | |
| and another for tool-use, or retrieval-augmented generation. In these cases, <code>tokenizer.chat_template</code> is a dictionary. | |
| This can cause some confusion, and where possible, we recommend using a single template for all use-cases. You can use | |
| Jinja statements like <code>if tools is defined</code> and <code>{% macro %}</code> definitions to easily wrap multiple code paths in a | |
| single template.</p> <p data-svelte-h="svelte-1u88h1j">When a tokenizer has multiple templates, <code>tokenizer.chat_template</code> will be a <code>dict</code>, where each key is the name | |
| of a template. The <code>apply_chat_template</code> method has special handling for certain template names: Specifically, it will | |
| look for a template named <code>default</code> in most cases, and will raise an error if it can’t find one. However, if a template | |
| named <code>tool_use</code> exists when the user has passed a <code>tools</code> argument, it will use that instead. To access templates | |
| with other names, pass the name of the template you want to the <code>chat_template</code> argument of | |
| <code>apply_chat_template()</code>.</p> <p data-svelte-h="svelte-1g7ri12">We find that this can be a bit confusing for users, though - so if you’re writing a template yourself, we recommend | |
| trying to put it all in a single template where possible!</p> <h2 class="relative group"><a id="what-template-should-i-use" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-template-should-i-use"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What template should I use?</span></h2> <p data-svelte-h="svelte-5u6sqi">When setting the template for a model that’s already been trained for chat, you should ensure that the template | |
| exactly matches the message formatting that the model saw during training, or else you will probably experience | |
| performance degradation. This is true even if you’re training the model further - you will probably get the best | |
| performance if you keep the chat tokens constant. This is very analogous to tokenization - you generally get the | |
| best performance for inference or fine-tuning when you precisely match the tokenization used during training.</p> <p data-svelte-h="svelte-ffreiw">If you’re training a model from scratch, or fine-tuning a base language model for chat, on the other hand, | |
| you have a lot of freedom to choose an appropriate template! LLMs are smart enough to learn to handle lots of different | |
| input formats. One popular choice is the <code>ChatML</code> format, and this is a good, flexible choice for many use-cases. | |
| It looks like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="language-xml">{%- for message in messages %} | |
| </span><span class="hljs-template-variable">{{<span class="hljs-name">-</span> <span class="hljs-string">'<|im_start|>'</span> + message['role'] + <span class="hljs-string">'\n'</span> + message['content'] + <span class="hljs-string">'<|im_end|>'</span> + <span class="hljs-string">'\n'</span> }}</span><span class="language-xml"> | |
| {%- endfor %}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1cj1ql7">If you like this one, here it is in one-liner form, ready to copy into your code. The one-liner also includes | |
| handy support for <a href="#what-are-generation-prompts">generation prompts</a>, but note that it doesn’t add BOS or EOS tokens! | |
| If your model expects those, they won’t be added automatically by <code>apply_chat_template</code> - in other words, the | |
| text will be tokenized with <code>add_special_tokens=False</code>. This is to avoid potential conflicts between the template and | |
| the <code>add_special_tokens</code> logic. If your model expects special tokens, make sure to add them to the template!</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->tokenizer.chat_template = <span class="hljs-string">"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-soh9qu">This template wraps each message in <code><|im_start|></code> and <code><|im_end|></code> tokens, and simply writes the role as a string, which | |
| allows for flexibility in the roles you train with. The output looks like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><|im_start|>system | |
| You are a helpful chatbot that will do its best not to say anything so stupid that people tweet about it.<|im_end|> | |
| <|im_start|>user | |
| How are you?<|im_end|> | |
| <|im_start|>assistant | |
| I'm doing great!<|im_end|><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-14p3dvw">The “user”, “system” and “assistant” roles are the standard for chat, and we recommend using them when it makes sense, | |
| particularly if you want your model to operate well with <a href="/docs/transformers/main/en/main_classes/pipelines#transformers.TextGenerationPipeline">TextGenerationPipeline</a>. However, you are not limited | |
| to these roles - templating is extremely flexible, and any string can be a role.</p> <h2 class="relative group"><a id="i-want-to-add-some-chat-templates-how-should-i-get-started" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#i-want-to-add-some-chat-templates-how-should-i-get-started"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>I want to add some chat templates! How should I get started?</span></h2> <p data-svelte-h="svelte-12av8vh">If you have any chat models, you should set their <code>tokenizer.chat_template</code> attribute and test it using | |
| <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template">apply_chat_template()</a>, then push the updated tokenizer to the Hub. This applies even if you’re | |
| not the model owner - if you’re using a model with an empty chat template, or one that’s still using the default class | |
| template, please open a <a href="https://huggingface.co/docs/hub/repositories-pull-requests-discussions" rel="nofollow">pull request</a> to the model repository so that this attribute can be set properly!</p> <p data-svelte-h="svelte-kn2i6o">Once the attribute is set, that’s it, you’re done! <code>tokenizer.apply_chat_template</code> will now work correctly for that | |
| model, which means it is also automatically supported in places like <code>TextGenerationPipeline</code>!</p> <p data-svelte-h="svelte-197jyne">By ensuring that models have this attribute, we can make sure that the whole community gets to use the full power of | |
| open-source models. Formatting mismatches have been haunting the field and silently harming performance for too long - | |
| it’s time to put an end to them!</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-qy60pj">The easiest way to get started with writing Jinja templates is to take a look at some existing ones. You can use | |
| <code>print(tokenizer.chat_template)</code> for any chat model to see what template it’s using. In general, models that support tool use have | |
| much more complex templates than other models - so when you’re just getting started, they’re probably a bad example | |
| to learn from! You can also take a look at the | |
| <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#synopsis" rel="nofollow">Jinja documentation</a> for details | |
| of general Jinja formatting and syntax.</p></div> <p data-svelte-h="svelte-1eu5v04">Jinja templates in <code>transformers</code> are identical to Jinja templates elsewhere. The main thing to know is that | |
| the conversation history will be accessible inside your template as a variable called <code>messages</code>.<br> | |
| You will be able to access <code>messages</code> in your template just like you can in Python, which means you can loop over | |
| it with <code>{% for message in messages %}</code> or access individual messages with <code>{{ messages[0] }}</code>, for example.</p> <p data-svelte-h="svelte-10wmjwo">You can also use the following tips to write clean, efficient Jinja templates:</p> <h3 class="relative group"><a id="trimming-whitespace" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#trimming-whitespace"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Trimming whitespace</span></h3> <p data-svelte-h="svelte-1ttgeg7">By default, Jinja will print any whitespace that comes before or after a block. This can be a problem for chat | |
| templates, which generally want to be very precise with whitespace! To avoid this, we strongly recommend writing | |
| your templates like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="language-xml">{%- for message in messages %} | |
| </span><span class="hljs-template-variable">{{<span class="hljs-name">-</span> message['role'] + message['content'] }}</span><span class="language-xml"> | |
| {%- endfor %}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qihux6">rather than like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-template-tag">{%</span> <span class="hljs-name">for</span> message <span class="hljs-keyword">in</span> messages <span class="hljs-template-tag">%}</span><span class="language-xml"> | |
| </span><span class="hljs-template-variable">{{ message[<span class="hljs-string">'role'</span>] + message[<span class="hljs-string">'content'</span>] }}</span><span class="language-xml"> | |
| </span><span class="hljs-template-tag">{%</span> <span class="hljs-name">endfor</span> <span class="hljs-template-tag">%}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-pefrh0">Adding <code>-</code> will strip any whitespace that comes before the block. The second example looks innocent, but the newline | |
| and indentation may end up being included in the output, which is probably not what you want!</p> <h3 class="relative group"><a id="special-variables" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#special-variables"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Special variables</span></h3> <p data-svelte-h="svelte-1s7e55r">Inside your template, you will have access several special variables. The most important of these is <code>messages</code>, | |
| which contains the chat history as a list of message dicts. However, there are several others. Not every | |
| variable will be used in every template. The most common other variables are:</p> <ul data-svelte-h="svelte-1rdzqgp"><li><code>tools</code> contains a list of tools in JSON schema format. Will be <code>None</code> or undefined if no tools are passed.</li> <li><code>documents</code> contains a list of documents in the format <code>{"title": "Title", "contents": "Contents"}</code>, used for retrieval-augmented generation. Will be <code>None</code> or undefined if no documents are passed.</li> <li><code>add_generation_prompt</code> is a bool that is <code>True</code> if the user has requested a generation prompt, and <code>False</code> otherwise. If this is set, your template should add the header for an assistant message to the end of the conversation. If your model doesn’t have a specific header for assistant messages, you can ignore this flag.</li> <li><strong>Special tokens</strong> like <code>bos_token</code> and <code>eos_token</code>. These are extracted from <code>tokenizer.special_tokens_map</code>. The exact tokens available inside each template will differ depending on the parent tokenizer.</li></ul> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-cvlh0x">You can actually pass any <code>kwarg</code> to <code>apply_chat_template</code>, and it will be accessible inside the template as a variable. In general, | |
| we recommend trying to stick to the core variables above, as it will make your model harder to use if users have | |
| to write custom code to pass model-specific <code>kwargs</code>. However, we’re aware that this field moves quickly, so if you | |
| have a new use-case that doesn’t fit in the core API, feel free to use a new <code>kwarg</code> for it! If a new <code>kwarg</code> | |
| becomes common we may promote it into the core API and create a standard, documented format for it.</p></div> <h3 class="relative group"><a id="callable-functions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#callable-functions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Callable functions</span></h3> <p data-svelte-h="svelte-w29vry">There is also a short list of callable functions available to you inside your templates. These are:</p> <ul data-svelte-h="svelte-1fmzhdy"><li><code>raise_exception(msg)</code>: Raises a <code>TemplateException</code>. This is useful for debugging, and for telling users when they’re | |
| doing something that your template doesn’t support.</li> <li><code>strftime_now(format_str)</code>: Equivalent to <code>datetime.now().strftime(format_str)</code> in Python. This is used for getting | |
| the current date/time in a specific format, which is sometimes included in system messages.</li></ul> <h3 class="relative group"><a id="compatibility-with-non-python-jinja" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compatibility-with-non-python-jinja"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compatibility with non-Python Jinja</span></h3> <p data-svelte-h="svelte-jvvtjt">There are multiple implementations of Jinja in various languages. They generally have the same syntax, | |
| but a key difference is that when you’re writing a template in Python you can use Python methods, such as | |
| <code>.lower()</code> on strings or <code>.items()</code> on dicts. This will break if someone tries to use your template on a non-Python | |
| implementation of Jinja. Non-Python implementations are particularly common in deployment environments, where JS | |
| and Rust are very popular.</p> <p data-svelte-h="svelte-f0ucf0">Don’t panic, though! There are a few easy changes you can make to your templates to ensure they’re compatible across | |
| all implementations of Jinja:</p> <ul data-svelte-h="svelte-doa6oc"><li>Replace Python methods with Jinja filters. These usually have the same name, for example <code>string.lower()</code> becomes | |
| <code>string|lower</code>, and <code>dict.items()</code> becomes <code>dict|items</code>. One notable change is that <code>string.strip()</code> becomes <code>string|trim</code>. | |
| See the <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters" rel="nofollow">list of built-in filters</a> | |
| in the Jinja documentation for more.</li> <li>Replace <code>True</code>, <code>False</code> and <code>None</code>, which are Python-specific, with <code>true</code>, <code>false</code> and <code>none</code>.</li> <li>Directly rendering a dict or list may give different results in other implementations (for example, string entries | |
| might change from single-quoted to double-quoted). Adding the <code>tojson</code> filter can help to ensure consistency here.</li></ul> <h3 class="relative group"><a id="writing-generation-prompts" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-generation-prompts"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Writing generation prompts</span></h3> <p data-svelte-h="svelte-13g6lvo">We mentioned above that <code>add_generation_prompt</code> is a special variable that will be accessible inside your template, | |
| and is controlled by the user setting the <code>add_generation_prompt</code> flag. If your model expects a header for | |
| assistant messages, then your template must support adding the header when <code>add_generation_prompt</code> is set.</p> <p data-svelte-h="svelte-i5qalm">Here is an example of a template that formats messages ChatML-style, with generation prompt support:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{{- bos_token }} | |
| {%- for message in messages %} | |
| {{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }} | |
| {%- endfor %} | |
| {%- if add_generation_prompt %} | |
| {{- '<|im_start|>assistant\n' }} | |
| {%- endif %}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-148o5ra">The exact content of the assistant header will depend on your specific model, but it should always be <strong>the string | |
| that represents the start of an assistant message</strong>, so that if the user applies your template with | |
| <code>add_generation_prompt=True</code> and then generates text, the model will write an assistant response. Also note that some | |
| models do not need a generation prompt, because assistant messages always begin immediately after user messages. | |
| This is particularly common for LLaMA and Mistral models, where assistant messages begin immediately after the <code>[/INST]</code> | |
| token that ends user messages. In these cases, the template can ignore the <code>add_generation_prompt</code> flag.</p> <p data-svelte-h="svelte-17es6d5">Generation prompts are important! If your model requires a generation prompt but it is not set in the template, then | |
| model generations will likely be severely degraded, or the model may display unusual behaviour like continuing | |
| the final user message!</p> <h3 class="relative group"><a id="writing-and-debugging-larger-templates" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-and-debugging-larger-templates"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Writing and debugging larger templates</span></h3> <p data-svelte-h="svelte-180ap61">When this feature was introduced, most templates were quite small, the Jinja equivalent of a “one-liner” script. | |
| However, with new models and features like tool-use and RAG, some templates can be 100 lines long or more. When | |
| writing templates like these, it’s a good idea to write them in a separate file, using a text editor. You can easily | |
| extract a chat template to a file:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">open</span>(<span class="hljs-string">"template.jinja"</span>, <span class="hljs-string">"w"</span>).write(tokenizer.chat_template)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1bv602">Or load the edited template back into the tokenizer:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->tokenizer.chat_template = <span class="hljs-built_in">open</span>(<span class="hljs-string">"template.jinja"</span>).read()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1mq7g8k">As an added bonus, when you write a long, multi-line template in a separate file, line numbers in that file will | |
| exactly correspond to line numbers in template parsing or execution errors. This will make it much easier to | |
| identify the source of issues.</p> <h2 class="relative group"><a id="writing-templates-for-tools" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-templates-for-tools"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Writing templates for tools</span></h2> <p data-svelte-h="svelte-kt3iym">Although chat templates do not enforce a specific API for tools (or for anything, really), we recommend | |
| template authors try to stick to a standard API where possible. The whole point of chat templates is to allow code | |
| to be transferable across models, so deviating from the standard tools API means users will have to write | |
| custom code to use tools with your model. Sometimes it’s unavoidable, but often with clever templating you can | |
| make the standard API work!</p> <p data-svelte-h="svelte-1nlwaki">Below, we’ll list the elements of the standard API, and give tips on writing templates that will work well with it.</p> <h3 class="relative group"><a id="tool-definitions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tool-definitions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Tool definitions</span></h3> <p data-svelte-h="svelte-wrq3v1">Your template should expect that the variable <code>tools</code> will either be null (if no tools are passed), or is a list | |
| of JSON schema dicts. Our chat template methods allow users to pass tools as either JSON schema or Python functions, but when | |
| functions are passed, we automatically generate JSON schema and pass that to your template. As a result, the | |
| <code>tools</code> variable that your template receives will always be a list of JSON schema. Here is | |
| a sample tool JSON schema:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"function"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"function"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"name"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"multiply"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"description"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"A function that multiplies two numbers"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"parameters"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"object"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"properties"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"a"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"number"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"description"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"The first number to multiply"</span> | |
| <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"b"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"number"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"description"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"The second number to multiply"</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"required"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">"a"</span><span class="hljs-punctuation">,</span> <span class="hljs-string">"b"</span><span class="hljs-punctuation">]</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1woymkp">And here is some example code for handling tools in your chat template. Remember, this is just an example for a | |
| specific format - your model will probably need different formatting!</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{%- if tools %} | |
| {%- for tool in tools %} | |
| {{- '<tool>' + tool['function']['name'] + '\n' }} | |
| {%- for argument in tool['function']['parameters']['properties'] %} | |
| {{- argument + ': ' + tool['function']['parameters']['properties'][argument]['description'] + '\n' }} | |
| {%- endfor %} | |
| {{- '\n</tool>' }} | |
| {%- endif %} | |
| {%- endif %}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1vngoc7">The specific tokens and tool descriptions your template renders should of course be chosen to match the ones your model | |
| was trained with. There is no requirement that your <strong>model</strong> understands JSON schema input, only that your template can translate | |
| JSON schema into your model’s format. For example, <a href="https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024" rel="nofollow">Command-R</a> | |
| was trained with tools defined using Python function headers, but the Command-R tool template accepts JSON schema, | |
| converts types internally and renders the input tools as Python headers. You can do a lot with templates!</p> <h3 class="relative group"><a id="tool-calls" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tool-calls"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Tool calls</span></h3> <p data-svelte-h="svelte-1x6y95z">Tool calls, if present, will be a list attached to a message with the “assistant” role. Note that <code>tool_calls</code> is | |
| always a list, even though most tool-calling models only support single tool calls at a time, which means | |
| the list will usually only have a single element. Here is a sample message dict containing a tool call:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"role"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"assistant"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"tool_calls"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span> | |
| <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"function"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"function"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"name"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"multiply"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"arguments"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"a"</span><span class="hljs-punctuation">:</span> <span class="hljs-number">5</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"b"</span><span class="hljs-punctuation">:</span> <span class="hljs-number">6</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">]</span> | |
| <span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-my9opo">And a common pattern for handling them would be something like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{%- if message['role'] == 'assistant' and 'tool_calls' in message %} | |
| {%- for tool_call in message['tool_calls'] %} | |
| {{- '<tool_call>' + tool_call['function']['name'] + '\n' + tool_call['function']['arguments']|tojson + '\n</tool_call>' }} | |
| {%- endif %} | |
| {%- endfor %} | |
| {%- endif %}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-hsoefm">Again, you should render the tool call with the formatting and special tokens that your model expects.</p> <h3 class="relative group"><a id="tool-responses" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tool-responses"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Tool responses</span></h3> <p data-svelte-h="svelte-1hx88do">Tool responses have a simple format: They are a message dict with the “tool” role, a “name” key giving the name | |
| of the called function, and a “content” key containing the result of the tool call. Here is a sample tool response:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"role"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"tool"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"name"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"multiply"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"content"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"30"</span> | |
| <span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1okuuze">You don’t need to use all of the keys in the tool response. For example, if your model doesn’t expect the function | |
| name to be included in the tool response, then rendering it can be as simple as:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{%- if message['role'] == 'tool' %} | |
| {{- "<tool_result>" + message['content'] + "</tool_result>" }} | |
| {%- endif %}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13n2q2c">Again, remember that the actual formatting and special tokens are model-specific - you should take a lot of care | |
| to ensure that tokens, whitespace and everything else exactly match the format your model was trained with!</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/chat_template_advanced.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1yq8tgy = { | |
| assets: "/docs/transformers/main/en", | |
| base: "/docs/transformers/main/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/transformers/main/en/_app/immutable/entry/start.8cba93b8.js"), | |
| import("/docs/transformers/main/en/_app/immutable/entry/app.00a1188c.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 11], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 84.5 kB
- Xet hash:
- a3295b25a36803cd726773f701f303a495ff51af23fce25271d572f2f7860cba
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.