Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Inference server backends","local":"inference-server-backends","sections":[{"title":"vLLM","local":"vllm","sections":[],"depth":2},{"title":"SGLang","local":"sglang","sections":[],"depth":2},{"title":"TGI","local":"tgi","sections":[],"depth":2},{"title":"Building a compatible model backend","local":"building-a-compatible-model-backend","sections":[{"title":"Multimodal models","local":"multimodal-models","sections":[],"depth":3}],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/transformers/pr_33962/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/entry/start.1776ac26.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/scheduler.31fdf58d.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/singletons.73ad6261.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/index.252883d5.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/paths.7e4d6052.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/entry/app.d2b9a7d8.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/preload-helper.769e5991.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/index.2f76fdf0.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/nodes/0.29a76cb1.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/nodes/588.7107f034.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/CopyLLMTxtMenu.9922688f.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/globals.7f7f1b26.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/IconCopy.ac192424.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.4975a63c.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_33962/en/_app/immutable/chunks/CodeBlock.ab12f8e1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Inference server backends","local":"inference-server-backends","sections":[{"title":"vLLM","local":"vllm","sections":[],"depth":2},{"title":"SGLang","local":"sglang","sections":[],"depth":2},{"title":"TGI","local":"tgi","sections":[],"depth":2},{"title":"Building a compatible model backend","local":"building-a-compatible-model-backend","sections":[{"title":"Multimodal models","local":"multimodal-models","sections":[],"depth":3}],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 max-sm:gap-0.5 h-6 max-sm:h-5 px-2 max-sm:px-1.5 text-[11px] max-sm:text-[9px] font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0"><svg class="w-3 h-3 max-sm:w-2.5 max-sm:h-2.5" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-6 max-sm:h-5 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible w-3 h-3 max-sm:w-2.5 max-sm:h-2.5 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="inference-server-backends" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inference-server-backends"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inference server backends</span></h1> <p data-svelte-h="svelte-14430n9">Transformers’ models are compatible with different inference servers like vLLM and SGLang. Instead of implementing a model for each inference server, you only need one model, which can be plugged into any inference server. It simplifies maintenance and makes it easy for users to use different inference servers for different use cases.</p> <p data-svelte-h="svelte-1vi4d07">With Transformers as a backend, you can also serve any model - including custom and Hub-hosted models - without waiting for native support.</p> <p data-svelte-h="svelte-dsndik">This guide shows how to use Transformers’ models as a backend to some popular inference servers and how to build a model that supports all inference servers.</p> <h2 class="relative group"><a id="vllm" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#vllm"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>vLLM</span></h2> <p data-svelte-h="svelte-1l34289"><a href="https://github.com/vllm-project/vllm" rel="nofollow">vLLM</a> is a high-performance inference engine optimized for serving LLMs at scale. It supports many Transformers’ models, including all decoder-only LLMs and several vision-language models (VLMs). VLMs currently support image inputs only, with video support planned.</p> <p data-svelte-h="svelte-13utaja">vLLM automatically selects the best backend, and if a model isn’t natively supported, it falls back to the Transformers model. To explicitly use a Transformers’ model, set <code>model_impl="transformers"</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> vllm <span class="hljs-keyword">import</span> LLM | |
| llm = LLM(model=<span class="hljs-string">"meta-llama/Llama-3.2-1B"</span>, model_impl=<span class="hljs-string">"transformers"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qdexcx">Add <code>--model-impl transformers</code> to <code>vllm serve</code> to launch a server with a Transformers’ model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->vllm serve meta-llama/Llama-3.2-1B \ | |
| --task generate \ | |
| --model-impl transformers<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-2bb0ov">Refer to the <a href="https://docs.vllm.ai/en/latest/models/supported_models.html#transformers" rel="nofollow">vLLM docs</a> for more usage examples and tips on using a Transformers as the backend.</p> <h2 class="relative group"><a id="sglang" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#sglang"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>SGLang</span></h2> <p data-svelte-h="svelte-11bfus"><a href="https://github.com/InternLM/sglang" rel="nofollow">SGLang</a> is a high-performance, OpenAI-compatible server and runtime designed for chat-based LLMs. It offers fast inference, role-based conversation handling, and support for custom pipelines, making it great for building real-world LLM apps.</p> <p data-svelte-h="svelte-drnrt0">SGLang automatically falls back to the Transformers backend if a model isn’t natively supported. To explicitly use a Transformers’ model, set <code>impl="transformers"</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> sglang <span class="hljs-keyword">as</span> sgl | |
| llm = sgl.Engine(<span class="hljs-string">"meta-llama/Llama-3.2-1B-Instruct"</span>, impl=<span class="hljs-string">"transformers"</span>) | |
| <span class="hljs-built_in">print</span>(llm.generate([<span class="hljs-string">"The capital of France is"</span>], {<span class="hljs-string">"max_new_tokens"</span>: <span class="hljs-number">20</span>})[<span class="hljs-number">0</span>])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1eej3vc">Add <code>impl transformers</code> to <code>sglang.launch_server</code> to launch a server with a Transformers’ model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python3 -m sglang.launch_server \ | |
| --model-path kyutai/helium-1-preview-2b \ | |
| --impl transformers \ | |
| --host 0.0.0.0 \ | |
| --port 30000<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-5f0kir">Refer to the <a href="https://docs.sglang.ai/supported_models/transformers_fallback.html" rel="nofollow">SGLang docs</a> for more usage examples and tips on using a Transformers as the backend.</p> <h2 class="relative group"><a id="tgi" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tgi"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>TGI</span></h2> <p data-svelte-h="svelte-1kihfet"><a href="https://huggingface.co/docs/text-generation-inference/index" rel="nofollow">TGI</a> can serve models that aren’t <a href="https://huggingface.co/docs/text-generation-inference/supported_models" rel="nofollow">natively implemented</a> by falling back on the Transformers implementation of the model. Some of TGIs high-performance features aren’t available in the Transformers implementation, but other features like continuous batching and streaming are still supported.</p> <blockquote class="tip" data-svelte-h="svelte-wr2gsi"><p>Refer to the <a href="https://huggingface.co/docs/text-generation-inference/basic_tutorials/non_core_models" rel="nofollow">Non-core model serving</a> guide for more details.</p></blockquote> <p data-svelte-h="svelte-84rke5">Serve a Transformers implementation the same way you’d serve a TGI model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->docker <span class="hljs-keyword">run</span><span class="language-bash"> --gpus all --shm-size 1g -p 8080:80 -v <span class="hljs-variable">$volume</span>:/data ghcr.io/huggingface/text-generation-inference:latest --model-id gpt2</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1h3jbpj">Add <code>--trust-remote_code</code> to the command to serve a custom Transformers model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->docker <span class="hljs-keyword">run</span><span class="language-bash"> --gpus all --shm-size 1g -p 8080:80 -v <span class="hljs-variable">$volume</span>:/data ghcr.io/huggingface/text-generation-inference:latest --model-id <CUSTOM_MODEL_ID> --trust-remote-code</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="building-a-compatible-model-backend" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#building-a-compatible-model-backend"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Building a compatible model backend</span></h2> <p data-svelte-h="svelte-bivmzw">To ensure a model is compatible as a backend to any inference server, make sure it is compatible with Transformers and supports the <a href="./attention_interface">AttentionInterface</a> class.</p> <ol data-svelte-h="svelte-1fp8uds"><li><p>A model must be Transformers-compatible following the model <a href="./add_new_model">contribution guidelines</a> or the <a href="./custom_models">custom model contribution guidelines</a>. Make sure the model has a valid <code>config.json</code> in its directory and a valid <code>auto_map</code> field pointing to the model class in the config.</p></li> <li><p>A model’s attentions needs to be configurable with the <a href="./attention_interface">AttentionInterface</a> to allow custom and optimized attention functions. This is important for enabling the performance features of the different inference servers. | |
| Use <code>ALL_ATTENTION_FUNCTIONS</code> when defining the attention layer and propagate <code>**kwargs**</code> from the base <code>MyModel</code> class to the attention layers. Set <code>_supports_attention_backend</code> to <code>True</code> in <a href="/docs/transformers/pr_33962/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Expand the code below for an example.</p></li></ol> <details><summary data-svelte-h="svelte-1j6wskj">modeling_my_model.py</summary> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --> | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedModel | |
| <span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyAttention</span>(nn.Module): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states, **kwargs</span>): | |
| ... | |
| attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] | |
| attn_output, attn_weights = attention_interface( | |
| self, | |
| query_states, | |
| key_states, | |
| value_states, | |
| **kwargs, | |
| ) | |
| ... | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(<span class="hljs-title class_ inherited__">PreTrainedModel</span>): | |
| _supports_attention_backend = <span class="hljs-literal">True</span><!-- HTML_TAG_END --></pre></div></details> <ol start="3" data-svelte-h="svelte-5gow24"><li><p>This step is optional, but if you want to support tensor parallel and/or pipeline parallel features, add the following keys to the config.</p> <ul><li><code>base_model_tp_plan</code> enables <a href="./perf_infer_gpu_multi">tensor parallelism</a> by mapping fully qualified layer name patterns to tensor parallel styles. Only the <code>"colwise"</code> and <code>"rowwise"</code> partitioning strategies are currently supported.</li> <li><code>base_model_pp_plan</code> enables pipeline parallelism by mapping direct child layer names to tuples of lists of strings. The list in the first element of the tuple contains the names of the input arguments. The list in the last element of the tuple contains the names of the variables the layer outputs to in the modeling code.</li></ul> <p>Expand the code below for an example.</p></li></ol> <details><summary data-svelte-h="svelte-1av8lv4">configuration_my_model.py</summary> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --> | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedConfig | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyConfig</span>(<span class="hljs-title class_ inherited__">PreTrainedConfig</span>): | |
| base_model_tp_plan = { | |
| <span class="hljs-string">"layers.*.self_attn.k_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.self_attn.v_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.self_attn.o_proj"</span>: <span class="hljs-string">"rowwise"</span>, | |
| <span class="hljs-string">"layers.*.mlp.gate_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.mlp.up_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.mlp.down_proj"</span>: <span class="hljs-string">"rowwise"</span>, | |
| } | |
| base_model_pp_plan = { | |
| <span class="hljs-string">"embed_tokens"</span>: ([<span class="hljs-string">"input_ids"</span>], [<span class="hljs-string">"inputs_embeds"</span>]), | |
| <span class="hljs-string">"layers"</span>: ([<span class="hljs-string">"hidden_states"</span>, <span class="hljs-string">"attention_mask"</span>], [<span class="hljs-string">"hidden_states"</span>]), | |
| <span class="hljs-string">"norm"</span>: ([<span class="hljs-string">"hidden_states"</span>], [<span class="hljs-string">"hidden_states"</span>]), | |
| }<!-- HTML_TAG_END --></pre></div></details> <h3 class="relative group"><a id="multimodal-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multimodal-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Multimodal models</span></h3> <p data-svelte-h="svelte-1j6ghl6">For multimodal models, you need to include a few more changes on top of the general recommendations. These rules ensure that your model integrates properly with multimodal data.</p> <ol data-svelte-h="svelte-lgotgp"><li><p>A multimodal model requires a base <code>MyMultiModalModel</code> class to handle multimodal fusion without a language modeling head and a separate generative class that adds a head.</p> <p>The base model needs to implement the <code>get_image_features()</code> method to accept image pixel values and return encoded outputs. These are later merged with the language embeddings and don’t require any postprocessing. The shape of the returned features must match the number of input images. If a vision encoder returns variable-length outputs (patch-based), return a list of 2D tensors of size <code>(image_seq_len, image_dim)</code> for each image.</p></li></ol> <p data-svelte-h="svelte-1kxgoyb">Expand the code below for an example.</p> <details><summary data-svelte-h="svelte-1uj7rw0">modeling_my_multimodal_model.py</summary> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.generation <span class="hljs-keyword">import</span> GenerationMixin | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyMultimodalModel</span>(<span class="hljs-title class_ inherited__">MyMultimodalPreTrainedModel</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>): | |
| <span class="hljs-built_in">super</span>().__init__(config) | |
| self.language_model = AutoModel.from_config(config.text_config) | |
| self.vision_tower = AutoModel.from_config(config.vision_config) | |
| self.multimodal_projection = nn.Linear(vision_dim, text_dim) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_image_features</span>(<span class="hljs-params">self, pixel_values</span>): | |
| <span class="hljs-keyword">return</span> self.vision_tower(pixel_values).last_hidden_states | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, input_ids, pixel_values, **kwargs</span>): | |
| <span class="hljs-comment"># process your inputs</span> | |
| <span class="hljs-keyword">return</span> MyModelOutputWithPast( | |
| last_hidden_state=last_hidden_state, | |
| image_hidden_states=image_features, | |
| [...] | |
| ) | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyMultimodalModelForConditionalGeneration</span>(MyMultimodalPreTrainedModel, GenerationMixin): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>): | |
| <span class="hljs-built_in">super</span>().__init__(config) | |
| self.model = MyMultimodalModel(config) | |
| self.lm_head = nn.Linear(hidden_dim, vocab_size)<!-- HTML_TAG_END --></pre></div></details> <ol start="2" data-svelte-h="svelte-1vm7hyy"><li><p>A multimodal model config must be nested with the following fields.</p> <ul><li>text_config: decoder language model config</li> <li>vision_config: vision encoder config</li> <li>image_token_id: ID of the image placeholder token used in the input to indicate image position</li></ul></li> <li><p>A multimodal model’s processing class must have the <code>self.image_token</code> and <code>self.image_token_ids</code> attributes. These are placeholder tokens used to indicate image positions in the input. The placeholder token is the same token used in the input prompt and to mask scatter image features.</p> <p>The processing class also needs <code>self._get_num_multimodal_tokens</code> method to compute the number of placeholder tokens needed for multimodal inputs with given sizes and to return a <code>MultiModalData</code> object. The placeholder for row and column tokens don’t count as image placeholders. Only the tokens that are actually replaced by image features are computed.</p></li></ol> <p data-svelte-h="svelte-1gwvbeh">Finally, when <code>return_mm_token_type_ids=True</code>, the class has to return <code>mm_token_type_ids</code> to indicate whether each position is a text token (<code>0</code>) or image placeholder token (<code>1</code>). Each image’s token type IDs must be contiguous with no breaks between consecutive ones.</p> <p data-svelte-h="svelte-1kxgoyb">Expand the code below for an example.</p> <details><summary data-svelte-h="svelte-1q6gcjs">processing_my_multimodal_model.py</summary> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyMultimodalProcessor</span>(<span class="hljs-title class_ inherited__">ProcessorMixin</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, images=<span class="hljs-literal">None</span>, text=<span class="hljs-literal">None</span>, **kwargs</span>): | |
| <span class="hljs-keyword">if</span> return_mm_token_type_ids: | |
| mm_token_type_ids = np.zeros_like(input_ids) | |
| mm_token_type_ids[input_ids == self.image_token_id] = <span class="hljs-number">1</span> | |
| text_inputs[<span class="hljs-string">"mm_token_type_ids"</span>] = mm_token_type_ids.tolist() | |
| <span class="hljs-keyword">return</span> BatchFeature(data={**text_inputs, **image_inputs}, tensor_type=return_tensors) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">_get_num_multimodal_tokens</span>(<span class="hljs-params">self, image_sizes=<span class="hljs-literal">None</span>, **kwargs</span>): | |
| <span class="hljs-string">""" | |
| Computes the number of placeholder tokens needed for multimodal inputs with the given sizes. | |
| Args: | |
| image_sizes (`list[list[int]]`, *optional*): | |
| The input sizes formatted as (height, width) per each image. | |
| Returns: | |
| `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided | |
| input modalities, along with other useful data. | |
| """</span> | |
| vision_data = {} | |
| <span class="hljs-keyword">if</span> image_sizes <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| num_image_tokens = [<span class="hljs-number">256</span>] * <span class="hljs-built_in">len</span>(image_sizes) <span class="hljs-comment"># 256 placeholder tokens for each image always</span> | |
| num_image_patches = [<span class="hljs-number">1</span>] * <span class="hljs-built_in">len</span>(image_sizes) <span class="hljs-comment"># no patching, thus each image is processed as a single base image</span> | |
| vision_data.update({<span class="hljs-string">"num_image_tokens"</span>: num_image_tokens, <span class="hljs-string">"num_image_patches"</span>: num_image_patches}) | |
| <span class="hljs-keyword">return</span> MultiModalData(**vision_data)<!-- HTML_TAG_END --></pre></div></details> <h2 class="relative group"><a id="resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Resources</span></h2> <ul data-svelte-h="svelte-5hiu2o"><li>Read the <a href="https://blog.vllm.ai/2025/04/11/transformers-backend.html" rel="nofollow">Transformers backend integration in vLLM</a> blog post for more details about the Transformers backend in vLLM.</li> <li>Read the <a href="https://huggingface.co/blog/transformers-backend-sglang" rel="nofollow">Transformers backend integration in SGLang</a> blog post for more details about the Transformers backend in SGLang.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/transformers_as_backend.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1yuubte = { | |
| assets: "/docs/transformers/pr_33962/en", | |
| base: "/docs/transformers/pr_33962/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/transformers/pr_33962/en/_app/immutable/entry/start.1776ac26.js"), | |
| import("/docs/transformers/pr_33962/en/_app/immutable/entry/app.d2b9a7d8.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 588], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 44.9 kB
- Xet hash:
- f33b3c5e6f2ad32abb805ed64835db58262804b8653abdfb361f1ce32a9593c3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.