Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers /pr_37082 /en /continuous_batching.html

rtrm

3 months ago

download

raw

35.6 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Continuous Batching","local":"continuous-batching","sections":[{"title":"API Reference","local":"api-reference","sections":[],"depth":2},{"title":"Usage Examples","local":"usage-examples","sections":[{"title":"generate_batch example","local":"generatebatch-example","sections":[],"depth":3},{"title":"ContinuousBatchingManager example","local":"continuousbatchingmanager-example","sections":[],"depth":3}],"depth":2},{"title":"Supported & Unsupported Features","local":"supported--unsupported-features","sections":[{"title":"Supported Features","local":"supported-features","sections":[],"depth":3},{"title":"Unsupported Features","local":"unsupported-features","sections":[],"depth":3}],"depth":2},{"title":"Performance Considerations","local":"performance-considerations","sections":[],"depth":2},{"title":"Integration with Serving","local":"integration-with-serving","sections":[],"depth":2},{"title":"Monitoring","local":"monitoring","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/transformers/pr_37082/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/entry/start.d6f96127.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/scheduler.31fdf58d.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/singletons.48fa53b0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/index.252883d5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/paths.a363795e.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/entry/app.6ba5e9c5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/preload-helper.48c71b3c.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/index.2f76fdf0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/nodes/0.6843006f.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/nodes/17.d98360da.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/CopyLLMTxtMenu.28924585.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.91dad97e.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/IconCopy.ac192424.js">
	<link rel="modulepreload" href="/docs/transformers/pr_37082/en/_app/immutable/chunks/CodeBlock.ab12f8e1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Continuous Batching","local":"continuous-batching","sections":[{"title":"API Reference","local":"api-reference","sections":[],"depth":2},{"title":"Usage Examples","local":"usage-examples","sections":[{"title":"generate_batch example","local":"generatebatch-example","sections":[],"depth":3},{"title":"ContinuousBatchingManager example","local":"continuousbatchingmanager-example","sections":[],"depth":3}],"depth":2},{"title":"Supported & Unsupported Features","local":"supported--unsupported-features","sections":[{"title":"Supported Features","local":"supported-features","sections":[],"depth":3},{"title":"Unsupported Features","local":"unsupported-features","sections":[],"depth":3}],"depth":2},{"title":"Performance Considerations","local":"performance-considerations","sections":[],"depth":2},{"title":"Integration with Serving","local":"integration-with-serving","sections":[],"depth":2},{"title":"Monitoring","local":"monitoring","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 max-sm:gap-0.5 h-6 max-sm:h-5 px-2 max-sm:px-1.5 text-[11px] max-sm:text-[9px] font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0"><svg class="w-3 h-3 max-sm:w-2.5 max-sm:h-2.5" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-6 max-sm:h-5 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible w-3 h-3 max-sm:w-2.5 max-sm:h-2.5 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="continuous-batching" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#continuous-batching"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Continuous Batching</span></h1> <p data-svelte-h="svelte-mmqn9p">Continuous Batching (CB) is an advanced technique to optimize the inference of transformer models by dynamically grouping multiple requests into batches. This approach maximizes GPU utilization and throughput, specifically for workloads with many variable-length inputs.</p> <p data-svelte-h="svelte-1i1iqo6">We are particularly interested in having Continuous Batching in transformers for the following use cases:</p> <ul data-svelte-h="svelte-1q6z4si"><li>Evaluation of models on large datasets with variable-length inputs</li> <li>Generating outputs for multiple sequences for GRPO policies</li></ul> <p data-svelte-h="svelte-owguw3">CB is what makes inference engines like vLLM or SGLang efficient. That being said, transformers does not aim to be a production-ready inference engine, but a complete framework for model development. For this reason, CB is available in <code>transformers serve</code>.</p> <p data-svelte-h="svelte-c224e3">If you are not familiar with some of the core concepts CB is built upon, we invite you to read the associated blog post: <a href="https://huggingface.co/blog/continuous-batching" rel="nofollow">Continuous Batching: Efficient Inference for Large Language Models</a>. <em>broken link for now</em></p> <h2 class="relative group"><a id="api-reference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#api-reference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>API Reference</span></h2> <h2 class="relative group"><a id="usage-examples" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#usage-examples"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Usage Examples</span></h2> <p data-svelte-h="svelte-2dtvgg">The main way to use CB in transformers is via the <code>generate_batch</code> method.</p> <p data-svelte-h="svelte-45ul6a">Unlike <code>generate</code>, CB takes already tokenized inputs, known as input IDs. Each sequence of input IDs is represented as a list of integers, in python: <code>list[int]</code>. Since</p> <p data-svelte-h="svelte-7qqpgi">For a more detailed example, please refer to: <a href="./path/to/example">examples/continuous_batching</a></p> <h3 class="relative group"><a id="generatebatch-example" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generatebatch-example"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>generate_batch example</span></h3> <p data-svelte-h="svelte-1hlsqpc">We have created a <code>ContinuousMixin</code> that is inherited by the <code>GenerationMixin</code> so that all auto regressive text models support CB.</p> <p data-svelte-h="svelte-1pe5ojz">This adds the <code>generate_batch</code> method to all models that inherit from <code>GenerationMixin</code>.</p> <p data-svelte-h="svelte-1lahfml">You can use it as follows:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> datasets
	<span class="hljs-keyword">import</span> torch

	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
	<span class="hljs-keyword">from</span> transformers.generation <span class="hljs-keyword">import</span> GenerationConfig

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-4B-Instruct-2507"</span>,
	attn_implementation=<span class="hljs-string">"spda_paged"</span>,
	device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># if you need cuda</span>
	dtype=torch.bfloat16,
	)
	tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, padding_side=<span class="hljs-string">"left"</span>)

	<span class="hljs-comment"># prepare a batch of inputs</span>
	dataset = datasets.load_dataset(<span class="hljs-string">"openai/gsm8k"</span>, <span class="hljs-string">"socratic"</span>, split=<span class="hljs-string">"test"</span>)
	dataset = dataset.select(<span class="hljs-built_in">range</span>(args.samples))
	tokenized_datasets = dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> x: tokenizer(x[<span class="hljs-string">"question"</span>]), batched=<span class="hljs-literal">True</span>)
	simple_batch_inputs = [item[<span class="hljs-string">"input_ids"</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> tokenized_datasets]

	generation_config = GenerationConfig(
	max_new_tokens=<span class="hljs-number">32</span>,
	use_cuda_graph=<span class="hljs-literal">False</span>, <span class="hljs-comment"># Not supported for simple version</span>
	eos_token_id=tokenizer.eos_token_id,
	pad_token_id=tokenizer.pad_token_id,
	do_sample=<span class="hljs-literal">False</span>,
	max_batch_tokens=<span class="hljs-number">512</span>, <span class="hljs-comment"># max number of tokens in a batch, this is just a default value you should tune based on your hardware</span>
	)

	batch_outputs = model.generate_batch(
	inputs=simple_batch_inputs,
	generation_config=generation_config,
	)

	<span class="hljs-keyword">for</span> request_id, output <span class="hljs-keyword">in</span> batch_outputs.items():
	generated_text = tokenizer.decode(output.generated_tokens, skip_special_tokens=<span class="hljs-literal">True</span>)
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"Request <span class="hljs-subst">{request_id}</span> output: <span class="hljs-subst">{generated_text}</span>"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="continuousbatchingmanager-example" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#continuousbatchingmanager-example"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ContinuousBatchingManager example</span></h3> <p data-svelte-h="svelte-1wa0j93">If you want more control w.r.t. how you want to schedule requests using CB, you can use the <code>ContinuousBatchingManager</code> class directly.</p> <p data-svelte-h="svelte-4sk78c">This is what we use in <code>transformers serve</code> because requests arrive asynchronously and we can leverage the asynchronous nature of the CB process to make things more efficient.</p> <p data-svelte-h="svelte-suwtjb">Under the hood, the <code>ContinuousBatchingManager</code> creates a background thread that receives inputs from a python <code>queue.Queue</code> which it uses to get requests to batch in each forward pass.</p> <p data-svelte-h="svelte-1oymyy5">Note that the manager is thread safe!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> datasets
	<span class="hljs-keyword">import</span> torch

	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
	<span class="hljs-keyword">from</span> transformers.generation <span class="hljs-keyword">import</span> GenerationConfig
	<span class="hljs-keyword">from</span> transformers.generation.continuous_batching <span class="hljs-keyword">import</span> RequestStatus

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-4B-Instruct-2507"</span>,
	attn_implementation=<span class="hljs-string">"spda_paged"</span>,
	device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># if you need cuda</span>
	dtype=torch.bfloat16,
	)
	tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, padding_side=<span class="hljs-string">"left"</span>)

	<span class="hljs-comment"># prepare a batch of inputs</span>
	dataset = datasets.load_dataset(<span class="hljs-string">"openai/gsm8k"</span>, <span class="hljs-string">"socratic"</span>, split=<span class="hljs-string">"test"</span>)
	dataset = dataset.select(<span class="hljs-built_in">range</span>(args.samples))
	tokenized_datasets = dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> x: tokenizer(x[<span class="hljs-string">"question"</span>]), batched=<span class="hljs-literal">True</span>)
	simple_batch_inputs = [item[<span class="hljs-string">"input_ids"</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> tokenized_datasets]

	<span class="hljs-comment"># initialize the manager, available method thanks to the `ContinuousMixin`</span>
	manager = model.init_continuous_batching(generation_config=generation_config)

	<span class="hljs-comment"># start the background thread</span>
	manager.start()

	<span class="hljs-comment"># this is for demonstration purposes only, in practice this is most useful to do concurrently</span>
	<span class="hljs-keyword">for</span> i, <span class="hljs-built_in">input</span> <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(simple_batch_inputs):
	request_id = manager.add_request(input_ids=<span class="hljs-built_in">input</span>, request_id=<span class="hljs-string">f"request_<span class="hljs-subst">{i}</span>"</span>) <span class="hljs-comment"># if you do not specify a request_id, one will be generated for you</span>

	<span class="hljs-comment"># Can be done in an other thread</span>
	<span class="hljs-keyword">for</span> <span class="hljs-built_in">id</span>, request <span class="hljs-keyword">in</span> manager.get_result():
	generated_text = tokenizer.decode(request.generated_tokens, skip_special_tokens=<span class="hljs-literal">True</span>)
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"Request <span class="hljs-subst">{<span class="hljs-built_in">id</span>}</span> output: <span class="hljs-subst">{generated_text}</span>"</span>)

	<span class="hljs-comment"># you can also get results for a specific request id</span>
	result = manager.get_result(request_id=<span class="hljs-string">"request_5"</span>) <span class="hljs-comment"># this is blocking and will wait for the result to be ready</span>

	<span class="hljs-comment"># or get results for a request that is streaming</span>
	manager.add_request(
	input_ids=<span class="hljs-built_in">input</span>,
	request_id=<span class="hljs-string">"streaming_request"</span>,
	stream=<span class="hljs-literal">True</span>,
	)
	<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> manager.request_id_iter(request_id=<span class="hljs-string">"streaming_request"</span>):
	generated_text = tokenizer.decode(chunk.generated_tokens, skip_special_tokens=<span class="hljs-literal">True</span>)
	<span class="hljs-built_in">print</span>(generated_text)
	<span class="hljs-comment"># <span class="hljs-doctag">FIXME:</span> stop iteration in `request_id_iter` when finished instead of doing it externally</span>
	<span class="hljs-keyword">if</span> chunk.status == RequestStatus.FINISHED:
	<span class="hljs-keyword">break</span>

	<span class="hljs-comment"># stop the background thread before exiting the process</span>
	manager.stop()<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="supported--unsupported-features" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#supported--unsupported-features"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Supported & Unsupported Features</span></h2> <h3 class="relative group"><a id="supported-features" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#supported-features"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Supported Features</span></h3> <ul data-svelte-h="svelte-1rrbmgu"><li>Dynamic scheduling of variable-length requests</li> <li>Chunked prefill</li> <li>Paged Attention Cache</li> <li>Sliding window attention</li> <li>Chat templates</li></ul> <h3 class="relative group"><a id="unsupported-features" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#unsupported-features"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Unsupported Features</span></h3> <p data-svelte-h="svelte-1fh45ju">At the moment, the following features are not supported with CB. We plan to add support to the following:</p> <ul data-svelte-h="svelte-8pjzfj"><li>Prefix caching</li> <li>Beam search</li> <li>tool calling</li></ul> <p data-svelte-h="svelte-1qhz94n">The others are unplanned, but depending on community requests we might consider adding them:</p> <ul data-svelte-h="svelte-ig7t7g"><li>MTP (multi token prediction)</li> <li>Medusa</li></ul> <h2 class="relative group"><a id="performance-considerations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#performance-considerations"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Performance Considerations</span></h2> <h2 class="relative group"><a id="integration-with-serving" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#integration-with-serving"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Integration with Serving</span></h2> <p data-svelte-h="svelte-195nnaw">You can use CB in <code>transformers serve</code> by passing the <code>--continuous-batching</code> flag when starting the server.</p> <h2 class="relative group"><a id="monitoring" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#monitoring"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Monitoring</span></h2> <p data-svelte-h="svelte-8uphkn">We have added <code>opentelemetry</code> support to Continuous Batching to help you monitor its performance in production. To enable it, you need to install the <code>opentelemetry</code> extra when installing <code>transformers</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># this installs `opentelemetry-api`, `opentelemetry-sdk` and `opentelemetry-exporter-otlp`</span>
	pip install transformers[open-telemetry]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-bp8c01">This will enable traces and metrics collection in CB. You will then have to setup the backend to collect and visualize the traces and metrics.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/continuous_batching.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_129pc60 = {
	assets: "/docs/transformers/pr_37082/en",
	base: "/docs/transformers/pr_37082/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers/pr_37082/en/_app/immutable/entry/start.d6f96127.js"),
	import("/docs/transformers/pr_37082/en/_app/immutable/entry/app.6ba5e9c5.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 17],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 35.6 kB
Xet hash:: 6d8a09ae43885fa7d9a1b91b13a20d06ab8e4d0ccd2fef6b3c5616faba5dd748

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.