transformers/pr_35010/en/perf_torch_compile.html

<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;torch.compile&quot;,&quot;local&quot;:&quot;torchcompile&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Modes&quot;,&quot;local&quot;:&quot;modes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Fullgraph&quot;,&quot;local&quot;:&quot;fullgraph&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Benchmarks&quot;,&quot;local&quot;:&quot;benchmarks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
		<link href="/docs/transformers/pr_35010/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/entry/start.80f0bf24.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/scheduler.01eeda35.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/singletons.2f9d3ffa.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/index.4862150a.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/paths.5ba5a4a8.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/entry/app.746bb5e5.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/index.6dd51b66.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/nodes/0.507a43fb.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/each.e59479a4.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/nodes/415.134bc278.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/Tip.de9bae2b.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/CodeBlock.864da1b0.js">
		<link rel="modulepreload" href="/docs/transformers/pr_35010/en/_app/immutable/chunks/EditOnGithub.7faefd25.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;torch.compile&quot;,&quot;local&quot;:&quot;torchcompile&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Modes&quot;,&quot;local&quot;:&quot;modes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Fullgraph&quot;,&quot;local&quot;:&quot;fullgraph&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Benchmarks&quot;,&quot;local&quot;:&quot;benchmarks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END -->      <p></p>   <h1 class="relative group"><a id="torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#torchcompile"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>torch.compile</span></h1> <p data-svelte-h="svelte-l62zdv"><a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> compiles PyTorch code into optimized kernels that significantly speed up inference. This feature relies on <a href="https://pytorch.org/docs/stable/torch.compiler_dynamo_overview.html" rel="nofollow">TorchDynamo</a> to compile the code into graphs and <a href="https://dev-discuss.pytorch.org/t/torchinductor-a-pytorch-native-compiler-with-define-by-run-ir-and-symbolic-shapes/747" rel="nofollow">TorchInductor</a> to further compile the graphs into optimized kernels. It is a powerful optimization tool, and in many cases, only requires adding a single line of code.</p> <p data-svelte-h="svelte-zm6ubd">Wrap a model with torch.compile to compile and return an optimized model.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
compiled_model = torch.<span class="hljs-built_in">compile</span>(model)<!-- HTML_TAG_END --></pre></div>  <div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1iqtu9r">The initial call to torch.compile is slow because the model needs to be compiled. Subsequent calls to the compiled model are much faster because it doesn’t need to compile again.</p></div> <p data-svelte-h="svelte-11wkqq2">There are several parameters to customize the compilation process. Two of the more important ones are listed below. For a full list of parameters, refer to the torch.compile <a href="https://pytorch.org/docs/stable/generated/torch.compile.html" rel="nofollow">documentation</a>.</p>  <h2 class="relative group"><a id="modes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Modes</span></h2> <p data-svelte-h="svelte-1xvw3go">The <code>mode</code> parameter offers several performance options for compiling. Try different modes to see which one works best for your use case.</p> <ul data-svelte-h="svelte-qtuy41"><li><code>default</code> is a balanced option between speed and memory.</li> <li><code>reduce-overhead</code> reduces the Python overhead at the expense of a little more memory, but it can be faster.</li> <li><code>max-autotune</code> offers the fastest speed, but compilation takes longer.</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
compiled_model = torch.<span class="hljs-built_in">compile</span>(model, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>)<!-- HTML_TAG_END --></pre></div>  <h2 class="relative group"><a id="fullgraph" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fullgraph"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Fullgraph</span></h2> <p data-svelte-h="svelte-fwkp52">Fullgraph attempts to compile the entire model into a single graph to maximize performance. torch.compile raises an error if it encounters a graph break, which means it can’t compile the model into a single graph.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
compiled_model = torch.<span class="hljs-built_in">compile</span>(model, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div>  <h2 class="relative group"><a id="benchmarks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#benchmarks"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Benchmarks</span></h2> <p data-svelte-h="svelte-1rdhs4z">Refer to the table below for performance benchmarks comparing the mean inference time in milliseconds with torch.compile enabled and disabled across various GPUs and batch sizes on the same image for different vision tasks.</p> <p data-svelte-h="svelte-fdi2jq">Select <strong>Subset</strong> in the table below to switch between different GPUs, as well as benchmarks on <a href="https://download.pytorch.org/whl/nightly/cu118" rel="nofollow">PyTorch nightly</a> 2.1.0dev and torch.compile with <code>reduce-overhead</code> mode enabled.</p> <iframe src="https://huggingface.co/datasets/stevhliu/compile-benchmarks/embed/viewer/t4/train" frameborder="0" width="100%" height="560px"></iframe> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/perf_torch_compile.md" target="_blank"><span data-svelte-h="svelte-1kd6by1">&lt;</span> <span data-svelte-h="svelte-x0xyl0">&gt;</span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a>  <p></p> 
			
			<script>
				{
					__sveltekit_14c1y9j = {
						assets: "/docs/transformers/pr_35010/en",
						base: "/docs/transformers/pr_35010/en",
						env: {}
					};

					const element = document.currentScript.parentElement;

					const data = [null,null];

					Promise.all([
						import("/docs/transformers/pr_35010/en/_app/immutable/entry/start.80f0bf24.js"),
						import("/docs/transformers/pr_35010/en/_app/immutable/entry/app.746bb5e5.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 415],
							data,
							form: null,
							error: null
						});
					});
				}
			</script>