Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / peft /main /en /developer_guides /torch_compile.html

rtrm

about 1 month ago

download

raw

12.2 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"torch.compile","local":"torchcompile","sections":[{"title":"Training and inference with torch.compile","local":"training-and-inference-with-torchcompile","sections":[],"depth":2},{"title":"Advanced PEFT features with torch.compile","local":"advanced-peft-features-with-torchcompile","sections":[],"depth":2},{"title":"Test cases","local":"test-cases","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/peft/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/entry/start.b1913684.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/scheduler.c57aa7ef.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/singletons.6b6c8b9d.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/index.c4ac7298.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/paths.62c814b0.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/entry/app.ca8eed50.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/index.c50cb18e.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/nodes/0.f0f7697b.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/nodes/16.a872e70d.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/Tip.9268b0ca.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/EditOnGithub.958a8a49.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"torch.compile","local":"torchcompile","sections":[{"title":"Training and inference with torch.compile","local":"training-and-inference-with-torchcompile","sections":[],"depth":2},{"title":"Advanced PEFT features with torch.compile","local":"advanced-peft-features-with-torchcompile","sections":[],"depth":2},{"title":"Test cases","local":"test-cases","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#torchcompile"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>torch.compile</span></h1> <p data-svelte-h="svelte-1epca12">In PEFT, <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> works for some but not all features. The reason why it won’t always work is because PEFT is highly dynamic in certain places (loading and switching between multiple adapters, for instance), which can cause trouble for <code>torch.compile</code>. In other places, <code>torch.compile</code> may work, but won’t be as fast as expected because of graph breaks.</p> <p data-svelte-h="svelte-1alfwjj">If you don’t see an error, it doesn’t necessarily mean that <code>torch.compile</code> worked correctly. It might give you an output, but the output is incorrect. This guide describes what works with <code>torch.compile</code> and what doesn’t.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-11mpuzm">Unless indicated otherwise, the default <code>torch.compile</code> settings were used.</p></div> <h2 class="relative group"><a id="training-and-inference-with-torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#training-and-inference-with-torchcompile"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Training and inference with torch.compile</span></h2> <p data-svelte-h="svelte-10kwh8m">These features <strong>work</strong> with <code>torch.compile</code>. Everything listed below was tested with a causal LM:</p> <ul data-svelte-h="svelte-1bf32c1"><li>Training with <code>Trainer</code> from 🤗 transformers</li> <li>Training with a custom PyTorch loop</li> <li>Inference</li> <li>Generation</li></ul> <p data-svelte-h="svelte-1u6npfd">The following adapters were tested successfully:</p> <ul data-svelte-h="svelte-8zg1lj"><li>AdaLoRA</li> <li>BOFT</li> <li>IA³</li> <li>Layer Norm Tuning</li> <li>LoHa</li> <li>LoRA</li> <li>LoRA + DoRA</li> <li>OFT</li> <li>VeRA</li> <li>HRA</li></ul> <p data-svelte-h="svelte-c97aay">The following adapters <strong>don’t work</strong> correctly for training or inference when using <code>torch.compile</code>:</p> <ul data-svelte-h="svelte-iv8nd2"><li>LoKr</li> <li>LoRA targeting embedding layers</li></ul> <h2 class="relative group"><a id="advanced-peft-features-with-torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#advanced-peft-features-with-torchcompile"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Advanced PEFT features with torch.compile</span></h2> <p data-svelte-h="svelte-gy3aka">Below are some of the more advanced PEFT features that <strong>work</strong>. They were all tested with LoRA.</p> <ul data-svelte-h="svelte-1oglq83"><li><code>modules_to_save</code> (i.e. <code>config = LoraConfig(..., modules_to_save=...)</code>)</li> <li>Merging adapters (one or multiple)</li> <li>Merging multiple adapters into one adapter (i.e. calling <code>model.add_weighted_adapter(...)</code>)</li></ul> <p data-svelte-h="svelte-x3yaa4">Generally, we can expect that if a feature works correctly with LoRA and is also supported by other adapter types, it should also work for that adapter type.</p> <p data-svelte-h="svelte-18hj1j5">The more advanced PEFT features below <strong>don’t work</strong> in conjunction with <code>torch.compile</code>. Tests were run with LoRA:</p> <ul data-svelte-h="svelte-58pk2s"><li>Using PEFT adapters with quantization (bitsandbytes)</li> <li>Inference with multiple adapters</li> <li>Unloading (i.e. calling <code>model.merge_and_unload()</code>)</li> <li>Disabling adapters (i.e. using <code>with model.disable_adapter()</code>)</li> <li>Mixed adapter batches (i.e. calling <code>model(batch, adapter_names=["__base__", "default", "other", ...])</code>)</li></ul> <h2 class="relative group"><a id="test-cases" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#test-cases"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Test cases</span></h2> <p data-svelte-h="svelte-1ks3zgv">All the use cases listed above are tested inside of <a href="https://github.com/huggingface/peft/blob/main/tests/test_torch_compile.py" rel="nofollow"><code>peft/tests/test_torch_compile.py</code></a>. If you want to check in more detail how we tested a certain feature, please go to that file and check the test that corresponds to your use case.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-fo45pn">If you have another use case where you know that <code>torch.compile</code> does or does not work with PEFT, please contribute by letting us know or by opening a PR to add this use case to the covered test cases.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/peft/blob/main/docs/source/developer_guides/torch_compile.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_nv4ir6 = {
	assets: "/docs/peft/main/en",
	base: "/docs/peft/main/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/peft/main/en/_app/immutable/entry/start.b1913684.js"),
	import("/docs/peft/main/en/_app/immutable/entry/app.ca8eed50.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 16],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 12.2 kB
Xet hash:: c5ce10a54edbdde483f6698a020cf610bef60e46c038e90eeb3f75906e7b5736

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.