Buckets:

hf-doc-build
/

doc-dev

Files
xet
hf-doc-build/doc-dev / transformers /pr_36708 /en /exporters_quantization.html
HuggingFaceDocBuilder
8 days ago
download
raw
39.3 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Quantizing exports&quot;,&quot;local&quot;:&quot;quantizing-exports&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;PT2E&quot;,&quot;local&quot;:&quot;pt2e&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;ONNX Runtime&quot;,&quot;local&quot;:&quot;onnx-runtime&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;NNCF&quot;,&quot;local&quot;:&quot;nncf&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Calibration&quot;,&quot;local&quot;:&quot;calibration&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;A different recipe per component&quot;,&quot;local&quot;:&quot;a-different-recipe-per-component&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Add a quantizer&quot;,&quot;local&quot;:&quot;add-a-quantizer&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/transformers/main/en/_app/immutable/entry/start.DZe-7K50.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/Cpjgof-f.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DEcOOaG_.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/entry/app.BtyGqhlx.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/BZy03ydf.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DOLBYqcr.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/BVXwUupI.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/nodes/0.D7-vU1VO.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/CYEh-5qn.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/nodes/3.DWPlqE4h.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/BpaAPiMq.js" rel="modulepreload">
		<!--3uy61z--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Quantizing exports&quot;,&quot;local&quot;:&quot;quantizing-exports&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;PT2E&quot;,&quot;local&quot;:&quot;pt2e&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;ONNX Runtime&quot;,&quot;local&quot;:&quot;onnx-runtime&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;NNCF&quot;,&quot;local&quot;:&quot;nncf&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Calibration&quot;,&quot;local&quot;:&quot;calibration&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;A different recipe per component&quot;,&quot;local&quot;:&quot;a-different-recipe-per-component&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Add a quantizer&quot;,&quot;local&quot;:&quot;add-a-quantizer&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/transformers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="quantizing-exports" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantizing-exports"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Quantizing exports</span></h1><!--]--><!----> <p>Every <a href="./exporters">export</a> config can take a <code>quantizer</code>, which quantizes the model during the export. The quantizers
live in <code>transformers.exporters.quantizers</code>, and each one runs at a fixed point of the export: on the <code>torch.export</code> FX graph, before any conversion, or on the model a backend converts it into.</p> <table><thead><tr><th>Quantizer</th><th>Quantizes</th><th>Dynamo</th><th>ONNX</th><th>ExecuTorch</th><th>OpenVINO</th></tr></thead><tbody><tr><td><a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.PT2EQuantizer">PT2EQuantizer</a></td><td>the FX graph, with PyTorch 2 Export (PT2E) quantization</td><td>🟢</td><td>🟢</td><td>🟢</td><td>🟢</td></tr><tr><td><a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.NNCFTorchFXQuantizer">NNCFTorchFXQuantizer</a></td><td>the FX graph, with <a href="https://github.com/openvinotoolkit/nncf" rel="nofollow">NNCF</a></td><td>🟢</td><td>🟢</td><td>🟢 (weight-only)</td><td>🟢</td></tr><tr><td><a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.OnnxRuntimeQuantizer">OnnxRuntimeQuantizer</a></td><td>the ONNX model, with <code>onnxruntime.quantization</code></td><td>🔴</td><td>🟢</td><td>🔴</td><td>🔴</td></tr><tr><td><a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.NNCFOnnxQuantizer">NNCFOnnxQuantizer</a></td><td>the ONNX model, with <a href="https://github.com/openvinotoolkit/nncf" rel="nofollow">NNCF</a></td><td>🔴</td><td>🟢</td><td>🔴</td><td>🔴</td></tr><tr><td><a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.NNCFOpenVINOQuantizer">NNCFOpenVINOQuantizer</a></td><td>the OpenVINO model, with <a href="https://github.com/openvinotoolkit/nncf" rel="nofollow">NNCF</a></td><td>🔴</td><td>🔴</td><td>🔴</td><td>🟢</td></tr></tbody></table> <p>The export checks the quantizer against its backend and installed packages before tracing, so a mismatch fails early.
Like the exporters, each quantizer lists the package versions it is tested on, and warns when yours differ.</p> <p>The example below quantizes a sentiment classifier to int8 with ONNX Runtime, then runs it.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification, AutoTokenizer
<span class="hljs-keyword">from</span> transformers.exporters <span class="hljs-keyword">import</span> OnnxConfig, OnnxExporter, OnnxRuntimeQuantizer

model_id = <span class="hljs-string">&quot;distilbert/distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForSequenceClassification.from_pretrained(model_id)

texts = [
    <span class="hljs-string">&quot;A wonderful, heartfelt film.&quot;</span>,
    <span class="hljs-string">&quot;The plot was dull and the acting worse.&quot;</span>,
    <span class="hljs-string">&quot;I would watch it again tomorrow.&quot;</span>,
    <span class="hljs-string">&quot;Two hours I will never get back.&quot;</span>,
]
calibration_dataset = [tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>) <span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> texts]
inputs = tokenizer(<span class="hljs-string">&quot;An absolute delight from start to finish.&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

config = OnnxConfig(dynamic=<span class="hljs-literal">True</span>, quantizer=OnnxRuntimeQuantizer(calibration_dataset=calibration_dataset))
onnx_program = OnnxExporter().export(model, inputs, config)

logits = onnx_program(**inputs)[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(model.config.id2label[<span class="hljs-built_in">int</span>(logits.argmax())])  <span class="hljs-comment"># POSITIVE</span><!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="pt2e" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pt2e"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>PT2E</span></h2><!--]--><!----> <p><a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.PT2EQuantizer">PT2EQuantizer</a> wraps a PT2E quantizer, from torchao or from the target runtime, and runs it
on the FX graph (<code>prepare_pt2e</code> → calibrate → <code>convert_pt2e</code>), so it works with every backend. Each PT2E quantizer
injects its own quantize/dequantize ops, so pick the one your runtime supports:</p> <table><thead><tr><th>Where you’ll run</th><th>PT2E quantizer</th><th>Import from</th></tr></thead><tbody><tr><td>PyTorch inductor, ONNX Runtime (QDQ), OpenVINO</td><td><code>X86InductorQuantizer</code></td><td><code>torchao.quantization.pt2e.quantizer.x86_inductor_quantizer</code></td></tr><tr><td>ExecuTorch XNNPACK backend</td><td><code>XNNPACKQuantizer</code></td><td><code>executorch.backends.xnnpack.quantizer.xnnpack_quantizer</code></td></tr></tbody></table> <p>For ExecuTorch, wrap ExecuTorch’s own <code>XNNPACKQuantizer</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> executorch.backends.xnnpack.quantizer.xnnpack_quantizer <span class="hljs-keyword">import</span> (
    XNNPACKQuantizer,
    get_symmetric_quantization_config,
)
<span class="hljs-keyword">from</span> transformers.exporters <span class="hljs-keyword">import</span> ExecutorchConfig, PT2EQuantizer

xnnpack = XNNPACKQuantizer().set_global(get_symmetric_quantization_config())
config = ExecutorchConfig(backend=<span class="hljs-string">&quot;xnnpack&quot;</span>, quantizer=PT2EQuantizer(xnnpack, calibration_dataset=calibration_dataset))<!----></pre></div><!----> <p>Quantize generative models through <code>~HfExporter.export_for_generation</code>. Here <code>calibration_dataset</code> holds generate
kwargs, like <code>inputs</code>, and each component calibrates on the inputs captured for it. A text-only model has a <code>prefill</code> and a <code>decode</code> component; a multimodal model splits its prefill further, into its modality encoders (such as <code>image_encoder</code>), its projector, its <code>language_model</code> and its <code>lm_head</code> (see <a href="./exporters#how-exportforgeneration-works">How <code>export_for_generation</code> works</a>).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> torchao.quantization.pt2e.quantizer.x86_inductor_quantizer <span class="hljs-keyword">import</span> (
    X86InductorQuantizer,
    get_default_x86_inductor_quantization_config,
)
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> transformers.exporters <span class="hljs-keyword">import</span> DynamoConfig, DynamoExporter, PT2EQuantizer

model_id = <span class="hljs-string">&quot;meta-llama/Llama-3.2-1B&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id)

prompts = [<span class="hljs-string">&quot;The capital of France is&quot;</span>, <span class="hljs-string">&quot;Photosynthesis turns sunlight into&quot;</span>, <span class="hljs-string">&quot;The first moon landing was in&quot;</span>]
calibration_dataset = [tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>) <span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> prompts]
inputs = tokenizer(<span class="hljs-string">&quot;Once upon a time&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

x86 = X86InductorQuantizer().set_global(get_default_x86_inductor_quantization_config())
config = DynamoConfig(dynamic=<span class="hljs-literal">True</span>, quantizer=PT2EQuantizer(x86, calibration_dataset=calibration_dataset))
components = DynamoExporter().export_for_generation(model, inputs, config)  <span class="hljs-comment"># {&quot;prefill&quot;: ..., &quot;decode&quot;: ...}</span><!----></pre></div><!----> <p>On OpenVINO, PT2E weights stay in full precision behind quantize/dequantize pairs, which OpenVINO converts to <code>FakeQuantize</code> and compresses to int8 itself.</p> <!--[1--><h2 class="relative group"><a id="onnx-runtime" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#onnx-runtime"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>ONNX Runtime</span></h2><!--]--><!----> <p><a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.OnnxRuntimeQuantizer">OnnxRuntimeQuantizer</a> runs <code>quantize_static</code> by default (int8 QDQ activations and weights,
calibrated on <code>calibration_dataset</code>), as in the example above, with the arguments of an <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.OnnxRuntimeStaticConfig">OnnxRuntimeStaticConfig</a>. With an <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.OnnxRuntimeDynamicConfig">OnnxRuntimeDynamicConfig</a> it runs <code>quantize_dynamic</code> (int8 weights, activations quantized at runtime, no calibration).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers.exporters <span class="hljs-keyword">import</span> OnnxRuntimeDynamicConfig, OnnxRuntimeStaticConfig

config = OnnxConfig(
    quantizer=OnnxRuntimeQuantizer(OnnxRuntimeStaticConfig(per_channel=<span class="hljs-literal">True</span>), calibration_dataset=calibration_dataset)
)
config = OnnxConfig(quantizer=OnnxRuntimeQuantizer(OnnxRuntimeDynamicConfig()))<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="nncf" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#nncf"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>NNCF</span></h2><!--]--><!----> <p>The three NNCF quantizers in the table above run <code>nncf.quantize</code> by default (int8 activations and weights, calibrated
on <code>calibration_dataset</code>, with <code>model_type=nncf.ModelType.TRANSFORMER</code> unless you pass another), with the arguments of
an <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.NNCFQuantizationConfig">NNCFQuantizationConfig</a>. With an <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.NNCFWeightCompressionConfig">NNCFWeightCompressionConfig</a> they
run <code>nncf.compress_weights</code> (weight-only int8 or int4, with AWQ and scale estimation for int4, plus GPTQ and LoRA
correction on OpenVINO). ExecuTorch lowers only <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.NNCFTorchFXQuantizer">NNCFTorchFXQuantizer</a>’s weight-only result.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> nncf
<span class="hljs-keyword">from</span> transformers.exporters <span class="hljs-keyword">import</span> (
    DynamoConfig,
    NNCFOnnxQuantizer,
    NNCFOpenVINOQuantizer,
    NNCFTorchFXQuantizer,
    NNCFWeightCompressionConfig,
    OnnxConfig,
    OpenVINOConfig,
    OpenVINOExporter,
)

int4 = NNCFWeightCompressionConfig(mode=nncf.CompressWeightsMode.INT4_SYM, group_size=<span class="hljs-number">128</span>)

<span class="hljs-comment"># OpenVINO: int8 activations and weights, or weight-only int4</span>
openvino_int8 = OpenVINOConfig(quantizer=NNCFOpenVINOQuantizer(calibration_dataset=calibration_dataset))
openvino_int4 = OpenVINOConfig(quantizer=NNCFOpenVINOQuantizer(int4))
<span class="hljs-comment"># ONNX: weight-only int4, as ONNX Runtime&#x27;s `MatMulNBits`</span>
onnx_int4 = OnnxConfig(quantizer=NNCFOnnxQuantizer(int4))
<span class="hljs-comment"># Dynamo: int8 on the FX graph</span>
dynamo_int8 = DynamoConfig(quantizer=NNCFTorchFXQuantizer(calibration_dataset=calibration_dataset))

ov_model = OpenVINOExporter().export(model, inputs, openvino_int4)<!----></pre></div><!----> <p>Int8 activations run int8 matmuls, which saturate on CPUs without VNNI (AVX2-only Intel and AMD Zen 2 and older), so
expect lower accuracy there. Weight-only compression keeps the matmuls in floating point.</p> <!--[1--><h2 class="relative group"><a id="calibration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#calibration"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Calibration</span></h2><!--]--><!----> <p>A quantizer’s <code>calibration_dataset</code> is an iterable of model inputs, such as a list of tokenizer outputs or a <code>DataLoader</code> whose batches collate to model inputs. It is a plain attribute, so you can also set it after creating
the quantizer (<code>quantizer.calibration_dataset = samples</code>). Omit it and a quantizer that needs data calibrates on the
export’s own sample inputs, with a warning, because one sample can skew the observed ranges. Quantizers that need no
data (dynamic ONNX Runtime quantization, int8 weight compression) never read it.</p> <!--[1--><h2 class="relative group"><a id="a-different-recipe-per-component" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#a-different-recipe-per-component"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>A different recipe per component</span></h2><!--]--><!----> <p>Pass a <code>{component: config}</code> dict to <code>~HfExporter.export_for_generation</code> to quantize each component on its own
terms. For the model above, static int8 on <code>prefill</code> and the lighter dynamic int8 on <code>decode</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->dynamic_x86 = X86InductorQuantizer().set_global(get_default_x86_inductor_quantization_config(is_dynamic=<span class="hljs-literal">True</span>))

config = {
    <span class="hljs-string">&quot;prefill&quot;</span>: DynamoConfig(dynamic=<span class="hljs-literal">True</span>, quantizer=PT2EQuantizer(x86)),
    <span class="hljs-string">&quot;decode&quot;</span>: DynamoConfig(dynamic=<span class="hljs-literal">True</span>, quantizer=PT2EQuantizer(dynamic_x86)),
}
components = DynamoExporter().export_for_generation(model, inputs, config)<!----></pre></div><!----> <p>The dict must name every component <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.utils.decompose_for_generation">decompose_for_generation()</a> produces (a multimodal model adds its
encoders, and may split out <code>language_model</code> and <code>lm_head</code>). A component whose config sets no quantizer stays in full
precision. In this dict, a quantizer’s <code>calibration_dataset</code> holds the component’s forward kwargs, not generate
kwargs.</p> <!--[1--><h2 class="relative group"><a id="add-a-quantizer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#add-a-quantizer"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Add a quantizer</span></h2><!--]--><!----> <p>Quantizers live in <code>transformers.exporters.quantizers</code>, one class per toolchain and representation. Subclass <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.ExportQuantizer">ExportQuantizer</a> and declare:</p> <ul><li><code>quantizes_fx_graph</code>: <code>True</code> for the <code>torch.export</code> FX graph, before any conversion (so every backend takes the
result), or <code>False</code> (the default) for the model a backend converts it into (<code>onnx_ir.Model</code>, <code>openvino.Model</code>).</li> <li><code>supported_formats</code>: the export formats it quantizes. The export rejects any other format before tracing.</li> <li><code>required_packages</code>, and optionally <code>min_versions</code> and <code>tested_versions</code>: what it imports. The export checks
they’re installed before tracing, raises below a minimum version, and warns when a version differs from the tested
one.</li></ul> <p>Then implement <code>quantize(model, sample_inputs, export_format)</code>. <code>model</code> is the FX graph or the backend model, and <code>sample_inputs</code> are the export’s own inputs. Calibrate on a <a href="/docs/transformers/pr_36708/en/main_classes/exporters#transformers.exporters.quantizers.CalibrationSet">CalibrationSet</a>: it reads the
quantizer’s <code>calibration_dataset</code> (or <code>sample_inputs</code> when there is none) and maps each sample to the model’s own
inputs with a feed from <code>transformers.exporters.quantizers.utils</code> (<code>traced_inputs</code> for the FX graph, <code>onnx_feed</code> for ONNX, <code>openvino_feed</code> for OpenVINO).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers.exporters <span class="hljs-keyword">import</span> ExportFormat
<span class="hljs-keyword">from</span> transformers.exporters.quantizers <span class="hljs-keyword">import</span> CalibrationSet, ExportQuantizer
<span class="hljs-keyword">from</span> transformers.exporters.quantizers.utils <span class="hljs-keyword">import</span> onnx_feed


<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyOnnxQuantizer</span>(<span class="hljs-title class_ inherited__">ExportQuantizer</span>):
    supported_formats = (ExportFormat.ONNX,)
    required_packages = (<span class="hljs-string">&quot;my_toolkit&quot;</span>,)
    tested_versions = {<span class="hljs-string">&quot;my_toolkit&quot;</span>: <span class="hljs-string">&quot;1.0.0&quot;</span>}

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">quantize</span>(<span class="hljs-params">self, model, sample_inputs, export_format</span>):
        <span class="hljs-keyword">import</span> my_toolkit

        names = [graph_input.name <span class="hljs-keyword">for</span> graph_input <span class="hljs-keyword">in</span> model.graph.inputs]
        calibration = CalibrationSet(<span class="hljs-variable language_">self</span>.calibration_dataset, sample_inputs, <span class="hljs-keyword">lambda</span> sample: onnx_feed(names, sample))
        <span class="hljs-keyword">return</span> my_toolkit.quantize(model, calibration_feeds=<span class="hljs-built_in">list</span>(calibration))<!----></pre></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/exporters_quantization.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_8fh4j = {
						base: "/docs/transformers/main/en",
						assets: "/docs/transformers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/transformers/main/en/_app/immutable/entry/start.DZe-7K50.js"),
						import("/docs/transformers/main/en/_app/immutable/entry/app.BtyGqhlx.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 3],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 39.3 kB
Xet hash:: 4b51ed7576e367af02b3e9fa0dbe6f0a12c9352a4c82ffae7f778064b0f890f8
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.