Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"compressed-tensors","local":"compressed-tensors","sections":[{"title":"Loading modes","local":"loading-modes","sections":[],"depth":2},{"title":"FP8 kernel acceleration","local":"fp8-kernel-acceleration","sections":[{"title":"Loading a pre-quantized FP8 model","local":"loading-a-pre-quantized-fp8-model","sections":[],"depth":3},{"title":"Dequantizing at load time","local":"dequantizing-at-load-time","sections":[],"depth":3}],"depth":2},{"title":"Model checkpoint","local":"model-checkpoint","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/transformers/main/en/_app/immutable/entry/start.CCacYxwz.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/CBn-xyeu.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/BhmhGQv5.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/entry/app.Can_yyrx.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/qMuWdE-Y.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/CfMEBQAN.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/BRRD8tqb.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/nodes/0.HSXqqoXV.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/B76T2TVW.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/nodes/656.Cs-fRhIR.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/vlAHDplH.js" rel="modulepreload"> | |
| <link href="/docs/transformers/main/en/_app/immutable/chunks/BpA0a5UK.js" rel="modulepreload"> | |
| <!--6pfp--><meta name="hf:doc:metadata" content="{"title":"compressed-tensors","local":"compressed-tensors","sections":[{"title":"Loading modes","local":"loading-modes","sections":[],"depth":2},{"title":"FP8 kernel acceleration","local":"fp8-kernel-acceleration","sections":[{"title":"Loading a pre-quantized FP8 model","local":"loading-a-pre-quantized-fp8-model","sections":[],"depth":3},{"title":"Dequantizing at load time","local":"dequantizing-at-load-time","sections":[],"depth":3}],"depth":2},{"title":"Model checkpoint","local":"model-checkpoint","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/transformers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="compressed-tensors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compressed-tensors"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>compressed-tensors</span></h1><!--]--><!----> <p><a href="https://github.com/neuralmagic/compressed-tensors" rel="nofollow">compressed-tensors</a> extends <a href="https://github.com/huggingface/safetensors" rel="nofollow">safetensors</a> files to compressed tensor data types to provide a unified checkpoint format for storing and loading various quantization formats such as dense, int-quantized (int8), float-quantized (fp8), and pack-quantized (int4 or int8 weight-quantized packed into int32).</p> <p>compressed-tensors supports fine-tuning with <a href="https://huggingface.co/docs/peft" rel="nofollow">PEFT</a> and includes the following features as well.</p> <ul><li>fp8, int4, int8 weight and activation precisions.</li> <li>Quantization scales and zero-points strategies for <a href="https://github.com/neuralmagic/compressed-tensors/blob/83b2e7a969d70606421a76b9a3d112646077c8de/src/compressed_tensors/quantization/quant_args.py#L43-L52" rel="nofollow">tensor, channel, group, block, token</a>.</li> <li>Dynamic per-token activation quantization (or any static strategy).</li> <li>Quantization of arbitrary modules, not just <a href="https://pytorch.org/docs/stable/generated/torch.nn.Linear.html" rel="nofollow">nn.Linear</a> modules.</li> <li>Targeted support for specific modules by name or class.</li></ul> <p>Install compressed-tensors from <a href="https://pypi.org/project/compressed-tensors" rel="nofollow">PyPI</a> to get the latest stable release (recommended) or install it from source to get the latest features.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">PyPI</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">source code</div><!--]--></div> <div class="language-select"><!--[0--><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->pip install compressed-tensors<!----></pre></div><!----><!--]--><!----> <!--[-1--><!--]--><!----><!----></div><!----> <p>Search using the compressed-tensors <a href="https://huggingface.co/models?other=compressed-tensors" rel="nofollow">tag</a> to find a compatible model on the Hugging Face Hub.</p> <p>Pre-quantized models can be loaded directly. To quantize a model into the compressed-tensors format, see <a href="https://github.com/vllm-project/llm-compressor" rel="nofollow">llm-compressor</a>. Alternatively, models can be created independently and serialized with a compressed-tensors config.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM | |
| ct_model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"nm-testing/Meta-Llama-3.1-8B-Instruct-FP8-hf"</span>, device_map=<span class="hljs-string">"auto"</span>) | |
| <span class="hljs-comment"># measure memory usage</span> | |
| mem_params = <span class="hljs-built_in">sum</span>([param.nelement()*param.element_size() <span class="hljs-keyword">for</span> param <span class="hljs-keyword">in</span> ct_model.parameters()]) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"<span class="hljs-subst">{mem_params/<span class="hljs-number">2</span>**<span class="hljs-number">30</span>:<span class="hljs-number">.4</span>f}</span> GB"</span>) | |
| <span class="hljs-comment"># 8.4575 GB</span><!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="loading-modes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-modes"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Loading modes</span></h2><!--]--><!----> <p>A compressed-tensors checkpoint stores its weights compressed (fp8, or packed int4/int8). How they are executed is up to two <a href="/docs/transformers/pr_48167/en/main_classes/quantization#transformers.CompressedTensorsConfig">CompressedTensorsConfig</a> arguments.</p> <table><thead><tr><th>Configuration</th><th>Weights after loading</th><th>Execution</th></tr></thead><tbody><tr><td>default</td><td>left compressed</td><td>compressed-tensors owns the layers and decompresses the model on the first forward pass</td></tr><tr><td><code>dequantize=True</code></td><td>dequantized to the model dtype (e.g. BF16)</td><td>regular dense matmuls, and the model can be fine-tuned or saved in that dtype</td></tr><tr><td><code>use_optimized_inference=True</code></td><td>kept quantized</td><td>layers whose scheme has a kernel run through it, currently W8A8 fp8; inference only</td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="fp8-kernel-acceleration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fp8-kernel-acceleration"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>FP8 kernel acceleration</span></h2><!--]--><!----> <p>Pass <code>use_optimized_inference=True</code> to keep an FP8 compressed-tensors model in FP8 and run its matmuls through hardware-accelerated FP8 kernels (<a href="https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_mm.html" rel="nofollow">torch.nn.functional.scaled_mm</a>, which dispatches to <code>torch._scaled_mm_v2</code>; older torch versions fall back to <code>torch._scaled_mm</code>), instead of dequantizing the weights back to BF16. Keeping weights in FP8 throughout inference lowers memory usage and speeds up computation. This is inference only, so leave it off to fine-tune.</p> <table><thead><tr><th>Device</th><th>Kernel</th><th>Notes</th></tr></thead><tbody><tr><td>Intel XPU</td><td><code>torch.nn.functional.scaled_mm</code></td><td>All XPU devices with FP8 support</td></tr><tr><td>NVIDIA CUDA (SM89+)</td><td><code>torch.nn.functional.scaled_mm</code></td><td>Ada Lovelace (L4, L40), Hopper (H100), Blackwell and newer</td></tr><tr><td>CPU / CUDA SM80 (A100)</td><td>Fallback</td><td><code>use_optimized_inference=True</code> is ignored, the model runs dequantized</td></tr></tbody></table> <p>The FP8 kernel path supports these quantization layouts.</p> <table><thead><tr><th>Strategy</th><th>Example model</th></tr></thead><tbody><tr><td>Per-channel dynamic</td><td><a href="https://huggingface.co/RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic" rel="nofollow">RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic</a></td></tr><tr><td>Per-tensor static</td><td><a href="https://huggingface.co/RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8" rel="nofollow">RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8</a></td></tr></tbody></table> <!--[2--><h3 class="relative group"><a id="loading-a-pre-quantized-fp8-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-a-pre-quantized-fp8-model"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Loading a pre-quantized FP8 model</span></h3><!--]--><!----> <p>The FP8 kernels are opt-in: ask for them with <code>use_optimized_inference=True</code>, and they are used when the model’s config specifies FP8 quantization and a supported GPU is available.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, CompressedTensorsConfig | |
| model = AutoModelForCausalLM.from_pretrained( | |
| <span class="hljs-string">"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic"</span>, | |
| quantization_config=CompressedTensorsConfig(use_optimized_inference=<span class="hljs-literal">True</span>), | |
| device_map=<span class="hljs-string">"auto"</span>, | |
| ) | |
| tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic"</span>) | |
| inputs = tokenizer(<span class="hljs-string">"Hello, how are you?"</span>, return_tensors=<span class="hljs-string">"pt"</span>).to(model.device) | |
| outputs = model.generate(**inputs, max_new_tokens=<span class="hljs-number">20</span>) | |
| <span class="hljs-built_in">print</span>(tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="dequantizing-at-load-time" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dequantizing-at-load-time"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Dequantizing at load time</span></h3><!--]--><!----> <p>Without <code>use_optimized_inference=True</code>, the model takes the regular compressed-tensors route: the weights are left compressed and compressed-tensors decompresses them on the first forward pass. Pass <code>dequantize=True</code> to dequantize them during loading instead, which is what you want to fine-tune the model or save it in its original precision (e.g. BF16).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, CompressedTensorsConfig | |
| model = AutoModelForCausalLM.from_pretrained( | |
| <span class="hljs-string">"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic"</span>, | |
| quantization_config=CompressedTensorsConfig(dequantize=<span class="hljs-literal">True</span>), | |
| device_map=<span class="hljs-string">"auto"</span>, | |
| )<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="model-checkpoint" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#model-checkpoint"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Model checkpoint</span></h2><!--]--><!----> <p>Compressed-tensor models are defined through its configuration entry. The following example is taken from the <a href="https://huggingface.co/nm-testing/Meta-Llama-3.1-8B-Instruct-FP8-hf/blob/main/config.json" rel="nofollow">nm-testing/Meta-Llama-3.1-8B-Instruct-FP8-hf</a> <code>config.json</code> file.</p> <p>There are a lot of entries to allow for flexible expression both during and after compression, but the entries for loading and inference can be simplified to focus on just a few key entries.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-json "><!----><span class="hljs-attr">"quantization_config"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"config_groups"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"group_0"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"input_activations"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"num_bits"</span><span class="hljs-punctuation">:</span> <span class="hljs-number">8</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"strategy"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"tensor"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"float"</span> | |
| <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"targets"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">"Linear"</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"weights"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"num_bits"</span><span class="hljs-punctuation">:</span> <span class="hljs-number">8</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"strategy"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"tensor"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"float"</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"format"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"naive-quantized"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"ignore"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">"lm_head"</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"quant_method"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"compressed-tensors"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"quantization_status"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"frozen"</span> | |
| <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span><!----></pre></div><!----> <p>The config file specifies the quantization of a config group (<code>group_0</code>), which includes weight and activation quantization to fp8 with a static per-tensor strategy. The <code>lm_head</code> module is unquantized as shown in the <code>ignore</code> key.</p> <p>For a more detailed look at the model weights, use the <a href="https://huggingface.co/nm-testing/Meta-Llama-3.1-8B-Instruct-FP8-hf?show_file_info=model.safetensors.index.json" rel="nofollow">safetensors viewer</a> on the model card to see the quantized weights, input scale, and weight scale for all <a href="https://pytorch.org/docs/stable/generated/torch.nn.Linear.html" rel="nofollow">nn.Linear</a> modules.</p> <table><thead><tr><th>Tensors</th><th>Shape</th><th>Precision</th></tr></thead><tbody><tr><td>model.layers.0.input_layernorm.weight</td><td>[4 096]</td><td>BF16</td></tr><tr><td>model.layers.0.mlp.down_proj.input_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.mlp.down_proj.weight</td><td>[4 096, 14 336]</td><td>F8_E4M3</td></tr><tr><td>model.layers.0.mlp.down_proj.weight_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.mlp.gate_proj.input_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.mlp.gate_proj.weight</td><td>[14 336, 4 096]</td><td>F8_E4M3</td></tr><tr><td>model.layers.0.mlp.gate_proj.weight_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.mlp.up_proj.input_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.mlp.up_proj.weight</td><td>[14 336, 4 096]</td><td>F8_E4M3</td></tr><tr><td>model.layers.0.mlp.up_proj.weight_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.post_attention_layernorm.weight</td><td>[4 096]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.k_proj.input_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.k_proj.weight</td><td>[1 024, 4 096]</td><td>F8_E4M3</td></tr><tr><td>model.layers.0.self_attn.k_proj.weight_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.o_proj.input_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.o_proj.weight</td><td>[4 096, 4 096]</td><td>F8_E4M3</td></tr><tr><td>model.layers.0.self_attn.o_proj.weight_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.q_proj.input_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.q_proj.weight</td><td>[4 096, 4 096]</td><td>F8_E4M3</td></tr><tr><td>model.layers.0.self_attn.q_proj.weight_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.v_proj.input_scale</td><td>[1]</td><td>BF16</td></tr><tr><td>model.layers.0.self_attn.v_proj.weight</td><td>[1 024, 4 096]</td><td>F8_E4M3</td></tr><tr><td>model.layers.0.self_attn.v_proj.weight_scale</td><td>[1]</td><td>BF16</td></tr></tbody></table> <p>When loading a compressed-tensors model with the <code>~quantizers.HFQuantizer</code> integration, the targeted modules are handed over to compressed-tensors: it attaches the resolved <code>quantization_scheme</code>, sets <code>quantization_status</code>, registers the parameters the checkpoint stores (<code>weight</code> in fp8, plus <code>weight_scale</code> and, for a static strategy, <code>input_scale</code>) and installs its own forward pass over them. They stay <a href="https://pytorch.org/docs/stable/generated/torch.nn.Linear.html" rel="nofollow">nn.Linear</a> instances, so that is what <code>print</code> shows — recent compressed-tensors versions no longer wrap them in a <code>CompressedLinear</code> subclass. Modules listed under <code>ignore</code>, such as <code>lm_head</code>, are left untouched.</p> <p>With <code>dequantize=False</code> (the default), the weights are still compressed once loading is over, and compressed-tensors decompresses the whole model on the first forward pass. <code>dequantize=True</code> does it during loading instead, so no forward pass is needed to get dense weights.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, CompressedTensorsConfig | |
| model_id = <span class="hljs-string">"nm-testing/Meta-Llama-3.1-8B-Instruct-FP8-hf"</span> | |
| ct_model = AutoModelForCausalLM.from_pretrained( | |
| model_id, | |
| quantization_config=CompressedTensorsConfig(dequantize=<span class="hljs-literal">False</span>), | |
| device_map=<span class="hljs-string">"auto"</span>, | |
| ) | |
| q_proj = ct_model.model.layers[<span class="hljs-number">0</span>].self_attn.q_proj | |
| <span class="hljs-built_in">print</span>(q_proj, q_proj.quantization_status) | |
| <span class="hljs-comment"># Linear(in_features=4096, out_features=4096, bias=False) QuantizationStatus.COMPRESSED</span> | |
| <span class="hljs-comment"># ^ compressed-tensors module: fp8 weight, weight_scale, and its own forward</span> | |
| ct_model(input_ids=torch.tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>]], device=ct_model.device)) | |
| <span class="hljs-built_in">print</span>(q_proj, q_proj.quantization_status) | |
| <span class="hljs-comment"># Linear(in_features=4096, out_features=4096, bias=False) QuantizationStatus.DECOMPRESSED</span> | |
| <span class="hljs-comment"># ^ weight is BF16 now, decompressed by that forward pass</span> | |
| ct_model = AutoModelForCausalLM.from_pretrained( | |
| model_id, | |
| quantization_config=CompressedTensorsConfig(dequantize=<span class="hljs-literal">True</span>), | |
| device_map=<span class="hljs-string">"auto"</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(ct_model.model.layers[<span class="hljs-number">0</span>].self_attn.q_proj) | |
| <span class="hljs-comment"># Linear(in_features=4096, out_features=4096, bias=False) weight: BF16</span><!----></pre></div><!----> <p>With <code>use_optimized_inference=True</code>, the layers covered by an fp8 config group are replaced by <code>CompressedTensorsFP8Linear</code>, which holds the fp8 weight and its scale in the layout its row-wise matmul kernel expects. Those weights stay in fp8, forward passes included.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, CompressedTensorsConfig | |
| ct_model = AutoModelForCausalLM.from_pretrained( | |
| <span class="hljs-string">"nm-testing/Meta-Llama-3.1-8B-Instruct-FP8-hf"</span>, | |
| quantization_config=CompressedTensorsConfig(use_optimized_inference=<span class="hljs-literal">True</span>), | |
| device_map=<span class="hljs-string">"auto"</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(ct_model.model.layers[<span class="hljs-number">0</span>].self_attn.q_proj) | |
| <span class="hljs-comment"># CompressedTensorsFP8Linear(in_features=4096, out_features=4096, bias=False) weight: F8_E4M3</span><!----></pre></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/compressed_tensors.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_1go0z3q = { | |
| base: "/docs/transformers/main/en", | |
| assets: "/docs/transformers/main/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/transformers/main/en/_app/immutable/entry/start.CCacYxwz.js"), | |
| import("/docs/transformers/main/en/_app/immutable/entry/app.Can_yyrx.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 656], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 40.5 kB
- Xet hash:
- 7478d489ee85f7daf878ab643d48ce10b4836caee2761bcb10d6a454670372d0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.