Buckets:

hf-doc-build
/

doc-dev

Files
xet
hf-doc-build/doc-dev / transformers /pr_45649 /en /quantization /concept_guide.html
HuggingFaceDocBuilder
about 18 hours ago
download
raw
55.6 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Quantization concepts&quot;,&quot;local&quot;:&quot;quantization-concepts&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Map values to fewer bits&quot;,&quot;local&quot;:&quot;map-values-to-fewer-bits&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Choose the granularity&quot;,&quot;local&quot;:&quot;choose-the-granularity&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Quantize weights or activations&quot;,&quot;local&quot;:&quot;quantize-weights-or-activations&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Data types&quot;,&quot;local&quot;:&quot;data-types&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;int8 and int4&quot;,&quot;local&quot;:&quot;int8-and-int4&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;FP8&quot;,&quot;local&quot;:&quot;fp8&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;FP4 with block scales&quot;,&quot;local&quot;:&quot;fp4-with-block-scales&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Codebooks&quot;,&quot;local&quot;:&quot;codebooks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;When quantization happens&quot;,&quot;local&quot;:&quot;when-quantization-happens&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Post-training quantization&quot;,&quot;local&quot;:&quot;post-training-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Quantization-aware training&quot;,&quot;local&quot;:&quot;quantization-aware-training&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/transformers/pr_45649/en/_app/immutable/entry/start.Cf6g_lQr.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/bcIanF0A.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/CBySwR1V.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/entry/app.Ckn5M_jH.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/Cdb1BouH.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/dUGfULTE.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/D5-p1Lz5.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/nodes/0.92KZZ6l3.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/Conj1S5Z.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/nodes/698.na8igYZi.js" rel="modulepreload">
		<link href="/docs/transformers/pr_45649/en/_app/immutable/chunks/DoQ7MZQh.js" rel="modulepreload">
		<!--85pub6--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Quantization concepts&quot;,&quot;local&quot;:&quot;quantization-concepts&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Map values to fewer bits&quot;,&quot;local&quot;:&quot;map-values-to-fewer-bits&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Choose the granularity&quot;,&quot;local&quot;:&quot;choose-the-granularity&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Quantize weights or activations&quot;,&quot;local&quot;:&quot;quantize-weights-or-activations&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Data types&quot;,&quot;local&quot;:&quot;data-types&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;int8 and int4&quot;,&quot;local&quot;:&quot;int8-and-int4&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;FP8&quot;,&quot;local&quot;:&quot;fp8&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;FP4 with block scales&quot;,&quot;local&quot;:&quot;fp4-with-block-scales&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Codebooks&quot;,&quot;local&quot;:&quot;codebooks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;When quantization happens&quot;,&quot;local&quot;:&quot;when-quantization-happens&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Post-training quantization&quot;,&quot;local&quot;:&quot;post-training-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Quantization-aware training&quot;,&quot;local&quot;:&quot;quantization-aware-training&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/transformers/pr_45649/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="quantization-concepts" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantization-concepts"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Quantization concepts<!----></span></h1><!--]--><!----> <p>A quantized number can only take one of a small set of values, 256 for int8 and 16 for int4, so every quantization method introduces rounding error. Methods differ in how they keep that error from hurting the model’s output. This page explains the choices behind those differences: how numbers are mapped to that small set, which tensors get quantized, which data types are used, and when quantization happens.</p> <p>If you’re new to quantization in Transformers, start with the <a href="./overview">Overview</a>.</p> <!--[1--><h2 class="relative group"><a id="map-values-to-fewer-bits" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#map-values-to-fewer-bits"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Map values to fewer bits<!----></span></h2><!--]--><!----> <p>Quantization maps a range of high-precision values onto the small set of integers a low-bit type can represent. int8 can represent 256 integers, <code>[-128, 127]</code>, and int4 can represent 16.</p> <p><em>Affine quantization</em> is the most common mapping. It takes the minimum and maximum values of a tensor and stretches the integer range over it. In the example below, a tensor whose values run from -1.8 to 2.4 is mapped to int8, and 0.0 lands on -19.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->bf16  -1.8             0.0                    2.4
       ├────────────────┼──────────────────────┤
       ▼                ▼                      ▼
int8  -128             -19                    127<!----></pre></div><!----> <p>Two parameters define the mapping.</p> <ul><li><p>The <em>scale</em> <code>S</code> is the size of one integer step in the original units. <!----><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>S</mi><mo>=</mo><mfrac><mrow><mi>v</mi><mi>a</mi><msub><mi>l</mi><mrow><mi>m</mi><mi>a</mi><mi>x</mi></mrow></msub><mo>−</mo><mi>v</mi><mi>a</mi><msub><mi>l</mi><mrow><mi>m</mi><mi>i</mi><mi>n</mi></mrow></msub></mrow><mrow><msub><mi>q</mi><mrow><mi>m</mi><mi>a</mi><mi>x</mi></mrow></msub><mo>−</mo><msub><mi>q</mi><mrow><mi>m</mi><mi>i</mi><mi>n</mi></mrow></msub></mrow></mfrac></mrow><annotation encoding="application/x-tex">
S = \frac{val_{max} - val_{min}}{q_{max} - q_{min}}
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal" style="margin-right:0.0576em;">S</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.2519em;vertical-align:-0.8804em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3714em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">q</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">ma</span><span class="mord mathnormal mtight">x</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">q</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">min</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">v</span><span class="mord mathnormal">a</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0197em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">ma</span><span class="mord mathnormal mtight">x</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal" style="margin-right:0.0359em;">v</span><span class="mord mathnormal">a</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0197em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">min</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.8804em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span><!----></p></li> <li><p>The <em>zero point</em> <code>Z</code> is the integer that represents <code>0.0</code>. <!----><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>Z</mi><mo>=</mo><msub><mi>q</mi><mrow><mi>m</mi><mi>i</mi><mi>n</mi></mrow></msub><mo>−</mo><mi>r</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>d</mi><mrow><mo fence="true">(</mo><mfrac><mrow><mi>v</mi><mi>a</mi><msub><mi>l</mi><mrow><mi>m</mi><mi>i</mi><mi>n</mi></mrow></msub></mrow><mi>S</mi></mfrac><mo fence="true">)</mo></mrow></mrow><annotation encoding="application/x-tex">
Z = q_{min} - round\left(\frac{val_{min}}{S}\right)
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal" style="margin-right:0.0715em;">Z</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.7778em;vertical-align:-0.1944em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">q</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">min</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.4em;vertical-align:-0.95em;"></span><span class="mord mathnormal" style="margin-right:0.0278em;">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">d</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3714em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0576em;">S</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">v</span><span class="mord mathnormal">a</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0197em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">min</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span></span></span></span></span><!----></p></li></ul> <p>A value <code>x</code> is quantized to the integer <code>q</code> by dividing by the scale and shifting by the zero point. <!----><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>q</mi><mo>=</mo><mi>r</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>d</mi><mrow><mo fence="true">(</mo><mfrac><mi>x</mi><mi>S</mi></mfrac><mo>+</mo><mi>Z</mi><mo fence="true">)</mo></mrow></mrow><annotation encoding="application/x-tex">
q = round\left(\frac{x}{S} + Z\right)
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.0359em;">q</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.836em;vertical-align:-0.686em;"></span><span class="mord mathnormal" style="margin-right:0.0278em;">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">d</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size2">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.1076em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0576em;">S</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">x</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal" style="margin-right:0.0715em;">Z</span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size2">)</span></span></span></span></span></span></span><!----></p> <p>Dequantizing reverses the mapping and gives back an approximation of <code>x</code>. The difference between <code>x</code> and its approximation is the rounding error. <!----><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>x</mi><mo>≈</mo><mi>S</mi><mo>⋅</mo><mo stretchy="false">(</mo><mi>q</mi><mo>−</mo><mi>Z</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">
x \approx S \cdot (q - Z)
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4831em;"></span><span class="mord mathnormal">x</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">≈</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal" style="margin-right:0.0576em;">S</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">⋅</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.0359em;">q</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.0715em;">Z</span><span class="mclose">)</span></span></span></span></span><!----></p> <p>In the example above, <code>S</code> is about 0.0165 and <code>Z</code> is -19. The value 0.5 quantizes to 11, and 11 dequantizes to about 0.494, a rounding error of about 0.006.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->┌─────┐  quantize   ┌────┐ dequantize   ┌───────┐
│ 0.5 │ ──────────▶ │ 11 │ ───────────▶ │ 0.494 │
└─────┘             └────┘              └───────┘<!----></pre></div><!----> <p>Affine quantization comes in two forms.</p> <ul><li><em>Asymmetric</em> quantization uses the formulas above. It maps the exact range, so no representable values go to waste when the weights aren’t centered on zero.</li> <li><em>Symmetric</em> quantization assumes the range is <code>[-a, a]</code> and fixes <code>Z</code> at 0, so only the scale is stored. The math is cheaper, but some representable values go unused when the weights lean to one side of zero.</li></ul> <!--[1--><h2 class="relative group"><a id="choose-the-granularity" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#choose-the-granularity"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Choose the granularity<!----></span></h2><!--]--><!----> <p>A single scale for a whole tensor is cheap to store, but one large outlier stretches the range and spreads the representable values farther apart for every other weight. Smaller groups give each scale a narrower range to cover.</p> <ul><li>Per-tensor uses one scale, and one zero point, for the entire tensor.</li> <li>Per-channel uses one scale for each output channel, which is each row of a weight matrix.</li> <li>Per-group, or per-block, uses one scale for each group of consecutive values, such as 128 weights. Each group costs extra storage for its scale.</li></ul> <p>The diagram below shows the three options on a weight matrix with three rows. Each <code>s</code> is one scale.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->per-tensor        per-channel           per-group
┌────────────┐    ┌────────────┐        ┌──────┬──────┐
│            │    │            │ s0     │  s0  │  s1  │
│            │    ├────────────┤        ├──────┼──────┤
│     s      │    │            │ s1     │  s2  │  s3  │
│            │    ├────────────┤        ├──────┼──────┤
│            │    │            │ s2     │  s4  │  s5  │
└────────────┘    └────────────┘        └──────┴──────┘<!----></pre></div><!----> <p>Most low-bit methods quantize per group. <a href="./gptq">GPTQ</a> defaults to groups of 128 weights, and <a href="./finegrained_fp8">Fine-grained FP8</a> defaults to 128x128 blocks. The FP4 formats in <a href="#data-types">Data types</a> build the block size into the format itself.</p> <!--[1--><h2 class="relative group"><a id="quantize-weights-or-activations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantize-weights-or-activations"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Quantize weights or activations<!----></span></h2><!--]--><!----> <p>Methods differ in which tensors they quantize, and that decides where the matrix multiplication runs.</p> <ul><li><em>Weight-only</em> methods store the weights in low precision and leave the activations in bf16 or fp16. Before each matmul, the kernel dequantizes the weights to a higher-precision compute dtype. bitsandbytes 4-bit, AWQ, GPTQ, and HQQ work this way. In bitsandbytes, <code>bnb_4bit_compute_dtype</code> sets that compute dtype.</li> <li><em>Weight-and-activation</em> methods quantize the activations as well, and the matmul runs directly in low precision. This needs hardware that supports the low-precision matmul, like FP8 on recent GPUs. The scheme is often written as W8A8 for 8-bit weights and activations.</li></ul> <p>Weight-only methods save memory but don’t always run faster, because dequantizing adds work to every matmul. They speed up generation when the model is <em>memory-bound</em>. Generating one token at a time, a GPU spends most of each step reading weights from memory, not computing. Reading 4-bit weights moves a quarter of the data that bf16 weights do, and that saving can outweigh the cost of dequantizing.</p> <p>Weight-and-activation methods help when the model is <em>compute-bound</em>, as in prefill or large batches, because the low-precision matmul itself is faster. Some methods switch between the two. NVFP4 runs a 4-bit-weight, 4-bit-activation matmul for prefill and a 4-bit-weight, 16-bit-activation matmul for decoding.</p> <!--[1--><h2 class="relative group"><a id="data-types" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#data-types"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Data types<!----></span></h2><!--]--><!----> <p>The data type sets which values a quantized number can represent.</p> <!--[2--><h3 class="relative group"><a id="int8-and-int4" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#int8-and-int4"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->int8 and int4<!----></span></h3><!--]--><!----> <p>int8 and int4 represent evenly spaced integers and use the affine mapping above. int4 can represent only 16 values, so it loses more accuracy than int8. Methods that use int4 typically quantize per group or use calibration to keep accuracy up.</p> <p>Most hardware has no 4-bit storage type, so two int4 values are <em>packed</em> into one byte. For example, the first value can take the lower 4 bits and the second value the upper 4 bits. Kernels unpack the values before computing with them.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->┌──────┐  ┌──────┐        ┌──────┬──────┐
│ val1 │  │ val2 │        │ val2 │ val1 │
│ 0101 │  │ 0011 │   →    │ 0011 │ 0101 │  1 byte
└──────┘  └──────┘        └──────┴──────┘
  int4      int4            high   low<!----></pre></div><!----> <p>Some 4-bit types space their representable values unevenly to match how weights are distributed. bitsandbytes NF4 puts more of its values near zero, where most weights are.</p> <!--[2--><h3 class="relative group"><a id="fp8" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fp8"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->FP8<!----></span></h3><!--]--><!----> <p>FP8 keeps the floating-point layout of a sign, an exponent, and a mantissa in 8 bits. Its representable values are closer together near zero and farther apart at large magnitudes, which suits the spread of values in weights and activations.</p> <p>FP8 comes in two variants.</p> <ul><li>E4M3 has 4 exponent bits and 3 mantissa bits. It’s more precise but covers a smaller range. It’s the usual choice for weights and activations during inference.</li> <li>E5M2 has 5 exponent bits and 2 mantissa bits. It covers a wider range with less precision.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->E4M3   ┌─┬───────┬─────┐
       │S│E E E E│M M M│
       └─┴───────┴─────┘
E5M2   ┌─┬─────────┬───┐
       │S│E E E E E│M M│
       └─┴─────────┴───┘
  S sign   E exponent   M mantissa<!----></pre></div><!----> <p>FP8 matmuls need hardware support, such as NVIDIA GPUs with compute capability 8.9 or higher (Ada, Hopper, Blackwell), AMD Instinct MI300, and Intel XPU. Transformers supports FP8 through <a href="./finegrained_fp8">Fine-grained FP8</a>, <a href="./fbgemm_fp8">FBGEMM FP8</a>, and <a href="./compressed_tensors">compressed-tensors</a>.</p> <!--[2--><h3 class="relative group"><a id="fp4-with-block-scales" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fp4-with-block-scales"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->FP4 with block scales<!----></span></h3><!--]--><!----> <p>FP4 has 1 sign bit, 2 exponent bits, and 1 mantissa bit (E2M1). It represents only 0, 0.5, 1, 1.5, 2, 3, 4, and 6, positive and negative. With so few values, FP4 formats can’t work with one scale per tensor. They attach a scale to every small block of values and store the scales as part of the format.</p> <table><thead><tr><th>Format</th><th>Block size</th><th>Scale per block</th><th>Extra scale</th></tr></thead><tbody><tr><td><a href="./mxfp4">MXFP4</a></td><td>32 values</td><td>8-bit power of two (E8M0)</td><td>None</td></tr><tr><td><a href="./nvfp4">NVFP4</a></td><td>16 values</td><td>FP8</td><td>One FP32 scale per tensor</td></tr></tbody></table> <p>MXFP4 is a microscaling format. A power-of-two scale only shifts the exponent, so applying it is cheap, but it can only land on powers of two. NVFP4 uses smaller blocks and an FP8 scale that can take values between powers of two. This keeps more accuracy, at the cost of a per-tensor scale that keeps the FP8 block scales in range. <a href="./fouroversix">Four Over Six</a> and <a href="./fp_quant">FP-Quant</a> also produce NVFP4 or MXFP4 weights.</p> <p>The same block idea shows up at 8 bits. MXFP8 stores FP8 values with power-of-two block scales, and <a href="./finegrained_fp8">Fine-grained FP8</a> loads it.</p> <!--[2--><h3 class="relative group"><a id="codebooks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#codebooks"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Codebooks<!----></span></h3><!--]--><!----> <p>Below 4 bits, some methods stop rounding each weight to the nearest representable value. They quantize groups of weights together as <em>vectors</em>. A codebook holds a set of learned vectors, and each group of weights is stored as the index of its closest vector. <a href="./aqlm">AQLM</a> defaults to groups of 8 weights and a 16-bit index, which works out to 2 bits per weight. <a href="./vptq">VPTQ</a> also uses codebooks. Finding good codebooks takes a long optimization over calibration data, so these models are quantized ahead of time and loaded as checkpoints.</p> <!--[1--><h2 class="relative group"><a id="when-quantization-happens" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#when-quantization-happens"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->When quantization happens<!----></span></h2><!--]--><!----> <p>A model is quantized either after training or during it. Most methods in Transformers quantize after training.</p> <!--[2--><h3 class="relative group"><a id="post-training-quantization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#post-training-quantization"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Post-training quantization<!----></span></h3><!--]--><!----> <p><em>Post-training quantization (PTQ)</em> quantizes a model that’s already trained, without changing what it learned. PTQ methods differ in whether they need data, and that decides which group a method falls into in the documentation’s table of contents.</p> <ul><li>Without calibration data, the scales come from the weights alone, and quantization is fast enough to run while the model loads. bitsandbytes, HQQ, and torchao work this way. These methods are listed under <em>Quantize while loading</em>.</li> <li>With calibration data, the method runs a small dataset through the model and uses the activations to decide how to quantize each layer. Calibration usually gives better accuracy at 4 bits and below, but it takes minutes to hours.</li></ul> <p>Calibration methods use the activations in different ways. GPTQ quantizes a layer one column at a time and adjusts the remaining columns to make up for each column’s rounding error. AWQ finds the weight channels with the largest activations and scales them up before rounding to protect them.</p> <p>Where the calibration runs decides how you use the method in Transformers.</p> <ul><li><a href="./gptq">GPTQ</a> calibrates inside <code>from_pretrained</code> when you pass a <code>dataset</code> to <a href="/docs/transformers/pr_45649/en/main_classes/quantization#transformers.GPTQConfig">GPTQConfig</a>, so it’s listed under <em>Quantize while loading</em>.</li> <li>Other calibration methods, like AWQ, AutoRound, AQLM, and VPTQ, calibrate in their own libraries. Transformers loads the result, so they’re listed under <em>Load pre-quantized checkpoints</em>.</li></ul> <!--[2--><h3 class="relative group"><a id="quantization-aware-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantization-aware-training"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Quantization-aware training<!----></span></h3><!--]--><!----> <p><em>Quantization-aware training (QAT)</em> simulates quantization during training. The forward pass rounds values as quantization would while gradients update the full-precision weights, so the model learns to tolerate the rounding error. QAT usually gives the best accuracy at very low bit widths, but it requires training the model.</p> <p><a href="./bitnet">BitNet</a> models are trained this way, with their 1.58-bit weights in the loop, and Transformers loads the trained checkpoints.</p> <!--[1--><h2 class="relative group"><a id="resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Resources<!----></span></h2><!--]--><!----> <ul><li><a href="https://www.deeplearning.ai/short-courses/quantization-fundamentals-with-hugging-face/" rel="nofollow">Quantization Fundamentals with Hugging Face</a></li> <li><a href="https://www.deeplearning.ai/short-courses/quantization-in-depth" rel="nofollow">Quantization in Depth</a></li> <li><a href="https://huggingface.co/blog/merve/quantization" rel="nofollow">Introduction to Quantization cooked in 🤗 with 💗🧑‍🍳</a></li> <li><a href="https://www.youtube.com/watch?v=RP23-dRVDWM" rel="nofollow">EfficientML.ai Lecture 5 - Quantization Part I</a></li> <li><a href="https://horace.io/brrr_intro.html" rel="nofollow">Making Deep Learning Go Brrrr From First Principles</a></li> <li><a href="https://pytorch.org/blog/accelerating-generative-ai-2/" rel="nofollow">Accelerating Generative AI with PyTorch Part 2: LLM Optimizations</a></li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/concept_guide.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_j5svul = {
						base: "/docs/transformers/pr_45649/en",
						assets: "/docs/transformers/pr_45649/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/transformers/pr_45649/en/_app/immutable/entry/start.Cf6g_lQr.js"),
						import("/docs/transformers/pr_45649/en/_app/immutable/entry/app.Ckn5M_jH.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 698],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 55.6 kB
Xet hash:: 996eb1c4934d50daddcf897ecf64755014ac647f04ed73fb6c37558f076543e5
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.