bitsandbytes/pr_2001/en/_app/immutable/nodes/2.--rchnfk.js

import"../chunks/DsnmJJEf.js";import{i as et,h as nt,C as ot,H as i,D as e,E as at,s as it}from"../chunks/DT0OpeMJ.js";import{p as st,o as rt,s as t,f as dt,a as F,b as ct,c as n,d as E,n as a,r as o}from"../chunks/Bb-LL0eD.js";const lt='{"title":"Overview","local":"overview","sections":[{"title":"When to Use bitsandbytes.functional","local":"when-to-use-bitsandbytesfunctional","sections":[],"depth":2},{"title":"LLM.int8()","local":"bitsandbytes.functional.int8_linear_matmul","sections":[],"depth":2},{"title":"4-bit","local":"bitsandbytes.functional.dequantize_4bit","sections":[],"depth":2},{"title":"Dynamic 8-bit Quantization","local":"bitsandbytes.functional.dequantize_blockwise","sections":[],"depth":2},{"title":"Utility","local":"bitsandbytes.functional.get_ptr","sections":[],"depth":2}],"depth":1}';var pt=E('<meta name="hf:doc:metadata"/>'),ut=E(`<p></p> <!> <!> <p>The <code>bitsandbytes.functional</code> API provides the low-level building blocks for the library’s features.</p> <!> <ul><li>When you need direct control over quantized operations and their parameters.</li> <li>To build custom layers or operations leveraging low-bit arithmetic.</li> <li>To integrate with other ecosystem tooling.</li> <li>For experimental or research purposes requiring non-standard quantization or performance optimizations.</li></ul> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Performs an 8-bit integer matrix multiplication.</p> <p>A linear transformation is applied such that <code>out = A @ B.T</code>. When possible, integer tensor core hardware is
utilized to accelerate the operation.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Performs dequantization on the result of a quantized int8 matrix multiplication.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dequantizes a tensor with dtype <code>torch.int8</code> to <code>torch.float32</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Quantizes a tensor with dtype <code>torch.float16</code> to <code>torch.int8</code> in accordance to the <code>LLM.int8()</code> algorithm.</p> <p>For more information, see the <a href="https://arxiv.org/abs/2208.07339" rel="nofollow">LLM.int8() paper</a>.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dequantizes a packed 4-bit quantized tensor.</p> <p>The input tensor is dequantized by dividing it into blocks of <code>blocksize</code> values.
The absolute maximum value within these blocks is used for scaling
the non-linear dequantization.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Quantize tensor A in blocks of 4-bit values.</p> <p>Quantizes tensor A by dividing it into blocks which are independently quantized.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>container for quantization state components to work with Params4bit and similar classes</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>returns dict of tensors and strings to use in serialization via _save_to_state_dict()
param: packed — returns dict[str, torch.Tensor] for state_dict fit for safetensors saving</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>unpacks components of state_dict into QuantState
where necessary, convert into strings, torch.dtype, ints, etc.</p> <p>qs_dict: based on state_dict, with only relevant keys, striped of prefixes.</p> <p>item with key <code>quant_state.bitsandbytes__[nf4/fp4]</code> may contain minor and non-tensor quant state items.</p></div></div> <!> <p>Primitives used in the 8-bit optimizer quantization.</p> <p>For more details see <a href="https://arxiv.org/abs/1511.04561" rel="nofollow">8-Bit Approximations for Parallelism in Deep Learning</a></p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dequantize a tensor in blocks of values.</p> <p>The input tensor is dequantized by dividing it into blocks of <code>blocksize</code> values.
The the absolute maximum value within these blocks is used for scaling
the non-linear dequantization.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Quantize a tensor in blocks of values.</p> <p>The input tensor is quantized by dividing it into blocks of <code>blocksize</code> values.
The the absolute maximum value within these blocks is calculated for scaling
the non-linear quantization.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Gets the memory address of the first element of a tenso</p></div> <!> <p></p>`,1);function yt(P,R){st(R,!1),rt(()=>{new URLSearchParams(window.location.search).get("fw")}),et();var T=ut();nt("wt8c2c",Q=>{var S=pt();it(S,"content",lt),F(Q,S)});var q=t(dt(T),2);ot(q,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var z=t(q,2);i(z,{title:"Overview",local:"overview",headingTag:"h1"});var w=t(z,4);i(w,{title:"When to Use bitsandbytes.functional",local:"when-to-use-bitsandbytesfunctional",headingTag:"h2"});var A=t(w,4);i(A,{title:"LLM.int8()",local:"bitsandbytes.functional.int8_linear_matmul",headingTag:"h2"});var s=t(A,2),V=n(s);e(V,{name:"bitsandbytes.functional.int8_linear_matmul",anchor:"bitsandbytes.functional.int8_linear_matmul",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L1532",parameters:[{name:"A",val:": Tensor"},{name:"B",val:": Tensor"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"dtype",val:" = torch.int32"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_linear_matmul.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The first matrix operand with the data type <code>torch.int8</code>.",name:"A"},{anchor:"bitsandbytes.functional.int8_linear_matmul.B",description:"<strong>B</strong> (<code>torch.Tensor</code>) &#x2014; The second matrix operand with the data type <code>torch.int8</code>.",name:"B"},{anchor:"bitsandbytes.functional.int8_linear_matmul.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A pre-allocated tensor used to store the result.",name:"out"},{anchor:"bitsandbytes.functional.int8_linear_matmul.dtype",description:"<strong>dtype</strong> (<code>torch.dtype</code>, <em>optional</em>) &#x2014; The expected data type of the output. Defaults to <code>torch.int32</code>.",name:"dtype"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The result of the operation.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>NotImplementedError</code> — The operation is not supported in the current environment.</li>
<li><code>RuntimeError</code> — Raised when the cannot be completed for any other reason.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>NotImplementedError</code> or <code>RuntimeError</code></p>
`}),a(4),o(s);var r=t(s,2),B=n(r);e(B,{name:"bitsandbytes.functional.int8_mm_dequant",anchor:"bitsandbytes.functional.int8_mm_dequant",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L1558",parameters:[{name:"A",val:": Tensor"},{name:"row_stats",val:": Tensor"},{name:"col_stats",val:": Tensor"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"bias",val:": typing.Optional[torch.Tensor] = None"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_mm_dequant.A",description:"<strong>A</strong> (<code>torch.Tensor</code> with dtype <code>torch.int32</code>) &#x2014; The result of a quantized int8 matrix multiplication.",name:"A"},{anchor:"bitsandbytes.functional.int8_mm_dequant.row_stats",description:"<strong>row_stats</strong> (<code>torch.Tensor</code>) &#x2014; The row-wise quantization statistics for the lhs operand of the matrix multiplication.",name:"row_stats"},{anchor:"bitsandbytes.functional.int8_mm_dequant.col_stats",description:"<strong>col_stats</strong> (<code>torch.Tensor</code>) &#x2014; The column-wise quantization statistics for the rhs operand of the matrix multiplication.",name:"col_stats"},{anchor:"bitsandbytes.functional.int8_mm_dequant.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A pre-allocated tensor to store the output of the operation.",name:"out"},{anchor:"bitsandbytes.functional.int8_mm_dequant.bias",description:"<strong>bias</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; An optional bias vector to add to the result.",name:"bias"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized result with an optional bias, with dtype <code>torch.float16</code>.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`}),a(2),o(r);var d=t(r,2),M=n(d);e(M,{name:"bitsandbytes.functional.int8_vectorwise_dequant",anchor:"bitsandbytes.functional.int8_vectorwise_dequant",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L1637",parameters:[{name:"A",val:": Tensor"},{name:"stats",val:": Tensor"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_vectorwise_dequant.A",description:"<strong>A</strong> (<code>torch.Tensor</code> with dtype <code>torch.int8</code>) &#x2014; The quantized int8 tensor.",name:"A"},{anchor:"bitsandbytes.functional.int8_vectorwise_dequant.stats",description:"<strong>stats</strong> (<code>torch.Tensor</code> with dtype <code>torch.float32</code>) &#x2014; The row-wise quantization statistics.",name:"stats"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized tensor.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code> with dtype <code>torch.float32</code></p>
`}),a(2),o(d);var c=t(d,2),W=n(c);e(W,{name:"bitsandbytes.functional.int8_vectorwise_quant",anchor:"bitsandbytes.functional.int8_vectorwise_quant",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L1651",parameters:[{name:"A",val:": Tensor"},{name:"threshold",val:" = 0.0"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_vectorwise_quant.A",description:"<strong>A</strong> (<code>torch.Tensor</code> with dtype <code>torch.float16</code>) &#x2014; The input tensor.",name:"A"},{anchor:"bitsandbytes.functional.int8_vectorwise_quant.threshold",description:`<strong>threshold</strong> (<code>float</code>, <em>optional</em>) &#x2014;
An optional threshold for sparse decomposition of outlier features.</p>
<p>No outliers are held back when 0.0. Defaults to 0.0.`,name:"threshold"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple containing the quantized tensor and relevant statistics.</p>
<ul>
<li><code>torch.Tensor</code> with dtype <code>torch.int8</code>: The quantized data.</li>
<li><code>torch.Tensor</code> with dtype <code>torch.float32</code>: The quantization scales.</li>
<li><code>torch.Tensor</code> with dtype <code>torch.int32</code>, <em>optional</em>: A list of column indices which contain outlier features.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]</code></p>
`}),a(4),o(c);var k=t(c,2);i(k,{title:"4-bit",local:"bitsandbytes.functional.dequantize_4bit",headingTag:"h2"});var l=t(k,2),I=n(l);e(I,{name:"bitsandbytes.functional.dequantize_4bit",anchor:"bitsandbytes.functional.dequantize_4bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L991",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": typing.Optional[int] = None"},{name:"quant_type",val:" = 'fp4'"}],parametersDescription:[{anchor:"bitsandbytes.functional.dequantize_4bit.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The quantized input tensor.",name:"A"},{anchor:"bitsandbytes.functional.dequantize_4bit.quant_state",description:`<strong>quant_state</strong> (<code>QuantState</code>, <em>optional</em>) &#x2014;
The quantization state as returned by <code>quantize_4bit</code>.
Required if <code>absmax</code> is not provided.`,name:"quant_state"},{anchor:"bitsandbytes.functional.dequantize_4bit.absmax",description:`<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A tensor containing the scaling values.
Required if <code>quant_state</code> is not provided and ignored otherwise.`,name:"absmax"},{anchor:"bitsandbytes.functional.dequantize_4bit.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.dequantize_4bit.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 64.
Valid values are 32, 64, 128, 256, 512, 1024, 2048, and 4096.`,name:"blocksize"},{anchor:"bitsandbytes.functional.dequantize_4bit.quant_type",description:"<strong>quant_type</strong> (<code>str</code>, <em>optional</em>) &#x2014; The data type to use: <code>nf4</code> or <code>fp4</code>. Defaults to <code>fp4</code>.",name:"quant_type"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized tensor.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type or blocksize is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}),a(4),o(l);var p=t(l,2),U=n(p);e(U,{name:"bitsandbytes.functional.dequantize_fp4",anchor:"bitsandbytes.functional.dequantize_fp4",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L971",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": typing.Optional[int] = None"}]}),o(p);var u=t(p,2),j=n(u);e(j,{name:"bitsandbytes.functional.dequantize_nf4",anchor:"bitsandbytes.functional.dequantize_nf4",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L981",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": typing.Optional[int] = None"}]}),o(u);var b=t(u,2),C=n(b);e(C,{name:"bitsandbytes.functional.gemv_4bit",anchor:"bitsandbytes.functional.gemv_4bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L1298",parameters:[{name:"A",val:": Tensor"},{name:"B",val:": Tensor"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"transposed_A",val:" = False"},{name:"transposed_B",val:" = False"},{name:"state",val:" = None"}]}),o(b);var m=t(b,2),G=n(m);e(G,{name:"bitsandbytes.functional.quantize_4bit",anchor:"bitsandbytes.functional.quantize_4bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L883",parameters:[{name:"A",val:": Tensor"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = None"},{name:"compress_statistics",val:" = False"},{name:"quant_type",val:" = 'fp4'"},{name:"quant_storage",val:" = torch.uint8"}],parametersDescription:[{anchor:"bitsandbytes.functional.quantize_4bit.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The input tensor. Supports <code>float16</code>, <code>bfloat16</code>, or <code>float32</code> datatypes.",name:"A"},{anchor:"bitsandbytes.functional.quantize_4bit.absmax",description:"<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the absmax values.",name:"absmax"},{anchor:"bitsandbytes.functional.quantize_4bit.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.quantize_4bit.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 64.
Valid values are 32, 64, 128, 256, 512, 1024, 2048, and 4096.`,name:"blocksize"},{anchor:"bitsandbytes.functional.quantize_4bit.compress_statistics",description:"<strong>compress_statistics</strong> (<code>bool</code>, <em>optional</em>) &#x2014; Whether to additionally quantize the absmax values. Defaults to False.",name:"compress_statistics"},{anchor:"bitsandbytes.functional.quantize_4bit.quant_type",description:"<strong>quant_type</strong> (<code>str</code>, <em>optional</em>) &#x2014; The data type to use: <code>nf4</code> or <code>fp4</code>. Defaults to <code>fp4</code>.",name:"quant_type"},{anchor:"bitsandbytes.functional.quantize_4bit.quant_storage",description:"<strong>quant_storage</strong> (<code>torch.dtype</code>, <em>optional</em>) &#x2014; The dtype of the tensor used to store the result. Defaults to <code>torch.uint8</code>.",name:"quant_storage"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple containing the quantization results.</p>
<ul>
<li><code>torch.Tensor</code>: The quantized tensor with packed 4-bit values.</li>
<li><code>QuantState</code>: The state object used to undo the quantization.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Tuple[<code>torch.Tensor</code>, <code>QuantState</code>]</p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}),a(4),o(m);var h=t(m,2),H=n(h);e(H,{name:"bitsandbytes.functional.quantize_fp4",anchor:"bitsandbytes.functional.quantize_fp4",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L861",parameters:[{name:"A",val:": Tensor"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = None"},{name:"compress_statistics",val:" = False"},{name:"quant_storage",val:" = torch.uint8"}]}),o(h);var y=t(h,2),J=n(y);e(J,{name:"bitsandbytes.functional.quantize_nf4",anchor:"bitsandbytes.functional.quantize_nf4",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L872",parameters:[{name:"A",val:": Tensor"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = None"},{name:"compress_statistics",val:" = False"},{name:"quant_storage",val:" = torch.uint8"}]}),o(y);var f=t(y,2),D=n(f);e(D,{name:"class bitsandbytes.functional.QuantState",anchor:"bitsandbytes.functional.QuantState",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L419",parameters:[{name:"absmax",val:""},{name:"shape",val:" = None"},{name:"code",val:" = None"},{name:"blocksize",val:" = None"},{name:"quant_type",val:" = None"},{name:"dtype",val:" = None"},{name:"offset",val:" = None"},{name:"state2",val:" = None"}]});var v=t(D,4),K=n(v);e(K,{name:"as_dict",anchor:"bitsandbytes.functional.QuantState.as_dict",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L544",parameters:[{name:"packed",val:": bool = False"}]}),a(2),o(v);var N=t(v,2),X=n(N);e(X,{name:"from_dict",anchor:"bitsandbytes.functional.QuantState.from_dict",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L492",parameters:[{name:"qs_dict",val:": dict"},{name:"device",val:": device"}]}),a(6),o(N),o(f);var O=t(f,2);i(O,{title:"Dynamic 8-bit Quantization",local:"bitsandbytes.functional.dequantize_blockwise",headingTag:"h2"});var g=t(O,6),Y=n(g);e(Y,{name:"bitsandbytes.functional.dequantize_blockwise",anchor:"bitsandbytes.functional.dequantize_blockwise",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L688",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"code",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": int = 4096"},{name:"nested",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.functional.dequantize_blockwise.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The quantized input tensor.",name:"A"},{anchor:"bitsandbytes.functional.dequantize_blockwise.quant_state",description:`<strong>quant_state</strong> (<code>QuantState</code>, <em>optional</em>) &#x2014;
The quantization state as returned by <code>quantize_blockwise</code>.
Required if <code>absmax</code> is not provided.`,name:"quant_state"},{anchor:"bitsandbytes.functional.dequantize_blockwise.absmax",description:`<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A tensor containing the scaling values.
Required if <code>quant_state</code> is not provided and ignored otherwise.`,name:"absmax"},{anchor:"bitsandbytes.functional.dequantize_blockwise.code",description:`<strong>code</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type.
For more details, see  (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561].
Ignored when <code>quant_state</code> is provided.`,name:"code"},{anchor:"bitsandbytes.functional.dequantize_blockwise.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.dequantize_blockwise.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 4096.
Valid values are 64, 128, 256, 512, 1024, 2048, and 4096.
Ignored when <code>quant_state</code> is provided.`,name:"blocksize"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized tensor. The datatype is indicated by <code>quant_state.dtype</code> and defaults to <code>torch.float32</code>.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}),a(4),o(g);var _=t(g,2),Z=n(_);e(Z,{name:"bitsandbytes.functional.quantize_blockwise",anchor:"bitsandbytes.functional.quantize_blockwise",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L612",parameters:[{name:"A",val:": Tensor"},{name:"code",val:": typing.Optional[torch.Tensor] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = 4096"},{name:"nested",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.functional.quantize_blockwise.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The input tensor. Supports <code>float16</code>, <code>bfloat16</code>, or <code>float32</code> datatypes.",name:"A"},{anchor:"bitsandbytes.functional.quantize_blockwise.code",description:`<strong>code</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type.
For more details, see  (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561].`,name:"code"},{anchor:"bitsandbytes.functional.quantize_blockwise.absmax",description:"<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the absmax values.",name:"absmax"},{anchor:"bitsandbytes.functional.quantize_blockwise.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.quantize_blockwise.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 4096.
Valid values are 64, 128, 256, 512, 1024, 2048, and 4096.`,name:"blocksize"},{anchor:"bitsandbytes.functional.quantize_blockwise.nested",description:"<strong>nested</strong> (<code>bool</code>, <em>optional</em>) &#x2014; Whether to additionally quantize the absmax values. Defaults to False.",name:"nested"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple containing the quantization results.</p>
<ul>
<li><code>torch.Tensor</code>: The quantized tensor.</li>
<li><code>QuantState</code>: The state object used to undo the quantization.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[torch.Tensor, QuantState]</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}),a(4),o(_);var L=t(_,2);i(L,{title:"Utility",local:"bitsandbytes.functional.get_ptr",headingTag:"h2"});var x=t(L,2),$=n(x);e($,{name:"bitsandbytes.functional.get_ptr",anchor:"bitsandbytes.functional.get_ptr",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2001/bitsandbytes/functional.py#L404",parameters:[{name:"A",val:": typing.Optional[torch.Tensor]"}],parametersDescription:[{anchor:"bitsandbytes.functional.get_ptr.A",description:"<strong>A</strong> (<code>Optional[Tensor]</code>) &#x2014; A PyTorch tensor.",name:"A"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A pointer to the underlying tensor data.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Optional[ct.c_void_p]</code></p>
`}),a(2),o(x);var tt=t(x,2);at(tt,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/functional.mdx"}),a(2),F(P,T),ct()}export{yt as component};