bitsandbytes/pr_1908/en/_app/immutable/nodes/13.6a37c99e.js

import{s as $n,n as Tn,o as qn}from"../chunks/scheduler.8a2cc2fa.js";import{S as zn,i as wn,e as r,s as o,c as l,h as kn,a as d,d as e,b as a,f as h,g as c,j as f,k as y,l as s,m as i,n as p,t as u,o as m,p as b}from"../chunks/index.7079e750.js";import{C as An,H as Qt,E as Dn}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2b7ce466.js";import{D as g}from"../chunks/Docstring.8c9a5003.js";function Ln(Ue){let k,Rt,St,Bt,O,Ut,E,Wt,M,We="The <code>bitsandbytes.functional</code> API provides the low-level building blocks for the library’s features.",jt,V,Gt,H,je="<li>When you need direct control over quantized operations and their parameters.</li> <li>To build custom layers or operations leveraging low-bit arithmetic.</li> <li>To integrate with other ecosystem tooling.</li> <li>For experimental or research purposes requiring non-standard quantization or performance optimizations.</li>",Jt,I,Kt,x,Q,Te,bt,Ge="Performs an 8-bit integer matrix multiplication.",qe,ft,Je=`A linear transformation is applied such that <code>out = A @ B.T</code>. When possible, integer tensor core hardware is
utilized to accelerate the operation.`,Xt,A,S,ze,ht,Ke="Performs dequantization on the result of a quantized int8 matrix multiplication.",Yt,D,F,we,yt,Xe="Dequantizes a tensor with dtype <code>torch.int8</code> to <code>torch.float32</code>.",Zt,$,R,ke,gt,Ye="Quantizes a tensor with dtype <code>torch.float16</code> to <code>torch.int8</code> in accordance to the <code>LLM.int8()</code> algorithm.",Ae,vt,Ze='For more information, see the <a href="https://arxiv.org/abs/2208.07339" rel="nofollow">LLM.int8() paper</a>.',te,B,ee,T,U,De,_t,tn="Dequantizes a packed 4-bit quantized tensor.",Le,xt,en=`The input tensor is dequantized by dividing it into blocks of <code>blocksize</code> values.
The absolute maximum value within these blocks is used for scaling
the non-linear dequantization.`,ne,W,j,oe,G,J,ae,K,X,ie,q,Y,Ce,$t,nn="Quantize tensor A in blocks of 4-bit values.",Ne,Tt,on="Quantizes tensor A by dividing it into blocks which are independently quantized.",se,Z,tt,re,et,nt,de,v,ot,Pe,qt,an="container for quantization state components to work with Params4bit and similar classes",Oe,C,at,Ee,zt,sn=`returns dict of tensors and strings to use in serialization via _save_to_state_dict()
param: packed — returns dict[str, torch.Tensor] for state_dict fit for safetensors saving`,Me,_,it,Ve,wt,rn=`unpacks components of state_dict into QuantState
where necessary, convert into strings, torch.dtype, ints, etc.`,He,kt,dn="qs_dict: based on state_dict, with only relevant keys, striped of prefixes.",Ie,At,ln="item with key <code>quant_state.bitsandbytes__[nf4/fp4]</code> may contain minor and non-tensor quant state items.",le,st,ce,rt,cn="Primitives used in the 8-bit optimizer quantization.",pe,dt,pn='For more details see <a href="https://arxiv.org/abs/1511.04561" rel="nofollow">8-Bit Approximations for Parallelism in Deep Learning</a>',ue,z,lt,Qe,Dt,un="Dequantize a tensor in blocks of values.",Se,Lt,mn=`The input tensor is dequantized by dividing it into blocks of <code>blocksize</code> values.
The the absolute maximum value within these blocks is used for scaling
the non-linear dequantization.`,me,w,ct,Fe,Ct,bn="Quantize a tensor in blocks of values.",Re,Nt,fn=`The input tensor is quantized by dividing it into blocks of <code>blocksize</code> values.
The the absolute maximum value within these blocks is calculated for scaling
the non-linear quantization.`,be,pt,fe,L,ut,Be,Pt,hn="Gets the memory address of the first element of a tenso",he,mt,ye,Ft,ge;return O=new An({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),E=new Qt({props:{title:"Overview",local:"overview",headingTag:"h1"}}),V=new Qt({props:{title:"When to Use bitsandbytes.functional",local:"when-to-use-bitsandbytesfunctional",headingTag:"h2"}}),I=new Qt({props:{title:"LLM.int8()",local:"bitsandbytes.functional.int8_linear_matmul",headingTag:"h2"}}),Q=new g({props:{name:"bitsandbytes.functional.int8_linear_matmul",anchor:"bitsandbytes.functional.int8_linear_matmul",parameters:[{name:"A",val:": Tensor"},{name:"B",val:": Tensor"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"dtype",val:" = torch.int32"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_linear_matmul.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The first matrix operand with the data type <code>torch.int8</code>.",name:"A"},{anchor:"bitsandbytes.functional.int8_linear_matmul.B",description:"<strong>B</strong> (<code>torch.Tensor</code>) &#x2014; The second matrix operand with the data type <code>torch.int8</code>.",name:"B"},{anchor:"bitsandbytes.functional.int8_linear_matmul.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A pre-allocated tensor used to store the result.",name:"out"},{anchor:"bitsandbytes.functional.int8_linear_matmul.dtype",description:"<strong>dtype</strong> (<code>torch.dtype</code>, <em>optional</em>) &#x2014; The expected data type of the output. Defaults to <code>torch.int32</code>.",name:"dtype"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L1503",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The result of the operation.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>NotImplementedError</code> — The operation is not supported in the current environment.</li>
<li><code>RuntimeError</code> — Raised when the cannot be completed for any other reason.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>NotImplementedError</code> or <code>RuntimeError</code></p>
`}}),S=new g({props:{name:"bitsandbytes.functional.int8_mm_dequant",anchor:"bitsandbytes.functional.int8_mm_dequant",parameters:[{name:"A",val:": Tensor"},{name:"row_stats",val:": Tensor"},{name:"col_stats",val:": Tensor"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"bias",val:": typing.Optional[torch.Tensor] = None"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_mm_dequant.A",description:"<strong>A</strong> (<code>torch.Tensor</code> with dtype <code>torch.int32</code>) &#x2014; The result of a quantized int8 matrix multiplication.",name:"A"},{anchor:"bitsandbytes.functional.int8_mm_dequant.row_stats",description:"<strong>row_stats</strong> (<code>torch.Tensor</code>) &#x2014; The row-wise quantization statistics for the lhs operand of the matrix multiplication.",name:"row_stats"},{anchor:"bitsandbytes.functional.int8_mm_dequant.col_stats",description:"<strong>col_stats</strong> (<code>torch.Tensor</code>) &#x2014; The column-wise quantization statistics for the rhs operand of the matrix multiplication.",name:"col_stats"},{anchor:"bitsandbytes.functional.int8_mm_dequant.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A pre-allocated tensor to store the output of the operation.",name:"out"},{anchor:"bitsandbytes.functional.int8_mm_dequant.bias",description:"<strong>bias</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; An optional bias vector to add to the result.",name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L1529",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized result with an optional bias, with dtype <code>torch.float16</code>.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`}}),F=new g({props:{name:"bitsandbytes.functional.int8_vectorwise_dequant",anchor:"bitsandbytes.functional.int8_vectorwise_dequant",parameters:[{name:"A",val:": Tensor"},{name:"stats",val:": Tensor"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_vectorwise_dequant.A",description:"<strong>A</strong> (<code>torch.Tensor</code> with dtype <code>torch.int8</code>) &#x2014; The quantized int8 tensor.",name:"A"},{anchor:"bitsandbytes.functional.int8_vectorwise_dequant.stats",description:"<strong>stats</strong> (<code>torch.Tensor</code> with dtype <code>torch.float32</code>) &#x2014; The row-wise quantization statistics.",name:"stats"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L1608",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized tensor.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code> with dtype <code>torch.float32</code></p>
`}}),R=new g({props:{name:"bitsandbytes.functional.int8_vectorwise_quant",anchor:"bitsandbytes.functional.int8_vectorwise_quant",parameters:[{name:"A",val:": Tensor"},{name:"threshold",val:" = 0.0"}],parametersDescription:[{anchor:"bitsandbytes.functional.int8_vectorwise_quant.A",description:"<strong>A</strong> (<code>torch.Tensor</code> with dtype <code>torch.float16</code>) &#x2014; The input tensor.",name:"A"},{anchor:"bitsandbytes.functional.int8_vectorwise_quant.threshold",description:`<strong>threshold</strong> (<code>float</code>, <em>optional</em>) &#x2014;
An optional threshold for sparse decomposition of outlier features.</p>
<p>No outliers are held back when 0.0. Defaults to 0.0.`,name:"threshold"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L1622",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple containing the quantized tensor and relevant statistics.</p>
<ul>
<li><code>torch.Tensor</code> with dtype <code>torch.int8</code>: The quantized data.</li>
<li><code>torch.Tensor</code> with dtype <code>torch.float32</code>: The quantization scales.</li>
<li><code>torch.Tensor</code> with dtype <code>torch.int32</code>, <em>optional</em>: A list of column indices which contain outlier features.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]</code></p>
`}}),B=new Qt({props:{title:"4-bit",local:"bitsandbytes.functional.dequantize_4bit",headingTag:"h2"}}),U=new g({props:{name:"bitsandbytes.functional.dequantize_4bit",anchor:"bitsandbytes.functional.dequantize_4bit",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": typing.Optional[int] = None"},{name:"quant_type",val:" = 'fp4'"}],parametersDescription:[{anchor:"bitsandbytes.functional.dequantize_4bit.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The quantized input tensor.",name:"A"},{anchor:"bitsandbytes.functional.dequantize_4bit.quant_state",description:`<strong>quant_state</strong> (<code>QuantState</code>, <em>optional</em>) &#x2014;
The quantization state as returned by <code>quantize_4bit</code>.
Required if <code>absmax</code> is not provided.`,name:"quant_state"},{anchor:"bitsandbytes.functional.dequantize_4bit.absmax",description:`<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A tensor containing the scaling values.
Required if <code>quant_state</code> is not provided and ignored otherwise.`,name:"absmax"},{anchor:"bitsandbytes.functional.dequantize_4bit.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.dequantize_4bit.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 64.
Valid values are 32, 64, 128, 256, 512, 1024, 2048, and 4096.`,name:"blocksize"},{anchor:"bitsandbytes.functional.dequantize_4bit.quant_type",description:"<strong>quant_type</strong> (<code>str</code>, <em>optional</em>) &#x2014; The data type to use: <code>nf4</code> or <code>fp4</code>. Defaults to <code>fp4</code>.",name:"quant_type"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L973",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized tensor.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type or blocksize is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}}),j=new g({props:{name:"bitsandbytes.functional.dequantize_fp4",anchor:"bitsandbytes.functional.dequantize_fp4",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": typing.Optional[int] = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L953"}}),J=new g({props:{name:"bitsandbytes.functional.dequantize_nf4",anchor:"bitsandbytes.functional.dequantize_nf4",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": typing.Optional[int] = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L963"}}),X=new g({props:{name:"bitsandbytes.functional.gemv_4bit",anchor:"bitsandbytes.functional.gemv_4bit",parameters:[{name:"A",val:": Tensor"},{name:"B",val:": Tensor"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"transposed_A",val:" = False"},{name:"transposed_B",val:" = False"},{name:"state",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L1269"}}),Y=new g({props:{name:"bitsandbytes.functional.quantize_4bit",anchor:"bitsandbytes.functional.quantize_4bit",parameters:[{name:"A",val:": Tensor"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = None"},{name:"compress_statistics",val:" = False"},{name:"quant_type",val:" = 'fp4'"},{name:"quant_storage",val:" = torch.uint8"}],parametersDescription:[{anchor:"bitsandbytes.functional.quantize_4bit.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The input tensor. Supports <code>float16</code>, <code>bfloat16</code>, or <code>float32</code> datatypes.",name:"A"},{anchor:"bitsandbytes.functional.quantize_4bit.absmax",description:"<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the absmax values.",name:"absmax"},{anchor:"bitsandbytes.functional.quantize_4bit.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.quantize_4bit.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 64.
Valid values are 32, 64, 128, 256, 512, 1024, 2048, and 4096.`,name:"blocksize"},{anchor:"bitsandbytes.functional.quantize_4bit.compress_statistics",description:"<strong>compress_statistics</strong> (<code>bool</code>, <em>optional</em>) &#x2014; Whether to additionally quantize the absmax values. Defaults to False.",name:"compress_statistics"},{anchor:"bitsandbytes.functional.quantize_4bit.quant_type",description:"<strong>quant_type</strong> (<code>str</code>, <em>optional</em>) &#x2014; The data type to use: <code>nf4</code> or <code>fp4</code>. Defaults to <code>fp4</code>.",name:"quant_type"},{anchor:"bitsandbytes.functional.quantize_4bit.quant_storage",description:"<strong>quant_storage</strong> (<code>torch.dtype</code>, <em>optional</em>) &#x2014; The dtype of the tensor used to store the result. Defaults to <code>torch.uint8</code>.",name:"quant_storage"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L872",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple containing the quantization results.</p>
<ul>
<li><code>torch.Tensor</code>: The quantized tensor with packed 4-bit values.</li>
<li><code>QuantState</code>: The state object used to undo the quantization.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Tuple[<code>torch.Tensor</code>, <code>QuantState</code>]</p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}}),tt=new g({props:{name:"bitsandbytes.functional.quantize_fp4",anchor:"bitsandbytes.functional.quantize_fp4",parameters:[{name:"A",val:": Tensor"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = None"},{name:"compress_statistics",val:" = False"},{name:"quant_storage",val:" = torch.uint8"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L850"}}),nt=new g({props:{name:"bitsandbytes.functional.quantize_nf4",anchor:"bitsandbytes.functional.quantize_nf4",parameters:[{name:"A",val:": Tensor"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = None"},{name:"compress_statistics",val:" = False"},{name:"quant_storage",val:" = torch.uint8"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L861"}}),ot=new g({props:{name:"class bitsandbytes.functional.QuantState",anchor:"bitsandbytes.functional.QuantState",parameters:[{name:"absmax",val:""},{name:"shape",val:" = None"},{name:"code",val:" = None"},{name:"blocksize",val:" = None"},{name:"quant_type",val:" = None"},{name:"dtype",val:" = None"},{name:"offset",val:" = None"},{name:"state2",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L419"}}),at=new g({props:{name:"as_dict",anchor:"bitsandbytes.functional.QuantState.as_dict",parameters:[{name:"packed",val:": bool = False"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L544"}}),it=new g({props:{name:"from_dict",anchor:"bitsandbytes.functional.QuantState.from_dict",parameters:[{name:"qs_dict",val:": dict"},{name:"device",val:": device"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L492"}}),st=new Qt({props:{title:"Dynamic 8-bit Quantization",local:"bitsandbytes.functional.dequantize_blockwise",headingTag:"h2"}}),lt=new g({props:{name:"bitsandbytes.functional.dequantize_blockwise",anchor:"bitsandbytes.functional.dequantize_blockwise",parameters:[{name:"A",val:": Tensor"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"code",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:": int = 4096"},{name:"nested",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.functional.dequantize_blockwise.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The quantized input tensor.",name:"A"},{anchor:"bitsandbytes.functional.dequantize_blockwise.quant_state",description:`<strong>quant_state</strong> (<code>QuantState</code>, <em>optional</em>) &#x2014;
The quantization state as returned by <code>quantize_blockwise</code>.
Required if <code>absmax</code> is not provided.`,name:"quant_state"},{anchor:"bitsandbytes.functional.dequantize_blockwise.absmax",description:`<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A tensor containing the scaling values.
Required if <code>quant_state</code> is not provided and ignored otherwise.`,name:"absmax"},{anchor:"bitsandbytes.functional.dequantize_blockwise.code",description:`<strong>code</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type.
For more details, see  (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561].
Ignored when <code>quant_state</code> is provided.`,name:"code"},{anchor:"bitsandbytes.functional.dequantize_blockwise.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.dequantize_blockwise.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 4096.
Valid values are 64, 128, 256, 512, 1024, 2048, and 4096.
Ignored when <code>quant_state</code> is provided.`,name:"blocksize"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L683",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The dequantized tensor. The datatype is indicated by <code>quant_state.dtype</code> and defaults to <code>torch.float32</code>.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.Tensor</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}}),ct=new g({props:{name:"bitsandbytes.functional.quantize_blockwise",anchor:"bitsandbytes.functional.quantize_blockwise",parameters:[{name:"A",val:": Tensor"},{name:"code",val:": typing.Optional[torch.Tensor] = None"},{name:"absmax",val:": typing.Optional[torch.Tensor] = None"},{name:"out",val:": typing.Optional[torch.Tensor] = None"},{name:"blocksize",val:" = 4096"},{name:"nested",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.functional.quantize_blockwise.A",description:"<strong>A</strong> (<code>torch.Tensor</code>) &#x2014; The input tensor. Supports <code>float16</code>, <code>bfloat16</code>, or <code>float32</code> datatypes.",name:"A"},{anchor:"bitsandbytes.functional.quantize_blockwise.code",description:`<strong>code</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type.
For more details, see  (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561].`,name:"code"},{anchor:"bitsandbytes.functional.quantize_blockwise.absmax",description:"<strong>absmax</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the absmax values.",name:"absmax"},{anchor:"bitsandbytes.functional.quantize_blockwise.out",description:"<strong>out</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; A tensor to use to store the result.",name:"out"},{anchor:"bitsandbytes.functional.quantize_blockwise.blocksize",description:`<strong>blocksize</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the blocks. Defaults to 4096.
Valid values are 64, 128, 256, 512, 1024, 2048, and 4096.`,name:"blocksize"},{anchor:"bitsandbytes.functional.quantize_blockwise.nested",description:"<strong>nested</strong> (<code>bool</code>, <em>optional</em>) &#x2014; Whether to additionally quantize the absmax values. Defaults to False.",name:"nested"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L612",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple containing the quantization results.</p>
<ul>
<li><code>torch.Tensor</code>: The quantized tensor.</li>
<li><code>QuantState</code>: The state object used to undo the quantization.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[torch.Tensor, QuantState]</code></p>
`,raiseDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<ul>
<li><code>ValueError</code> — Raised when the input data type is not supported.</li>
</ul>
`,raiseType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>ValueError</code></p>
`}}),pt=new Qt({props:{title:"Utility",local:"bitsandbytes.functional.get_ptr",headingTag:"h2"}}),ut=new g({props:{name:"bitsandbytes.functional.get_ptr",anchor:"bitsandbytes.functional.get_ptr",parameters:[{name:"A",val:": typing.Optional[torch.Tensor]"}],parametersDescription:[{anchor:"bitsandbytes.functional.get_ptr.A",description:"<strong>A</strong> (<code>Optional[Tensor]</code>) &#x2014; A PyTorch tensor.",name:"A"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/functional.py#L404",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A pointer to the underlying tensor data.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Optional[ct.c_void_p]</code></p>
`}}),mt=new Dn({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/functional.mdx"}}),{c(){k=r("meta"),Rt=o(),St=r("p"),Bt=o(),l(O.$$.fragment),Ut=o(),l(E.$$.fragment),Wt=o(),M=r("p"),M.innerHTML=We,jt=o(),l(V.$$.fragment),Gt=o(),H=r("ul"),H.innerHTML=je,Jt=o(),l(I.$$.fragment),Kt=o(),x=r("div"),l(Q.$$.fragment),Te=o(),bt=r("p"),bt.textContent=Ge,qe=o(),ft=r("p"),ft.innerHTML=Je,Xt=o(),A=r("div"),l(S.$$.fragment),ze=o(),ht=r("p"),ht.textContent=Ke,Yt=o(),D=r("div"),l(F.$$.fragment),we=o(),yt=r("p"),yt.innerHTML=Xe,Zt=o(),$=r("div"),l(R.$$.fragment),ke=o(),gt=r("p"),gt.innerHTML=Ye,Ae=o(),vt=r("p"),vt.innerHTML=Ze,te=o(),l(B.$$.fragment),ee=o(),T=r("div"),l(U.$$.fragment),De=o(),_t=r("p"),_t.textContent=tn,Le=o(),xt=r("p"),xt.innerHTML=en,ne=o(),W=r("div"),l(j.$$.fragment),oe=o(),G=r("div"),l(J.$$.fragment),ae=o(),K=r("div"),l(X.$$.fragment),ie=o(),q=r("div"),l(Y.$$.fragment),Ce=o(),$t=r("p"),$t.textContent=nn,Ne=o(),Tt=r("p"),Tt.textContent=on,se=o(),Z=r("div"),l(tt.$$.fragment),re=o(),et=r("div"),l(nt.$$.fragment),de=o(),v=r("div"),l(ot.$$.fragment),Pe=o(),qt=r("p"),qt.textContent=an,Oe=o(),C=r("div"),l(at.$$.fragment),Ee=o(),zt=r("p"),zt.textContent=sn,Me=o(),_=r("div"),l(it.$$.fragment),Ve=o(),wt=r("p"),wt.textContent=rn,He=o(),kt=r("p"),kt.textContent=dn,Ie=o(),At=r("p"),At.innerHTML=ln,le=o(),l(st.$$.fragment),ce=o(),rt=r("p"),rt.textContent=cn,pe=o(),dt=r("p"),dt.innerHTML=pn,ue=o(),z=r("div"),l(lt.$$.fragment),Qe=o(),Dt=r("p"),Dt.textContent=un,Se=o(),Lt=r("p"),Lt.innerHTML=mn,me=o(),w=r("div"),l(ct.$$.fragment),Fe=o(),Ct=r("p"),Ct.textContent=bn,Re=o(),Nt=r("p"),Nt.innerHTML=fn,be=o(),l(pt.$$.fragment),fe=o(),L=r("div"),l(ut.$$.fragment),Be=o(),Pt=r("p"),Pt.textContent=hn,he=o(),l(mt.$$.fragment),ye=o(),Ft=r("p"),this.h()},l(t){const n=kn("svelte-u9bgzb",document.head);k=d(n,"META",{name:!0,content:!0}),n.forEach(e),Rt=a(t),St=d(t,"P",{}),h(St).forEach(e),Bt=a(t),c(O.$$.fragment,t),Ut=a(t),c(E.$$.fragment,t),Wt=a(t),M=d(t,"P",{"data-svelte-h":!0}),f(M)!=="svelte-donk0z"&&(M.innerHTML=We),jt=a(t),c(V.$$.fragment,t),Gt=a(t),H=d(t,"UL",{"data-svelte-h":!0}),f(H)!=="svelte-ug6xw5"&&(H.innerHTML=je),Jt=a(t),c(I.$$.fragment,t),Kt=a(t),x=d(t,"DIV",{class:!0});var Ot=h(x);c(Q.$$.fragment,Ot),Te=a(Ot),bt=d(Ot,"P",{"data-svelte-h":!0}),f(bt)!=="svelte-1phi9i6"&&(bt.textContent=Ge),qe=a(Ot),ft=d(Ot,"P",{"data-svelte-h":!0}),f(ft)!=="svelte-q2aa2u"&&(ft.innerHTML=Je),Ot.forEach(e),Xt=a(t),A=d(t,"DIV",{class:!0});var ve=h(A);c(S.$$.fragment,ve),ze=a(ve),ht=d(ve,"P",{"data-svelte-h":!0}),f(ht)!=="svelte-1lqdwfe"&&(ht.textContent=Ke),ve.forEach(e),Yt=a(t),D=d(t,"DIV",{class:!0});var _e=h(D);c(F.$$.fragment,_e),we=a(_e),yt=d(_e,"P",{"data-svelte-h":!0}),f(yt)!=="svelte-15q912e"&&(yt.innerHTML=Xe),_e.forEach(e),Zt=a(t),$=d(t,"DIV",{class:!0});var Et=h($);c(R.$$.fragment,Et),ke=a(Et),gt=d(Et,"P",{"data-svelte-h":!0}),f(gt)!=="svelte-1u2p684"&&(gt.innerHTML=Ye),Ae=a(Et),vt=d(Et,"P",{"data-svelte-h":!0}),f(vt)!=="svelte-1f18irr"&&(vt.innerHTML=Ze),Et.forEach(e),te=a(t),c(B.$$.fragment,t),ee=a(t),T=d(t,"DIV",{class:!0});var Mt=h(T);c(U.$$.fragment,Mt),De=a(Mt),_t=d(Mt,"P",{"data-svelte-h":!0}),f(_t)!=="svelte-1o0c7r0"&&(_t.textContent=tn),Le=a(Mt),xt=d(Mt,"P",{"data-svelte-h":!0}),f(xt)!=="svelte-yn8gwu"&&(xt.innerHTML=en),Mt.forEach(e),ne=a(t),W=d(t,"DIV",{class:!0});var yn=h(W);c(j.$$.fragment,yn),yn.forEach(e),oe=a(t),G=d(t,"DIV",{class:!0});var gn=h(G);c(J.$$.fragment,gn),gn.forEach(e),ae=a(t),K=d(t,"DIV",{class:!0});var vn=h(K);c(X.$$.fragment,vn),vn.forEach(e),ie=a(t),q=d(t,"DIV",{class:!0});var Vt=h(q);c(Y.$$.fragment,Vt),Ce=a(Vt),$t=d(Vt,"P",{"data-svelte-h":!0}),f($t)!=="svelte-1n8tbt5"&&($t.textContent=nn),Ne=a(Vt),Tt=d(Vt,"P",{"data-svelte-h":!0}),f(Tt)!=="svelte-1ucdexx"&&(Tt.textContent=on),Vt.forEach(e),se=a(t),Z=d(t,"DIV",{class:!0});var _n=h(Z);c(tt.$$.fragment,_n),_n.forEach(e),re=a(t),et=d(t,"DIV",{class:!0});var xn=h(et);c(nt.$$.fragment,xn),xn.forEach(e),de=a(t),v=d(t,"DIV",{class:!0});var N=h(v);c(ot.$$.fragment,N),Pe=a(N),qt=d(N,"P",{"data-svelte-h":!0}),f(qt)!=="svelte-1ec4axr"&&(qt.textContent=an),Oe=a(N),C=d(N,"DIV",{class:!0});var xe=h(C);c(at.$$.fragment,xe),Ee=a(xe),zt=d(xe,"P",{"data-svelte-h":!0}),f(zt)!=="svelte-1ubgx6o"&&(zt.textContent=sn),xe.forEach(e),Me=a(N),_=d(N,"DIV",{class:!0});var P=h(_);c(it.$$.fragment,P),Ve=a(P),wt=d(P,"P",{"data-svelte-h":!0}),f(wt)!=="svelte-1k7tn2t"&&(wt.textContent=rn),He=a(P),kt=d(P,"P",{"data-svelte-h":!0}),f(kt)!=="svelte-11ui0wm"&&(kt.textContent=dn),Ie=a(P),At=d(P,"P",{"data-svelte-h":!0}),f(At)!=="svelte-1ykfpyf"&&(At.innerHTML=ln),P.forEach(e),N.forEach(e),le=a(t),c(st.$$.fragment,t),ce=a(t),rt=d(t,"P",{"data-svelte-h":!0}),f(rt)!=="svelte-7lntof"&&(rt.textContent=cn),pe=a(t),dt=d(t,"P",{"data-svelte-h":!0}),f(dt)!=="svelte-1bzp8dj"&&(dt.innerHTML=pn),ue=a(t),z=d(t,"DIV",{class:!0});var Ht=h(z);c(lt.$$.fragment,Ht),Qe=a(Ht),Dt=d(Ht,"P",{"data-svelte-h":!0}),f(Dt)!=="svelte-sv0f0s"&&(Dt.textContent=un),Se=a(Ht),Lt=d(Ht,"P",{"data-svelte-h":!0}),f(Lt)!=="svelte-1oor9gf"&&(Lt.innerHTML=mn),Ht.forEach(e),me=a(t),w=d(t,"DIV",{class:!0});var It=h(w);c(ct.$$.fragment,It),Fe=a(It),Ct=d(It,"P",{"data-svelte-h":!0}),f(Ct)!=="svelte-g7axkd"&&(Ct.textContent=bn),Re=a(It),Nt=d(It,"P",{"data-svelte-h":!0}),f(Nt)!=="svelte-1e3tiho"&&(Nt.innerHTML=fn),It.forEach(e),be=a(t),c(pt.$$.fragment,t),fe=a(t),L=d(t,"DIV",{class:!0});var $e=h(L);c(ut.$$.fragment,$e),Be=a($e),Pt=d($e,"P",{"data-svelte-h":!0}),f(Pt)!=="svelte-mjy6qu"&&(Pt.textContent=hn),$e.forEach(e),he=a(t),c(mt.$$.fragment,t),ye=a(t),Ft=d(t,"P",{}),h(Ft).forEach(e),this.h()},h(){y(k,"name","hf:doc:metadata"),y(k,"content",Cn),y(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(et,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,n){s(document.head,k),i(t,Rt,n),i(t,St,n),i(t,Bt,n),p(O,t,n),i(t,Ut,n),p(E,t,n),i(t,Wt,n),i(t,M,n),i(t,jt,n),p(V,t,n),i(t,Gt,n),i(t,H,n),i(t,Jt,n),p(I,t,n),i(t,Kt,n),i(t,x,n),p(Q,x,null),s(x,Te),s(x,bt),s(x,qe),s(x,ft),i(t,Xt,n),i(t,A,n),p(S,A,null),s(A,ze),s(A,ht),i(t,Yt,n),i(t,D,n),p(F,D,null),s(D,we),s(D,yt),i(t,Zt,n),i(t,$,n),p(R,$,null),s($,ke),s($,gt),s($,Ae),s($,vt),i(t,te,n),p(B,t,n),i(t,ee,n),i(t,T,n),p(U,T,null),s(T,De),s(T,_t),s(T,Le),s(T,xt),i(t,ne,n),i(t,W,n),p(j,W,null),i(t,oe,n),i(t,G,n),p(J,G,null),i(t,ae,n),i(t,K,n),p(X,K,null),i(t,ie,n),i(t,q,n),p(Y,q,null),s(q,Ce),s(q,$t),s(q,Ne),s(q,Tt),i(t,se,n),i(t,Z,n),p(tt,Z,null),i(t,re,n),i(t,et,n),p(nt,et,null),i(t,de,n),i(t,v,n),p(ot,v,null),s(v,Pe),s(v,qt),s(v,Oe),s(v,C),p(at,C,null),s(C,Ee),s(C,zt),s(v,Me),s(v,_),p(it,_,null),s(_,Ve),s(_,wt),s(_,He),s(_,kt),s(_,Ie),s(_,At),i(t,le,n),p(st,t,n),i(t,ce,n),i(t,rt,n),i(t,pe,n),i(t,dt,n),i(t,ue,n),i(t,z,n),p(lt,z,null),s(z,Qe),s(z,Dt),s(z,Se),s(z,Lt),i(t,me,n),i(t,w,n),p(ct,w,null),s(w,Fe),s(w,Ct),s(w,Re),s(w,Nt),i(t,be,n),p(pt,t,n),i(t,fe,n),i(t,L,n),p(ut,L,null),s(L,Be),s(L,Pt),i(t,he,n),p(mt,t,n),i(t,ye,n),i(t,Ft,n),ge=!0},p:Tn,i(t){ge||(u(O.$$.fragment,t),u(E.$$.fragment,t),u(V.$$.fragment,t),u(I.$$.fragment,t),u(Q.$$.fragment,t),u(S.$$.fragment,t),u(F.$$.fragment,t),u(R.$$.fragment,t),u(B.$$.fragment,t),u(U.$$.fragment,t),u(j.$$.fragment,t),u(J.$$.fragment,t),u(X.$$.fragment,t),u(Y.$$.fragment,t),u(tt.$$.fragment,t),u(nt.$$.fragment,t),u(ot.$$.fragment,t),u(at.$$.fragment,t),u(it.$$.fragment,t),u(st.$$.fragment,t),u(lt.$$.fragment,t),u(ct.$$.fragment,t),u(pt.$$.fragment,t),u(ut.$$.fragment,t),u(mt.$$.fragment,t),ge=!0)},o(t){m(O.$$.fragment,t),m(E.$$.fragment,t),m(V.$$.fragment,t),m(I.$$.fragment,t),m(Q.$$.fragment,t),m(S.$$.fragment,t),m(F.$$.fragment,t),m(R.$$.fragment,t),m(B.$$.fragment,t),m(U.$$.fragment,t),m(j.$$.fragment,t),m(J.$$.fragment,t),m(X.$$.fragment,t),m(Y.$$.fragment,t),m(tt.$$.fragment,t),m(nt.$$.fragment,t),m(ot.$$.fragment,t),m(at.$$.fragment,t),m(it.$$.fragment,t),m(st.$$.fragment,t),m(lt.$$.fragment,t),m(ct.$$.fragment,t),m(pt.$$.fragment,t),m(ut.$$.fragment,t),m(mt.$$.fragment,t),ge=!1},d(t){t&&(e(Rt),e(St),e(Bt),e(Ut),e(Wt),e(M),e(jt),e(Gt),e(H),e(Jt),e(Kt),e(x),e(Xt),e(A),e(Yt),e(D),e(Zt),e($),e(te),e(ee),e(T),e(ne),e(W),e(oe),e(G),e(ae),e(K),e(ie),e(q),e(se),e(Z),e(re),e(et),e(de),e(v),e(le),e(ce),e(rt),e(pe),e(dt),e(ue),e(z),e(me),e(w),e(be),e(fe),e(L),e(he),e(ye),e(Ft)),e(k),b(O,t),b(E,t),b(V,t),b(I,t),b(Q),b(S),b(F),b(R),b(B,t),b(U),b(j),b(J),b(X),b(Y),b(tt),b(nt),b(ot),b(at),b(it),b(st,t),b(lt),b(ct),b(pt,t),b(ut),b(mt,t)}}}const Cn='{"title":"Overview","local":"overview","sections":[{"title":"When to Use bitsandbytes.functional","local":"when-to-use-bitsandbytesfunctional","sections":[],"depth":2},{"title":"LLM.int8()","local":"bitsandbytes.functional.int8_linear_matmul","sections":[],"depth":2},{"title":"4-bit","local":"bitsandbytes.functional.dequantize_4bit","sections":[],"depth":2},{"title":"Dynamic 8-bit Quantization","local":"bitsandbytes.functional.dequantize_blockwise","sections":[],"depth":2},{"title":"Utility","local":"bitsandbytes.functional.get_ptr","sections":[],"depth":2}],"depth":1}';function Nn(Ue){return qn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Vn extends zn{constructor(k){super(),wn(this,k,Nn,Ln,$n,{})}}export{Vn as component};