bitsandbytes/pr_2012/en/_app/immutable/nodes/3.C-3FwHZW.js

import"../chunks/DsnmJJEf.js";import{i as Z,h as k,C as z,H as r,D as a,E as W,s as U,a as G}from"../chunks/DT0OpeMJ.js";import{p as O,o as V,s as e,f as w,a as m,b as F,c as t,d as l,r as d,n as j}from"../chunks/Bb-LL0eD.js";import{E as R}from"../chunks/-HcYSbRW.js";const q='{"title":"Embedding","local":"embedding","sections":[{"title":"Embedding","local":"bitsandbytes.nn.Embedding","sections":[],"depth":2},{"title":"StableEmbedding","local":"bitsandbytes.nn.StableEmbedding","sections":[],"depth":2}],"depth":1}';var C=l('<meta name="hf:doc:metadata"/>'),I=l("<p>Example:</p> <!>",1),X=l(`<p></p> <!> <!> <p>The embedding class is used to store and retrieve word embeddings from their indices. There are two types of embeddings in bitsandbytes, the standard PyTorch <code>Embedding</code> class and the <code>StableEmbedding</code> class.</p> <p>The <code>StableEmbedding</code> class was introduced in the <a href="https://hf.co/papers/2110.02861" rel="nofollow">8-bit Optimizers via Block-wise Quantization</a> paper to reduce gradient variance as a result of the non-uniform distribution of input tokens. This class is designed to support quantization.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Embedding class to store and retrieve word embeddings from their indices.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Custom embedding layer designed to improve stability during training for NLP tasks by using 32-bit optimizer states. It is designed to reduce gradient variations that can result from quantization. This embedding layer is initialized with Xavier uniform initialization followed by layer normalization.</p> <!> <p>Methods:
reset_parameters(): Reset embedding parameters using Xavier uniform initialization.
forward(input: Tensor) -> Tensor: Forward pass through the stable embedding layer.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div></div> <!> <p></p>`,1);function D(E,T){O(T,!1),V(()=>{new URLSearchParams(window.location.search).get("fw")}),Z();var b=X();k("12bz8xe",n=>{var s=C();U(s,"content",q),m(n,s)});var c=e(w(b),2);z(c,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var p=e(c,2);r(p,{title:"Embedding",local:"embedding",headingTag:"h1"});var g=e(p,6);r(g,{title:"Embedding",local:"bitsandbytes.nn.Embedding",headingTag:"h2"});var i=e(g,2),_=t(i);a(_,{name:"class bitsandbytes.nn.Embedding",anchor:"bitsandbytes.nn.Embedding",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2012/bitsandbytes/nn/modules.py#L134",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:": typing.Optional[torch.device] = None"}]});var y=e(_,4),M=t(y);a(M,{name:"__init__",anchor:"bitsandbytes.nn.Embedding.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2012/bitsandbytes/nn/modules.py#L139",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:": typing.Optional[torch.device] = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.Embedding.__init__.num_embeddings",description:`<strong>num_embeddings</strong> (<code>int</code>) &#x2014;
The number of unique embeddings (vocabulary size).`,name:"num_embeddings"},{anchor:"bitsandbytes.nn.Embedding.__init__.embedding_dim",description:`<strong>embedding_dim</strong> (<code>int</code>) &#x2014;
The dimensionality of the embedding.`,name:"embedding_dim"},{anchor:"bitsandbytes.nn.Embedding.__init__.padding_idx",description:`<strong>padding_idx</strong> (<code>Optional[int]</code>) &#x2014;
Pads the output with zeros at the given index.`,name:"padding_idx"},{anchor:"bitsandbytes.nn.Embedding.__init__.max_norm",description:`<strong>max_norm</strong> (<code>Optional[float]</code>) &#x2014;
Renormalizes embeddings to have a maximum L2 norm.`,name:"max_norm"},{anchor:"bitsandbytes.nn.Embedding.__init__.norm_type",description:`<strong>norm_type</strong> (<code>float</code>, defaults to <code>2.0</code>) &#x2014;
The p-norm to compute for the <code>max_norm</code> option.`,name:"norm_type"},{anchor:"bitsandbytes.nn.Embedding.__init__.scale_grad_by_freq",description:`<strong>scale_grad_by_freq</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Scale gradient by frequency during backpropagation.`,name:"scale_grad_by_freq"},{anchor:"bitsandbytes.nn.Embedding.__init__.sparse",description:`<strong>sparse</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Computes dense gradients. Set to <code>True</code> to compute sparse gradients instead.`,name:"sparse"},{anchor:"bitsandbytes.nn.Embedding.__init__._weight",description:`<strong>_weight</strong> (<code>Optional[Tensor]</code>) &#x2014;
Pretrained embeddings.`,name:"_weight"}]}),d(y),d(i);var h=e(i,2);r(h,{title:"StableEmbedding",local:"bitsandbytes.nn.StableEmbedding",headingTag:"h2"});var o=e(h,2),u=t(o);a(u,{name:"class bitsandbytes.nn.StableEmbedding",anchor:"bitsandbytes.nn.StableEmbedding",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2012/bitsandbytes/nn/modules.py#L28",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:" = None"},{name:"dtype",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.StableEmbedding.norm",description:"<strong>norm</strong> (<code>torch.nn.LayerNorm</code>) &#x2014; Layer normalization applied after the embedding.",name:"norm"}]});var v=e(u,4);R(v,{anchor:"bitsandbytes.nn.StableEmbedding.example",children:(n,s)=>{var x=I(),N=e(w(x),2);G(N,{code:"JTIzJTIwSW5pdGlhbGl6ZSUyMFN0YWJsZUVtYmVkZGluZyUyMGxheWVyJTIwd2l0aCUyMHZvY2FidWxhcnklMjBzaXplJTIwMTAwMCUyQyUyMGVtYmVkZGluZyUyMGRpbWVuc2lvbiUyMDMwMCUwQWVtYmVkZGluZ19sYXllciUyMCUzRCUyMFN0YWJsZUVtYmVkZGluZyhudW1fZW1iZWRkaW5ncyUzRDEwMDAlMkMlMjBlbWJlZGRpbmdfZGltJTNEMzAwKSUwQSUwQSUyMyUyMFJlc2V0JTIwZW1iZWRkaW5nJTIwcGFyYW1ldGVycyUwQWVtYmVkZGluZ19sYXllci5yZXNldF9wYXJhbWV0ZXJzKCklMEElMEElMjMlMjBQZXJmb3JtJTIwYSUyMGZvcndhcmQlMjBwYXNzJTIwd2l0aCUyMGlucHV0JTIwdGVuc29yJTBBaW5wdXRfdGVuc29yJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1QjElMkMlMjAyJTJDJTIwMyU1RCklMEFvdXRwdXRfZW1iZWRkaW5nJTIwJTNEJTIwZW1iZWRkaW5nX2xheWVyKGlucHV0X3RlbnNvcik=",highlighted:`# Initialize StableEmbedding layer <span class="hljs-keyword">with</span> vocabulary size <span class="hljs-number">1000</span>, embedding dimension <span class="hljs-number">300</span>
embedding_layer = StableEmbedding(num_embeddings=<span class="hljs-number">1000</span>, embedding_dim=<span class="hljs-number">300</span>)

# <span class="hljs-keyword">Reset</span> embedding parameters
embedding_layer.reset_parameters()

# <span class="hljs-keyword">Perform</span> a forward pass <span class="hljs-keyword">with</span> <span class="hljs-keyword">input</span> tensor
input_tensor = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>])
output_embedding = embedding_layer(input_tensor)`,lang:"",wrap:!1}),m(n,x)},$$slots:{default:!0}});var f=e(v,4),S=t(f);a(S,{name:"__init__",anchor:"bitsandbytes.nn.StableEmbedding.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2012/bitsandbytes/nn/modules.py#L54",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:" = None"},{name:"dtype",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.StableEmbedding.__init__.num_embeddings",description:`<strong>num_embeddings</strong> (<code>int</code>) &#x2014;
The number of unique embeddings (vocabulary size).`,name:"num_embeddings"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.embedding_dim",description:`<strong>embedding_dim</strong> (<code>int</code>) &#x2014;
The dimensionality of the embedding.`,name:"embedding_dim"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.padding_idx",description:`<strong>padding_idx</strong> (<code>Optional[int]</code>) &#x2014;
Pads the output with zeros at the given index.`,name:"padding_idx"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.max_norm",description:`<strong>max_norm</strong> (<code>Optional[float]</code>) &#x2014;
Renormalizes embeddings to have a maximum L2 norm.`,name:"max_norm"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.norm_type",description:`<strong>norm_type</strong> (<code>float</code>, defaults to <code>2.0</code>) &#x2014;
The p-norm to compute for the <code>max_norm</code> option.`,name:"norm_type"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.scale_grad_by_freq",description:`<strong>scale_grad_by_freq</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Scale gradient by frequency during backpropagation.`,name:"scale_grad_by_freq"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.sparse",description:`<strong>sparse</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Computes dense gradients. Set to <code>True</code> to compute sparse gradients instead.`,name:"sparse"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__._weight",description:`<strong>_weight</strong> (<code>Optional[Tensor]</code>) &#x2014;
Pretrained embeddings.`,name:"_weight"}]}),d(f),d(o);var J=e(o,2);W(J,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/nn/embeddings.mdx"}),j(2),m(E,b),F()}export{D as component};