bitsandbytes/pr_1908/en/_app/immutable/nodes/14.598a3f56.js

import{s as ue,o as ye,n as fe}from"../chunks/scheduler.8a2cc2fa.js";import{S as he,i as ve,e as m,s as d,c as u,h as $e,a as l,d as t,b as s,f as I,g as y,j as q,k as P,l as g,m as i,n as f,t as h,o as v,p as $}from"../chunks/index.7079e750.js";import{C as xe,H as re,E as we}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2b7ce466.js";import{D as ne}from"../chunks/Docstring.8c9a5003.js";import{C as Ee}from"../chunks/CodeBlock.a326412a.js";import{E as Te}from"../chunks/ExampleCodeBlock.7664d7e9.js";function Me(L){let o,M="Example:",x,b,p;return b=new Ee({props:{code:"JTIzJTIwSW5pdGlhbGl6ZSUyMFN0YWJsZUVtYmVkZGluZyUyMGxheWVyJTIwd2l0aCUyMHZvY2FidWxhcnklMjBzaXplJTIwMTAwMCUyQyUyMGVtYmVkZGluZyUyMGRpbWVuc2lvbiUyMDMwMCUwQWVtYmVkZGluZ19sYXllciUyMCUzRCUyMFN0YWJsZUVtYmVkZGluZyhudW1fZW1iZWRkaW5ncyUzRDEwMDAlMkMlMjBlbWJlZGRpbmdfZGltJTNEMzAwKSUwQSUwQSUyMyUyMFJlc2V0JTIwZW1iZWRkaW5nJTIwcGFyYW1ldGVycyUwQWVtYmVkZGluZ19sYXllci5yZXNldF9wYXJhbWV0ZXJzKCklMEElMEElMjMlMjBQZXJmb3JtJTIwYSUyMGZvcndhcmQlMjBwYXNzJTIwd2l0aCUyMGlucHV0JTIwdGVuc29yJTBBaW5wdXRfdGVuc29yJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1QjElMkMlMjAyJTJDJTIwMyU1RCklMEFvdXRwdXRfZW1iZWRkaW5nJTIwJTNEJTIwZW1iZWRkaW5nX2xheWVyKGlucHV0X3RlbnNvcik=",highlighted:`# Initialize StableEmbedding layer <span class="hljs-keyword">with</span> vocabulary size <span class="hljs-number">1000</span>, embedding dimension <span class="hljs-number">300</span>
embedding_layer = StableEmbedding(num_embeddings=<span class="hljs-number">1000</span>, embedding_dim=<span class="hljs-number">300</span>)

# <span class="hljs-keyword">Reset</span> embedding parameters
embedding_layer.reset_parameters()

# <span class="hljs-keyword">Perform</span> a forward pass <span class="hljs-keyword">with</span> <span class="hljs-keyword">input</span> tensor
input_tensor = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>])
output_embedding = embedding_layer(input_tensor)`,wrap:!1}}),{c(){o=m("p"),o.textContent=M,x=d(),u(b.$$.fragment)},l(a){o=l(a,"P",{"data-svelte-h":!0}),q(o)!=="svelte-11lpom8"&&(o.textContent=M),x=s(a),y(b.$$.fragment,a)},m(a,c){i(a,o,c),i(a,x,c),f(b,a,c),p=!0},p:fe,i(a){p||(h(b.$$.fragment,a),p=!0)},o(a){v(b.$$.fragment,a),p=!1},d(a){a&&(t(o),t(x)),$(b,a)}}}function Se(L){let o,M,x,b,p,a,c,X,S,me="The embedding class is used to store and retrieve word embeddings from their indices. There are two types of embeddings in bitsandbytes, the standard PyTorch <code>Embedding</code> class and the <code>StableEmbedding</code> class.",Y,k,le='The <code>StableEmbedding</code> class was introduced in the <a href="https://hf.co/papers/2110.02861" rel="nofollow">8-bit Optimizers via Block-wise Quantization</a> paper to reduce gradient variance as a result of the non-uniform distribution of input tokens. This class is designed to support quantization.',D,C,B,_,J,te,U,be="Embedding class to store and retrieve word embeddings from their indices.",ae,G,N,H,Z,Q,r,z,ie,O,pe="Custom embedding layer designed to improve stability during training for NLP tasks by using 32-bit optimizer states. It is designed to reduce gradient variations that can result from quantization. This embedding layer is initialized with Xavier uniform initialization followed by layer normalization.",de,T,se,j,ge=`Methods:
reset_parameters(): Reset embedding parameters using Xavier uniform initialization.
forward(input: Tensor) -> Tensor: Forward pass through the stable embedding layer.`,oe,F,W,A,V,K,R,ee;return p=new xe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),c=new re({props:{title:"Embedding",local:"embedding",headingTag:"h1"}}),C=new re({props:{title:"Embedding",local:"bitsandbytes.nn.Embedding",headingTag:"h2"}}),J=new ne({props:{name:"class bitsandbytes.nn.Embedding",anchor:"bitsandbytes.nn.Embedding",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:": typing.Optional[torch.device] = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L134"}}),N=new ne({props:{name:"__init__",anchor:"bitsandbytes.nn.Embedding.__init__",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:": typing.Optional[torch.device] = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.Embedding.__init__.num_embeddings",description:`<strong>num_embeddings</strong> (<code>int</code>) &#x2014;
The number of unique embeddings (vocabulary size).`,name:"num_embeddings"},{anchor:"bitsandbytes.nn.Embedding.__init__.embedding_dim",description:`<strong>embedding_dim</strong> (<code>int</code>) &#x2014;
The dimensionality of the embedding.`,name:"embedding_dim"},{anchor:"bitsandbytes.nn.Embedding.__init__.padding_idx",description:`<strong>padding_idx</strong> (<code>Optional[int]</code>) &#x2014;
Pads the output with zeros at the given index.`,name:"padding_idx"},{anchor:"bitsandbytes.nn.Embedding.__init__.max_norm",description:`<strong>max_norm</strong> (<code>Optional[float]</code>) &#x2014;
Renormalizes embeddings to have a maximum L2 norm.`,name:"max_norm"},{anchor:"bitsandbytes.nn.Embedding.__init__.norm_type",description:`<strong>norm_type</strong> (<code>float</code>, defaults to <code>2.0</code>) &#x2014;
The p-norm to compute for the <code>max_norm</code> option.`,name:"norm_type"},{anchor:"bitsandbytes.nn.Embedding.__init__.scale_grad_by_freq",description:`<strong>scale_grad_by_freq</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Scale gradient by frequency during backpropagation.`,name:"scale_grad_by_freq"},{anchor:"bitsandbytes.nn.Embedding.__init__.sparse",description:`<strong>sparse</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Computes dense gradients. Set to <code>True</code> to compute sparse gradients instead.`,name:"sparse"},{anchor:"bitsandbytes.nn.Embedding.__init__._weight",description:`<strong>_weight</strong> (<code>Optional[Tensor]</code>) &#x2014;
Pretrained embeddings.`,name:"_weight"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L139"}}),Z=new re({props:{title:"StableEmbedding",local:"bitsandbytes.nn.StableEmbedding",headingTag:"h2"}}),z=new ne({props:{name:"class bitsandbytes.nn.StableEmbedding",anchor:"bitsandbytes.nn.StableEmbedding",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:" = None"},{name:"dtype",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.StableEmbedding.norm",description:"<strong>norm</strong> (<code>torch.nn.LayerNorm</code>) &#x2014; Layer normalization applied after the embedding.",name:"norm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L28"}}),T=new Te({props:{anchor:"bitsandbytes.nn.StableEmbedding.example",$$slots:{default:[Me]},$$scope:{ctx:L}}}),W=new ne({props:{name:"__init__",anchor:"bitsandbytes.nn.StableEmbedding.__init__",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:" = None"},{name:"dtype",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.StableEmbedding.__init__.num_embeddings",description:`<strong>num_embeddings</strong> (<code>int</code>) &#x2014;
The number of unique embeddings (vocabulary size).`,name:"num_embeddings"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.embedding_dim",description:`<strong>embedding_dim</strong> (<code>int</code>) &#x2014;
The dimensionality of the embedding.`,name:"embedding_dim"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.padding_idx",description:`<strong>padding_idx</strong> (<code>Optional[int]</code>) &#x2014;
Pads the output with zeros at the given index.`,name:"padding_idx"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.max_norm",description:`<strong>max_norm</strong> (<code>Optional[float]</code>) &#x2014;
Renormalizes embeddings to have a maximum L2 norm.`,name:"max_norm"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.norm_type",description:`<strong>norm_type</strong> (<code>float</code>, defaults to <code>2.0</code>) &#x2014;
The p-norm to compute for the <code>max_norm</code> option.`,name:"norm_type"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.scale_grad_by_freq",description:`<strong>scale_grad_by_freq</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Scale gradient by frequency during backpropagation.`,name:"scale_grad_by_freq"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.sparse",description:`<strong>sparse</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Computes dense gradients. Set to <code>True</code> to compute sparse gradients instead.`,name:"sparse"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__._weight",description:`<strong>_weight</strong> (<code>Optional[Tensor]</code>) &#x2014;
Pretrained embeddings.`,name:"_weight"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L54"}}),V=new we({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/nn/embeddings.mdx"}}),{c(){o=m("meta"),M=d(),x=m("p"),b=d(),u(p.$$.fragment),a=d(),u(c.$$.fragment),X=d(),S=m("p"),S.innerHTML=me,Y=d(),k=m("p"),k.innerHTML=le,D=d(),u(C.$$.fragment),B=d(),_=m("div"),u(J.$$.fragment),te=d(),U=m("p"),U.textContent=be,ae=d(),G=m("div"),u(N.$$.fragment),H=d(),u(Z.$$.fragment),Q=d(),r=m("div"),u(z.$$.fragment),ie=d(),O=m("p"),O.textContent=pe,de=d(),u(T.$$.fragment),se=d(),j=m("p"),j.textContent=ge,oe=d(),F=m("div"),u(W.$$.fragment),A=d(),u(V.$$.fragment),K=d(),R=m("p"),this.h()},l(e){const n=$e("svelte-u9bgzb",document.head);o=l(n,"META",{name:!0,content:!0}),n.forEach(t),M=s(e),x=l(e,"P",{}),I(x).forEach(t),b=s(e),y(p.$$.fragment,e),a=s(e),y(c.$$.fragment,e),X=s(e),S=l(e,"P",{"data-svelte-h":!0}),q(S)!=="svelte-hjtctc"&&(S.innerHTML=me),Y=s(e),k=l(e,"P",{"data-svelte-h":!0}),q(k)!=="svelte-1vvt62i"&&(k.innerHTML=le),D=s(e),y(C.$$.fragment,e),B=s(e),_=l(e,"DIV",{class:!0});var E=I(_);y(J.$$.fragment,E),te=s(E),U=l(E,"P",{"data-svelte-h":!0}),q(U)!=="svelte-1tikli9"&&(U.textContent=be),ae=s(E),G=l(E,"DIV",{class:!0});var ce=I(G);y(N.$$.fragment,ce),ce.forEach(t),E.forEach(t),H=s(e),y(Z.$$.fragment,e),Q=s(e),r=l(e,"DIV",{class:!0});var w=I(r);y(z.$$.fragment,w),ie=s(w),O=l(w,"P",{"data-svelte-h":!0}),q(O)!=="svelte-15pctl3"&&(O.textContent=pe),de=s(w),y(T.$$.fragment,w),se=s(w),j=l(w,"P",{"data-svelte-h":!0}),q(j)!=="svelte-wb92uq"&&(j.textContent=ge),oe=s(w),F=l(w,"DIV",{class:!0});var _e=I(F);y(W.$$.fragment,_e),_e.forEach(t),w.forEach(t),A=s(e),y(V.$$.fragment,e),K=s(e),R=l(e,"P",{}),I(R).forEach(t),this.h()},h(){P(o,"name","hf:doc:metadata"),P(o,"content",ke),P(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(r,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){g(document.head,o),i(e,M,n),i(e,x,n),i(e,b,n),f(p,e,n),i(e,a,n),f(c,e,n),i(e,X,n),i(e,S,n),i(e,Y,n),i(e,k,n),i(e,D,n),f(C,e,n),i(e,B,n),i(e,_,n),f(J,_,null),g(_,te),g(_,U),g(_,ae),g(_,G),f(N,G,null),i(e,H,n),f(Z,e,n),i(e,Q,n),i(e,r,n),f(z,r,null),g(r,ie),g(r,O),g(r,de),f(T,r,null),g(r,se),g(r,j),g(r,oe),g(r,F),f(W,F,null),i(e,A,n),f(V,e,n),i(e,K,n),i(e,R,n),ee=!0},p(e,[n]){const E={};n&2&&(E.$$scope={dirty:n,ctx:e}),T.$set(E)},i(e){ee||(h(p.$$.fragment,e),h(c.$$.fragment,e),h(C.$$.fragment,e),h(J.$$.fragment,e),h(N.$$.fragment,e),h(Z.$$.fragment,e),h(z.$$.fragment,e),h(T.$$.fragment,e),h(W.$$.fragment,e),h(V.$$.fragment,e),ee=!0)},o(e){v(p.$$.fragment,e),v(c.$$.fragment,e),v(C.$$.fragment,e),v(J.$$.fragment,e),v(N.$$.fragment,e),v(Z.$$.fragment,e),v(z.$$.fragment,e),v(T.$$.fragment,e),v(W.$$.fragment,e),v(V.$$.fragment,e),ee=!1},d(e){e&&(t(M),t(x),t(b),t(a),t(X),t(S),t(Y),t(k),t(D),t(B),t(_),t(H),t(Q),t(r),t(A),t(K),t(R)),t(o),$(p,e),$(c,e),$(C,e),$(J),$(N),$(Z,e),$(z),$(T),$(W),$(V,e)}}}const ke='{"title":"Embedding","local":"embedding","sections":[{"title":"Embedding","local":"bitsandbytes.nn.Embedding","sections":[],"depth":2},{"title":"StableEmbedding","local":"bitsandbytes.nn.StableEmbedding","sections":[],"depth":2}],"depth":1}';function Ce(L){return ye(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ue extends he{constructor(o){super(),ve(this,o,Ce,Se,ue,{})}}export{Ue as component};