accelerate/pr_4073/en/_app/immutable/nodes/53.799a42e7.js

import{s as Te,o as we,n as ve}from"../chunks/scheduler.b9285784.js";import{S as ge,i as Le,e as d,s as a,c as m,h as He,a as r,d as l,b as n,f as be,g as f,j as o,k as Nt,l as _e,m as i,n as c,t as p,o as h,p as u}from"../chunks/index.26bc89a1.js";import{T as Be}from"../chunks/Tip.e4eba3d6.js";import{C as xe,H as it,E as Ce}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.ca265e81.js";import{C as Jt}from"../chunks/CodeBlock.7808cca4.js";function ke(nt){let s,T=`This API will load the model into memory on the <code>meta</code> device, so we are not actually downloading
and loading the full weights of the model into memory, nor do we need to. As a result it’s
perfectly fine to measure 8 billion parameter models (or more), without having to worry about
if your CPU can handle it!`;return{c(){s=d("p"),s.innerHTML=T},l(y){s=r(y,"P",{"data-svelte-h":!0}),o(s)!=="svelte-1byyf03"&&(s.innerHTML=T)},m(y,lt){i(y,s,lt)},p:ve,d(y){y&&l(s)}}}function Ge(nt){let s,T,y,lt,w,dt,v,rt,g,It="One very difficult aspect when exploring potential models to use on your machine is knowing just how big of a model will <em>fit</em> into memory with your current device (such as loading the model onto CUDA or XPU).",ot,L,Rt=`To help alleviate this, Accelerate has a CLI interface through <code>accelerate estimate-memory</code>. This tutorial will
help walk you through using it, what to expect, and at the end link to the interactive demo hosted on the Hub which will
even let you post those results directly on the model repo!`,st,H,qt="Currently we support searching for models that can be used in <code>timm</code> and <code>transformers</code>.",mt,M,ft,_,ct,B,Dt="Below are a few gradio demos related to what was described above. The first is the official Hugging Face memory estimation space, utilizing Accelerate directly:",pt,$,Qt='<iframe src="https://hf-accelerate-model-memory-usage.hf.space?__theme=light" width="850" height="1600"></iframe>',ht,b,Ot='<iframe src="https://hf-accelerate-model-memory-usage.hf.space?__theme=dark" width="850" height="1600"></iframe>',ut,x,Kt='A community member has taken the idea and expanded it further, allowing you to filter models directly and see if you can run a particular LLM given GPU constraints and LoRA configurations. To play with it, see <a href="https://huggingface.co/spaces/Vokturz/can-it-run-llm" rel="nofollow">here</a> for more details.',yt,C,Mt,k,te=`When using <code>accelerate estimate-memory</code>, you need to pass in the name of the model you want to use, potentially the framework
that model utilizing (if it can’t be found automatically), and the data types you want the model to be loaded in with.`,$t,G,ee="For example, here is how we can calculate the memory footprint for <code>bert-base-cased</code>:",bt,P,Tt,W,le=`This will download the <code>config.json</code> for <code>bert-based-cased</code>, load the model on the <code>meta</code> device, and report back how much space
it will use:`,wt,S,ie="Memory Usage for loading <code>bert-base-cased</code>:",vt,z,ae="<thead><tr><th>dtype</th> <th>Largest Layer</th> <th>Total Size</th> <th>Training using Adam</th></tr></thead> <tbody><tr><td>float32</td> <td>84.95 MB</td> <td>418.18 MB</td> <td>1.61 GB</td></tr> <tr><td>float16</td> <td>42.47 MB</td> <td>206.59 MB</td> <td>826.36 MB</td></tr> <tr><td>int8</td> <td>21.24 MB</td> <td>103.29 MB</td> <td>413.18 MB</td></tr> <tr><td>int4</td> <td>10.62 MB</td> <td>51.65 MB</td> <td>206.59 MB</td></tr></tbody>",gt,U,ne="By default it will return all the supported dtypes (<code>int4</code> through <code>float32</code>), but if you are interested in specific ones these can be filtered.",Lt,Z,Ht,A,de=`If the source library cannot be determined automatically (like it could in the case of <code>bert-base-cased</code>), a library name can
be passed in.`,_t,j,Bt,Y,re="Memory Usage for loading <code>HuggingFaceM4/idefics-80b-instruct</code>:",xt,F,oe="<thead><tr><th>dtype</th> <th>Largest Layer</th> <th>Total Size</th> <th>Training using Adam</th></tr></thead> <tbody><tr><td>float32</td> <td>3.02 GB</td> <td>297.12 GB</td> <td>1.16 TB</td></tr> <tr><td>float16</td> <td>1.51 GB</td> <td>148.56 GB</td> <td>594.24 GB</td></tr> <tr><td>int8</td> <td>772.52 MB</td> <td>74.28 GB</td> <td>297.12 GB</td></tr> <tr><td>int4</td> <td>386.26 MB</td> <td>37.14 GB</td> <td>148.56 GB</td></tr></tbody>",Ct,X,kt,E,se="Memory Usage for loading <code>timm/resnet50.a1_in1k</code>:",Gt,V,me="<thead><tr><th>dtype</th> <th>Largest Layer</th> <th>Total Size</th> <th>Training using Adam</th></tr></thead> <tbody><tr><td>float32</td> <td>9.0 MB</td> <td>97.7 MB</td> <td>390.78 MB</td></tr> <tr><td>float16</td> <td>4.5 MB</td> <td>48.85 MB</td> <td>195.39 MB</td></tr> <tr><td>int8</td> <td>2.25 MB</td> <td>24.42 MB</td> <td>97.7 MB</td></tr> <tr><td>int4</td> <td>1.12 MB</td> <td>12.21 MB</td> <td>48.85 MB</td></tr></tbody>",Pt,N,Wt,J,fe="As mentioned earlier, while we return <code>int4</code> through <code>float32</code> by default, any dtype can be used from <code>float32</code>, <code>float16</code>, <code>int8</code>, and <code>int4</code>.",St,I,ce="To do so, pass them in after specifying <code>--dtypes</code>:",zt,R,Ut,q,pe="Memory Usage for loading <code>bert-base-cased</code>:",Zt,D,he="<thead><tr><th>dtype</th> <th>Largest Layer</th> <th>Total Size</th> <th>Training using Adam</th></tr></thead> <tbody><tr><td>float32</td> <td>84.95 MB</td> <td>413.18 MB</td> <td>1.61 GB</td></tr> <tr><td>float16</td> <td>42.47 MB</td> <td>206.59 MB</td> <td>826.36 MB</td></tr></tbody>",At,Q,jt,O,ue="This calculator will tell you how much memory is needed to purely load the model in, <em>not</em> to perform inference.",Yt,K,ye="This calculation is accurate within a few % of the actual value, so it is a very good view of just how much memory it will take. For instance loading <code>bert-base-cased</code> actually takes <code>413.68 MB</code> when loaded on CUDA in full precision, and the calculator estimates <code>413.18 MB</code>.",Ft,tt,Me=`When performing inference you can expect to add up to an additional 20% as found by <a href="https://blog.eleuther.ai/transformer-math/" rel="nofollow">EleutherAI</a>. We’ll be conducting research into finding a more accurate estimate to these values, and will update
this calculator once done.`,Xt,et,Et,at,Vt;return w=new xe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),v=new it({props:{title:"Model memory estimator",local:"model-memory-estimator",headingTag:"h1"}}),M=new Be({props:{$$slots:{default:[ke]},$$scope:{ctx:nt}}}),_=new it({props:{title:"Gradio Demos",local:"gradio-demos",headingTag:"h2"}}),C=new it({props:{title:"The Command",local:"the-command",headingTag:"h2"}}),P=new Jt({props:{code:"YWNjZWxlcmF0ZSUyMGVzdGltYXRlLW1lbW9yeSUyMGJlcnQtYmFzZS1jYXNlZA==",highlighted:"accelerate estimate-memory bert-base-cased",lang:"bash",wrap:!1}}),Z=new it({props:{title:"Specific libraries",local:"specific-libraries",headingTag:"h3"}}),j=new Jt({props:{code:"YWNjZWxlcmF0ZSUyMGVzdGltYXRlLW1lbW9yeSUyMEh1Z2dpbmdGYWNlTTQlMkZpZGVmaWNzLTgwYi1pbnN0cnVjdCUyMC0tbGlicmFyeV9uYW1lJTIwdHJhbnNmb3JtZXJz",highlighted:"accelerate estimate-memory HuggingFaceM4/idefics-80b-instruct --library_name transformers",lang:"bash",wrap:!1}}),X=new Jt({props:{code:"YWNjZWxlcmF0ZSUyMGVzdGltYXRlLW1lbW9yeSUyMHRpbW0lMkZyZXNuZXQ1MC5hMV9pbjFrJTIwLS1saWJyYXJ5X25hbWUlMjB0aW1t",highlighted:"accelerate estimate-memory timm/resnet50.a1_in1k --library_name timm",lang:"bash",wrap:!1}}),N=new it({props:{title:"Specific dtypes",local:"specific-dtypes",headingTag:"h3"}}),R=new Jt({props:{code:"YWNjZWxlcmF0ZSUyMGVzdGltYXRlLW1lbW9yeSUyMGJlcnQtYmFzZS1jYXNlZCUyMC0tZHR5cGVzJTIwZmxvYXQzMiUyMGZsb2F0MTY=",highlighted:"accelerate estimate-memory bert-base-cased --dtypes float32 float16",lang:"bash",wrap:!1}}),Q=new it({props:{title:"Caveats with this calculator",local:"caveats-with-this-calculator",headingTag:"h2"}}),et=new Ce({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/model_size_estimator.md"}}),{c(){s=d("meta"),T=a(),y=d("p"),lt=a(),m(w.$$.fragment),dt=a(),m(v.$$.fragment),rt=a(),g=d("p"),g.innerHTML=It,ot=a(),L=d("p"),L.innerHTML=Rt,st=a(),H=d("p"),H.innerHTML=qt,mt=a(),m(M.$$.fragment),ft=a(),m(_.$$.fragment),ct=a(),B=d("p"),B.textContent=Dt,pt=a(),$=d("div"),$.innerHTML=Qt,ht=a(),b=d("div"),b.innerHTML=Ot,ut=a(),x=d("p"),x.innerHTML=Kt,yt=a(),m(C.$$.fragment),Mt=a(),k=d("p"),k.innerHTML=te,$t=a(),G=d("p"),G.innerHTML=ee,bt=a(),m(P.$$.fragment),Tt=a(),W=d("p"),W.innerHTML=le,wt=a(),S=d("p"),S.innerHTML=ie,vt=a(),z=d("table"),z.innerHTML=ae,gt=a(),U=d("p"),U.innerHTML=ne,Lt=a(),m(Z.$$.fragment),Ht=a(),A=d("p"),A.innerHTML=de,_t=a(),m(j.$$.fragment),Bt=a(),Y=d("p"),Y.innerHTML=re,xt=a(),F=d("table"),F.innerHTML=oe,Ct=a(),m(X.$$.fragment),kt=a(),E=d("p"),E.innerHTML=se,Gt=a(),V=d("table"),V.innerHTML=me,Pt=a(),m(N.$$.fragment),Wt=a(),J=d("p"),J.innerHTML=fe,St=a(),I=d("p"),I.innerHTML=ce,zt=a(),m(R.$$.fragment),Ut=a(),q=d("p"),q.innerHTML=pe,Zt=a(),D=d("table"),D.innerHTML=he,At=a(),m(Q.$$.fragment),jt=a(),O=d("p"),O.innerHTML=ue,Yt=a(),K=d("p"),K.innerHTML=ye,Ft=a(),tt=d("p"),tt.innerHTML=Me,Xt=a(),m(et.$$.fragment),Et=a(),at=d("p"),this.h()},l(t){const e=He("svelte-u9bgzb",document.head);s=r(e,"META",{name:!0,content:!0}),e.forEach(l),T=n(t),y=r(t,"P",{}),be(y).forEach(l),lt=n(t),f(w.$$.fragment,t),dt=n(t),f(v.$$.fragment,t),rt=n(t),g=r(t,"P",{"data-svelte-h":!0}),o(g)!=="svelte-4pj9jb"&&(g.innerHTML=It),ot=n(t),L=r(t,"P",{"data-svelte-h":!0}),o(L)!=="svelte-1kup5k5"&&(L.innerHTML=Rt),st=n(t),H=r(t,"P",{"data-svelte-h":!0}),o(H)!=="svelte-w4272k"&&(H.innerHTML=qt),mt=n(t),f(M.$$.fragment,t),ft=n(t),f(_.$$.fragment,t),ct=n(t),B=r(t,"P",{"data-svelte-h":!0}),o(B)!=="svelte-1lyji5c"&&(B.textContent=Dt),pt=n(t),$=r(t,"DIV",{class:!0,"data-svelte-h":!0}),o($)!=="svelte-1pqs7ii"&&($.innerHTML=Qt),ht=n(t),b=r(t,"DIV",{class:!0,"data-svelte-h":!0}),o(b)!=="svelte-ocgkm1"&&(b.innerHTML=Ot),ut=n(t),x=r(t,"P",{"data-svelte-h":!0}),o(x)!=="svelte-7cbi9c"&&(x.innerHTML=Kt),yt=n(t),f(C.$$.fragment,t),Mt=n(t),k=r(t,"P",{"data-svelte-h":!0}),o(k)!=="svelte-45l1al"&&(k.innerHTML=te),$t=n(t),G=r(t,"P",{"data-svelte-h":!0}),o(G)!=="svelte-h4f2xa"&&(G.innerHTML=ee),bt=n(t),f(P.$$.fragment,t),Tt=n(t),W=r(t,"P",{"data-svelte-h":!0}),o(W)!=="svelte-ils653"&&(W.innerHTML=le),wt=n(t),S=r(t,"P",{"data-svelte-h":!0}),o(S)!=="svelte-1nfe19i"&&(S.innerHTML=ie),vt=n(t),z=r(t,"TABLE",{"data-svelte-h":!0}),o(z)!=="svelte-1555s6h"&&(z.innerHTML=ae),gt=n(t),U=r(t,"P",{"data-svelte-h":!0}),o(U)!=="svelte-1yqm0m1"&&(U.innerHTML=ne),Lt=n(t),f(Z.$$.fragment,t),Ht=n(t),A=r(t,"P",{"data-svelte-h":!0}),o(A)!=="svelte-njwxoj"&&(A.innerHTML=de),_t=n(t),f(j.$$.fragment,t),Bt=n(t),Y=r(t,"P",{"data-svelte-h":!0}),o(Y)!=="svelte-15w8ahh"&&(Y.innerHTML=re),xt=n(t),F=r(t,"TABLE",{"data-svelte-h":!0}),o(F)!=="svelte-18sp6z1"&&(F.innerHTML=oe),Ct=n(t),f(X.$$.fragment,t),kt=n(t),E=r(t,"P",{"data-svelte-h":!0}),o(E)!=="svelte-1mr9yqw"&&(E.innerHTML=se),Gt=n(t),V=r(t,"TABLE",{"data-svelte-h":!0}),o(V)!=="svelte-1kjhg7d"&&(V.innerHTML=me),Pt=n(t),f(N.$$.fragment,t),Wt=n(t),J=r(t,"P",{"data-svelte-h":!0}),o(J)!=="svelte-15o0uxk"&&(J.innerHTML=fe),St=n(t),I=r(t,"P",{"data-svelte-h":!0}),o(I)!=="svelte-9rym4t"&&(I.innerHTML=ce),zt=n(t),f(R.$$.fragment,t),Ut=n(t),q=r(t,"P",{"data-svelte-h":!0}),o(q)!=="svelte-1nfe19i"&&(q.innerHTML=pe),Zt=n(t),D=r(t,"TABLE",{"data-svelte-h":!0}),o(D)!=="svelte-1hk5v9f"&&(D.innerHTML=he),At=n(t),f(Q.$$.fragment,t),jt=n(t),O=r(t,"P",{"data-svelte-h":!0}),o(O)!=="svelte-gbqkuz"&&(O.innerHTML=ue),Yt=n(t),K=r(t,"P",{"data-svelte-h":!0}),o(K)!=="svelte-wpwqvl"&&(K.innerHTML=ye),Ft=n(t),tt=r(t,"P",{"data-svelte-h":!0}),o(tt)!=="svelte-1x4mgsf"&&(tt.innerHTML=Me),Xt=n(t),f(et.$$.fragment,t),Et=n(t),at=r(t,"P",{}),be(at).forEach(l),this.h()},h(){Nt(s,"name","hf:doc:metadata"),Nt(s,"content",Pe),Nt($,"class","block dark:hidden"),Nt(b,"class","hidden dark:block")},m(t,e){_e(document.head,s),i(t,T,e),i(t,y,e),i(t,lt,e),c(w,t,e),i(t,dt,e),c(v,t,e),i(t,rt,e),i(t,g,e),i(t,ot,e),i(t,L,e),i(t,st,e),i(t,H,e),i(t,mt,e),c(M,t,e),i(t,ft,e),c(_,t,e),i(t,ct,e),i(t,B,e),i(t,pt,e),i(t,$,e),i(t,ht,e),i(t,b,e),i(t,ut,e),i(t,x,e),i(t,yt,e),c(C,t,e),i(t,Mt,e),i(t,k,e),i(t,$t,e),i(t,G,e),i(t,bt,e),c(P,t,e),i(t,Tt,e),i(t,W,e),i(t,wt,e),i(t,S,e),i(t,vt,e),i(t,z,e),i(t,gt,e),i(t,U,e),i(t,Lt,e),c(Z,t,e),i(t,Ht,e),i(t,A,e),i(t,_t,e),c(j,t,e),i(t,Bt,e),i(t,Y,e),i(t,xt,e),i(t,F,e),i(t,Ct,e),c(X,t,e),i(t,kt,e),i(t,E,e),i(t,Gt,e),i(t,V,e),i(t,Pt,e),c(N,t,e),i(t,Wt,e),i(t,J,e),i(t,St,e),i(t,I,e),i(t,zt,e),c(R,t,e),i(t,Ut,e),i(t,q,e),i(t,Zt,e),i(t,D,e),i(t,At,e),c(Q,t,e),i(t,jt,e),i(t,O,e),i(t,Yt,e),i(t,K,e),i(t,Ft,e),i(t,tt,e),i(t,Xt,e),c(et,t,e),i(t,Et,e),i(t,at,e),Vt=!0},p(t,[e]){const $e={};e&2&&($e.$$scope={dirty:e,ctx:t}),M.$set($e)},i(t){Vt||(p(w.$$.fragment,t),p(v.$$.fragment,t),p(M.$$.fragment,t),p(_.$$.fragment,t),p(C.$$.fragment,t),p(P.$$.fragment,t),p(Z.$$.fragment,t),p(j.$$.fragment,t),p(X.$$.fragment,t),p(N.$$.fragment,t),p(R.$$.fragment,t),p(Q.$$.fragment,t),p(et.$$.fragment,t),Vt=!0)},o(t){h(w.$$.fragment,t),h(v.$$.fragment,t),h(M.$$.fragment,t),h(_.$$.fragment,t),h(C.$$.fragment,t),h(P.$$.fragment,t),h(Z.$$.fragment,t),h(j.$$.fragment,t),h(X.$$.fragment,t),h(N.$$.fragment,t),h(R.$$.fragment,t),h(Q.$$.fragment,t),h(et.$$.fragment,t),Vt=!1},d(t){t&&(l(T),l(y),l(lt),l(dt),l(rt),l(g),l(ot),l(L),l(st),l(H),l(mt),l(ft),l(ct),l(B),l(pt),l($),l(ht),l(b),l(ut),l(x),l(yt),l(Mt),l(k),l($t),l(G),l(bt),l(Tt),l(W),l(wt),l(S),l(vt),l(z),l(gt),l(U),l(Lt),l(Ht),l(A),l(_t),l(Bt),l(Y),l(xt),l(F),l(Ct),l(kt),l(E),l(Gt),l(V),l(Pt),l(Wt),l(J),l(St),l(I),l(zt),l(Ut),l(q),l(Zt),l(D),l(At),l(jt),l(O),l(Yt),l(K),l(Ft),l(tt),l(Xt),l(Et),l(at)),l(s),u(w,t),u(v,t),u(M,t),u(_,t),u(C,t),u(P,t),u(Z,t),u(j,t),u(X,t),u(N,t),u(R,t),u(Q,t),u(et,t)}}}const Pe='{"title":"Model memory estimator","local":"model-memory-estimator","sections":[{"title":"Gradio Demos","local":"gradio-demos","sections":[],"depth":2},{"title":"The Command","local":"the-command","sections":[{"title":"Specific libraries","local":"specific-libraries","sections":[],"depth":3},{"title":"Specific dtypes","local":"specific-dtypes","sections":[],"depth":3}],"depth":2},{"title":"Caveats with this calculator","local":"caveats-with-this-calculator","sections":[],"depth":2}],"depth":1}';function We(nt){return we(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class je extends ge{constructor(s){super(),Le(this,s,We,Ge,Te,{})}}export{je as component};