Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / transformers /pr_33962 /ko /_app /immutable /nodes /120.f8dfd1fa.js
rtrm's picture
rtrm
download
raw
31.3 kB
import{s as Xt,o as Et,n as ut}from"../chunks/scheduler.bdbef820.js";import{S as Rt,i as xt,g as h,s as o,r as y,A as Ft,h as c,f as l,c as p,j as kt,u as w,x as M,k as rt,y as Ht,a as e,v as b,d as j,t as U,w as $}from"../chunks/index.33f81d56.js";import{T as It}from"../chunks/Tip.34194030.js";import{C as x}from"../chunks/CodeBlock.362b34a4.js";import{H as qt,E as zt}from"../chunks/EditOnGithub.a9246e21.js";import{H as Lt,a as Vt}from"../chunks/HfOption.6b792247.js";function Nt(C){let a,g='이 <a href="https://colab.research.google.com/drive/1HzZH89yAXJaZgwJDhQj9LqSBux932BvY" rel="nofollow">노트북</a> 으로 AWQ 양자화를 실습해보세요 !';return{c(){a=h("p"),a.innerHTML=g},l(i){a=c(i,"P",{"data-svelte-h":!0}),M(a)!=="svelte-cfteqc"&&(a.innerHTML=g)},m(i,d){e(i,a,d)},p:ut,d(i){i&&l(a)}}}function St(C){let a,g="퓨즈된 모듈은 FlashAttention-2와 같은 다른 최적화 기술과 결합할 수 없습니다.";return{c(){a=h("p"),a.textContent=g},l(i){a=c(i,"P",{"data-svelte-h":!0}),M(a)!=="svelte-epevub"&&(a.textContent=g)},m(i,d){e(i,a,d)},p:ut,d(i){i&&l(a)}}}function Yt(C){let a,g='지원되는 아키텍처에서 퓨즈된 모듈을 활성화하려면, <a href="/docs/transformers/pr_33962/ko/main_classes/quantization#transformers.AwqConfig">AwqConfig</a> 를 생성하고 매개변수 <code>fuse_max_seq_len</code> 과 <code>do_fuse=True</code>를 설정해야 합니다. <code>fuse_max_seq_len</code> 매개변수는 전체 시퀀스 길이로, 컨텍스트 길이와 예상 생성 길이를 포함해야 합니다. 안전하게 사용하기 위해 더 큰 값으로 설정할 수 있습니다.',i,d,u='예를 들어, <a href="https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-AWQ" rel="nofollow">TheBloke/Mistral-7B-OpenOrca-AWQ</a> 모델의 AWQ 모듈을 퓨즈해보겠습니다.',m,T,_,f,F='<a href="https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-AWQ" rel="nofollow">TheBloke/Mistral-7B-OpenOrca-AWQ</a> 모델은 퓨즈된 모듈이 있는 경우와 없는 경우 모두 <code>batch_size=1</code> 로 성능 평가되었습니다.',v,r,B="퓨즈되지 않은 모듈",X,q,ot='<thead><tr><th align="right">배치 크기</th> <th align="right">프리필 길이</th> <th align="right">디코드 길이</th> <th align="right">프리필 토큰/초</th> <th align="right">디코드 토큰/초</th> <th align="left">메모리 (VRAM)</th></tr></thead> <tbody><tr><td align="right">1</td> <td align="right">32</td> <td align="right">32</td> <td align="right">60.0984</td> <td align="right">38.4537</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">64</td> <td align="right">64</td> <td align="right">1333.67</td> <td align="right">31.6604</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">128</td> <td align="right">128</td> <td align="right">2434.06</td> <td align="right">31.6272</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">256</td> <td align="right">256</td> <td align="right">3072.26</td> <td align="right">38.1731</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">512</td> <td align="right">512</td> <td align="right">3184.74</td> <td align="right">31.6819</td> <td align="left">4.59 GB (5.80%)</td></tr> <tr><td align="right">1</td> <td align="right">1024</td> <td align="right">1024</td> <td align="right">3148.18</td> <td align="right">36.8031</td> <td align="left">4.81 GB (6.07%)</td></tr> <tr><td align="right">1</td> <td align="right">2048</td> <td align="right">2048</td> <td align="right">2927.33</td> <td align="right">35.2676</td> <td align="left">5.73 GB (7.23%)</td></tr></tbody>',E,I,N="퓨즈된 모듈",W,G,S='<thead><tr><th align="right">배치 크기</th> <th align="right">프리필 길이</th> <th align="right">디코드 길이</th> <th align="right">프리필 토큰/초</th> <th align="right">디코드 토큰/초</th> <th align="left">메모리 (VRAM)</th></tr></thead> <tbody><tr><td align="right">1</td> <td align="right">32</td> <td align="right">32</td> <td align="right">81.4899</td> <td align="right">80.2569</td> <td align="left">4.00 GB (5.05%)</td></tr> <tr><td align="right">1</td> <td align="right">64</td> <td align="right">64</td> <td align="right">1756.1</td> <td align="right">106.26</td> <td align="left">4.00 GB (5.05%)</td></tr> <tr><td align="right">1</td> <td align="right">128</td> <td align="right">128</td> <td align="right">2479.32</td> <td align="right">105.631</td> <td align="left">4.00 GB (5.06%)</td></tr> <tr><td align="right">1</td> <td align="right">256</td> <td align="right">256</td> <td align="right">1813.6</td> <td align="right">85.7485</td> <td align="left">4.01 GB (5.06%)</td></tr> <tr><td align="right">1</td> <td align="right">512</td> <td align="right">512</td> <td align="right">2848.9</td> <td align="right">97.701</td> <td align="left">4.11 GB (5.19%)</td></tr> <tr><td align="right">1</td> <td align="right">1024</td> <td align="right">1024</td> <td align="right">3044.35</td> <td align="right">87.7323</td> <td align="left">4.41 GB (5.57%)</td></tr> <tr><td align="right">1</td> <td align="right">2048</td> <td align="right">2048</td> <td align="right">2715.11</td> <td align="right">89.4709</td> <td align="left">5.57 GB (7.04%)</td></tr></tbody>',A,Z,k='퓨즈된 모듈 및 퓨즈되지 않은 모듈의 속도와 처리량은 <a href="https://github.com/huggingface/optimum-benchmark" rel="nofollow">optimum-benchmark</a>라이브러리를 사용하여 테스트 되었습니다.',Y,Q,V='<div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/fused_forward_memory_plot.png" alt="generate throughput per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">포워드 피크 메모리 (forward peak memory)/배치 크기</figcaption></div> <div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/fused_generate_throughput_plot.png" alt="forward latency per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">생성 처리량/배치크기</figcaption></div>',R;return T=new x({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXdxQ29uZmlnJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMlRoZUJsb2tlJTJGTWlzdHJhbC03Qi1PcGVuT3JjYS1BV1ElMjIlMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQXdxQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGJpdHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZnVzZV9tYXhfc2VxX2xlbiUzRDUxMiUyQyUwQSUyMCUyMCUyMCUyMGRvX2Z1c2UlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpLnRvKDAp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AwqConfig, AutoModelForCausalLM
model_id = <span class="hljs-string">&quot;TheBloke/Mistral-7B-OpenOrca-AWQ&quot;</span>
quantization_config = AwqConfig(
 bits=<span class="hljs-number">4</span>,
 fuse_max_seq_len=<span class="hljs-number">512</span>,
 do_fuse=<span class="hljs-literal">True</span>,
)
model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=quantization_config).to(<span class="hljs-number">0</span>)`,wrap:!1}}),{c(){a=h("p"),a.innerHTML=g,i=o(),d=h("p"),d.innerHTML=u,m=o(),y(T.$$.fragment),_=o(),f=h("p"),f.innerHTML=F,v=o(),r=h("figcaption"),r.textContent=B,X=o(),q=h("table"),q.innerHTML=ot,E=o(),I=h("figcaption"),I.textContent=N,W=o(),G=h("table"),G.innerHTML=S,A=o(),Z=h("p"),Z.innerHTML=k,Y=o(),Q=h("div"),Q.innerHTML=V,this.h()},l(n){a=c(n,"P",{"data-svelte-h":!0}),M(a)!=="svelte-1rjiain"&&(a.innerHTML=g),i=p(n),d=c(n,"P",{"data-svelte-h":!0}),M(d)!=="svelte-16m2a83"&&(d.innerHTML=u),m=p(n),w(T.$$.fragment,n),_=p(n),f=c(n,"P",{"data-svelte-h":!0}),M(f)!=="svelte-1utzrt0"&&(f.innerHTML=F),v=p(n),r=c(n,"FIGCAPTION",{class:!0,"data-svelte-h":!0}),M(r)!=="svelte-uubs04"&&(r.textContent=B),X=p(n),q=c(n,"TABLE",{"data-svelte-h":!0}),M(q)!=="svelte-ty84pu"&&(q.innerHTML=ot),E=p(n),I=c(n,"FIGCAPTION",{class:!0,"data-svelte-h":!0}),M(I)!=="svelte-1fro5y6"&&(I.textContent=N),W=p(n),G=c(n,"TABLE",{"data-svelte-h":!0}),M(G)!=="svelte-1irqixs"&&(G.innerHTML=S),A=p(n),Z=c(n,"P",{"data-svelte-h":!0}),M(Z)!=="svelte-1fux1xr"&&(Z.innerHTML=k),Y=p(n),Q=c(n,"DIV",{class:!0,"data-svelte-h":!0}),M(Q)!=="svelte-f84qab"&&(Q.innerHTML=V),this.h()},h(){rt(r,"class","text-center text-gray-500 text-lg"),rt(I,"class","text-center text-gray-500 text-lg"),rt(Q,"class","flex gap-4")},m(n,J){e(n,a,J),e(n,i,J),e(n,d,J),e(n,m,J),b(T,n,J),e(n,_,J),e(n,f,J),e(n,v,J),e(n,r,J),e(n,X,J),e(n,q,J),e(n,E,J),e(n,I,J),e(n,W,J),e(n,G,J),e(n,A,J),e(n,Z,J),e(n,Y,J),e(n,Q,J),R=!0},p:ut,i(n){R||(j(T.$$.fragment,n),R=!0)},o(n){U(T.$$.fragment,n),R=!1},d(n){n&&(l(a),l(i),l(d),l(m),l(_),l(f),l(v),l(r),l(X),l(q),l(E),l(I),l(W),l(G),l(A),l(Z),l(Y),l(Q)),$(T,n)}}}function Dt(C){let a,g='퓨즈된 모듈을 지원하지 않는 아키텍처의 경우, <code>modules_to_fuse</code> 매개변수를 사용해 직접 퓨즈 매핑을 만들어 어떤 모듈을 퓨즈할지 정의해야합니다. 예로, <a href="https://huggingface.co/TheBloke/Yi-34B-AWQ" rel="nofollow">TheBloke/Yi-34B-AWQ</a> 모델의 AWQ 모듈을 퓨즈하는 방법입니다.',i,d,u,m,T="<code>modules_to_fuse</code> 매개변수는 다음을 포함해야 합니다:",_,f,F="<li><code>&quot;attention&quot;</code>: 어텐션 레이어는 다음 순서로 퓨즈하세요 : 쿼리 (query), 키 (key), 값 (value) , 출력 프로젝션 계층 (output projection layer). 해당 레이어를 퓨즈하지 않으려면 빈 리스트를 전달하세요.</li> <li><code>&quot;layernorm&quot;</code>: 사용자 정의 퓨즈 레이어 정규화로 교할 레이어 정규화 레이어명. 해당 레이어를 퓨즈하지 않으려면 빈 리스트를 전달하세요.</li> <li><code>&quot;mlp&quot;</code>: 단일 MLP 레이어로 퓨즈할 MLP 레이어 순서 : (게이트 (gate) (덴스(dense), 레이어(layer), 포스트 어텐션(post-attention)) / 위 / 아래 레이어).</li> <li><code>&quot;use_alibi&quot;</code>: 모델이 ALiBi positional embedding을 사용할 경우 설정합니다.</li> <li><code>&quot;num_attention_heads&quot;</code>: 어텐션 헤드 (attention heads)의 수를 설정합니다.</li> <li><code>&quot;num_key_value_heads&quot;</code>: 그룹화 쿼리 어텐션 (GQA)을 구현하는데 사용되는 키 값 헤드의 수를 설정합니다. <code>num_key_value_heads=num_attention_heads</code>로 설정할 경우, 모델은 다중 헤드 어텐션 (MHA)가 사용되며, <code>num_key_value_heads=1</code> 는 다중 쿼리 어텐션 (MQA)가, 나머지는 GQA가 사용됩니다.</li> <li><code>&quot;hidden_size&quot;</code>: 숨겨진 표현(hidden representations)의 차원을 설정합니다.</li>",v;return d=new x({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXdxQ29uZmlnJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMlRoZUJsb2tlJTJGWWktMzRCLUFXUSUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBBd3FDb25maWcoJTBBJTIwJTIwJTIwJTIwYml0cyUzRDQlMkMlMEElMjAlMjAlMjAlMjBmdXNlX21heF9zZXFfbGVuJTNENTEyJTJDJTBBJTIwJTIwJTIwJTIwbW9kdWxlc190b19mdXNlJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYXR0ZW50aW9uJTIyJTNBJTIwJTVCJTIycV9wcm9qJTIyJTJDJTIwJTIya19wcm9qJTIyJTJDJTIwJTIydl9wcm9qJTIyJTJDJTIwJTIyb19wcm9qJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybGF5ZXJub3JtJTIyJTNBJTIwJTVCJTIybG4xJTIyJTJDJTIwJTIybG4yJTIyJTJDJTIwJTIybm9ybSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1scCUyMiUzQSUyMCU1QiUyMmdhdGVfcHJvaiUyMiUyQyUyMCUyMnVwX3Byb2olMjIlMkMlMjAlMjJkb3duX3Byb2olMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ1c2VfYWxpYmklMjIlM0ElMjBGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm51bV9hdHRlbnRpb25faGVhZHMlMjIlM0ElMjA1NiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm51bV9rZXlfdmFsdWVfaGVhZHMlMjIlM0ElMjA4JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyaGlkZGVuX3NpemUlMjIlM0ElMjA3MTY4JTBBJTIwJTIwJTIwJTIwJTdEJTBBKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpLnRvKDAp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AwqConfig, AutoModelForCausalLM
model_id = <span class="hljs-string">&quot;TheBloke/Yi-34B-AWQ&quot;</span>
quantization_config = AwqConfig(
 bits=<span class="hljs-number">4</span>,
 fuse_max_seq_len=<span class="hljs-number">512</span>,
 modules_to_fuse={
 <span class="hljs-string">&quot;attention&quot;</span>: [<span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;o_proj&quot;</span>],
 <span class="hljs-string">&quot;layernorm&quot;</span>: [<span class="hljs-string">&quot;ln1&quot;</span>, <span class="hljs-string">&quot;ln2&quot;</span>, <span class="hljs-string">&quot;norm&quot;</span>],
 <span class="hljs-string">&quot;mlp&quot;</span>: [<span class="hljs-string">&quot;gate_proj&quot;</span>, <span class="hljs-string">&quot;up_proj&quot;</span>, <span class="hljs-string">&quot;down_proj&quot;</span>],
 <span class="hljs-string">&quot;use_alibi&quot;</span>: <span class="hljs-literal">False</span>,
 <span class="hljs-string">&quot;num_attention_heads&quot;</span>: <span class="hljs-number">56</span>,
 <span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">8</span>,
 <span class="hljs-string">&quot;hidden_size&quot;</span>: <span class="hljs-number">7168</span>
 }
)
model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=quantization_config).to(<span class="hljs-number">0</span>)`,wrap:!1}}),{c(){a=h("p"),a.innerHTML=g,i=o(),y(d.$$.fragment),u=o(),m=h("p"),m.innerHTML=T,_=o(),f=h("ul"),f.innerHTML=F},l(r){a=c(r,"P",{"data-svelte-h":!0}),M(a)!=="svelte-oiiobr"&&(a.innerHTML=g),i=p(r),w(d.$$.fragment,r),u=p(r),m=c(r,"P",{"data-svelte-h":!0}),M(m)!=="svelte-1bbowmb"&&(m.innerHTML=T),_=p(r),f=c(r,"UL",{"data-svelte-h":!0}),M(f)!=="svelte-dexwhb"&&(f.innerHTML=F)},m(r,B){e(r,a,B),e(r,i,B),b(d,r,B),e(r,u,B),e(r,m,B),e(r,_,B),e(r,f,B),v=!0},p:ut,i(r){v||(j(d.$$.fragment,r),v=!0)},o(r){U(d.$$.fragment,r),v=!1},d(r){r&&(l(a),l(i),l(u),l(m),l(_),l(f)),$(d,r)}}}function Pt(C){let a,g,i,d;return a=new Vt({props:{id:"fuse",option:"supported architectures",$$slots:{default:[Yt]},$$scope:{ctx:C}}}),i=new Vt({props:{id:"fuse",option:"unsupported architectures",$$slots:{default:[Dt]},$$scope:{ctx:C}}}),{c(){y(a.$$.fragment),g=o(),y(i.$$.fragment)},l(u){w(a.$$.fragment,u),g=p(u),w(i.$$.fragment,u)},m(u,m){b(a,u,m),e(u,g,m),b(i,u,m),d=!0},p(u,m){const T={};m&2&&(T.$$scope={dirty:m,ctx:u}),a.$set(T);const _={};m&2&&(_.$$scope={dirty:m,ctx:u}),i.$set(_)},i(u){d||(j(a.$$.fragment,u),j(i.$$.fragment,u),d=!0)},o(u){U(a.$$.fragment,u),U(i.$$.fragment,u),d=!1},d(u){u&&l(g),$(a,u),$(i,u)}}}function Kt(C){let a,g="이 기능은 AMD GPUs에서 지원됩니다.";return{c(){a=h("p"),a.textContent=g},l(i){a=c(i,"P",{"data-svelte-h":!0}),M(a)!=="svelte-1y7b24l"&&(a.textContent=g)},m(i,d){e(i,a,d)},p:ut,d(i){i&&l(a)}}}function Ot(C){let a,g,i,d,u,m,T,_,f,F='<a href="https://hf.co/papers/2306.00978" rel="nofollow">Activation-aware Weight Quantization (AWQ)</a>은 모델의 모든 가중치를 양자화하지 않고, LLM 성능에 중요한 가중치를 유지합니다. 이로써 4비트 정밀도로 모델을 실행해도 성능 저하 없이 양자화 손실을 크게 줄일 수 있습니다.',v,r,B='AWQ 알고리즘을 사용하여 모델을 양자화할 수 있는 여러 라이브러리가 있습니다. 예를 들어 <a href="https://github.com/mit-han-lab/llm-awq" rel="nofollow">llm-awq</a>, <a href="https://github.com/casper-hansen/AutoAWQ" rel="nofollow">autoawq</a> , <a href="https://huggingface.co/docs/optimum/main/en/intel/optimization_inc" rel="nofollow">optimum-intel</a> 등이 있습니다. Transformers는 llm-awq, autoawq 라이브러리를 이용해 양자화된 모델을 가져올 수 있도록 지원합니다. 이 가이드에서는 autoawq로 양자화된 모델을 가져오는 방법을 보여드리나, llm-awq로 양자화된 모델의 경우도 유사한 절차를 따릅니다.',X,q,ot="autoawq가 설치되어 있는지 확인하세요:",E,I,N,W,G='AWQ 양자화된 모델은 해당 모델의 <a href="https://huggingface.co/TheBloke/zephyr-7B-alpha-AWQ/blob/main/config.json" rel="nofollow">config.json</a> 파일의 <code>quantization_config</code> 속성을 통해 식별할 수 있습니다.:',S,A,Z,k,Y='양자화된 모델은 <a href="/docs/transformers/pr_33962/ko/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> 메서드를 사용하여 가져옵니다. 모델을 CPU에 가져왔다면, 먼저 모델을 GPU 장치로 옮겨야 합니다. <code>device_map</code> 파라미터를 사용하여 모델을 배치할 위치를 지정하세요:',Q,V,R,n,J="AWQ 양자화 모델을 가져오면 자동으로 성능상의 이유로 인해 가중치들의 기본값이 fp16으로 설정됩니다. 가중치를 다른 형식으로 가져오려면, <code>torch_dtype</code> 파라미터를 사용하세요:",dt,D,ht,P,Ct='추론을 더욱 가속화하기 위해 AWQ 양자화와 <a href="../perf_infer_gpu_one#flashattention-2">FlashAttention-2</a> 를 결합 할 수 있습니다:',ct,K,mt,O,Mt,tt,vt='퓨즈된 모듈은 정확도와 성능을 개선합니다. 퓨즈된 모듈은 <a href="https://huggingface.co/meta-llama" rel="nofollow">Llama</a> 아키텍처와 <a href="https://huggingface.co/mistralai/Mistral-7B-v0.1" rel="nofollow">Mistral</a> 아키텍처의 AWQ모듈에 기본적으로 지원됩니다. 그러나 지원되지 않는 아키텍처에 대해서도 AWQ 모듈을 퓨즈할 수 있습니다.',gt,H,ft,z,Tt,lt,Jt,et,Bt="최신 버전 <code>autoawq</code>는 빠른 프리필과 디코딩을 위해 ExLlama-v2 커널을 지원합니다. 시작하기 위해 먼저 최신 버전 <code>autoawq</code> 를 설치하세요 :",yt,st,wt,at,Qt="매개변수를 <code>version=&quot;exllama&quot;</code>로 설정해 <code>AwqConfig()</code>를 생성하고 모델에 넘겨주세요.",bt,nt,jt,L,Ut,it,$t,pt,_t;return u=new qt({props:{title:"AWQ",local:"awq",headingTag:"h1"}}),T=new It({props:{$$slots:{default:[Nt]},$$scope:{ctx:C}}}),I=new x({props:{code:"cGlwJTIwaW5zdGFsbCUyMGF1dG9hd3E=",highlighted:"pip install autoawq",wrap:!1}}),A=new x({props:{code:"JTdCJTBBJTIwJTIwJTIyX25hbWVfb3JfcGF0aCUyMiUzQSUyMCUyMiUyRndvcmtzcGFjZSUyRnByb2Nlc3MlMkZodWdnaW5nZmFjZWg0X3plcGh5ci03Yi1hbHBoYSUyRnNvdXJjZSUyMiUyQyUwQSUyMCUyMCUyMmFyY2hpdGVjdHVyZXMlMjIlM0ElMjAlNUIlMEElMjAlMjAlMjAlMjAlMjJNaXN0cmFsRm9yQ2F1c2FsTE0lMjIlMEElMjAlMjAlNUQlMkMlMEElMjAlMjAuLi4lMEElMjAlMjAuLi4lMEElMjAlMjAuLi4lMEElMjAlMjAlMjJxdWFudGl6YXRpb25fY29uZmlnJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycXVhbnRfbWV0aG9kJTIyJTNBJTIwJTIyYXdxJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19wb2ludCUyMiUzQSUyMHRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjJncm91cF9zaXplJTIyJTNBJTIwMTI4JTJDJTBBJTIwJTIwJTIwJTIwJTIyYml0cyUyMiUzQSUyMDQlMkMlMEElMjAlMjAlMjAlMjAlMjJ2ZXJzaW9uJTIyJTNBJTIwJTIyZ2VtbSUyMiUwQSUyMCUyMCU3RCUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
 <span class="hljs-attr">&quot;_name_or_path&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;/workspace/process/huggingfaceh4_zephyr-7b-alpha/source&quot;</span><span class="hljs-punctuation">,</span>
 <span class="hljs-attr">&quot;architectures&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span>
 <span class="hljs-string">&quot;MistralForCausalLM&quot;</span>
 <span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
 ...
 ...
 ...
 <span class="hljs-attr">&quot;quantization_config&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
 <span class="hljs-attr">&quot;quant_method&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;awq&quot;</span><span class="hljs-punctuation">,</span>
 <span class="hljs-attr">&quot;zero_point&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
 <span class="hljs-attr">&quot;group_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">128</span><span class="hljs-punctuation">,</span>
 <span class="hljs-attr">&quot;bits&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">4</span><span class="hljs-punctuation">,</span>
 <span class="hljs-attr">&quot;version&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;gemm&quot;</span>
 <span class="hljs-punctuation">}</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),V=new x({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyVGhlQmxva2UlMkZ6ZXBoeXItN0ItYWxwaGEtQVdRJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyY3VkYSUzQTAlMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
model_id = <span class="hljs-string">&quot;TheBloke/zephyr-7B-alpha-AWQ&quot;</span>
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;cuda:0&quot;</span>)`,wrap:!1}}),D=new x({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyVGhlQmxva2UlMkZ6ZXBoeXItN0ItYWxwaGEtQVdRJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
model_id = <span class="hljs-string">&quot;TheBloke/zephyr-7B-alpha-AWQ&quot;</span>
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32)`,wrap:!1}}),K=new x({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMlRoZUJsb2tlJTJGemVwaHlyLTdCLWFscGhhLUFXUSUyMiUyQyUyMGF0dG5faW1wbGVtZW50YXRpb24lM0QlMjJmbGFzaF9hdHRlbnRpb25fMiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJjdWRhJTNBMCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;TheBloke/zephyr-7B-alpha-AWQ&quot;</span>, attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>, device_map=<span class="hljs-string">&quot;cuda:0&quot;</span>)`,wrap:!1}}),O=new qt({props:{title:"퓨즈된 모듈",local:"fused-modules",headingTag:"h2"}}),H=new It({props:{warning:!0,$$slots:{default:[St]},$$scope:{ctx:C}}}),z=new Lt({props:{id:"fuse",options:["supported architectures","unsupported architectures"],$$slots:{default:[Pt]},$$scope:{ctx:C}}}),lt=new qt({props:{title:"ExLlama-v2 서포트",local:"exllama-v2-support",headingTag:"h2"}}),st=new x({props:{code:"cGlwJTIwaW5zdGFsbCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRmNhc3Blci1oYW5zZW4lMkZBdXRvQVdRLmdpdA==",highlighted:"pip install git+https://github.com/casper-hansen/AutoAWQ.git",wrap:!1}}),nt=new x({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXdxQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEF3cUNvbmZpZyh2ZXJzaW9uJTNEJTIyZXhsbGFtYSUyMiklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJUaGVCbG9rZSUyRk1pc3RyYWwtN0ItSW5zdHJ1Y3QtdjAuMS1BV1ElMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQWlucHV0X2lkcyUyMCUzRCUyMHRvcmNoLnJhbmRpbnQoMCUyQyUyMDEwMCUyQyUyMCgxJTJDJTIwMTI4KSUyQyUyMGR0eXBlJTNEdG9yY2gubG9uZyUyQyUyMGRldmljZSUzRCUyMmN1ZGElMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoaW5wdXRfaWRzKSUwQXByaW50KG91dHB1dC5sb2dpdHMpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyVGhlQmxva2UlMkZNaXN0cmFsLTdCLUluc3RydWN0LXYwLjEtQVdRJTIyKSUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplci5lbmNvZGUoJTIySG93JTIwdG8lMjBtYWtlJTIwYSUyMGNha2UlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoaW5wdXRfaWRzJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSUyQyUyMG1heF9sZW5ndGglM0Q1MCUyQyUyMHBhZF90b2tlbl9pZCUzRDUwMjU2KSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, AwqConfig
quantization_config = AwqConfig(version=<span class="hljs-string">&quot;exllama&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(
 <span class="hljs-string">&quot;TheBloke/Mistral-7B-Instruct-v0.1-AWQ&quot;</span>,
 quantization_config=quantization_config,
 device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)
input_ids = torch.randint(<span class="hljs-number">0</span>, <span class="hljs-number">100</span>, (<span class="hljs-number">1</span>, <span class="hljs-number">128</span>), dtype=torch.long, device=<span class="hljs-string">&quot;cuda&quot;</span>)
output = model(input_ids)
<span class="hljs-built_in">print</span>(output.logits)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;TheBloke/Mistral-7B-Instruct-v0.1-AWQ&quot;</span>)
input_ids = tokenizer.encode(<span class="hljs-string">&quot;How to make a cake&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
output = model.generate(input_ids, do_sample=<span class="hljs-literal">True</span>, max_length=<span class="hljs-number">50</span>, pad_token_id=<span class="hljs-number">50256</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),L=new It({props:{warning:!0,$$slots:{default:[Kt]},$$scope:{ctx:C}}}),it=new zt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/ko/quantization/awq.md"}}),{c(){a=h("meta"),g=o(),i=h("p"),d=o(),y(u.$$.fragment),m=o(),y(T.$$.fragment),_=o(),f=h("p"),f.innerHTML=F,v=o(),r=h("p"),r.innerHTML=B,X=o(),q=h("p"),q.textContent=ot,E=o(),y(I.$$.fragment),N=o(),W=h("p"),W.innerHTML=G,S=o(),y(A.$$.fragment),Z=o(),k=h("p"),k.innerHTML=Y,Q=o(),y(V.$$.fragment),R=o(),n=h("p"),n.innerHTML=J,dt=o(),y(D.$$.fragment),ht=o(),P=h("p"),P.innerHTML=Ct,ct=o(),y(K.$$.fragment),mt=o(),y(O.$$.fragment),Mt=o(),tt=h("p"),tt.innerHTML=vt,gt=o(),y(H.$$.fragment),ft=o(),y(z.$$.fragment),Tt=o(),y(lt.$$.fragment),Jt=o(),et=h("p"),et.innerHTML=Bt,yt=o(),y(st.$$.fragment),wt=o(),at=h("p"),at.innerHTML=Qt,bt=o(),y(nt.$$.fragment),jt=o(),y(L.$$.fragment),Ut=o(),y(it.$$.fragment),$t=o(),pt=h("p"),this.h()},l(t){const s=Ft("svelte-u9bgzb",document.head);a=c(s,"META",{name:!0,content:!0}),s.forEach(l),g=p(t),i=c(t,"P",{}),kt(i).forEach(l),d=p(t),w(u.$$.fragment,t),m=p(t),w(T.$$.fragment,t),_=p(t),f=c(t,"P",{"data-svelte-h":!0}),M(f)!=="svelte-50rz6s"&&(f.innerHTML=F),v=p(t),r=c(t,"P",{"data-svelte-h":!0}),M(r)!=="svelte-ibfq68"&&(r.innerHTML=B),X=p(t),q=c(t,"P",{"data-svelte-h":!0}),M(q)!=="svelte-13tyifb"&&(q.textContent=ot),E=p(t),w(I.$$.fragment,t),N=p(t),W=c(t,"P",{"data-svelte-h":!0}),M(W)!=="svelte-127ay4z"&&(W.innerHTML=G),S=p(t),w(A.$$.fragment,t),Z=p(t),k=c(t,"P",{"data-svelte-h":!0}),M(k)!=="svelte-uh9olj"&&(k.innerHTML=Y),Q=p(t),w(V.$$.fragment,t),R=p(t),n=c(t,"P",{"data-svelte-h":!0}),M(n)!=="svelte-pmm406"&&(n.innerHTML=J),dt=p(t),w(D.$$.fragment,t),ht=p(t),P=c(t,"P",{"data-svelte-h":!0}),M(P)!=="svelte-111qm6v"&&(P.innerHTML=Ct),ct=p(t),w(K.$$.fragment,t),mt=p(t),w(O.$$.fragment,t),Mt=p(t),tt=c(t,"P",{"data-svelte-h":!0}),M(tt)!=="svelte-g2x0tg"&&(tt.innerHTML=vt),gt=p(t),w(H.$$.fragment,t),ft=p(t),w(z.$$.fragment,t),Tt=p(t),w(lt.$$.fragment,t),Jt=p(t),et=c(t,"P",{"data-svelte-h":!0}),M(et)!=="svelte-189uhby"&&(et.innerHTML=Bt),yt=p(t),w(st.$$.fragment,t),wt=p(t),at=c(t,"P",{"data-svelte-h":!0}),M(at)!=="svelte-1up0kbc"&&(at.innerHTML=Qt),bt=p(t),w(nt.$$.fragment,t),jt=p(t),w(L.$$.fragment,t),Ut=p(t),w(it.$$.fragment,t),$t=p(t),pt=c(t,"P",{}),kt(pt).forEach(l),this.h()},h(){rt(a,"name","hf:doc:metadata"),rt(a,"content",tl)},m(t,s){Ht(document.head,a),e(t,g,s),e(t,i,s),e(t,d,s),b(u,t,s),e(t,m,s),b(T,t,s),e(t,_,s),e(t,f,s),e(t,v,s),e(t,r,s),e(t,X,s),e(t,q,s),e(t,E,s),b(I,t,s),e(t,N,s),e(t,W,s),e(t,S,s),b(A,t,s),e(t,Z,s),e(t,k,s),e(t,Q,s),b(V,t,s),e(t,R,s),e(t,n,s),e(t,dt,s),b(D,t,s),e(t,ht,s),e(t,P,s),e(t,ct,s),b(K,t,s),e(t,mt,s),b(O,t,s),e(t,Mt,s),e(t,tt,s),e(t,gt,s),b(H,t,s),e(t,ft,s),b(z,t,s),e(t,Tt,s),b(lt,t,s),e(t,Jt,s),e(t,et,s),e(t,yt,s),b(st,t,s),e(t,wt,s),e(t,at,s),e(t,bt,s),b(nt,t,s),e(t,jt,s),b(L,t,s),e(t,Ut,s),b(it,t,s),e(t,$t,s),e(t,pt,s),_t=!0},p(t,[s]){const Wt={};s&2&&(Wt.$$scope={dirty:s,ctx:t}),T.$set(Wt);const At={};s&2&&(At.$$scope={dirty:s,ctx:t}),H.$set(At);const Zt={};s&2&&(Zt.$$scope={dirty:s,ctx:t}),z.$set(Zt);const Gt={};s&2&&(Gt.$$scope={dirty:s,ctx:t}),L.$set(Gt)},i(t){_t||(j(u.$$.fragment,t),j(T.$$.fragment,t),j(I.$$.fragment,t),j(A.$$.fragment,t),j(V.$$.fragment,t),j(D.$$.fragment,t),j(K.$$.fragment,t),j(O.$$.fragment,t),j(H.$$.fragment,t),j(z.$$.fragment,t),j(lt.$$.fragment,t),j(st.$$.fragment,t),j(nt.$$.fragment,t),j(L.$$.fragment,t),j(it.$$.fragment,t),_t=!0)},o(t){U(u.$$.fragment,t),U(T.$$.fragment,t),U(I.$$.fragment,t),U(A.$$.fragment,t),U(V.$$.fragment,t),U(D.$$.fragment,t),U(K.$$.fragment,t),U(O.$$.fragment,t),U(H.$$.fragment,t),U(z.$$.fragment,t),U(lt.$$.fragment,t),U(st.$$.fragment,t),U(nt.$$.fragment,t),U(L.$$.fragment,t),U(it.$$.fragment,t),_t=!1},d(t){t&&(l(g),l(i),l(d),l(m),l(_),l(f),l(v),l(r),l(X),l(q),l(E),l(N),l(W),l(S),l(Z),l(k),l(Q),l(R),l(n),l(dt),l(ht),l(P),l(ct),l(mt),l(Mt),l(tt),l(gt),l(ft),l(Tt),l(Jt),l(et),l(yt),l(wt),l(at),l(bt),l(jt),l(Ut),l($t),l(pt)),l(a),$(u,t),$(T,t),$(I,t),$(A,t),$(V,t),$(D,t),$(K,t),$(O,t),$(H,t),$(z,t),$(lt,t),$(st,t),$(nt,t),$(L,t),$(it,t)}}}const tl='{"title":"AWQ","local":"awq","sections":[{"title":"퓨즈된 모듈","local":"fused-modules","sections":[],"depth":2},{"title":"ExLlama-v2 서포트","local":"exllama-v2-support","sections":[],"depth":2}],"depth":1}';function ll(C){return Et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pl extends Rt{constructor(a){super(),xt(this,a,ll,Ot,Xt,{})}}export{pl as component};

Xet Storage Details

Size:
31.3 kB
·
Xet hash:
2f23eb692afa70d50d285b9baf97d32cc9f72191b5e54521f719ffa387631179

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.