transformers/pr_35010/zh/_app/immutable/nodes/16.8ea021dd.js

import{s as Je,n as ke,o as He}from"../chunks/scheduler.9991993c.js";import{S as Xe,i as Be,g as s,s as n,r as o,A as Ie,h as f,f as l,c as a,j as Ze,u as r,x as m,k as We,y as Pe,a as i,v as u,d as g,t as d,w as _}from"../chunks/index.ed60ef0f.js";import{C as Ue}from"../chunks/CodeBlock.a73b7ee1.js";import{H as F,E as Se}from"../chunks/EditOnGithub.ba269039.js";function Fe(de){let p,E,P,V,$,j,T,_e='GGUF文件格式用于存储模型，以便通过<a href="https://github.com/ggerganov/ggml" rel="nofollow">GGML</a>和其他依赖它的库进行推理，例如非常流行的<a href="https://github.com/ggerganov/llama.cpp" rel="nofollow">llama.cpp</a>或<a href="https://github.com/ggerganov/whisper.cpp" rel="nofollow">whisper.cpp</a>。',z,c,$e='该文件格式<a href="https://huggingface.co/docs/hub/en/gguf" rel="nofollow">由抱抱脸支持</a>，可用于快速检查文件中张量和元数据。',Y,h,Te='该文件格式是一种“单文件格式”，通常单个文件就包含了配置属性、分词器词汇表和其他属性，同时还有模型中要加载的所有张量。这些文件根据文件的量化类型有不同的格式。我们在<a href="https://huggingface.co/docs/hub/en/gguf#quantization-types" rel="nofollow">这里</a>进行了简要介绍。',q,v,R,M,ce="我们在 transformers 中添加了加载 gguf 文件的功能，这样可以对 GGUF 模型进行进一步的训练或微调，然后再将模型转换回 GGUF 格式，以便在 ggml 生态系统中使用。加载模型时，我们首先将其反量化为 FP32，然后再加载权重以在 PyTorch 中使用。",K,b,he=`<p>[!注意]
目前这个功能还处于探索阶段，欢迎大家贡献力量，以便在不同量化类型和模型架构之间更好地完善这一功能。</p>`,A,x,ve="目前，支持的模型架构和量化类型如下：",N,w,O,y,Me="根据分享在 Hub 上的较为热门的量化文件，初步支持以下量化类型：",D,C,be="<li>F32</li> <li>F16</li> <li>BF16</li> <li>Q4_0</li> <li>Q4_1</li> <li>Q5_0</li> <li>Q5_1</li> <li>Q8_0</li> <li>Q2_K</li> <li>Q3_K</li> <li>Q4_K</li> <li>Q5_K</li> <li>Q6_K</li> <li>IQ1_S</li> <li>IQ1_M</li> <li>IQ2_XXS</li> <li>IQ2_XS</li> <li>IQ2_S</li> <li>IQ3_XXS</li> <li>IQ3_S</li> <li>IQ4_XS</li> <li>IQ4_NL</li>",ee,G,xe=`<p>[!注意]
为了支持 gguf 反量化，需要安装 <code>gguf&gt;=0.10.0</code>。</p>`,te,L,le,Q,we="目前支持以下在 Hub 上非常热门的模型架构：",ie,Z,ye="<li>LLaMa</li> <li>Mistral</li> <li>Qwen2</li> <li>Qwen2Moe</li> <li>Phi3</li> <li>Bloom</li> <li>Falcon</li> <li>StableLM</li> <li>GPT2</li> <li>Starcoder2</li>",ne,W,ae,U,Ce="为了在<code>transformers</code>中加载<code>gguf</code>文件，你需要在 <code>from_pretrained</code>方法中为分词器和模型指定 <code>gguf_file</code>参数。下面是从同一个文件中加载分词器和模型的示例：",se,J,fe,k,Ge="现在，你就已经可以结合 PyTorch 生态系统中的一系列其他工具，来使用完整的、未量化的模型了。",me,H,Le='为了将模型转换回<code>gguf</code>文件，我们建议使用<code>llama.cpp</code>中的<a href="https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py" rel="nofollow"><code>convert-hf-to-gguf.py</code>文件</a>。',pe,X,Qe="以下是如何补充上面的脚本，以保存模型并将其导出回 <code>gguf</code>的示例：",oe,B,re,I,ue,S,ge;return $=new F({props:{title:"GGUF 和 Transformers 的交互",local:"gguf-和-transformers-的交互",headingTag:"h1"}}),v=new F({props:{title:"在 Transformers 中的支持",local:"在-transformers-中的支持",headingTag:"h2"}}),w=new F({props:{title:"支持的量化类型",local:"支持的量化类型",headingTag:"h3"}}),L=new F({props:{title:"支持的模型架构",local:"支持的模型架构",headingTag:"h3"}}),W=new F({props:{title:"使用示例",local:"使用示例",headingTag:"h2"}}),J=new Ue({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyVGhlQmxva2UlMkZUaW55TGxhbWEtMS4xQi1DaGF0LXYxLjAtR0dVRiUyMiUwQWZpbGVuYW1lJTIwJTNEJTIwJTIydGlueWxsYW1hLTEuMWItY2hhdC12MS4wLlE2X0suZ2d1ZiUyMiUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZ2d1Zl9maWxlJTNEZmlsZW5hbWUpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBnZ3VmX2ZpbGUlM0RmaWxlbmFtZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

model_id = <span class="hljs-string">&quot;TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF&quot;</span>
filename = <span class="hljs-string">&quot;tinyllama-1.1b-chat-v1.0.Q6_K.gguf&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename)`,wrap:!1}}),B=new Ue({props:{code:"dG9rZW5pemVyLnNhdmVfcHJldHJhaW5lZCgnZGlyZWN0b3J5JyklMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJ2RpcmVjdG9yeScpJTBBJTBBIXB5dGhvbiUyMCUyNCU3QnBhdGhfdG9fbGxhbWFfY3BwJTdEJTJGY29udmVydC1oZi10by1nZ3VmLnB5JTIwJTI0JTdCZGlyZWN0b3J5JTdE",highlighted:`tokenizer.save_pretrained(<span class="hljs-string">&#x27;directory&#x27;</span>)
model.save_pretrained(<span class="hljs-string">&#x27;directory&#x27;</span>)

!python \${path_to_llama_cpp}/convert-hf-to-gguf.py \${directory}`,wrap:!1}}),I=new Se({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/gguf.md"}}),{c(){p=s("meta"),E=n(),P=s("p"),V=n(),o($.$$.fragment),j=n(),T=s("p"),T.innerHTML=_e,z=n(),c=s("p"),c.innerHTML=$e,Y=n(),h=s("p"),h.innerHTML=Te,q=n(),o(v.$$.fragment),R=n(),M=s("p"),M.textContent=ce,K=n(),b=s("blockquote"),b.innerHTML=he,A=n(),x=s("p"),x.textContent=ve,N=n(),o(w.$$.fragment),O=n(),y=s("p"),y.textContent=Me,D=n(),C=s("ul"),C.innerHTML=be,ee=n(),G=s("blockquote"),G.innerHTML=xe,te=n(),o(L.$$.fragment),le=n(),Q=s("p"),Q.textContent=we,ie=n(),Z=s("ul"),Z.innerHTML=ye,ne=n(),o(W.$$.fragment),ae=n(),U=s("p"),U.innerHTML=Ce,se=n(),o(J.$$.fragment),fe=n(),k=s("p"),k.textContent=Ge,me=n(),H=s("p"),H.innerHTML=Le,pe=n(),X=s("p"),X.innerHTML=Qe,oe=n(),o(B.$$.fragment),re=n(),o(I.$$.fragment),ue=n(),S=s("p"),this.h()},l(e){const t=Ie("svelte-u9bgzb",document.head);p=f(t,"META",{name:!0,content:!0}),t.forEach(l),E=a(e),P=f(e,"P",{}),Ze(P).forEach(l),V=a(e),r($.$$.fragment,e),j=a(e),T=f(e,"P",{"data-svelte-h":!0}),m(T)!=="svelte-mdnd46"&&(T.innerHTML=_e),z=a(e),c=f(e,"P",{"data-svelte-h":!0}),m(c)!=="svelte-twtju8"&&(c.innerHTML=$e),Y=a(e),h=f(e,"P",{"data-svelte-h":!0}),m(h)!=="svelte-1ccuwz8"&&(h.innerHTML=Te),q=a(e),r(v.$$.fragment,e),R=a(e),M=f(e,"P",{"data-svelte-h":!0}),m(M)!=="svelte-1olmn47"&&(M.textContent=ce),K=a(e),b=f(e,"BLOCKQUOTE",{"data-svelte-h":!0}),m(b)!=="svelte-1hw6ivd"&&(b.innerHTML=he),A=a(e),x=f(e,"P",{"data-svelte-h":!0}),m(x)!=="svelte-5gud0f"&&(x.textContent=ve),N=a(e),r(w.$$.fragment,e),O=a(e),y=f(e,"P",{"data-svelte-h":!0}),m(y)!=="svelte-7f6b8c"&&(y.textContent=Me),D=a(e),C=f(e,"UL",{"data-svelte-h":!0}),m(C)!=="svelte-1f29qgp"&&(C.innerHTML=be),ee=a(e),G=f(e,"BLOCKQUOTE",{"data-svelte-h":!0}),m(G)!=="svelte-el52vx"&&(G.innerHTML=xe),te=a(e),r(L.$$.fragment,e),le=a(e),Q=f(e,"P",{"data-svelte-h":!0}),m(Q)!=="svelte-5qsmy1"&&(Q.textContent=we),ie=a(e),Z=f(e,"UL",{"data-svelte-h":!0}),m(Z)!=="svelte-1bkwewe"&&(Z.innerHTML=ye),ne=a(e),r(W.$$.fragment,e),ae=a(e),U=f(e,"P",{"data-svelte-h":!0}),m(U)!=="svelte-1lqi9da"&&(U.innerHTML=Ce),se=a(e),r(J.$$.fragment,e),fe=a(e),k=f(e,"P",{"data-svelte-h":!0}),m(k)!=="svelte-5jxp91"&&(k.textContent=Ge),me=a(e),H=f(e,"P",{"data-svelte-h":!0}),m(H)!=="svelte-v2zjkj"&&(H.innerHTML=Le),pe=a(e),X=f(e,"P",{"data-svelte-h":!0}),m(X)!=="svelte-1yxp62b"&&(X.innerHTML=Qe),oe=a(e),r(B.$$.fragment,e),re=a(e),r(I.$$.fragment,e),ue=a(e),S=f(e,"P",{}),Ze(S).forEach(l),this.h()},h(){We(p,"name","hf:doc:metadata"),We(p,"content",Ee)},m(e,t){Pe(document.head,p),i(e,E,t),i(e,P,t),i(e,V,t),u($,e,t),i(e,j,t),i(e,T,t),i(e,z,t),i(e,c,t),i(e,Y,t),i(e,h,t),i(e,q,t),u(v,e,t),i(e,R,t),i(e,M,t),i(e,K,t),i(e,b,t),i(e,A,t),i(e,x,t),i(e,N,t),u(w,e,t),i(e,O,t),i(e,y,t),i(e,D,t),i(e,C,t),i(e,ee,t),i(e,G,t),i(e,te,t),u(L,e,t),i(e,le,t),i(e,Q,t),i(e,ie,t),i(e,Z,t),i(e,ne,t),u(W,e,t),i(e,ae,t),i(e,U,t),i(e,se,t),u(J,e,t),i(e,fe,t),i(e,k,t),i(e,me,t),i(e,H,t),i(e,pe,t),i(e,X,t),i(e,oe,t),u(B,e,t),i(e,re,t),u(I,e,t),i(e,ue,t),i(e,S,t),ge=!0},p:ke,i(e){ge||(g($.$$.fragment,e),g(v.$$.fragment,e),g(w.$$.fragment,e),g(L.$$.fragment,e),g(W.$$.fragment,e),g(J.$$.fragment,e),g(B.$$.fragment,e),g(I.$$.fragment,e),ge=!0)},o(e){d($.$$.fragment,e),d(v.$$.fragment,e),d(w.$$.fragment,e),d(L.$$.fragment,e),d(W.$$.fragment,e),d(J.$$.fragment,e),d(B.$$.fragment,e),d(I.$$.fragment,e),ge=!1},d(e){e&&(l(E),l(P),l(V),l(j),l(T),l(z),l(c),l(Y),l(h),l(q),l(R),l(M),l(K),l(b),l(A),l(x),l(N),l(O),l(y),l(D),l(C),l(ee),l(G),l(te),l(le),l(Q),l(ie),l(Z),l(ne),l(ae),l(U),l(se),l(fe),l(k),l(me),l(H),l(pe),l(X),l(oe),l(re),l(ue),l(S)),l(p),_($,e),_(v,e),_(w,e),_(L,e),_(W,e),_(J,e),_(B,e),_(I,e)}}}const Ee='{"title":"GGUF 和 Transformers 的交互","local":"gguf-和-transformers-的交互","sections":[{"title":"在 Transformers 中的支持","local":"在-transformers-中的支持","sections":[{"title":"支持的量化类型","local":"支持的量化类型","sections":[],"depth":3},{"title":"支持的模型架构","local":"支持的模型架构","sections":[],"depth":3}],"depth":2},{"title":"使用示例","local":"使用示例","sections":[],"depth":2}],"depth":1}';function Ve(de){return He(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Re extends Xe{constructor(p){super(),Be(this,p,Ve,Fe,Je,{})}}export{Re as component};