transformers/pr_37350/zh/_app/immutable/nodes/70.fcff2d40.js

import{s as re,n as ae,o as ie}from"../chunks/scheduler.25b97de1.js";import{S as me,i as pe,g as a,s as o,r as m,A as fe,h as i,f as n,c as l,j as oe,u as p,x as J,k as le,y as ke,a as s,v as f,d as k,t as c,w as d}from"../chunks/index.d9030fc9.js";import{C as N}from"../chunks/CodeBlock.b38ef023.js";import{H as S,E as ce}from"../chunks/index.676ef4be.js";function de(K){let r,B,U,H,M,W,$,O='在🤗 transformers中，当使用<code>from_pretrained</code>方法从Hub加载模型时，如果模型包含tiktoken格式的<code>tokenizer.model</code>文件，框架可以无缝支持tiktoken模型文件，并自动将其转换为我们的<a href="https://huggingface.co/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast" rel="nofollow">快速词符化器</a>。',E,u,G,b,D="<li>gpt2</li> <li>llama3</li>",C,h,L,y,ee="为了在transformers中正确加载<code>tiktoken</code>文件，请确保<code>tiktoken.model</code>文件是tiktoken格式的，并且会在加载<code>from_pretrained</code>时自动加载。以下展示如何从同一个文件中加载词符化器(tokenizer)和模型：",X,T,F,g,I,_,te='<code>tokenizer.model</code>文件中不包含任何额外的词符(token)或模式字符串(pattern strings)的信息。如果这些信息很重要，需要将词符化器(tokenizer)转换为适用于<a href="/docs/transformers/pr_37350/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>类的<code>tokenizer.json</code>格式。',P,j,ne='使用<a href="https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/tiktoken/registry.py#L63" rel="nofollow">tiktoken.get_encoding</a>生成<code>tokenizer.model</code>文件，再使用<code>convert_tiktoken_to_fast</code>函数将其转换为<code>tokenizer.json</code>文件。',V,Z,q,v,se='生成的<code>tokenizer.json</code>文件将被保存到指定的目录，并且可以通过<a href="/docs/transformers/pr_37350/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>类来加载。',x,z,Q,w,A,R,Y;return M=new S({props:{title:"Transformers与Tiktonken的互操作性",local:"transformers与tiktonken的互操作性",headingTag:"h1"}}),u=new S({props:{title:"已知包含 tiktoken.model 文件发布的模型：",local:"已知包含-tiktokenmodel-文件发布的模型",headingTag:"h3"}}),h=new S({props:{title:"使用示例",local:"使用示例",headingTag:"h2"}}),T=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMHN1YmZvbGRlciUzRCUyMm9yaWdpbmFsJTIyKSUyMA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id, subfolder=<span class="hljs-string">&quot;original&quot;</span>) `,wrap:!1}}),g=new S({props:{title:"创建tiktoken词符化器(tokenizer)",local:"创建tiktoken词符化器tokenizer",headingTag:"h2"}}),Z=new N({props:{code:"JTBBZnJvbSUyMHRyYW5zZm9ybWVycy5pbnRlZ3JhdGlvbnMudGlrdG9rZW4lMjBpbXBvcnQlMjBjb252ZXJ0X3Rpa3Rva2VuX3RvX2Zhc3QlMEFmcm9tJTIwdGlrdG9rZW4lMjBpbXBvcnQlMjBnZXRfZW5jb2RpbmclMEElMEElMjMlMjBZb3UlMjBjYW4lMjBsb2FkJTIweW91ciUyMGN1c3RvbSUyMGVuY29kaW5nJTIwb3IlMjB0aGUlMjBvbmUlMjBwcm92aWRlZCUyMGJ5JTIwT3BlbkFJJTBBZW5jb2RpbmclMjAlM0QlMjBnZXRfZW5jb2RpbmcoJTIyZ3B0MiUyMiklMEFjb252ZXJ0X3Rpa3Rva2VuX3RvX2Zhc3QoZW5jb2RpbmclMkMlMjAlMjJjb25maWclMkZzYXZlJTJGZGlyJTIyKQ==",highlighted:`
<span class="hljs-keyword">from</span> transformers.integrations.tiktoken <span class="hljs-keyword">import</span> convert_tiktoken_to_fast
<span class="hljs-keyword">from</span> tiktoken <span class="hljs-keyword">import</span> get_encoding

<span class="hljs-comment"># You can load your custom encoding or the one provided by OpenAI</span>
encoding = get_encoding(<span class="hljs-string">&quot;gpt2&quot;</span>)
convert_tiktoken_to_fast(encoding, <span class="hljs-string">&quot;config/save/dir&quot;</span>)`,wrap:!1}}),z=new N({props:{code:"dG9rZW5pemVyJTIwJTNEJTIwUHJlVHJhaW5lZFRva2VuaXplckZhc3QuZnJvbV9wcmV0cmFpbmVkKCUyMmNvbmZpZyUyRnNhdmUlMkZkaXIlMjIp",highlighted:'tokenizer = PreTrainedTokenizerFast.from_pretrained(<span class="hljs-string">&quot;config/save/dir&quot;</span>)',wrap:!1}}),w=new ce({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/tiktoken.md"}}),{c(){r=a("meta"),B=o(),U=a("p"),H=o(),m(M.$$.fragment),W=o(),$=a("p"),$.innerHTML=O,E=o(),m(u.$$.fragment),G=o(),b=a("ul"),b.innerHTML=D,C=o(),m(h.$$.fragment),L=o(),y=a("p"),y.innerHTML=ee,X=o(),m(T.$$.fragment),F=o(),m(g.$$.fragment),I=o(),_=a("p"),_.innerHTML=te,P=o(),j=a("p"),j.innerHTML=ne,V=o(),m(Z.$$.fragment),q=o(),v=a("p"),v.innerHTML=se,x=o(),m(z.$$.fragment),Q=o(),m(w.$$.fragment),A=o(),R=a("p"),this.h()},l(e){const t=fe("svelte-u9bgzb",document.head);r=i(t,"META",{name:!0,content:!0}),t.forEach(n),B=l(e),U=i(e,"P",{}),oe(U).forEach(n),H=l(e),p(M.$$.fragment,e),W=l(e),$=i(e,"P",{"data-svelte-h":!0}),J($)!=="svelte-24plml"&&($.innerHTML=O),E=l(e),p(u.$$.fragment,e),G=l(e),b=i(e,"UL",{"data-svelte-h":!0}),J(b)!=="svelte-tocjh9"&&(b.innerHTML=D),C=l(e),p(h.$$.fragment,e),L=l(e),y=i(e,"P",{"data-svelte-h":!0}),J(y)!=="svelte-1i81m0l"&&(y.innerHTML=ee),X=l(e),p(T.$$.fragment,e),F=l(e),p(g.$$.fragment,e),I=l(e),_=i(e,"P",{"data-svelte-h":!0}),J(_)!=="svelte-iefr4h"&&(_.innerHTML=te),P=l(e),j=i(e,"P",{"data-svelte-h":!0}),J(j)!=="svelte-1bwb20c"&&(j.innerHTML=ne),V=l(e),p(Z.$$.fragment,e),q=l(e),v=i(e,"P",{"data-svelte-h":!0}),J(v)!=="svelte-3wr7qd"&&(v.innerHTML=se),x=l(e),p(z.$$.fragment,e),Q=l(e),p(w.$$.fragment,e),A=l(e),R=i(e,"P",{}),oe(R).forEach(n),this.h()},h(){le(r,"name","hf:doc:metadata"),le(r,"content",Me)},m(e,t){ke(document.head,r),s(e,B,t),s(e,U,t),s(e,H,t),f(M,e,t),s(e,W,t),s(e,$,t),s(e,E,t),f(u,e,t),s(e,G,t),s(e,b,t),s(e,C,t),f(h,e,t),s(e,L,t),s(e,y,t),s(e,X,t),f(T,e,t),s(e,F,t),f(g,e,t),s(e,I,t),s(e,_,t),s(e,P,t),s(e,j,t),s(e,V,t),f(Z,e,t),s(e,q,t),s(e,v,t),s(e,x,t),f(z,e,t),s(e,Q,t),f(w,e,t),s(e,A,t),s(e,R,t),Y=!0},p:ae,i(e){Y||(k(M.$$.fragment,e),k(u.$$.fragment,e),k(h.$$.fragment,e),k(T.$$.fragment,e),k(g.$$.fragment,e),k(Z.$$.fragment,e),k(z.$$.fragment,e),k(w.$$.fragment,e),Y=!0)},o(e){c(M.$$.fragment,e),c(u.$$.fragment,e),c(h.$$.fragment,e),c(T.$$.fragment,e),c(g.$$.fragment,e),c(Z.$$.fragment,e),c(z.$$.fragment,e),c(w.$$.fragment,e),Y=!1},d(e){e&&(n(B),n(U),n(H),n(W),n($),n(E),n(G),n(b),n(C),n(L),n(y),n(X),n(F),n(I),n(_),n(P),n(j),n(V),n(q),n(v),n(x),n(Q),n(A),n(R)),n(r),d(M,e),d(u,e),d(h,e),d(T,e),d(g,e),d(Z,e),d(z,e),d(w,e)}}}const Me='{"title":"Transformers与Tiktonken的互操作性","local":"transformers与tiktonken的互操作性","sections":[{"title":"已知包含 tiktoken.model 文件发布的模型：","local":"已知包含-tiktokenmodel-文件发布的模型","sections":[],"depth":3},{"title":"使用示例","local":"使用示例","sections":[],"depth":2},{"title":"创建tiktoken词符化器(tokenizer)","local":"创建tiktoken词符化器tokenizer","sections":[],"depth":2}],"depth":1}';function $e(K){return ie(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Te extends me{constructor(r){super(),pe(this,r,$e,de,re,{})}}export{Te as component};