transformers/pr_34606/zh/_app/immutable/nodes/11.dfd03167.js

import{s as ce,n as Me,o as ke}from"../chunks/scheduler.9991993c.js";import{S as ge,i as ye,g as r,s as n,r as m,A as he,h as p,f as s,c as a,j as oe,u as o,x as f,k as fe,y as je,a as l,v as c,d as M,t as k,w as g}from"../chunks/index.7fc9a5e7.js";import{C as D}from"../chunks/CodeBlock.e11cba92.js";import{H as ee,E as Te}from"../chunks/EditOnGithub.84ab7f0e.js";function ue(te){let i,C,F,I,y,P,h,se='<a href="/docs/transformers/pr_34606/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> 依赖于 <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a> 库。从 🤗 Tokenizers 库获得的分词器可以被轻松地加载到 🤗 Transformers 中。',N,j,le="在了解具体内容之前，让我们先用几行代码创建一个虚拟的分词器：",Q,T,W,u,ne="现在，我们拥有了一个针对我们定义的文件进行训练的分词器。我们可以在当前运行时中继续使用它，或者将其保存到一个 JSON 文件以供将来重复使用。",E,$,G,d,ae='让我们看看如何利用 🤗 Transformers 库中的这个分词器对象。<a href="/docs/transformers/pr_34606/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> 类允许通过接受已实例化的 <em>tokenizer</em> 对象作为参数，进行轻松实例化：',X,z,R,b,re='现在可以使用这个对象，使用 🤗 Transformers 分词器共享的所有方法！前往<a href="main_classes/tokenizer">分词器页面</a>了解更多信息。',q,U,x,Z,pe="为了从 JSON 文件中加载分词器，让我们先保存我们的分词器：",H,w,S,_,ie='我们保存此文件的路径可以通过 <code>tokenizer_file</code> 参数传递给 <a href="/docs/transformers/pr_34606/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> 初始化方法：',L,J,A,V,me='现在可以使用这个对象，使用 🤗 Transformers 分词器共享的所有方法！前往<a href="main_classes/tokenizer">分词器页面</a>了解更多信息。',Y,B,K,v,O;return y=new ee({props:{title:"使用 🤗 Tokenizers 中的分词器",local:"使用--tokenizers-中的分词器",headingTag:"h1"}}),T=new D({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`,wrap:!1}}),$=new ee({props:{title:"直接从分词器对象加载",local:"直接从分词器对象加载",headingTag:"h2"}}),z=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`,wrap:!1}}),U=new ee({props:{title:"从 JSON 文件加载",local:"从-json-文件加载",headingTag:"h2"}}),w=new D({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)',wrap:!1}}),J=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`,wrap:!1}}),B=new Te({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/fast_tokenizers.md"}}),{c(){i=r("meta"),C=n(),F=r("p"),I=n(),m(y.$$.fragment),P=n(),h=r("p"),h.innerHTML=se,N=n(),j=r("p"),j.textContent=le,Q=n(),m(T.$$.fragment),W=n(),u=r("p"),u.textContent=ne,E=n(),m($.$$.fragment),G=n(),d=r("p"),d.innerHTML=ae,X=n(),m(z.$$.fragment),R=n(),b=r("p"),b.innerHTML=re,q=n(),m(U.$$.fragment),x=n(),Z=r("p"),Z.textContent=pe,H=n(),m(w.$$.fragment),S=n(),_=r("p"),_.innerHTML=ie,L=n(),m(J.$$.fragment),A=n(),V=r("p"),V.innerHTML=me,Y=n(),m(B.$$.fragment),K=n(),v=r("p"),this.h()},l(e){const t=he("svelte-u9bgzb",document.head);i=p(t,"META",{name:!0,content:!0}),t.forEach(s),C=a(e),F=p(e,"P",{}),oe(F).forEach(s),I=a(e),o(y.$$.fragment,e),P=a(e),h=p(e,"P",{"data-svelte-h":!0}),f(h)!=="svelte-845tf0"&&(h.innerHTML=se),N=a(e),j=p(e,"P",{"data-svelte-h":!0}),f(j)!=="svelte-opvc5k"&&(j.textContent=le),Q=a(e),o(T.$$.fragment,e),W=a(e),u=p(e,"P",{"data-svelte-h":!0}),f(u)!=="svelte-17x9tu2"&&(u.textContent=ne),E=a(e),o($.$$.fragment,e),G=a(e),d=p(e,"P",{"data-svelte-h":!0}),f(d)!=="svelte-jy2fwq"&&(d.innerHTML=ae),X=a(e),o(z.$$.fragment,e),R=a(e),b=p(e,"P",{"data-svelte-h":!0}),f(b)!=="svelte-tb93lc"&&(b.innerHTML=re),q=a(e),o(U.$$.fragment,e),x=a(e),Z=p(e,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-lr3arf"&&(Z.textContent=pe),H=a(e),o(w.$$.fragment,e),S=a(e),_=p(e,"P",{"data-svelte-h":!0}),f(_)!=="svelte-jv9c5r"&&(_.innerHTML=ie),L=a(e),o(J.$$.fragment,e),A=a(e),V=p(e,"P",{"data-svelte-h":!0}),f(V)!=="svelte-tb93lc"&&(V.innerHTML=me),Y=a(e),o(B.$$.fragment,e),K=a(e),v=p(e,"P",{}),oe(v).forEach(s),this.h()},h(){fe(i,"name","hf:doc:metadata"),fe(i,"content",$e)},m(e,t){je(document.head,i),l(e,C,t),l(e,F,t),l(e,I,t),c(y,e,t),l(e,P,t),l(e,h,t),l(e,N,t),l(e,j,t),l(e,Q,t),c(T,e,t),l(e,W,t),l(e,u,t),l(e,E,t),c($,e,t),l(e,G,t),l(e,d,t),l(e,X,t),c(z,e,t),l(e,R,t),l(e,b,t),l(e,q,t),c(U,e,t),l(e,x,t),l(e,Z,t),l(e,H,t),c(w,e,t),l(e,S,t),l(e,_,t),l(e,L,t),c(J,e,t),l(e,A,t),l(e,V,t),l(e,Y,t),c(B,e,t),l(e,K,t),l(e,v,t),O=!0},p:Me,i(e){O||(M(y.$$.fragment,e),M(T.$$.fragment,e),M($.$$.fragment,e),M(z.$$.fragment,e),M(U.$$.fragment,e),M(w.$$.fragment,e),M(J.$$.fragment,e),M(B.$$.fragment,e),O=!0)},o(e){k(y.$$.fragment,e),k(T.$$.fragment,e),k($.$$.fragment,e),k(z.$$.fragment,e),k(U.$$.fragment,e),k(w.$$.fragment,e),k(J.$$.fragment,e),k(B.$$.fragment,e),O=!1},d(e){e&&(s(C),s(F),s(I),s(P),s(h),s(N),s(j),s(Q),s(W),s(u),s(E),s(G),s(d),s(X),s(R),s(b),s(q),s(x),s(Z),s(H),s(S),s(_),s(L),s(A),s(V),s(Y),s(K),s(v)),s(i),g(y,e),g(T,e),g($,e),g(z,e),g(U,e),g(w,e),g(J,e),g(B,e)}}}const $e='{"title":"使用 🤗 Tokenizers 中的分词器","local":"使用--tokenizers-中的分词器","sections":[{"title":"直接从分词器对象加载","local":"直接从分词器对象加载","sections":[],"depth":2},{"title":"从 JSON 文件加载","local":"从-json-文件加载","sections":[],"depth":2}],"depth":1}';function de(te){return ke(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class we extends ge{constructor(i){super(),ye(this,i,de,ue,ce,{})}}export{we as component};