tokenizers/pr_2136/en/_app/immutable/nodes/11.a294167e.js

import{s as Ys,o as Rs,n as K}from"../chunks/scheduler.7c59faff.js";import{S as Ks,i as Os,e as i,s as n,c as m,h as ea,a as d,d as u,b as o,f as v,g as h,j as x,k as e,l as w,m as g,t as f,n as k,o as _,p}from"../chunks/index.09bb5655.js";import{C as ta,H as Gs,E as na}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f3e3904a.js";import{D as y}from"../chunks/Docstring.c9b33828.js";import{C as Mt}from"../chunks/CodeBlock.f6d09e51.js";import{T as oa,M as Qr}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";import{E as yt}from"../chunks/ExampleCodeBlock.2993e81d.js";function ra(M){let r,T="Example:",t,c,$;return c=new Mt({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy5ub3JtYWxpemVycyUyMGltcG9ydCUyMExvd2VyY2FzZSUwQWZyb20lMjB0b2tlbml6ZXJzLnByZV90b2tlbml6ZXJzJTIwaW1wb3J0JTIwV2hpdGVzcGFjZSUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplcihCUEUodW5rX3Rva2VuJTNEJTIyJTNDdW5rJTNFJTIyKSklMEF0b2tlbml6ZXIubm9ybWFsaXplciUyMCUzRCUyMExvd2VyY2FzZSgpJTBBdG9rZW5pemVyLnByZV90b2tlbml6ZXIlMjAlM0QlMjBXaGl0ZXNwYWNlKCklMEElMjMlMjBMb2FkJTIwYSUyMHByZS1idWlsdCUyMHRva2VuaXplciUyMGZyb20lMjBIdWdnaW5nRmFjZSUyMEh1YiUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYmVydC1iYXNlLXVuY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> Lowercase
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.normalizer = Lowercase()
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Load a pre-built tokenizer from HuggingFace Hub</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)`,lang:"python",wrap:!1}}),{c(){r=i("p"),r.textContent=T,t=n(),m(c.$$.fragment)},l(s){r=d(s,"P",{"data-svelte-h":!0}),p(r)!=="svelte-11lpom8"&&(r.textContent=T),t=o(s),h(c.$$.fragment,s)},m(s,b){w(s,r,b),w(s,t,b),g(c,s,b),$=!0},p:K,i(s){$||(f(c.$$.fragment,s),$=!0)},o(s){k(c.$$.fragment,s),$=!1},d(s){s&&(u(r),u(t)),_(c,s)}}}function sa(M){let r,T="Here are some examples of the inputs that are accepted:",t,c,$;return c=new Mt({props:{code:"YXdhaXQlMjBhc3luY19lbmNvZGUoJTIyQSUyMHNpbmdsZSUyMHNlcXVlbmNlJTIyKQ==",highlighted:'<span class="hljs-keyword">await</span> async_encode(<span class="hljs-string">&quot;A single sequence&quot;</span>)',lang:"python",wrap:!1}}),{c(){r=i("p"),r.textContent=T,t=n(),m(c.$$.fragment)},l(s){r=d(s,"P",{"data-svelte-h":!0}),p(r)!=="svelte-fwz8ui"&&(r.textContent=T),t=o(s),h(c.$$.fragment,s)},m(s,b){w(s,r,b),w(s,t,b),g(c,s,b),$=!0},p:K,i(s){$||(f(c.$$.fragment,s),$=!0)},o(s){k(c.$$.fragment,s),$=!1},d(s){s&&(u(r),u(t)),_(c,s)}}}function aa(M){let r,T="Here are some examples of the inputs that are accepted:",t,c,$;return c=new Mt({props:{code:"YXdhaXQlMjBhc3luY19lbmNvZGVfYmF0Y2goJTVCJTBBJTIyQSUyMHNpbmdsZSUyMHNlcXVlbmNlJTIyJTJDJTBBKCUyMkElMjB0dXBsZSUyMHdpdGglMjBhJTIwc2VxdWVuY2UlMjIlMkMlMjAlMjJBbmQlMjBpdHMlMjBwYWlyJTIyKSUyQyUwQSU1QiUyMCUyMkElMjIlMkMlMjAlMjJwcmUlMjIlMkMlMjAlMjJ0b2tlbml6ZWQlMjIlMkMlMjAlMjJzZXF1ZW5jZSUyMiUyMCU1RCUyQyUwQSglNUIlMjAlMjJBJTIyJTJDJTIwJTIycHJlJTIyJTJDJTIwJTIydG9rZW5pemVkJTIyJTJDJTIwJTIyc2VxdWVuY2UlMjIlMjAlNUQlMkMlMjAlMjJBbmQlMjBpdHMlMjBwYWlyJTIyKSUwQSU1RCk=",highlighted:`<span class="hljs-keyword">await</span> async_encode_batch([
<span class="hljs-string">&quot;A single sequence&quot;</span>,
(<span class="hljs-string">&quot;A tuple with a sequence&quot;</span>, <span class="hljs-string">&quot;And its pair&quot;</span>),
[ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ],
([ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ], <span class="hljs-string">&quot;And its pair&quot;</span>)
])`,lang:"python",wrap:!1}}),{c(){r=i("p"),r.textContent=T,t=n(),m(c.$$.fragment)},l(s){r=d(s,"P",{"data-svelte-h":!0}),p(r)!=="svelte-fwz8ui"&&(r.textContent=T),t=o(s),h(c.$$.fragment,s)},m(s,b){w(s,r,b),w(s,t,b),g(c,s,b),$=!0},p:K,i(s){$||(f(c.$$.fragment,s),$=!0)},o(s){k(c.$$.fragment,s),$=!1},d(s){s&&(u(r),u(t)),_(c,s)}}}function ia(M){let r,T="Here are some examples of the inputs that are accepted:",t,c,$;return c=new Mt({props:{code:"YXdhaXQlMjBhc3luY19lbmNvZGVfYmF0Y2hfZmFzdCglNUIlMEElMjJBJTIwc2luZ2xlJTIwc2VxdWVuY2UlMjIlMkMlMEEoJTIyQSUyMHR1cGxlJTIwd2l0aCUyMGElMjBzZXF1ZW5jZSUyMiUyQyUyMCUyMkFuZCUyMGl0cyUyMHBhaXIlMjIpJTJDJTBBJTVCJTIwJTIyQSUyMiUyQyUyMCUyMnByZSUyMiUyQyUyMCUyMnRva2VuaXplZCUyMiUyQyUyMCUyMnNlcXVlbmNlJTIyJTIwJTVEJTJDJTBBKCU1QiUyMCUyMkElMjIlMkMlMjAlMjJwcmUlMjIlMkMlMjAlMjJ0b2tlbml6ZWQlMjIlMkMlMjAlMjJzZXF1ZW5jZSUyMiUyMCU1RCUyQyUyMCUyMkFuZCUyMGl0cyUyMHBhaXIlMjIpJTBBJTVEKQ==",highlighted:`<span class="hljs-keyword">await</span> async_encode_batch_fast([
<span class="hljs-string">&quot;A single sequence&quot;</span>,
(<span class="hljs-string">&quot;A tuple with a sequence&quot;</span>, <span class="hljs-string">&quot;And its pair&quot;</span>),
[ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ],
([ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ], <span class="hljs-string">&quot;And its pair&quot;</span>)
])`,lang:"python",wrap:!1}}),{c(){r=i("p"),r.textContent=T,t=n(),m(c.$$.fragment)},l(s){r=d(s,"P",{"data-svelte-h":!0}),p(r)!=="svelte-fwz8ui"&&(r.textContent=T),t=o(s),h(c.$$.fragment,s)},m(s,b){w(s,r,b),w(s,t,b),g(c,s,b),$=!0},p:K,i(s){$||(f(c.$$.fragment,s),$=!0)},o(s){k(c.$$.fragment,s),$=!1},d(s){s&&(u(r),u(t)),_(c,s)}}}function da(M){let r,T="Here are some examples of the inputs that are accepted:",t,c,$;return c=new Mt({props:{code:"ZW5jb2RlKCUyMkElMjBzaW5nbGUlMjBzZXF1ZW5jZSUyMikqJTBBZW5jb2RlKCUyMkElMjBzZXF1ZW5jZSUyMiUyQyUyMCUyMkFuZCUyMGl0cyUyMHBhaXIlMjIpKiUwQWVuY29kZSglNUIlMjAlMjJBJTIyJTJDJTIwJTIycHJlJTIyJTJDJTIwJTIydG9rZW5pemVkJTIyJTJDJTIwJTIyc2VxdWVuY2UlMjIlMjAlNUQlMkMlMjBpc19wcmV0b2tlbml6ZWQlM0RUcnVlKSU2MCUwQWVuY29kZSglMEElNUIlMjAlMjJBJTIyJTJDJTIwJTIycHJlJTIyJTJDJTIwJTIydG9rZW5pemVkJTIyJTJDJTIwJTIyc2VxdWVuY2UlMjIlMjAlNUQlMkMlMjAlNUIlMjAlMjJBbmQlMjIlMkMlMjAlMjJpdHMlMjIlMkMlMjAlMjJwYWlyJTIyJTIwJTVEJTJDJTBBaXNfcHJldG9rZW5pemVkJTNEVHJ1ZSUwQSk=",highlighted:`encode(<span class="hljs-string">&quot;A single sequence&quot;</span>)*
encode(<span class="hljs-string">&quot;A sequence&quot;</span>, <span class="hljs-string">&quot;And its pair&quot;</span>)*
encode([ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ], is_pretokenized=<span class="hljs-literal">True</span>)\`
encode(
[ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ], [ <span class="hljs-string">&quot;And&quot;</span>, <span class="hljs-string">&quot;its&quot;</span>, <span class="hljs-string">&quot;pair&quot;</span> ],
is_pretokenized=<span class="hljs-literal">True</span>
)`,lang:"python",wrap:!1}}),{c(){r=i("p"),r.textContent=T,t=n(),m(c.$$.fragment)},l(s){r=d(s,"P",{"data-svelte-h":!0}),p(r)!=="svelte-fwz8ui"&&(r.textContent=T),t=o(s),h(c.$$.fragment,s)},m(s,b){w(s,r,b),w(s,t,b),g(c,s,b),$=!0},p:K,i(s){$||(f(c.$$.fragment,s),$=!0)},o(s){k(c.$$.fragment,s),$=!1},d(s){s&&(u(r),u(t)),_(c,s)}}}function la(M){let r,T="Here are some examples of the inputs that are accepted:",t,c,$;return c=new Mt({props:{code:"ZW5jb2RlX2JhdGNoKCU1QiUwQSUyMkElMjBzaW5nbGUlMjBzZXF1ZW5jZSUyMiUyQyUwQSglMjJBJTIwdHVwbGUlMjB3aXRoJTIwYSUyMHNlcXVlbmNlJTIyJTJDJTIwJTIyQW5kJTIwaXRzJTIwcGFpciUyMiklMkMlMEElNUIlMjAlMjJBJTIyJTJDJTIwJTIycHJlJTIyJTJDJTIwJTIydG9rZW5pemVkJTIyJTJDJTIwJTIyc2VxdWVuY2UlMjIlMjAlNUQlMkMlMEEoJTVCJTIwJTIyQSUyMiUyQyUyMCUyMnByZSUyMiUyQyUyMCUyMnRva2VuaXplZCUyMiUyQyUyMCUyMnNlcXVlbmNlJTIyJTIwJTVEJTJDJTIwJTIyQW5kJTIwaXRzJTIwcGFpciUyMiklMEElNUQp",highlighted:`encode_batch([
<span class="hljs-string">&quot;A single sequence&quot;</span>,
(<span class="hljs-string">&quot;A tuple with a sequence&quot;</span>, <span class="hljs-string">&quot;And its pair&quot;</span>),
[ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ],
([ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ], <span class="hljs-string">&quot;And its pair&quot;</span>)
])`,lang:"python",wrap:!1}}),{c(){r=i("p"),r.textContent=T,t=n(),m(c.$$.fragment)},l(s){r=d(s,"P",{"data-svelte-h":!0}),p(r)!=="svelte-fwz8ui"&&(r.textContent=T),t=o(s),h(c.$$.fragment,s)},m(s,b){w(s,r,b),w(s,t,b),g(c,s,b),$=!0},p:K,i(s){$||(f(c.$$.fragment,s),$=!0)},o(s){k(c.$$.fragment,s),$=!1},d(s){s&&(u(r),u(t)),_(c,s)}}}function ca(M){let r,T="Here are some examples of the inputs that are accepted:",t,c,$;return c=new Mt({props:{code:"ZW5jb2RlX2JhdGNoX2Zhc3QoJTVCJTBBJTIyQSUyMHNpbmdsZSUyMHNlcXVlbmNlJTIyJTJDJTBBKCUyMkElMjB0dXBsZSUyMHdpdGglMjBhJTIwc2VxdWVuY2UlMjIlMkMlMjAlMjJBbmQlMjBpdHMlMjBwYWlyJTIyKSUyQyUwQSU1QiUyMCUyMkElMjIlMkMlMjAlMjJwcmUlMjIlMkMlMjAlMjJ0b2tlbml6ZWQlMjIlMkMlMjAlMjJzZXF1ZW5jZSUyMiUyMCU1RCUyQyUwQSglNUIlMjAlMjJBJTIyJTJDJTIwJTIycHJlJTIyJTJDJTIwJTIydG9rZW5pemVkJTIyJTJDJTIwJTIyc2VxdWVuY2UlMjIlMjAlNUQlMkMlMjAlMjJBbmQlMjBpdHMlMjBwYWlyJTIyKSUwQSU1RCk=",highlighted:`encode_batch_fast([
<span class="hljs-string">&quot;A single sequence&quot;</span>,
(<span class="hljs-string">&quot;A tuple with a sequence&quot;</span>, <span class="hljs-string">&quot;And its pair&quot;</span>),
[ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ],
([ <span class="hljs-string">&quot;A&quot;</span>, <span class="hljs-string">&quot;pre&quot;</span>, <span class="hljs-string">&quot;tokenized&quot;</span>, <span class="hljs-string">&quot;sequence&quot;</span> ], <span class="hljs-string">&quot;And its pair&quot;</span>)
])`,lang:"python",wrap:!1}}),{c(){r=i("p"),r.textContent=T,t=n(),m(c.$$.fragment)},l(s){r=d(s,"P",{"data-svelte-h":!0}),p(r)!=="svelte-fwz8ui"&&(r.textContent=T),t=o(s),h(c.$$.fragment,s)},m(s,b){w(s,r,b),w(s,t,b),g(c,s,b),$=!0},p:K,i(s){$||(f(c.$$.fragment,s),$=!0)},o(s){k(c.$$.fragment,s),$=!1},d(s){s&&(u(r),u(t)),_(c,s)}}}function pa(M){let r,T,t,c,$,s,b=`A <code>Tokenizer</code> works as a pipeline. It processes some raw text as input
and outputs an <a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a>.`,O,j,Ae="The pipeline is structured as follows:",H,B,ee=`<li>The <a href="/docs/tokenizers/pr_2136/en/api/normalizers#tokenizers.normalizers.Normalizer">Normalizer</a> normalizes the raw input text.</li> <li>The <a href="/docs/tokenizers/pr_2136/en/api/pre-tokenizers#tokenizers.pre_tokenizers.PreTokenizer">PreTokenizer</a> splits the normalized text
into word-level tokens.</li> <li>The <a href="/docs/tokenizers/pr_2136/en/api/models#tokenizers.models.Model">Model</a> tokenizes each word into subword tokens
and maps them to IDs.</li> <li>The <code>PostProcessor</code> applies any final
transformations (e.g., adding special tokens like <code>[CLS]</code> and <code>[SEP]</code>).</li>`,te,z,C,W,Ve,On,wt,Nr="The <em>optional</em> <code>Decoder</code> in use by the Tokenizer",eo,ne,Pe,to,Ct,Xr='The <a href="/docs/tokenizers/pr_2136/en/api/models#tokenizers.models.Model">Model</a> in use by the Tokenizer',no,oe,He,oo,jt,Fr='The <em>optional</em> <a href="/docs/tokenizers/pr_2136/en/api/normalizers#tokenizers.normalizers.Normalizer">Normalizer</a> in use by the Tokenizer',ro,Z,Be,so,qt,Gr="Get the current padding parameters",ao,It,Yr="<em>Cannot be set, use</em> <code>enable_padding()</code> <em>instead</em>",io,re,We,lo,Jt,Rr="The <em>optional</em> <code>PostProcessor</code> in use by the Tokenizer",co,se,Ze,po,Ut,Kr='The <em>optional</em> <a href="/docs/tokenizers/pr_2136/en/api/pre-tokenizers#tokenizers.pre_tokenizers.PreTokenizer">PreTokenizer</a> in use by the Tokenizer',uo,L,Le,mo,Et,Or="Get the currently set truncation parameters",ho,Dt,es="<em>Cannot set, use</em> <code>enable_truncation()</code> <em>instead</em>",go,U,Se,fo,At,ts="Add the given special tokens to the Tokenizer.",ko,Vt,ns=`If these tokens are already part of the vocabulary, it just let the Tokenizer know about
them. If they don’t exist, the Tokenizer creates them, giving them a new id.`,_o,Pt,os=`These special tokens will never be processed by the model (ie won’t be split into
multiple tokens), and they can be removed from the output when decoding.`,$o,S,Qe,zo,Ht,rs="Add the given tokens to the vocabulary",To,Bt,ss=`The given tokens are added only if they don’t already exist in the vocabulary.
Each token then gets a new attributed id.`,vo,ae,Ne,xo,Wt,as="Decode a batch of ids back to their corresponding string",bo,q,Xe,yo,Zt,is="Asynchronously encode the given input with character offsets.",Mo,Lt,ds="This is an async version of encode that can be awaited in async Python code.",wo,St,ls="Example:",Co,ie,jo,I,Fe,qo,Qt,cs="Asynchronously encode the given batch of inputs with character offsets.",Io,Nt,ps="This is an async version of encode_batch that can be awaited in async Python code.",Jo,Xt,us="Example:",Uo,de,Eo,J,Ge,Do,Ft,ms="Asynchronously encode the given batch of inputs without tracking character offsets.",Ao,Gt,hs="This is an async version of encode_batch_fast that can be awaited in async Python code.",Vo,Yt,gs="Example:",Po,le,Ho,Q,Ye,Bo,Rt,fs="Decode the given list of ids back to a string",Wo,Kt,ks="This is used to decode anything coming back from a Language Model",Zo,ce,Re,Lo,Ot,_s="Decode a batch of ids back to their corresponding string",So,pe,Ke,Qo,en,$s="Enable the padding",No,ue,Oe,Xo,tn,zs="Enable truncation",Fo,E,et,Go,nn,Ts=`Encode the given sequence and pair. This method can process raw text sequences
as well as already pre-tokenized sequences.`,Yo,on,vs="Example:",Ro,me,Ko,D,tt,Oo,rn,xs=`Encode the given batch of inputs. This method accept both raw text sequences
as well as already pre-tokenized sequences. The reason we use <em>PySequence</em> is
because it allows type checking with zero-cost (according to PyO3) as we don’t
have to convert to check.`,er,sn,bs="Example:",tr,he,nr,A,nt,or,an,ys=`Encode the given batch of inputs. This method is faster than <em>encode_batch</em>
because it doesn’t keep track of offsets, they will be all zeros.`,rr,dn,Ms="Example:",sr,ge,ar,fe,ot,ir,ln,ws='Instantiate a new <a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> from the given buffer.',dr,ke,rt,lr,cn,Cs='Instantiate a new <a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> from the file at the given path.',cr,_e,st,pr,pn,js=`Instantiate a new <a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> from an existing file on the
Hugging Face Hub.`,ur,$e,at,mr,un,qs='Instantiate a new <a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> from the given JSON string.',hr,ze,it,gr,mn,Is="Get the underlying vocabulary",fr,Te,dt,kr,hn,Js="Get the underlying vocabulary",_r,ve,lt,$r,gn,Us="Get the size of the underlying vocabulary",zr,xe,ct,Tr,fn,Es="Convert the given id to its corresponding token if it exists",vr,be,pt,xr,kn,Ds="Disable padding",br,ye,ut,yr,_n,As="Disable truncation",Mr,Me,mt,wr,$n,Vs=`Return the number of special tokens that would be added for single/pair sentences.
:param is_pair: Boolean indicating if the input would be a single sentence or a pair
:return:`,Cr,V,ht,jr,zn,Ps="Apply all the post-processing steps to the given encodings.",qr,Tn,Hs="The various steps are:",Ir,vn,Bs=`<li>Truncate according to the set truncation params (provided with
<code>enable_truncation()</code>)</li> <li>Apply the <code>PostProcessor</code></li> <li>Pad according to the set padding params (provided with
<code>enable_padding()</code>)</li>`,Jr,we,gt,Ur,xn,Ws='Save the <a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> to the file at the given path.',Er,Ce,ft,Dr,bn,Zs='Gets a serialized string representing this <a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>.',Ar,je,kt,Vr,yn,Ls="Convert the given token to its corresponding id if it exists",Pr,N,_t,Hr,Mn,Ss="Train the Tokenizer using the given files.",Br,wn,Qs=`Reads the files line by line, while keeping all the whitespace, even new lines.
If you want to train from data store in-memory, you can check
<code>train_from_iterator()</code>`,Wr,P,$t,Zr,Cn,Ns="Train the Tokenizer using the provided iterator.",Lr,jn,Xs="You can provide anything that is a Python Iterator",Sr,qn,Fs="<li>A list of sequences <code>List[str]</code></li> <li>A generator that yields <code>str</code> or <code>List[str]</code></li> <li>A Numpy array of strings</li> <li>…</li>",En;return r=new Gs({props:{title:"Tokenizer",local:"tokenizers.Tokenizer",headingTag:"h2"}}),c=new y({props:{name:"class tokenizers.Tokenizer",anchor:"tokenizers.Tokenizer",parameters:[{name:"model",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.model",description:`<strong>model</strong> (<a href="/docs/tokenizers/pr_2136/en/api/models#tokenizers.models.Model">Model</a>) &#x2014;
The core algorithm that this <code>Tokenizer</code> should be using.`,name:"model"}]}}),z=new yt({props:{anchor:"tokenizers.Tokenizer.example",$$slots:{default:[ra]},$$scope:{ctx:M}}}),Ve=new y({props:{name:"decoder",anchor:"tokenizers.Tokenizer.decoder",parameters:[],isGetSetDescriptor:!0}}),Pe=new y({props:{name:"model",anchor:"tokenizers.Tokenizer.model",parameters:[],isGetSetDescriptor:!0}}),He=new y({props:{name:"normalizer",anchor:"tokenizers.Tokenizer.normalizer",parameters:[],isGetSetDescriptor:!0}}),Be=new y({props:{name:"padding",anchor:"tokenizers.Tokenizer.padding",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A dict with the current padding parameters if padding is enabled</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>(<code>dict</code>, <em>optional</em>)</p>
`,isGetSetDescriptor:!0}}),We=new y({props:{name:"post_processor",anchor:"tokenizers.Tokenizer.post_processor",parameters:[],isGetSetDescriptor:!0}}),Ze=new y({props:{name:"pre_tokenizer",anchor:"tokenizers.Tokenizer.pre_tokenizer",parameters:[],isGetSetDescriptor:!0}}),Le=new y({props:{name:"truncation",anchor:"tokenizers.Tokenizer.truncation",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A dict with the current truncation parameters if truncation is enabled</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>(<code>dict</code>, <em>optional</em>)</p>
`,isGetSetDescriptor:!0}}),Se=new y({props:{name:"add_special_tokens",anchor:"tokenizers.Tokenizer.add_special_tokens",parameters:[{name:"tokens",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.add_special_tokens.tokens",description:`<strong>tokens</strong> (A <code>List</code> of <a href="/docs/tokenizers/pr_2136/en/api/added-tokens#tokenizers.AddedToken">AddedToken</a> or <code>str</code>) &#x2014;
The list of special tokens we want to add to the vocabulary. Each token can either
be a string or an instance of <a href="/docs/tokenizers/pr_2136/en/api/added-tokens#tokenizers.AddedToken">AddedToken</a> for more
customization.`,name:"tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The number of tokens that were created in the vocabulary</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`}}),Qe=new y({props:{name:"add_tokens",anchor:"tokenizers.Tokenizer.add_tokens",parameters:[{name:"tokens",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.add_tokens.tokens",description:`<strong>tokens</strong> (A <code>List</code> of <a href="/docs/tokenizers/pr_2136/en/api/added-tokens#tokenizers.AddedToken">AddedToken</a> or <code>str</code>) &#x2014;
The list of tokens we want to add to the vocabulary. Each token can be either a
string or an instance of <a href="/docs/tokenizers/pr_2136/en/api/added-tokens#tokenizers.AddedToken">AddedToken</a> for more customization.`,name:"tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The number of tokens that were created in the vocabulary</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`}}),Ne=new y({props:{name:"async_decode_batch",anchor:"tokenizers.Tokenizer.async_decode_batch",parameters:[{name:"sequences",val:""},{name:"skip_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.async_decode_batch.sequences",description:`<strong>sequences</strong> (<code>List</code> of <code>List[int]</code>) &#x2014;
The batch of sequences we want to decode`,name:"sequences"},{anchor:"tokenizers.Tokenizer.async_decode_batch.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the special tokens should be removed from the decoded strings`,name:"skip_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of decoded strings</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[str]</code></p>
`}}),Xe=new y({props:{name:"async_encode",anchor:"tokenizers.Tokenizer.async_encode",parameters:[{name:"sequence",val:""},{name:"pair",val:" = None"},{name:"is_pretokenized",val:" = False"},{name:"add_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.async_encode.sequence",description:`<strong>sequence</strong> (<code>~tokenizers.InputSequence</code>) &#x2014;
The main input sequence we want to encode. This sequence can be either raw
text or pre-tokenized, according to the <code>is_pretokenized</code> argument:</p>
<ul>
<li>If <code>is_pretokenized=False</code>: <code>TextInputSequence</code></li>
<li>If <code>is_pretokenized=True</code>: <code>PreTokenizedInputSequence()</code></li>
</ul>`,name:"sequence"},{anchor:"tokenizers.Tokenizer.async_encode.pair",description:`<strong>pair</strong> (<code>~tokenizers.InputSequence</code>, <em>optional</em>) &#x2014;
An optional input sequence. The expected format is the same that for <code>sequence</code>.`,name:"pair"},{anchor:"tokenizers.Tokenizer.async_encode.is_pretokenized",description:`<strong>is_pretokenized</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the input is already pre-tokenized`,name:"is_pretokenized"},{anchor:"tokenizers.Tokenizer.async_encode.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to add the special tokens`,name:"add_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The encoded result</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a></p>
`}}),ie=new yt({props:{anchor:"tokenizers.Tokenizer.async_encode.example",$$slots:{default:[sa]},$$scope:{ctx:M}}}),Fe=new y({props:{name:"async_encode_batch",anchor:"tokenizers.Tokenizer.async_encode_batch",parameters:[{name:"input",val:""},{name:"is_pretokenized",val:" = False"},{name:"add_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.async_encode_batch.input",description:`<strong>input</strong> (A <code>List</code>/\`<code>Tuple</code> of <code>~tokenizers.EncodeInput</code>) &#x2014;
A list of single sequences or pair sequences to encode. Each sequence
can be either raw text or pre-tokenized, according to the <code>is_pretokenized</code>
argument:</p>
<ul>
<li>If <code>is_pretokenized=False</code>: <code>TextEncodeInput()</code></li>
<li>If <code>is_pretokenized=True</code>: <code>PreTokenizedEncodeInput()</code></li>
</ul>`,name:"input"},{anchor:"tokenizers.Tokenizer.async_encode_batch.is_pretokenized",description:`<strong>is_pretokenized</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the input is already pre-tokenized`,name:"is_pretokenized"},{anchor:"tokenizers.Tokenizer.async_encode_batch.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to add the special tokens`,name:"add_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The encoded batch</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of [\`~tokenizers.Encoding“]</p>
`}}),de=new yt({props:{anchor:"tokenizers.Tokenizer.async_encode_batch.example",$$slots:{default:[aa]},$$scope:{ctx:M}}}),Ge=new y({props:{name:"async_encode_batch_fast",anchor:"tokenizers.Tokenizer.async_encode_batch_fast",parameters:[{name:"input",val:""},{name:"is_pretokenized",val:" = False"},{name:"add_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.async_encode_batch_fast.input",description:`<strong>input</strong> (A <code>List</code>/\`<code>Tuple</code> of <code>~tokenizers.EncodeInput</code>) &#x2014;
A list of single sequences or pair sequences to encode. Each sequence
can be either raw text or pre-tokenized, according to the <code>is_pretokenized</code>
argument:</p>
<ul>
<li>If <code>is_pretokenized=False</code>: <code>TextEncodeInput()</code></li>
<li>If <code>is_pretokenized=True</code>: <code>PreTokenizedEncodeInput()</code></li>
</ul>`,name:"input"},{anchor:"tokenizers.Tokenizer.async_encode_batch_fast.is_pretokenized",description:`<strong>is_pretokenized</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the input is already pre-tokenized`,name:"is_pretokenized"},{anchor:"tokenizers.Tokenizer.async_encode_batch_fast.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to add the special tokens`,name:"add_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The encoded batch</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of [\`~tokenizers.Encoding“]</p>
`}}),le=new yt({props:{anchor:"tokenizers.Tokenizer.async_encode_batch_fast.example",$$slots:{default:[ia]},$$scope:{ctx:M}}}),Ye=new y({props:{name:"decode",anchor:"tokenizers.Tokenizer.decode",parameters:[{name:"ids",val:""},{name:"skip_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.decode.ids",description:`<strong>ids</strong> (A <code>List/Tuple</code> of <code>int</code>) &#x2014;
The list of ids that we want to decode`,name:"ids"},{anchor:"tokenizers.Tokenizer.decode.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the special tokens should be removed from the decoded string`,name:"skip_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The decoded string</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>str</code></p>
`}}),Re=new y({props:{name:"decode_batch",anchor:"tokenizers.Tokenizer.decode_batch",parameters:[{name:"sequences",val:""},{name:"skip_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.decode_batch.sequences",description:`<strong>sequences</strong> (<code>List</code> of <code>List[int]</code>) &#x2014;
The batch of sequences we want to decode`,name:"sequences"},{anchor:"tokenizers.Tokenizer.decode_batch.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the special tokens should be removed from the decoded strings`,name:"skip_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of decoded strings</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[str]</code></p>
`}}),Ke=new y({props:{name:"enable_padding",anchor:"tokenizers.Tokenizer.enable_padding",parameters:[{name:"direction",val:" = 'right'"},{name:"pad_id",val:" = 0"},{name:"pad_type_id",val:" = 0"},{name:"pad_token",val:" = '[PAD]'"},{name:"length",val:" = None"},{name:"pad_to_multiple_of",val:" = None"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.enable_padding.direction",description:`<strong>direction</strong> (<code>str</code>, <em>optional</em>, defaults to <code>right</code>) &#x2014;
The direction in which to pad. Can be either <code>right</code> or <code>left</code>`,name:"direction"},{anchor:"tokenizers.Tokenizer.enable_padding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If specified, the padding length should always snap to the next multiple of the
given value. For example if we were going to pad witha length of 250 but
<code>pad_to_multiple_of=8</code> then we will pad to 256.`,name:"pad_to_multiple_of"},{anchor:"tokenizers.Tokenizer.enable_padding.pad_id",description:`<strong>pad_id</strong> (<code>int</code>, defaults to 0) &#x2014;
The id to be used when padding`,name:"pad_id"},{anchor:"tokenizers.Tokenizer.enable_padding.pad_type_id",description:`<strong>pad_type_id</strong> (<code>int</code>, defaults to 0) &#x2014;
The type id to be used when padding`,name:"pad_type_id"},{anchor:"tokenizers.Tokenizer.enable_padding.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, defaults to <code>[PAD]</code>) &#x2014;
The pad token to be used when padding`,name:"pad_token"},{anchor:"tokenizers.Tokenizer.enable_padding.length",description:`<strong>length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If specified, the length at which to pad. If not specified we pad using the size of
the longest sequence in a batch.`,name:"length"}]}}),Oe=new y({props:{name:"enable_truncation",anchor:"tokenizers.Tokenizer.enable_truncation",parameters:[{name:"max_length",val:""},{name:"stride",val:" = 0"},{name:"strategy",val:" = 'longest_first'"},{name:"direction",val:" = 'right'"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.enable_truncation.max_length",description:`<strong>max_length</strong> (<code>int</code>) &#x2014;
The max length at which to truncate`,name:"max_length"},{anchor:"tokenizers.Tokenizer.enable_truncation.stride",description:`<strong>stride</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The length of the previous first sequence to be included in the overflowing
sequence`,name:"stride"},{anchor:"tokenizers.Tokenizer.enable_truncation.strategy",description:`<strong>strategy</strong> (<code>str</code>, <em>optional</em>, defaults to <code>longest_first</code>) &#x2014;
The strategy used to truncation. Can be one of <code>longest_first</code>, <code>only_first</code> or
<code>only_second</code>.`,name:"strategy"},{anchor:"tokenizers.Tokenizer.enable_truncation.direction",description:`<strong>direction</strong> (<code>str</code>, defaults to <code>right</code>) &#x2014;
Truncate direction`,name:"direction"}]}}),et=new y({props:{name:"encode",anchor:"tokenizers.Tokenizer.encode",parameters:[{name:"sequence",val:""},{name:"pair",val:" = None"},{name:"is_pretokenized",val:" = False"},{name:"add_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.encode.sequence",description:`<strong>sequence</strong> (<code>~tokenizers.InputSequence</code>) &#x2014;
The main input sequence we want to encode. This sequence can be either raw
text or pre-tokenized, according to the <code>is_pretokenized</code> argument:</p>
<ul>
<li>If <code>is_pretokenized=False</code>: <code>TextInputSequence</code></li>
<li>If <code>is_pretokenized=True</code>: <code>PreTokenizedInputSequence()</code></li>
</ul>`,name:"sequence"},{anchor:"tokenizers.Tokenizer.encode.pair",description:`<strong>pair</strong> (<code>~tokenizers.InputSequence</code>, <em>optional</em>) &#x2014;
An optional input sequence. The expected format is the same that for <code>sequence</code>.`,name:"pair"},{anchor:"tokenizers.Tokenizer.encode.is_pretokenized",description:`<strong>is_pretokenized</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the input is already pre-tokenized`,name:"is_pretokenized"},{anchor:"tokenizers.Tokenizer.encode.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to add the special tokens`,name:"add_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The encoded result</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a></p>
`}}),me=new yt({props:{anchor:"tokenizers.Tokenizer.encode.example",$$slots:{default:[da]},$$scope:{ctx:M}}}),tt=new y({props:{name:"encode_batch",anchor:"tokenizers.Tokenizer.encode_batch",parameters:[{name:"input",val:""},{name:"is_pretokenized",val:" = False"},{name:"add_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.encode_batch.input",description:`<strong>input</strong> (A <code>List</code>/\`<code>Tuple</code> of <code>~tokenizers.EncodeInput</code>) &#x2014;
A list of single sequences or pair sequences to encode. Each sequence
can be either raw text or pre-tokenized, according to the <code>is_pretokenized</code>
argument:</p>
<ul>
<li>If <code>is_pretokenized=False</code>: <code>TextEncodeInput()</code></li>
<li>If <code>is_pretokenized=True</code>: <code>PreTokenizedEncodeInput()</code></li>
</ul>`,name:"input"},{anchor:"tokenizers.Tokenizer.encode_batch.is_pretokenized",description:`<strong>is_pretokenized</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the input is already pre-tokenized`,name:"is_pretokenized"},{anchor:"tokenizers.Tokenizer.encode_batch.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to add the special tokens`,name:"add_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The encoded batch</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of [\`~tokenizers.Encoding“]</p>
`}}),he=new yt({props:{anchor:"tokenizers.Tokenizer.encode_batch.example",$$slots:{default:[la]},$$scope:{ctx:M}}}),nt=new y({props:{name:"encode_batch_fast",anchor:"tokenizers.Tokenizer.encode_batch_fast",parameters:[{name:"input",val:""},{name:"is_pretokenized",val:" = False"},{name:"add_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.encode_batch_fast.input",description:`<strong>input</strong> (A <code>List</code>/\`<code>Tuple</code> of <code>~tokenizers.EncodeInput</code>) &#x2014;
A list of single sequences or pair sequences to encode. Each sequence
can be either raw text or pre-tokenized, according to the <code>is_pretokenized</code>
argument:</p>
<ul>
<li>If <code>is_pretokenized=False</code>: <code>TextEncodeInput()</code></li>
<li>If <code>is_pretokenized=True</code>: <code>PreTokenizedEncodeInput()</code></li>
</ul>`,name:"input"},{anchor:"tokenizers.Tokenizer.encode_batch_fast.is_pretokenized",description:`<strong>is_pretokenized</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the input is already pre-tokenized`,name:"is_pretokenized"},{anchor:"tokenizers.Tokenizer.encode_batch_fast.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to add the special tokens`,name:"add_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The encoded batch</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of [\`~tokenizers.Encoding“]</p>
`}}),ge=new yt({props:{anchor:"tokenizers.Tokenizer.encode_batch_fast.example",$$slots:{default:[ca]},$$scope:{ctx:M}}}),ot=new y({props:{name:"from_buffer",anchor:"tokenizers.Tokenizer.from_buffer",parameters:[{name:"buffer",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.from_buffer.buffer",description:`<strong>buffer</strong> (<code>bytes</code>) &#x2014;
A buffer containing a previously serialized <a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>`,name:"buffer"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The new tokenizer</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer"
>Tokenizer</a></p>
`}}),rt=new y({props:{name:"from_file",anchor:"tokenizers.Tokenizer.from_file",parameters:[{name:"path",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.from_file.path",description:`<strong>path</strong> (<code>str</code>) &#x2014;
A path to a local JSON file representing a previously serialized
<a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>`,name:"path"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The new tokenizer</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer"
>Tokenizer</a></p>
`}}),st=new y({props:{name:"from_pretrained",anchor:"tokenizers.Tokenizer.from_pretrained",parameters:[{name:"identifier",val:""},{name:"revision",val:" = 'main'"},{name:"token",val:" = None"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.from_pretrained.identifier",description:`<strong>identifier</strong> (<code>str</code>) &#x2014;
The identifier of a Model on the Hugging Face Hub, that contains
a tokenizer.json file`,name:"identifier"},{anchor:"tokenizers.Tokenizer.from_pretrained.revision",description:`<strong>revision</strong> (<code>str</code>, defaults to <em>main</em>) &#x2014;
A branch or commit id`,name:"revision"},{anchor:"tokenizers.Tokenizer.from_pretrained.token",description:`<strong>token</strong> (<code>str</code>, <em>optional</em>, defaults to <em>None</em>) &#x2014;
An optional auth token used to access private repositories on the
Hugging Face Hub`,name:"token"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The new tokenizer</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer"
>Tokenizer</a></p>
`}}),at=new y({props:{name:"from_str",anchor:"tokenizers.Tokenizer.from_str",parameters:[{name:"json",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.from_str.json",description:`<strong>json</strong> (<code>str</code>) &#x2014;
A valid JSON string representing a previously serialized
<a href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>`,name:"json"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The new tokenizer</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/tokenizers/pr_2136/en/api/tokenizer#tokenizers.Tokenizer"
>Tokenizer</a></p>
`}}),it=new y({props:{name:"get_added_tokens_decoder",anchor:"tokenizers.Tokenizer.get_added_tokens_decoder",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The vocabulary</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Dict[int, AddedToken]</code></p>
`}}),dt=new y({props:{name:"get_vocab",anchor:"tokenizers.Tokenizer.get_vocab",parameters:[{name:"with_added_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.get_vocab.with_added_tokens",description:`<strong>with_added_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to include the added tokens`,name:"with_added_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The vocabulary</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Dict[str, int]</code></p>
`}}),lt=new y({props:{name:"get_vocab_size",anchor:"tokenizers.Tokenizer.get_vocab_size",parameters:[{name:"with_added_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.get_vocab_size.with_added_tokens",description:`<strong>with_added_tokens</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to include the added tokens`,name:"with_added_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The size of the vocabulary</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`}}),ct=new y({props:{name:"id_to_token",anchor:"tokenizers.Tokenizer.id_to_token",parameters:[{name:"id",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.id_to_token.id",description:`<strong>id</strong> (<code>int</code>) &#x2014;
The id to convert`,name:"id"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>An optional token, <code>None</code> if out of vocabulary</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Optional[str]</code></p>
`}}),pt=new y({props:{name:"no_padding",anchor:"tokenizers.Tokenizer.no_padding",parameters:[]}}),ut=new y({props:{name:"no_truncation",anchor:"tokenizers.Tokenizer.no_truncation",parameters:[]}}),mt=new y({props:{name:"num_special_tokens_to_add",anchor:"tokenizers.Tokenizer.num_special_tokens_to_add",parameters:[{name:"is_pair",val:""}]}}),ht=new y({props:{name:"post_process",anchor:"tokenizers.Tokenizer.post_process",parameters:[{name:"encoding",val:""},{name:"pair",val:" = None"},{name:"add_special_tokens",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.post_process.encoding",description:`<strong>encoding</strong> (<a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a>) &#x2014;
The <a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a> corresponding to the main sequence.`,name:"encoding"},{anchor:"tokenizers.Tokenizer.post_process.pair",description:`<strong>pair</strong> (<a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a>, <em>optional</em>) &#x2014;
An optional <a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a> corresponding to the pair sequence.`,name:"pair"},{anchor:"tokenizers.Tokenizer.post_process.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>) &#x2014;
Whether to add the special tokens`,name:"add_special_tokens"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The final post-processed encoding</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a href="/docs/tokenizers/pr_2136/en/api/encoding#tokenizers.Encoding">Encoding</a></p>
`}}),gt=new y({props:{name:"save",anchor:"tokenizers.Tokenizer.save",parameters:[{name:"path",val:""},{name:"pretty",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.save.path",description:`<strong>path</strong> (<code>str</code>) &#x2014;
A path to a file in which to save the serialized tokenizer.`,name:"path"},{anchor:"tokenizers.Tokenizer.save.pretty",description:`<strong>pretty</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the JSON file should be pretty formatted.`,name:"pretty"}]}}),ft=new y({props:{name:"to_str",anchor:"tokenizers.Tokenizer.to_str",parameters:[{name:"pretty",val:" = False"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.to_str.pretty",description:`<strong>pretty</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the JSON string should be pretty formatted.`,name:"pretty"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A string representing the serialized Tokenizer</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>str</code></p>
`}}),kt=new y({props:{name:"token_to_id",anchor:"tokenizers.Tokenizer.token_to_id",parameters:[{name:"token",val:""}],parametersDescription:[{anchor:"tokenizers.Tokenizer.token_to_id.token",description:`<strong>token</strong> (<code>str</code>) &#x2014;
The token to convert`,name:"token"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>An optional id, <code>None</code> if out of vocabulary</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Optional[int]</code></p>
`}}),_t=new y({props:{name:"train",anchor:"tokenizers.Tokenizer.train",parameters:[{name:"files",val:""},{name:"trainer",val:" = None"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.train.files",description:`<strong>files</strong> (<code>List[str]</code>) &#x2014;
A list of path to the files that we should use for training`,name:"files"},{anchor:"tokenizers.Tokenizer.train.trainer",description:`<strong>trainer</strong> (<code>~tokenizers.trainers.Trainer</code>, <em>optional</em>) &#x2014;
An optional trainer that should be used to train our Model`,name:"trainer"}]}}),$t=new y({props:{name:"train_from_iterator",anchor:"tokenizers.Tokenizer.train_from_iterator",parameters:[{name:"iterator",val:""},{name:"trainer",val:" = None"},{name:"length",val:" = None"}],parametersDescription:[{anchor:"tokenizers.Tokenizer.train_from_iterator.iterator",description:`<strong>iterator</strong> (<code>Iterator</code>) &#x2014;
Any iterator over strings or list of strings`,name:"iterator"},{anchor:"tokenizers.Tokenizer.train_from_iterator.trainer",description:`<strong>trainer</strong> (<code>~tokenizers.trainers.Trainer</code>, <em>optional</em>) &#x2014;
An optional trainer that should be used to train our Model`,name:"trainer"},{anchor:"tokenizers.Tokenizer.train_from_iterator.length",description:`<strong>length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The total number of sequences in the iterator. This is used to
provide meaningful progress tracking`,name:"length"}]}}),{c(){m(r.$$.fragment),T=n(),t=i("div"),m(c.$$.fragment),$=n(),s=i("p"),s.innerHTML=b,O=n(),j=i("p"),j.textContent=Ae,H=n(),B=i("ol"),B.innerHTML=ee,te=n(),m(z.$$.fragment),C=n(),W=i("div"),m(Ve.$$.fragment),On=n(),wt=i("p"),wt.innerHTML=Nr,eo=n(),ne=i("div"),m(Pe.$$.fragment),to=n(),Ct=i("p"),Ct.innerHTML=Xr,no=n(),oe=i("div"),m(He.$$.fragment),oo=n(),jt=i("p"),jt.innerHTML=Fr,ro=n(),Z=i("div"),m(Be.$$.fragment),so=n(),qt=i("p"),qt.textContent=Gr,ao=n(),It=i("p"),It.innerHTML=Yr,io=n(),re=i("div"),m(We.$$.fragment),lo=n(),Jt=i("p"),Jt.innerHTML=Rr,co=n(),se=i("div"),m(Ze.$$.fragment),po=n(),Ut=i("p"),Ut.innerHTML=Kr,uo=n(),L=i("div"),m(Le.$$.fragment),mo=n(),Et=i("p"),Et.textContent=Or,ho=n(),Dt=i("p"),Dt.innerHTML=es,go=n(),U=i("div"),m(Se.$$.fragment),fo=n(),At=i("p"),At.textContent=ts,ko=n(),Vt=i("p"),Vt.textContent=ns,_o=n(),Pt=i("p"),Pt.textContent=os,$o=n(),S=i("div"),m(Qe.$$.fragment),zo=n(),Ht=i("p"),Ht.textContent=rs,To=n(),Bt=i("p"),Bt.textContent=ss,vo=n(),ae=i("div"),m(Ne.$$.fragment),xo=n(),Wt=i("p"),Wt.textContent=as,bo=n(),q=i("div"),m(Xe.$$.fragment),yo=n(),Zt=i("p"),Zt.textContent=is,Mo=n(),Lt=i("p"),Lt.textContent=ds,wo=n(),St=i("p"),St.textContent=ls,Co=n(),m(ie.$$.fragment),jo=n(),I=i("div"),m(Fe.$$.fragment),qo=n(),Qt=i("p"),Qt.textContent=cs,Io=n(),Nt=i("p"),Nt.textContent=ps,Jo=n(),Xt=i("p"),Xt.textContent=us,Uo=n(),m(de.$$.fragment),Eo=n(),J=i("div"),m(Ge.$$.fragment),Do=n(),Ft=i("p"),Ft.textContent=ms,Ao=n(),Gt=i("p"),Gt.textContent=hs,Vo=n(),Yt=i("p"),Yt.textContent=gs,Po=n(),m(le.$$.fragment),Ho=n(),Q=i("div"),m(Ye.$$.fragment),Bo=n(),Rt=i("p"),Rt.textContent=fs,Wo=n(),Kt=i("p"),Kt.textContent=ks,Zo=n(),ce=i("div"),m(Re.$$.fragment),Lo=n(),Ot=i("p"),Ot.textContent=_s,So=n(),pe=i("div"),m(Ke.$$.fragment),Qo=n(),en=i("p"),en.textContent=$s,No=n(),ue=i("div"),m(Oe.$$.fragment),Xo=n(),tn=i("p"),tn.textContent=zs,Fo=n(),E=i("div"),m(et.$$.fragment),Go=n(),nn=i("p"),nn.textContent=Ts,Yo=n(),on=i("p"),on.textContent=vs,Ro=n(),m(me.$$.fragment),Ko=n(),D=i("div"),m(tt.$$.fragment),Oo=n(),rn=i("p"),rn.innerHTML=xs,er=n(),sn=i("p"),sn.textContent=bs,tr=n(),m(he.$$.fragment),nr=n(),A=i("div"),m(nt.$$.fragment),or=n(),an=i("p"),an.innerHTML=ys,rr=n(),dn=i("p"),dn.textContent=Ms,sr=n(),m(ge.$$.fragment),ar=n(),fe=i("div"),m(ot.$$.fragment),ir=n(),ln=i("p"),ln.innerHTML=ws,dr=n(),ke=i("div"),m(rt.$$.fragment),lr=n(),cn=i("p"),cn.innerHTML=Cs,cr=n(),_e=i("div"),m(st.$$.fragment),pr=n(),pn=i("p"),pn.innerHTML=js,ur=n(),$e=i("div"),m(at.$$.fragment),mr=n(),un=i("p"),un.innerHTML=qs,hr=n(),ze=i("div"),m(it.$$.fragment),gr=n(),mn=i("p"),mn.textContent=Is,fr=n(),Te=i("div"),m(dt.$$.fragment),kr=n(),hn=i("p"),hn.textContent=Js,_r=n(),ve=i("div"),m(lt.$$.fragment),$r=n(),gn=i("p"),gn.textContent=Us,zr=n(),xe=i("div"),m(ct.$$.fragment),Tr=n(),fn=i("p"),fn.textContent=Es,vr=n(),be=i("div"),m(pt.$$.fragment),xr=n(),kn=i("p"),kn.textContent=Ds,br=n(),ye=i("div"),m(ut.$$.fragment),yr=n(),_n=i("p"),_n.textContent=As,Mr=n(),Me=i("div"),m(mt.$$.fragment),wr=n(),$n=i("p"),$n.textContent=Vs,Cr=n(),V=i("div"),m(ht.$$.fragment),jr=n(),zn=i("p"),zn.textContent=Ps,qr=n(),Tn=i("p"),Tn.textContent=Hs,Ir=n(),vn=i("ol"),vn.innerHTML=Bs,Jr=n(),we=i("div"),m(gt.$$.fragment),Ur=n(),xn=i("p"),xn.innerHTML=Ws,Er=n(),Ce=i("div"),m(ft.$$.fragment),Dr=n(),bn=i("p"),bn.innerHTML=Zs,Ar=n(),je=i("div"),m(kt.$$.fragment),Vr=n(),yn=i("p"),yn.textContent=Ls,Pr=n(),N=i("div"),m(_t.$$.fragment),Hr=n(),Mn=i("p"),Mn.textContent=Ss,Br=n(),wn=i("p"),wn.innerHTML=Qs,Wr=n(),P=i("div"),m($t.$$.fragment),Zr=n(),Cn=i("p"),Cn.textContent=Ns,Lr=n(),jn=i("p"),jn.textContent=Xs,Sr=n(),qn=i("ul"),qn.innerHTML=Fs,this.h()},l(a){h(r.$$.fragment,a),T=o(a),t=d(a,"DIV",{class:!0});var l=v(t);h(c.$$.fragment,l),$=o(l),s=d(l,"P",{"data-svelte-h":!0}),p(s)!=="svelte-4mkpnp"&&(s.innerHTML=b),O=o(l),j=d(l,"P",{"data-svelte-h":!0}),p(j)!=="svelte-llmwxo"&&(j.textContent=Ae),H=o(l),B=d(l,"OL",{"data-svelte-h":!0}),p(B)!=="svelte-1pva9t6"&&(B.innerHTML=ee),te=o(l),h(z.$$.fragment,l),C=o(l),W=d(l,"DIV",{class:!0});var zt=v(W);h(Ve.$$.fragment,zt),On=o(zt),wt=d(zt,"P",{"data-svelte-h":!0}),p(wt)!=="svelte-yaaz6s"&&(wt.innerHTML=Nr),zt.forEach(u),eo=o(l),ne=d(l,"DIV",{class:!0});var Tt=v(ne);h(Pe.$$.fragment,Tt),to=o(Tt),Ct=d(Tt,"P",{"data-svelte-h":!0}),p(Ct)!=="svelte-crugxn"&&(Ct.innerHTML=Xr),Tt.forEach(u),no=o(l),oe=d(l,"DIV",{class:!0});var vt=v(oe);h(He.$$.fragment,vt),oo=o(vt),jt=d(vt,"P",{"data-svelte-h":!0}),p(jt)!=="svelte-1gxtj0o"&&(jt.innerHTML=Fr),vt.forEach(u),ro=o(l),Z=d(l,"DIV",{class:!0});var Y=v(Z);h(Be.$$.fragment,Y),so=o(Y),qt=d(Y,"P",{"data-svelte-h":!0}),p(qt)!=="svelte-gtzyvb"&&(qt.textContent=Gr),ao=o(Y),It=d(Y,"P",{"data-svelte-h":!0}),p(It)!=="svelte-1gay38a"&&(It.innerHTML=Yr),Y.forEach(u),io=o(l),re=d(l,"DIV",{class:!0});var xt=v(re);h(We.$$.fragment,xt),lo=o(xt),Jt=d(xt,"P",{"data-svelte-h":!0}),p(Jt)!=="svelte-19nemma"&&(Jt.innerHTML=Rr),xt.forEach(u),co=o(l),se=d(l,"DIV",{class:!0});var bt=v(se);h(Ze.$$.fragment,bt),po=o(bt),Ut=d(bt,"P",{"data-svelte-h":!0}),p(Ut)!=="svelte-1yiitxu"&&(Ut.innerHTML=Kr),bt.forEach(u),uo=o(l),L=d(l,"DIV",{class:!0});var R=v(L);h(Le.$$.fragment,R),mo=o(R),Et=d(R,"P",{"data-svelte-h":!0}),p(Et)!=="svelte-1hd7lgk"&&(Et.textContent=Or),ho=o(R),Dt=d(R,"P",{"data-svelte-h":!0}),p(Dt)!=="svelte-mi9sfj"&&(Dt.innerHTML=es),R.forEach(u),go=o(l),U=d(l,"DIV",{class:!0});var qe=v(U);h(Se.$$.fragment,qe),fo=o(qe),At=d(qe,"P",{"data-svelte-h":!0}),p(At)!=="svelte-nd99ct"&&(At.textContent=ts),ko=o(qe),Vt=d(qe,"P",{"data-svelte-h":!0}),p(Vt)!=="svelte-1m4l01u"&&(Vt.textContent=ns),_o=o(qe),Pt=d(qe,"P",{"data-svelte-h":!0}),p(Pt)!=="svelte-luq3fs"&&(Pt.textContent=os),qe.forEach(u),$o=o(l),S=d(l,"DIV",{class:!0});var In=v(S);h(Qe.$$.fragment,In),zo=o(In),Ht=d(In,"P",{"data-svelte-h":!0}),p(Ht)!=="svelte-6gm3at"&&(Ht.textContent=rs),To=o(In),Bt=d(In,"P",{"data-svelte-h":!0}),p(Bt)!=="svelte-a0i6ys"&&(Bt.textContent=ss),In.forEach(u),vo=o(l),ae=d(l,"DIV",{class:!0});var Dn=v(ae);h(Ne.$$.fragment,Dn),xo=o(Dn),Wt=d(Dn,"P",{"data-svelte-h":!0}),p(Wt)!=="svelte-1cyd6v4"&&(Wt.textContent=as),Dn.forEach(u),bo=o(l),q=d(l,"DIV",{class:!0});var X=v(q);h(Xe.$$.fragment,X),yo=o(X),Zt=d(X,"P",{"data-svelte-h":!0}),p(Zt)!=="svelte-2n9uve"&&(Zt.textContent=is),Mo=o(X),Lt=d(X,"P",{"data-svelte-h":!0}),p(Lt)!=="svelte-64cdxx"&&(Lt.textContent=ds),wo=o(X),St=d(X,"P",{"data-svelte-h":!0}),p(St)!=="svelte-11lpom8"&&(St.textContent=ls),Co=o(X),h(ie.$$.fragment,X),X.forEach(u),jo=o(l),I=d(l,"DIV",{class:!0});var F=v(I);h(Fe.$$.fragment,F),qo=o(F),Qt=d(F,"P",{"data-svelte-h":!0}),p(Qt)!=="svelte-1g96r8c"&&(Qt.textContent=cs),Io=o(F),Nt=d(F,"P",{"data-svelte-h":!0}),p(Nt)!=="svelte-1fy9f34"&&(Nt.textContent=ps),Jo=o(F),Xt=d(F,"P",{"data-svelte-h":!0}),p(Xt)!=="svelte-11lpom8"&&(Xt.textContent=us),Uo=o(F),h(de.$$.fragment,F),F.forEach(u),Eo=o(l),J=d(l,"DIV",{class:!0});var G=v(J);h(Ge.$$.fragment,G),Do=o(G),Ft=d(G,"P",{"data-svelte-h":!0}),p(Ft)!=="svelte-iggd9z"&&(Ft.textContent=ms),Ao=o(G),Gt=d(G,"P",{"data-svelte-h":!0}),p(Gt)!=="svelte-1lojrmd"&&(Gt.textContent=hs),Vo=o(G),Yt=d(G,"P",{"data-svelte-h":!0}),p(Yt)!=="svelte-11lpom8"&&(Yt.textContent=gs),Po=o(G),h(le.$$.fragment,G),G.forEach(u),Ho=o(l),Q=d(l,"DIV",{class:!0});var Jn=v(Q);h(Ye.$$.fragment,Jn),Bo=o(Jn),Rt=d(Jn,"P",{"data-svelte-h":!0}),p(Rt)!=="svelte-levlgd"&&(Rt.textContent=fs),Wo=o(Jn),Kt=d(Jn,"P",{"data-svelte-h":!0}),p(Kt)!=="svelte-1yx4e76"&&(Kt.textContent=ks),Jn.forEach(u),Zo=o(l),ce=d(l,"DIV",{class:!0});var An=v(ce);h(Re.$$.fragment,An),Lo=o(An),Ot=d(An,"P",{"data-svelte-h":!0}),p(Ot)!=="svelte-1cyd6v4"&&(Ot.textContent=_s),An.forEach(u),So=o(l),pe=d(l,"DIV",{class:!0});var Vn=v(pe);h(Ke.$$.fragment,Vn),Qo=o(Vn),en=d(Vn,"P",{"data-svelte-h":!0}),p(en)!=="svelte-1istcmb"&&(en.textContent=$s),Vn.forEach(u),No=o(l),ue=d(l,"DIV",{class:!0});var Pn=v(ue);h(Oe.$$.fragment,Pn),Xo=o(Pn),tn=d(Pn,"P",{"data-svelte-h":!0}),p(tn)!=="svelte-312cgu"&&(tn.textContent=zs),Pn.forEach(u),Fo=o(l),E=d(l,"DIV",{class:!0});var Ie=v(E);h(et.$$.fragment,Ie),Go=o(Ie),nn=d(Ie,"P",{"data-svelte-h":!0}),p(nn)!=="svelte-1prk1bw"&&(nn.textContent=Ts),Yo=o(Ie),on=d(Ie,"P",{"data-svelte-h":!0}),p(on)!=="svelte-11lpom8"&&(on.textContent=vs),Ro=o(Ie),h(me.$$.fragment,Ie),Ie.forEach(u),Ko=o(l),D=d(l,"DIV",{class:!0});var Je=v(D);h(tt.$$.fragment,Je),Oo=o(Je),rn=d(Je,"P",{"data-svelte-h":!0}),p(rn)!=="svelte-1m0dehc"&&(rn.innerHTML=xs),er=o(Je),sn=d(Je,"P",{"data-svelte-h":!0}),p(sn)!=="svelte-11lpom8"&&(sn.textContent=bs),tr=o(Je),h(he.$$.fragment,Je),Je.forEach(u),nr=o(l),A=d(l,"DIV",{class:!0});var Ue=v(A);h(nt.$$.fragment,Ue),or=o(Ue),an=d(Ue,"P",{"data-svelte-h":!0}),p(an)!=="svelte-romm74"&&(an.innerHTML=ys),rr=o(Ue),dn=d(Ue,"P",{"data-svelte-h":!0}),p(dn)!=="svelte-11lpom8"&&(dn.textContent=Ms),sr=o(Ue),h(ge.$$.fragment,Ue),Ue.forEach(u),ar=o(l),fe=d(l,"DIV",{class:!0});var Hn=v(fe);h(ot.$$.fragment,Hn),ir=o(Hn),ln=d(Hn,"P",{"data-svelte-h":!0}),p(ln)!=="svelte-lccmsr"&&(ln.innerHTML=ws),Hn.forEach(u),dr=o(l),ke=d(l,"DIV",{class:!0});var Bn=v(ke);h(rt.$$.fragment,Bn),lr=o(Bn),cn=d(Bn,"P",{"data-svelte-h":!0}),p(cn)!=="svelte-1d7067i"&&(cn.innerHTML=Cs),Bn.forEach(u),cr=o(l),_e=d(l,"DIV",{class:!0});var Wn=v(_e);h(st.$$.fragment,Wn),pr=o(Wn),pn=d(Wn,"P",{"data-svelte-h":!0}),p(pn)!=="svelte-nnzfmy"&&(pn.innerHTML=js),Wn.forEach(u),ur=o(l),$e=d(l,"DIV",{class:!0});var Zn=v($e);h(at.$$.fragment,Zn),mr=o(Zn),un=d(Zn,"P",{"data-svelte-h":!0}),p(un)!=="svelte-ofqao0"&&(un.innerHTML=qs),Zn.forEach(u),hr=o(l),ze=d(l,"DIV",{class:!0});var Ln=v(ze);h(it.$$.fragment,Ln),gr=o(Ln),mn=d(Ln,"P",{"data-svelte-h":!0}),p(mn)!=="svelte-1cm0zfu"&&(mn.textContent=Is),Ln.forEach(u),fr=o(l),Te=d(l,"DIV",{class:!0});var Sn=v(Te);h(dt.$$.fragment,Sn),kr=o(Sn),hn=d(Sn,"P",{"data-svelte-h":!0}),p(hn)!=="svelte-1cm0zfu"&&(hn.textContent=Js),Sn.forEach(u),_r=o(l),ve=d(l,"DIV",{class:!0});var Qn=v(ve);h(lt.$$.fragment,Qn),$r=o(Qn),gn=d(Qn,"P",{"data-svelte-h":!0}),p(gn)!=="svelte-1qhxekf"&&(gn.textContent=Us),Qn.forEach(u),zr=o(l),xe=d(l,"DIV",{class:!0});var Nn=v(xe);h(ct.$$.fragment,Nn),Tr=o(Nn),fn=d(Nn,"P",{"data-svelte-h":!0}),p(fn)!=="svelte-rcsbf7"&&(fn.textContent=Es),Nn.forEach(u),vr=o(l),be=d(l,"DIV",{class:!0});var Xn=v(be);h(pt.$$.fragment,Xn),xr=o(Xn),kn=d(Xn,"P",{"data-svelte-h":!0}),p(kn)!=="svelte-4tet3p"&&(kn.textContent=Ds),Xn.forEach(u),br=o(l),ye=d(l,"DIV",{class:!0});var Fn=v(ye);h(ut.$$.fragment,Fn),yr=o(Fn),_n=d(Fn,"P",{"data-svelte-h":!0}),p(_n)!=="svelte-po2r3l"&&(_n.textContent=As),Fn.forEach(u),Mr=o(l),Me=d(l,"DIV",{class:!0});var Gn=v(Me);h(mt.$$.fragment,Gn),wr=o(Gn),$n=d(Gn,"P",{"data-svelte-h":!0}),p($n)!=="svelte-mtgjrv"&&($n.textContent=Vs),Gn.forEach(u),Cr=o(l),V=d(l,"DIV",{class:!0});var Ee=v(V);h(ht.$$.fragment,Ee),jr=o(Ee),zn=d(Ee,"P",{"data-svelte-h":!0}),p(zn)!=="svelte-1gcsw4k"&&(zn.textContent=Ps),qr=o(Ee),Tn=d(Ee,"P",{"data-svelte-h":!0}),p(Tn)!=="svelte-nfoxmp"&&(Tn.textContent=Hs),Ir=o(Ee),vn=d(Ee,"OL",{"data-svelte-h":!0}),p(vn)!=="svelte-pkckkg"&&(vn.innerHTML=Bs),Ee.forEach(u),Jr=o(l),we=d(l,"DIV",{class:!0});var Yn=v(we);h(gt.$$.fragment,Yn),Ur=o(Yn),xn=d(Yn,"P",{"data-svelte-h":!0}),p(xn)!=="svelte-156j6bq"&&(xn.innerHTML=Ws),Yn.forEach(u),Er=o(l),Ce=d(l,"DIV",{class:!0});var Rn=v(Ce);h(ft.$$.fragment,Rn),Dr=o(Rn),bn=d(Rn,"P",{"data-svelte-h":!0}),p(bn)!=="svelte-h7c9fx"&&(bn.innerHTML=Zs),Rn.forEach(u),Ar=o(l),je=d(l,"DIV",{class:!0});var Kn=v(je);h(kt.$$.fragment,Kn),Vr=o(Kn),yn=d(Kn,"P",{"data-svelte-h":!0}),p(yn)!=="svelte-1ncl685"&&(yn.textContent=Ls),Kn.forEach(u),Pr=o(l),N=d(l,"DIV",{class:!0});var Un=v(N);h(_t.$$.fragment,Un),Hr=o(Un),Mn=d(Un,"P",{"data-svelte-h":!0}),p(Mn)!=="svelte-181dzjr"&&(Mn.textContent=Ss),Br=o(Un),wn=d(Un,"P",{"data-svelte-h":!0}),p(wn)!=="svelte-15uw3am"&&(wn.innerHTML=Qs),Un.forEach(u),Wr=o(l),P=d(l,"DIV",{class:!0});var De=v(P);h($t.$$.fragment,De),Zr=o(De),Cn=d(De,"P",{"data-svelte-h":!0}),p(Cn)!=="svelte-1nul94m"&&(Cn.textContent=Ns),Lr=o(De),jn=d(De,"P",{"data-svelte-h":!0}),p(jn)!=="svelte-b4tu8e"&&(jn.textContent=Xs),Sr=o(De),qn=d(De,"UL",{"data-svelte-h":!0}),p(qn)!=="svelte-nlm24s"&&(qn.innerHTML=Fs),De.forEach(u),l.forEach(u),this.h()},h(){x(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(oe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(re,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ae,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ce,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(fe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ke,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(_e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ze,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(Te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(Me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(we,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(Ce,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(je,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(t,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(a,l){g(r,a,l),w(a,T,l),w(a,t,l),g(c,t,null),e(t,$),e(t,s),e(t,O),e(t,j),e(t,H),e(t,B),e(t,te),g(z,t,null),e(t,C),e(t,W),g(Ve,W,null),e(W,On),e(W,wt),e(t,eo),e(t,ne),g(Pe,ne,null),e(ne,to),e(ne,Ct),e(t,no),e(t,oe),g(He,oe,null),e(oe,oo),e(oe,jt),e(t,ro),e(t,Z),g(Be,Z,null),e(Z,so),e(Z,qt),e(Z,ao),e(Z,It),e(t,io),e(t,re),g(We,re,null),e(re,lo),e(re,Jt),e(t,co),e(t,se),g(Ze,se,null),e(se,po),e(se,Ut),e(t,uo),e(t,L),g(Le,L,null),e(L,mo),e(L,Et),e(L,ho),e(L,Dt),e(t,go),e(t,U),g(Se,U,null),e(U,fo),e(U,At),e(U,ko),e(U,Vt),e(U,_o),e(U,Pt),e(t,$o),e(t,S),g(Qe,S,null),e(S,zo),e(S,Ht),e(S,To),e(S,Bt),e(t,vo),e(t,ae),g(Ne,ae,null),e(ae,xo),e(ae,Wt),e(t,bo),e(t,q),g(Xe,q,null),e(q,yo),e(q,Zt),e(q,Mo),e(q,Lt),e(q,wo),e(q,St),e(q,Co),g(ie,q,null),e(t,jo),e(t,I),g(Fe,I,null),e(I,qo),e(I,Qt),e(I,Io),e(I,Nt),e(I,Jo),e(I,Xt),e(I,Uo),g(de,I,null),e(t,Eo),e(t,J),g(Ge,J,null),e(J,Do),e(J,Ft),e(J,Ao),e(J,Gt),e(J,Vo),e(J,Yt),e(J,Po),g(le,J,null),e(t,Ho),e(t,Q),g(Ye,Q,null),e(Q,Bo),e(Q,Rt),e(Q,Wo),e(Q,Kt),e(t,Zo),e(t,ce),g(Re,ce,null),e(ce,Lo),e(ce,Ot),e(t,So),e(t,pe),g(Ke,pe,null),e(pe,Qo),e(pe,en),e(t,No),e(t,ue),g(Oe,ue,null),e(ue,Xo),e(ue,tn),e(t,Fo),e(t,E),g(et,E,null),e(E,Go),e(E,nn),e(E,Yo),e(E,on),e(E,Ro),g(me,E,null),e(t,Ko),e(t,D),g(tt,D,null),e(D,Oo),e(D,rn),e(D,er),e(D,sn),e(D,tr),g(he,D,null),e(t,nr),e(t,A),g(nt,A,null),e(A,or),e(A,an),e(A,rr),e(A,dn),e(A,sr),g(ge,A,null),e(t,ar),e(t,fe),g(ot,fe,null),e(fe,ir),e(fe,ln),e(t,dr),e(t,ke),g(rt,ke,null),e(ke,lr),e(ke,cn),e(t,cr),e(t,_e),g(st,_e,null),e(_e,pr),e(_e,pn),e(t,ur),e(t,$e),g(at,$e,null),e($e,mr),e($e,un),e(t,hr),e(t,ze),g(it,ze,null),e(ze,gr),e(ze,mn),e(t,fr),e(t,Te),g(dt,Te,null),e(Te,kr),e(Te,hn),e(t,_r),e(t,ve),g(lt,ve,null),e(ve,$r),e(ve,gn),e(t,zr),e(t,xe),g(ct,xe,null),e(xe,Tr),e(xe,fn),e(t,vr),e(t,be),g(pt,be,null),e(be,xr),e(be,kn),e(t,br),e(t,ye),g(ut,ye,null),e(ye,yr),e(ye,_n),e(t,Mr),e(t,Me),g(mt,Me,null),e(Me,wr),e(Me,$n),e(t,Cr),e(t,V),g(ht,V,null),e(V,jr),e(V,zn),e(V,qr),e(V,Tn),e(V,Ir),e(V,vn),e(t,Jr),e(t,we),g(gt,we,null),e(we,Ur),e(we,xn),e(t,Er),e(t,Ce),g(ft,Ce,null),e(Ce,Dr),e(Ce,bn),e(t,Ar),e(t,je),g(kt,je,null),e(je,Vr),e(je,yn),e(t,Pr),e(t,N),g(_t,N,null),e(N,Hr),e(N,Mn),e(N,Br),e(N,wn),e(t,Wr),e(t,P),g($t,P,null),e(P,Zr),e(P,Cn),e(P,Lr),e(P,jn),e(P,Sr),e(P,qn),En=!0},p(a,l){const zt={};l&2&&(zt.$$scope={dirty:l,ctx:a}),z.$set(zt);const Tt={};l&2&&(Tt.$$scope={dirty:l,ctx:a}),ie.$set(Tt);const vt={};l&2&&(vt.$$scope={dirty:l,ctx:a}),de.$set(vt);const Y={};l&2&&(Y.$$scope={dirty:l,ctx:a}),le.$set(Y);const xt={};l&2&&(xt.$$scope={dirty:l,ctx:a}),me.$set(xt);const bt={};l&2&&(bt.$$scope={dirty:l,ctx:a}),he.$set(bt);const R={};l&2&&(R.$$scope={dirty:l,ctx:a}),ge.$set(R)},i(a){En||(f(r.$$.fragment,a),f(c.$$.fragment,a),f(z.$$.fragment,a),f(Ve.$$.fragment,a),f(Pe.$$.fragment,a),f(He.$$.fragment,a),f(Be.$$.fragment,a),f(We.$$.fragment,a),f(Ze.$$.fragment,a),f(Le.$$.fragment,a),f(Se.$$.fragment,a),f(Qe.$$.fragment,a),f(Ne.$$.fragment,a),f(Xe.$$.fragment,a),f(ie.$$.fragment,a),f(Fe.$$.fragment,a),f(de.$$.fragment,a),f(Ge.$$.fragment,a),f(le.$$.fragment,a),f(Ye.$$.fragment,a),f(Re.$$.fragment,a),f(Ke.$$.fragment,a),f(Oe.$$.fragment,a),f(et.$$.fragment,a),f(me.$$.fragment,a),f(tt.$$.fragment,a),f(he.$$.fragment,a),f(nt.$$.fragment,a),f(ge.$$.fragment,a),f(ot.$$.fragment,a),f(rt.$$.fragment,a),f(st.$$.fragment,a),f(at.$$.fragment,a),f(it.$$.fragment,a),f(dt.$$.fragment,a),f(lt.$$.fragment,a),f(ct.$$.fragment,a),f(pt.$$.fragment,a),f(ut.$$.fragment,a),f(mt.$$.fragment,a),f(ht.$$.fragment,a),f(gt.$$.fragment,a),f(ft.$$.fragment,a),f(kt.$$.fragment,a),f(_t.$$.fragment,a),f($t.$$.fragment,a),En=!0)},o(a){k(r.$$.fragment,a),k(c.$$.fragment,a),k(z.$$.fragment,a),k(Ve.$$.fragment,a),k(Pe.$$.fragment,a),k(He.$$.fragment,a),k(Be.$$.fragment,a),k(We.$$.fragment,a),k(Ze.$$.fragment,a),k(Le.$$.fragment,a),k(Se.$$.fragment,a),k(Qe.$$.fragment,a),k(Ne.$$.fragment,a),k(Xe.$$.fragment,a),k(ie.$$.fragment,a),k(Fe.$$.fragment,a),k(de.$$.fragment,a),k(Ge.$$.fragment,a),k(le.$$.fragment,a),k(Ye.$$.fragment,a),k(Re.$$.fragment,a),k(Ke.$$.fragment,a),k(Oe.$$.fragment,a),k(et.$$.fragment,a),k(me.$$.fragment,a),k(tt.$$.fragment,a),k(he.$$.fragment,a),k(nt.$$.fragment,a),k(ge.$$.fragment,a),k(ot.$$.fragment,a),k(rt.$$.fragment,a),k(st.$$.fragment,a),k(at.$$.fragment,a),k(it.$$.fragment,a),k(dt.$$.fragment,a),k(lt.$$.fragment,a),k(ct.$$.fragment,a),k(pt.$$.fragment,a),k(ut.$$.fragment,a),k(mt.$$.fragment,a),k(ht.$$.fragment,a),k(gt.$$.fragment,a),k(ft.$$.fragment,a),k(kt.$$.fragment,a),k(_t.$$.fragment,a),k($t.$$.fragment,a),En=!1},d(a){a&&(u(T),u(t)),_(r,a),_(c),_(z),_(Ve),_(Pe),_(He),_(Be),_(We),_(Ze),_(Le),_(Se),_(Qe),_(Ne),_(Xe),_(ie),_(Fe),_(de),_(Ge),_(le),_(Ye),_(Re),_(Ke),_(Oe),_(et),_(me),_(tt),_(he),_(nt),_(ge),_(ot),_(rt),_(st),_(at),_(it),_(dt),_(lt),_(ct),_(pt),_(ut),_(mt),_(ht),_(gt),_(ft),_(kt),_(_t),_($t)}}}function ua(M){let r,T;return r=new Qr({props:{$$slots:{default:[pa]},$$scope:{ctx:M}}}),{c(){m(r.$$.fragment)},l(t){h(r.$$.fragment,t)},m(t,c){g(r,t,c),T=!0},p(t,c){const $={};c&2&&($.$$scope={dirty:c,ctx:t}),r.$set($)},i(t){T||(f(r.$$.fragment,t),T=!0)},o(t){k(r.$$.fragment,t),T=!1},d(t){_(r,t)}}}function ma(M){let r,T='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){r=i("p"),r.innerHTML=T},l(t){r=d(t,"P",{"data-svelte-h":!0}),p(r)!=="svelte-4ytcyb"&&(r.innerHTML=T)},m(t,c){w(t,r,c)},p:K,d(t){t&&u(r)}}}function ha(M){let r,T;return r=new Qr({props:{$$slots:{default:[ma]},$$scope:{ctx:M}}}),{c(){m(r.$$.fragment)},l(t){h(r.$$.fragment,t)},m(t,c){g(r,t,c),T=!0},p(t,c){const $={};c&2&&($.$$scope={dirty:c,ctx:t}),r.$set($)},i(t){T||(f(r.$$.fragment,t),T=!0)},o(t){k(r.$$.fragment,t),T=!1},d(t){_(r,t)}}}function ga(M){let r,T="The node API has not been documented yet.";return{c(){r=i("p"),r.textContent=T},l(t){r=d(t,"P",{"data-svelte-h":!0}),p(r)!=="svelte-1mrchm6"&&(r.textContent=T)},m(t,c){w(t,r,c)},p:K,d(t){t&&u(r)}}}function fa(M){let r,T;return r=new Qr({props:{$$slots:{default:[ga]},$$scope:{ctx:M}}}),{c(){m(r.$$.fragment)},l(t){h(r.$$.fragment,t)},m(t,c){g(r,t,c),T=!0},p(t,c){const $={};c&2&&($.$$scope={dirty:c,ctx:t}),r.$set($)},i(t){T||(f(r.$$.fragment,t),T=!0)},o(t){k(r.$$.fragment,t),T=!1},d(t){_(r,t)}}}function ka(M){let r,T,t,c,$,s,b,O,j,Ae,H,B,ee,te;return $=new ta({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),b=new Gs({props:{title:"Tokenizer",local:"tokenizer",headingTag:"h1"}}),j=new oa({props:{python:!0,rust:!0,node:!0,$$slots:{node:[fa],rust:[ha],python:[ua]},$$scope:{ctx:M}}}),H=new na({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/tokenizer.mdx"}}),{c(){r=i("meta"),T=n(),t=i("p"),c=n(),m($.$$.fragment),s=n(),m(b.$$.fragment),O=n(),m(j.$$.fragment),Ae=n(),m(H.$$.fragment),B=n(),ee=i("p"),this.h()},l(z){const C=ea("svelte-u9bgzb",document.head);r=d(C,"META",{name:!0,content:!0}),C.forEach(u),T=o(z),t=d(z,"P",{}),v(t).forEach(u),c=o(z),h($.$$.fragment,z),s=o(z),h(b.$$.fragment,z),O=o(z),h(j.$$.fragment,z),Ae=o(z),h(H.$$.fragment,z),B=o(z),ee=d(z,"P",{}),v(ee).forEach(u),this.h()},h(){x(r,"name","hf:doc:metadata"),x(r,"content",_a)},m(z,C){e(document.head,r),w(z,T,C),w(z,t,C),w(z,c,C),g($,z,C),w(z,s,C),g(b,z,C),w(z,O,C),g(j,z,C),w(z,Ae,C),g(H,z,C),w(z,B,C),w(z,ee,C),te=!0},p(z,[C]){const W={};C&2&&(W.$$scope={dirty:C,ctx:z}),j.$set(W)},i(z){te||(f($.$$.fragment,z),f(b.$$.fragment,z),f(j.$$.fragment,z),f(H.$$.fragment,z),te=!0)},o(z){k($.$$.fragment,z),k(b.$$.fragment,z),k(j.$$.fragment,z),k(H.$$.fragment,z),te=!1},d(z){z&&(u(T),u(t),u(c),u(s),u(O),u(Ae),u(B),u(ee)),u(r),_($,z),_(b,z),_(j,z),_(H,z)}}}const _a='{"title":"Tokenizer","local":"tokenizer","sections":[{"title":"Tokenizer","local":"tokenizers.Tokenizer","sections":[],"depth":2}],"depth":1}';function $a(M){return Rs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class wa extends Ks{constructor(r){super(),Os(this,r,$a,ka,Ys,{})}}export{wa as component};