tokenizers/pr_2049/en/_app/immutable/nodes/5.3ffa942e.js

import{s as Mr,o as Pr,n as It}from"../chunks/scheduler.7c59faff.js";import{S as Lr,i as Ir,e as s,s as r,c as h,h as jr,a as i,d as l,b as o,f as b,g,j as T,k as e,l as q,m as f,t as x,n as $,o as _,p}from"../chunks/index.09bb5655.js";import{T as Hr}from"../chunks/Tip.25c348e8.js";import{C as Vr,H as Dr,E as Gr}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.7f4075ae.js";import{D as w}from"../chunks/Docstring.0f892337.js";import{C as Sr}from"../chunks/CodeBlock.61865591.js";import{T as Zr,M as Wn}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";import{E as Ar}from"../chunks/ExampleCodeBlock.66eafe9b.js";function Ur(E){let n,m="Example:",t,u,k;return u=new Sr({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmJlcnQtYmFzZS11bmNhc2VkJTIyKSUwQWVuY29kaW5nJTIwJTNEJTIwdG9rZW5pemVyLmVuY29kZSglMjJIZWxsbyUyQyUyMHdvcmxkISUyMiklMEFlbmNvZGluZy5pZHMlMEFlbmNvZGluZy50b2tlbnMlMEFlbmNvZGluZy5vZmZzZXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>encoding = tokenizer.encode(<span class="hljs-string">&quot;Hello, world!&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>encoding.ids
[<span class="hljs-number">101</span>, <span class="hljs-number">7592</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2088</span>, <span class="hljs-number">999</span>, <span class="hljs-number">102</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoding.tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;hello&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;world&#x27;</span>, <span class="hljs-string">&#x27;!&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoding.offsets
[(<span class="hljs-number">0</span>, <span class="hljs-number">0</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>), (<span class="hljs-number">5</span>, <span class="hljs-number">6</span>), (<span class="hljs-number">7</span>, <span class="hljs-number">12</span>), (<span class="hljs-number">12</span>, <span class="hljs-number">13</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">0</span>)]`,lang:"python",wrap:!1}}),{c(){n=s("p"),n.textContent=m,t=r(),h(u.$$.fragment)},l(v){n=i(v,"P",{"data-svelte-h":!0}),p(n)!=="svelte-11lpom8"&&(n.textContent=m),t=o(v),g(u.$$.fragment,v)},m(v,D){q(v,n,D),q(v,t,D),f(u,v,D),k=!0},p:It,i(v){k||(x(u.$$.fragment,v),k=!0)},o(v){$(u.$$.fragment,v),k=!1},d(v){v&&(l(n),l(t)),_(u,v)}}}function Wr(E){let n,m=`This is deprecated and will be removed in a future version.
Please use <code>~tokenizers.Encoding.word_ids</code> instead.`;return{c(){n=s("p"),n.innerHTML=m},l(t){n=i(t,"P",{"data-svelte-h":!0}),p(n)!=="svelte-1tdbt0g"&&(n.innerHTML=m)},m(t,u){q(t,n,u)},p:It,d(t){t&&l(n)}}}function Fr(E){let n,m,t,u,k,v,D='The <a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a> represents the output of a <a href="/docs/tokenizers/pr_2049/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>.',Y,C,ce=`It holds all the information about the tokenized input, including the token IDs,
token strings, attention masks, offsets, and more. This is the main data structure
returned by <code>encode()</code> and
<code>encode_batch()</code>.`,j,P,X,y,c,z,Q,Fn="The attention mask",jt,Me,Jn=`This indicates to the LM which tokens should be attended to, and which should not.
This is especially important when batching sequences, where we need to applying
padding.`,Ht,H,pe,Vt,Pe,Nn="The generated IDs",Gt,Le,Bn=`The IDs are the main input to a Language Model. They are the token indices,
the numerical representations that a LM understands.`,St,K,le,Zt,Ie,Xn="The number of sequences represented",At,V,ue,Ut,je,Qn="The offsets associated to each token",Wt,He,Rn=`These offsets let’s you slice the input string, and thus retrieve the original
part that led to producing the corresponding token.`,Ft,L,me,Jt,Ve,On='A <code>List</code> of overflowing <a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a>',Nt,Ge,Yn=`When using truncation, the <a href="/docs/tokenizers/pr_2049/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> takes care of splitting
the output into as many pieces as required to match the specified maximum length.
This field lets you retrieve all the subsequent pieces.`,Bt,Se,Kn=`When you use pairs of sequences, the overflowing pieces will contain enough
variations to cover all the possible combinations, while respecting the provided
maximum length.`,Xt,G,he,Qt,Ze,er="The generated sequence indices.",Rt,Ae,tr=`They represent the index of the input sequence associated to each token.
The sequence id can be None if the token is not related to any input sequence,
like for example with special tokens.`,Ot,S,ge,Yt,Ue,nr="The special token mask",Kt,We,rr="This indicates which tokens are special tokens, and which are not.",en,Z,fe,tn,Fe,or="The generated tokens",nn,Je,sr="They are the string representation of the IDs.",rn,A,xe,on,Ne,ir="The generated type IDs",sn,Be,ar=`Generally used for tasks like sequence classification or question answering,
these tokens let the LM know which input sequence corresponds to each tokens.`,an,I,$e,dn,Xe,dr="The generated word indices.",cn,Qe,cr=`They represent the index of the word associated to each token.
When the input is pre-tokenized, they correspond to the ID of the given input label,
otherwise they correspond to the words indices as defined by the
<a href="/docs/tokenizers/pr_2049/en/api/pre-tokenizers#tokenizers.pre_tokenizers.PreTokenizer">PreTokenizer</a> that was used.`,pn,Re,pr=`For special tokens and such (any token that was generated from something that was
not part of the input), the output is <code>None</code>`,ln,M,_e,un,Oe,lr="The generated word indices.",mn,ee,hn,Ye,ur=`They represent the index of the word associated to each token.
When the input is pre-tokenized, they correspond to the ID of the given input label,
otherwise they correspond to the words indices as defined by the
<a href="/docs/tokenizers/pr_2049/en/api/pre-tokenizers#tokenizers.pre_tokenizers.PreTokenizer">PreTokenizer</a> that was used.`,gn,Ke,mr=`For special tokens and such (any token that was generated from something that was
not part of the input), the output is <code>None</code>`,fn,te,ve,xn,et,hr="Get the token that contains the char at the given position in the input sequence.",$n,ne,ke,_n,tt,gr="Get the word that contains the char at the given position in the input sequence.",vn,re,be,kn,nt,fr='Merge the list of encodings into one final <a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a>',bn,oe,Te,Tn,rt,xr='Pad the <a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a> at the given length',wn,U,we,zn,ot,$r="Set the given sequence index",En,st,_r=`Set the given sequence index for the whole range of tokens contained in this
<a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a>.`,Cn,W,ze,yn,it,vr="Get the offsets of the token at the given index.",qn,at,kr=`The returned offsets are related to the input sequence that contains the
token.  In order to determine in which input sequence it belongs, you
must call <code>~tokenizers.Encoding.token_to_sequence()</code>.`,Dn,F,Ee,Mn,dt,br="Get the index of the sequence represented by the given token.",Pn,ct,Tr=`In the general use case, this method returns <code>0</code> for a single sequence or
the first sequence of a pair, and <code>1</code> for the second sequence of a pair`,Ln,J,Ce,In,pt,wr="Get the index of the word that contains the token in one of the input sequences.",jn,lt,zr=`The returned word index is related to the input sequence that contains
the token.  In order to determine in which input sequence it belongs, you
must call <code>~tokenizers.Encoding.token_to_sequence()</code>.`,Hn,N,ye,Vn,ut,Er='Truncate the <a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a> at the given length',Gn,mt,Cr=`If this <a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a> represents multiple sequences, when truncating
this information is lost. It will be considered as representing a single sequence.`,Sn,se,qe,Zn,ht,yr="Get the offsets of the word at the given index in one of the input sequences.",An,ie,De,Un,gt,qr=`Get the encoded tokens corresponding to the word at the given index
in one of the input sequences.`,Et;return n=new Dr({props:{title:"Encoding",local:"tokenizers.Encoding",headingTag:"h2"}}),u=new w({props:{name:"class tokenizers.Encoding",anchor:"tokenizers.Encoding",parameters:[]}}),P=new Ar({props:{anchor:"tokenizers.Encoding.example",$$slots:{default:[Ur]},$$scope:{ctx:E}}}),c=new w({props:{name:"attention_mask",anchor:"tokenizers.Encoding.attention_mask",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The attention mask</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`,isGetSetDescriptor:!0}}),pe=new w({props:{name:"ids",anchor:"tokenizers.Encoding.ids",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The list of IDs</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`,isGetSetDescriptor:!0}}),le=new w({props:{name:"n_sequences",anchor:"tokenizers.Encoding.n_sequences",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The number of sequences in this <a
  href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding"
>Encoding</a></p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`,isGetSetDescriptor:!0}}),ue=new w({props:{name:"offsets",anchor:"tokenizers.Encoding.offsets",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The list of offsets</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of <code>Tuple[int, int]</code></p>
`,isGetSetDescriptor:!0}}),me=new w({props:{name:"overflowing",anchor:"tokenizers.Encoding.overflowing",parameters:[],isGetSetDescriptor:!0}}),he=new w({props:{name:"sequence_ids",anchor:"tokenizers.Encoding.sequence_ids",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of optional sequence index.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of <code>Optional[int]</code></p>
`,isGetSetDescriptor:!0}}),ge=new w({props:{name:"special_tokens_mask",anchor:"tokenizers.Encoding.special_tokens_mask",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The special tokens mask</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`,isGetSetDescriptor:!0}}),fe=new w({props:{name:"tokens",anchor:"tokenizers.Encoding.tokens",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The list of tokens</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[str]</code></p>
`,isGetSetDescriptor:!0}}),xe=new w({props:{name:"type_ids",anchor:"tokenizers.Encoding.type_ids",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The list of type ids</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`,isGetSetDescriptor:!0}}),$e=new w({props:{name:"word_ids",anchor:"tokenizers.Encoding.word_ids",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of optional word index.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of <code>Optional[int]</code></p>
`,isGetSetDescriptor:!0}}),_e=new w({props:{name:"words",anchor:"tokenizers.Encoding.words",parameters:[],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of optional word index.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>List</code> of <code>Optional[int]</code></p>
`,isGetSetDescriptor:!0}}),ee=new Hr({props:{warning:!0,$$slots:{default:[Wr]},$$scope:{ctx:E}}}),ve=new w({props:{name:"char_to_token",anchor:"tokenizers.Encoding.char_to_token",parameters:[{name:"char_pos",val:""},{name:"sequence_index",val:" = 0"}],parametersDescription:[{anchor:"tokenizers.Encoding.char_to_token.char_pos",description:`<strong>char_pos</strong> (<code>int</code>) &#x2014;
The position of a char in the input string`,name:"char_pos"},{anchor:"tokenizers.Encoding.char_to_token.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The index of the sequence that contains the target char`,name:"sequence_index"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The index of the token that contains this char in the encoded sequence</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`}}),ke=new w({props:{name:"char_to_word",anchor:"tokenizers.Encoding.char_to_word",parameters:[{name:"char_pos",val:""},{name:"sequence_index",val:" = 0"}],parametersDescription:[{anchor:"tokenizers.Encoding.char_to_word.char_pos",description:`<strong>char_pos</strong> (<code>int</code>) &#x2014;
The position of a char in the input string`,name:"char_pos"},{anchor:"tokenizers.Encoding.char_to_word.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The index of the sequence that contains the target char`,name:"sequence_index"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The index of the word that contains this char in the input sequence</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`}}),be=new w({props:{name:"merge",anchor:"tokenizers.Encoding.merge",parameters:[{name:"encodings",val:""},{name:"growing_offsets",val:" = True"}],parametersDescription:[{anchor:"tokenizers.Encoding.merge.encodings",description:`<strong>encodings</strong> (A <code>List</code> of <a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a>) &#x2014;
The list of encodings that should be merged in one`,name:"encodings"},{anchor:"tokenizers.Encoding.merge.growing_offsets",description:`<strong>growing_offsets</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the offsets should accumulate while merging`,name:"growing_offsets"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The resulting Encoding</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a href="/docs/tokenizers/pr_2049/en/api/encoding#tokenizers.Encoding">Encoding</a></p>
`}}),Te=new w({props:{name:"pad",anchor:"tokenizers.Encoding.pad",parameters:[{name:"length",val:""},{name:"direction",val:" = 'right'"},{name:"pad_id",val:" = 0"},{name:"pad_type_id",val:" = 0"},{name:"pad_token",val:" = '[PAD]'"}],parametersDescription:[{anchor:"tokenizers.Encoding.pad.length",description:`<strong>length</strong> (<code>int</code>) &#x2014;
The desired length`,name:"length"},{anchor:"tokenizers.Encoding.pad.direction",description:`<strong>direction</strong> &#x2014; (<code>str</code>, defaults to <code>right</code>):
The expected padding direction. Can be either <code>right</code> or <code>left</code>`,name:"direction"},{anchor:"tokenizers.Encoding.pad.pad_id",description:`<strong>pad_id</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The ID corresponding to the padding token`,name:"pad_id"},{anchor:"tokenizers.Encoding.pad.pad_type_id",description:`<strong>pad_type_id</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The type ID corresponding to the padding token`,name:"pad_type_id"},{anchor:"tokenizers.Encoding.pad.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, defaults to <em>[PAD]</em>) &#x2014;
The pad token to use`,name:"pad_token"}]}}),we=new w({props:{name:"set_sequence_id",anchor:"tokenizers.Encoding.set_sequence_id",parameters:[{name:"sequence_id",val:""}]}}),ze=new w({props:{name:"token_to_chars",anchor:"tokenizers.Encoding.token_to_chars",parameters:[{name:"token_index",val:""}],parametersDescription:[{anchor:"tokenizers.Encoding.token_to_chars.token_index",description:`<strong>token_index</strong> (<code>int</code>) &#x2014;
The index of a token in the encoded sequence.`,name:"token_index"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The token offsets <code>(first, last + 1)</code></p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[int, int]</code></p>
`}}),Ee=new w({props:{name:"token_to_sequence",anchor:"tokenizers.Encoding.token_to_sequence",parameters:[{name:"token_index",val:""}],parametersDescription:[{anchor:"tokenizers.Encoding.token_to_sequence.token_index",description:`<strong>token_index</strong> (<code>int</code>) &#x2014;
The index of a token in the encoded sequence.`,name:"token_index"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The sequence id of the given token</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`}}),Ce=new w({props:{name:"token_to_word",anchor:"tokenizers.Encoding.token_to_word",parameters:[{name:"token_index",val:""}],parametersDescription:[{anchor:"tokenizers.Encoding.token_to_word.token_index",description:`<strong>token_index</strong> (<code>int</code>) &#x2014;
The index of a token in the encoded sequence.`,name:"token_index"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The index of the word in the relevant input sequence.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>int</code></p>
`}}),ye=new w({props:{name:"truncate",anchor:"tokenizers.Encoding.truncate",parameters:[{name:"max_length",val:""},{name:"stride",val:" = 0"},{name:"direction",val:" = 'right'"}],parametersDescription:[{anchor:"tokenizers.Encoding.truncate.max_length",description:`<strong>max_length</strong> (<code>int</code>) &#x2014;
The desired length`,name:"max_length"},{anchor:"tokenizers.Encoding.truncate.stride",description:`<strong>stride</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The length of previous content to be included in each overflowing piece`,name:"stride"},{anchor:"tokenizers.Encoding.truncate.direction",description:`<strong>direction</strong> (<code>str</code>, defaults to <code>right</code>) &#x2014;
Truncate direction`,name:"direction"}]}}),qe=new w({props:{name:"word_to_chars",anchor:"tokenizers.Encoding.word_to_chars",parameters:[{name:"word_index",val:""},{name:"sequence_index",val:" = 0"}],parametersDescription:[{anchor:"tokenizers.Encoding.word_to_chars.word_index",description:`<strong>word_index</strong> (<code>int</code>) &#x2014;
The index of a word in one of the input sequences.`,name:"word_index"},{anchor:"tokenizers.Encoding.word_to_chars.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The index of the sequence that contains the target word`,name:"sequence_index"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The range of characters (span) <code>(first, last + 1)</code></p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[int, int]</code></p>
`}}),De=new w({props:{name:"word_to_tokens",anchor:"tokenizers.Encoding.word_to_tokens",parameters:[{name:"word_index",val:""},{name:"sequence_index",val:" = 0"}],parametersDescription:[{anchor:"tokenizers.Encoding.word_to_tokens.word_index",description:`<strong>word_index</strong> (<code>int</code>) &#x2014;
The index of a word in one of the input sequences.`,name:"word_index"},{anchor:"tokenizers.Encoding.word_to_tokens.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The index of the sequence that contains the target word`,name:"sequence_index"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The range of tokens: <code>(first, last + 1)</code></p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[int, int]</code></p>
`}}),{c(){h(n.$$.fragment),m=r(),t=s("div"),h(u.$$.fragment),k=r(),v=s("p"),v.innerHTML=D,Y=r(),C=s("p"),C.innerHTML=ce,j=r(),h(P.$$.fragment),X=r(),y=s("div"),h(c.$$.fragment),z=r(),Q=s("p"),Q.textContent=Fn,jt=r(),Me=s("p"),Me.textContent=Jn,Ht=r(),H=s("div"),h(pe.$$.fragment),Vt=r(),Pe=s("p"),Pe.textContent=Nn,Gt=r(),Le=s("p"),Le.textContent=Bn,St=r(),K=s("div"),h(le.$$.fragment),Zt=r(),Ie=s("p"),Ie.textContent=Xn,At=r(),V=s("div"),h(ue.$$.fragment),Ut=r(),je=s("p"),je.textContent=Qn,Wt=r(),He=s("p"),He.textContent=Rn,Ft=r(),L=s("div"),h(me.$$.fragment),Jt=r(),Ve=s("p"),Ve.innerHTML=On,Nt=r(),Ge=s("p"),Ge.innerHTML=Yn,Bt=r(),Se=s("p"),Se.textContent=Kn,Xt=r(),G=s("div"),h(he.$$.fragment),Qt=r(),Ze=s("p"),Ze.textContent=er,Rt=r(),Ae=s("p"),Ae.textContent=tr,Ot=r(),S=s("div"),h(ge.$$.fragment),Yt=r(),Ue=s("p"),Ue.textContent=nr,Kt=r(),We=s("p"),We.textContent=rr,en=r(),Z=s("div"),h(fe.$$.fragment),tn=r(),Fe=s("p"),Fe.textContent=or,nn=r(),Je=s("p"),Je.textContent=sr,rn=r(),A=s("div"),h(xe.$$.fragment),on=r(),Ne=s("p"),Ne.textContent=ir,sn=r(),Be=s("p"),Be.textContent=ar,an=r(),I=s("div"),h($e.$$.fragment),dn=r(),Xe=s("p"),Xe.textContent=dr,cn=r(),Qe=s("p"),Qe.innerHTML=cr,pn=r(),Re=s("p"),Re.innerHTML=pr,ln=r(),M=s("div"),h(_e.$$.fragment),un=r(),Oe=s("p"),Oe.textContent=lr,mn=r(),h(ee.$$.fragment),hn=r(),Ye=s("p"),Ye.innerHTML=ur,gn=r(),Ke=s("p"),Ke.innerHTML=mr,fn=r(),te=s("div"),h(ve.$$.fragment),xn=r(),et=s("p"),et.textContent=hr,$n=r(),ne=s("div"),h(ke.$$.fragment),_n=r(),tt=s("p"),tt.textContent=gr,vn=r(),re=s("div"),h(be.$$.fragment),kn=r(),nt=s("p"),nt.innerHTML=fr,bn=r(),oe=s("div"),h(Te.$$.fragment),Tn=r(),rt=s("p"),rt.innerHTML=xr,wn=r(),U=s("div"),h(we.$$.fragment),zn=r(),ot=s("p"),ot.textContent=$r,En=r(),st=s("p"),st.innerHTML=_r,Cn=r(),W=s("div"),h(ze.$$.fragment),yn=r(),it=s("p"),it.textContent=vr,qn=r(),at=s("p"),at.innerHTML=kr,Dn=r(),F=s("div"),h(Ee.$$.fragment),Mn=r(),dt=s("p"),dt.textContent=br,Pn=r(),ct=s("p"),ct.innerHTML=Tr,Ln=r(),J=s("div"),h(Ce.$$.fragment),In=r(),pt=s("p"),pt.textContent=wr,jn=r(),lt=s("p"),lt.innerHTML=zr,Hn=r(),N=s("div"),h(ye.$$.fragment),Vn=r(),ut=s("p"),ut.innerHTML=Er,Gn=r(),mt=s("p"),mt.innerHTML=Cr,Sn=r(),se=s("div"),h(qe.$$.fragment),Zn=r(),ht=s("p"),ht.textContent=yr,An=r(),ie=s("div"),h(De.$$.fragment),Un=r(),gt=s("p"),gt.textContent=qr,this.h()},l(a){g(n.$$.fragment,a),m=o(a),t=i(a,"DIV",{class:!0});var d=b(t);g(u.$$.fragment,d),k=o(d),v=i(d,"P",{"data-svelte-h":!0}),p(v)!=="svelte-z2kbek"&&(v.innerHTML=D),Y=o(d),C=i(d,"P",{"data-svelte-h":!0}),p(C)!=="svelte-1qmm8yo"&&(C.innerHTML=ce),j=o(d),g(P.$$.fragment,d),X=o(d),y=i(d,"DIV",{class:!0});var R=b(y);g(c.$$.fragment,R),z=o(R),Q=i(R,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-nafsul"&&(Q.textContent=Fn),jt=o(R),Me=i(R,"P",{"data-svelte-h":!0}),p(Me)!=="svelte-2cl9fs"&&(Me.textContent=Jn),R.forEach(l),Ht=o(d),H=i(d,"DIV",{class:!0});var O=b(H);g(pe.$$.fragment,O),Vt=o(O),Pe=i(O,"P",{"data-svelte-h":!0}),p(Pe)!=="svelte-13hu4qs"&&(Pe.textContent=Nn),Gt=o(O),Le=i(O,"P",{"data-svelte-h":!0}),p(Le)!=="svelte-3xucbe"&&(Le.textContent=Bn),O.forEach(l),St=o(d),K=i(d,"DIV",{class:!0});var Ct=b(K);g(le.$$.fragment,Ct),Zt=o(Ct),Ie=i(Ct,"P",{"data-svelte-h":!0}),p(Ie)!=="svelte-128n7bm"&&(Ie.textContent=Xn),Ct.forEach(l),At=o(d),V=i(d,"DIV",{class:!0});var ft=b(V);g(ue.$$.fragment,ft),Ut=o(ft),je=i(ft,"P",{"data-svelte-h":!0}),p(je)!=="svelte-1jccc0s"&&(je.textContent=Qn),Wt=o(ft),He=i(ft,"P",{"data-svelte-h":!0}),p(He)!=="svelte-rqkny6"&&(He.textContent=Rn),ft.forEach(l),Ft=o(d),L=i(d,"DIV",{class:!0});var ae=b(L);g(me.$$.fragment,ae),Jt=o(ae),Ve=i(ae,"P",{"data-svelte-h":!0}),p(Ve)!=="svelte-1f6mthi"&&(Ve.innerHTML=On),Nt=o(ae),Ge=i(ae,"P",{"data-svelte-h":!0}),p(Ge)!=="svelte-1pajss8"&&(Ge.innerHTML=Yn),Bt=o(ae),Se=i(ae,"P",{"data-svelte-h":!0}),p(Se)!=="svelte-77thau"&&(Se.textContent=Kn),ae.forEach(l),Xt=o(d),G=i(d,"DIV",{class:!0});var xt=b(G);g(he.$$.fragment,xt),Qt=o(xt),Ze=i(xt,"P",{"data-svelte-h":!0}),p(Ze)!=="svelte-olyk0g"&&(Ze.textContent=er),Rt=o(xt),Ae=i(xt,"P",{"data-svelte-h":!0}),p(Ae)!=="svelte-1w4yrxp"&&(Ae.textContent=tr),xt.forEach(l),Ot=o(d),S=i(d,"DIV",{class:!0});var $t=b(S);g(ge.$$.fragment,$t),Yt=o($t),Ue=i($t,"P",{"data-svelte-h":!0}),p(Ue)!=="svelte-1kqf199"&&(Ue.textContent=nr),Kt=o($t),We=i($t,"P",{"data-svelte-h":!0}),p(We)!=="svelte-1vxwt8f"&&(We.textContent=rr),$t.forEach(l),en=o(d),Z=i(d,"DIV",{class:!0});var _t=b(Z);g(fe.$$.fragment,_t),tn=o(_t),Fe=i(_t,"P",{"data-svelte-h":!0}),p(Fe)!=="svelte-10jm6aw"&&(Fe.textContent=or),nn=o(_t),Je=i(_t,"P",{"data-svelte-h":!0}),p(Je)!=="svelte-d380jx"&&(Je.textContent=sr),_t.forEach(l),rn=o(d),A=i(d,"DIV",{class:!0});var vt=b(A);g(xe.$$.fragment,vt),on=o(vt),Ne=i(vt,"P",{"data-svelte-h":!0}),p(Ne)!=="svelte-1q6qddw"&&(Ne.textContent=ir),sn=o(vt),Be=i(vt,"P",{"data-svelte-h":!0}),p(Be)!=="svelte-r57n1l"&&(Be.textContent=ar),vt.forEach(l),an=o(d),I=i(d,"DIV",{class:!0});var de=b(I);g($e.$$.fragment,de),dn=o(de),Xe=i(de,"P",{"data-svelte-h":!0}),p(Xe)!=="svelte-7pgnnj"&&(Xe.textContent=dr),cn=o(de),Qe=i(de,"P",{"data-svelte-h":!0}),p(Qe)!=="svelte-f7npf0"&&(Qe.innerHTML=cr),pn=o(de),Re=i(de,"P",{"data-svelte-h":!0}),p(Re)!=="svelte-o7jdc6"&&(Re.innerHTML=pr),de.forEach(l),ln=o(d),M=i(d,"DIV",{class:!0});var B=b(M);g(_e.$$.fragment,B),un=o(B),Oe=i(B,"P",{"data-svelte-h":!0}),p(Oe)!=="svelte-7pgnnj"&&(Oe.textContent=lr),mn=o(B),g(ee.$$.fragment,B),hn=o(B),Ye=i(B,"P",{"data-svelte-h":!0}),p(Ye)!=="svelte-f7npf0"&&(Ye.innerHTML=ur),gn=o(B),Ke=i(B,"P",{"data-svelte-h":!0}),p(Ke)!=="svelte-o7jdc6"&&(Ke.innerHTML=mr),B.forEach(l),fn=o(d),te=i(d,"DIV",{class:!0});var yt=b(te);g(ve.$$.fragment,yt),xn=o(yt),et=i(yt,"P",{"data-svelte-h":!0}),p(et)!=="svelte-1ffrb0"&&(et.textContent=hr),yt.forEach(l),$n=o(d),ne=i(d,"DIV",{class:!0});var qt=b(ne);g(ke.$$.fragment,qt),_n=o(qt),tt=i(qt,"P",{"data-svelte-h":!0}),p(tt)!=="svelte-1ykgp61"&&(tt.textContent=gr),qt.forEach(l),vn=o(d),re=i(d,"DIV",{class:!0});var Dt=b(re);g(be.$$.fragment,Dt),kn=o(Dt),nt=i(Dt,"P",{"data-svelte-h":!0}),p(nt)!=="svelte-19r1aq3"&&(nt.innerHTML=fr),Dt.forEach(l),bn=o(d),oe=i(d,"DIV",{class:!0});var Mt=b(oe);g(Te.$$.fragment,Mt),Tn=o(Mt),rt=i(Mt,"P",{"data-svelte-h":!0}),p(rt)!=="svelte-1tbd8n6"&&(rt.innerHTML=xr),Mt.forEach(l),wn=o(d),U=i(d,"DIV",{class:!0});var kt=b(U);g(we.$$.fragment,kt),zn=o(kt),ot=i(kt,"P",{"data-svelte-h":!0}),p(ot)!=="svelte-1k2furt"&&(ot.textContent=$r),En=o(kt),st=i(kt,"P",{"data-svelte-h":!0}),p(st)!=="svelte-zc653d"&&(st.innerHTML=_r),kt.forEach(l),Cn=o(d),W=i(d,"DIV",{class:!0});var bt=b(W);g(ze.$$.fragment,bt),yn=o(bt),it=i(bt,"P",{"data-svelte-h":!0}),p(it)!=="svelte-1ax7nb7"&&(it.textContent=vr),qn=o(bt),at=i(bt,"P",{"data-svelte-h":!0}),p(at)!=="svelte-rntbkv"&&(at.innerHTML=kr),bt.forEach(l),Dn=o(d),F=i(d,"DIV",{class:!0});var Tt=b(F);g(Ee.$$.fragment,Tt),Mn=o(Tt),dt=i(Tt,"P",{"data-svelte-h":!0}),p(dt)!=="svelte-gsdag5"&&(dt.textContent=br),Pn=o(Tt),ct=i(Tt,"P",{"data-svelte-h":!0}),p(ct)!=="svelte-s4bu5e"&&(ct.innerHTML=Tr),Tt.forEach(l),Ln=o(d),J=i(d,"DIV",{class:!0});var wt=b(J);g(Ce.$$.fragment,wt),In=o(wt),pt=i(wt,"P",{"data-svelte-h":!0}),p(pt)!=="svelte-r4v2i2"&&(pt.textContent=wr),jn=o(wt),lt=i(wt,"P",{"data-svelte-h":!0}),p(lt)!=="svelte-10g7eg5"&&(lt.innerHTML=zr),wt.forEach(l),Hn=o(d),N=i(d,"DIV",{class:!0});var zt=b(N);g(ye.$$.fragment,zt),Vn=o(zt),ut=i(zt,"P",{"data-svelte-h":!0}),p(ut)!=="svelte-1579b7n"&&(ut.innerHTML=Er),Gn=o(zt),mt=i(zt,"P",{"data-svelte-h":!0}),p(mt)!=="svelte-lpda5a"&&(mt.innerHTML=Cr),zt.forEach(l),Sn=o(d),se=i(d,"DIV",{class:!0});var Pt=b(se);g(qe.$$.fragment,Pt),Zn=o(Pt),ht=i(Pt,"P",{"data-svelte-h":!0}),p(ht)!=="svelte-ftg3o5"&&(ht.textContent=yr),Pt.forEach(l),An=o(d),ie=i(d,"DIV",{class:!0});var Lt=b(ie);g(De.$$.fragment,Lt),Un=o(Lt),gt=i(Lt,"P",{"data-svelte-h":!0}),p(gt)!=="svelte-17aw0gq"&&(gt.textContent=qr),Lt.forEach(l),d.forEach(l),this.h()},h(){T(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(re,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(oe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(t,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(a,d){f(n,a,d),q(a,m,d),q(a,t,d),f(u,t,null),e(t,k),e(t,v),e(t,Y),e(t,C),e(t,j),f(P,t,null),e(t,X),e(t,y),f(c,y,null),e(y,z),e(y,Q),e(y,jt),e(y,Me),e(t,Ht),e(t,H),f(pe,H,null),e(H,Vt),e(H,Pe),e(H,Gt),e(H,Le),e(t,St),e(t,K),f(le,K,null),e(K,Zt),e(K,Ie),e(t,At),e(t,V),f(ue,V,null),e(V,Ut),e(V,je),e(V,Wt),e(V,He),e(t,Ft),e(t,L),f(me,L,null),e(L,Jt),e(L,Ve),e(L,Nt),e(L,Ge),e(L,Bt),e(L,Se),e(t,Xt),e(t,G),f(he,G,null),e(G,Qt),e(G,Ze),e(G,Rt),e(G,Ae),e(t,Ot),e(t,S),f(ge,S,null),e(S,Yt),e(S,Ue),e(S,Kt),e(S,We),e(t,en),e(t,Z),f(fe,Z,null),e(Z,tn),e(Z,Fe),e(Z,nn),e(Z,Je),e(t,rn),e(t,A),f(xe,A,null),e(A,on),e(A,Ne),e(A,sn),e(A,Be),e(t,an),e(t,I),f($e,I,null),e(I,dn),e(I,Xe),e(I,cn),e(I,Qe),e(I,pn),e(I,Re),e(t,ln),e(t,M),f(_e,M,null),e(M,un),e(M,Oe),e(M,mn),f(ee,M,null),e(M,hn),e(M,Ye),e(M,gn),e(M,Ke),e(t,fn),e(t,te),f(ve,te,null),e(te,xn),e(te,et),e(t,$n),e(t,ne),f(ke,ne,null),e(ne,_n),e(ne,tt),e(t,vn),e(t,re),f(be,re,null),e(re,kn),e(re,nt),e(t,bn),e(t,oe),f(Te,oe,null),e(oe,Tn),e(oe,rt),e(t,wn),e(t,U),f(we,U,null),e(U,zn),e(U,ot),e(U,En),e(U,st),e(t,Cn),e(t,W),f(ze,W,null),e(W,yn),e(W,it),e(W,qn),e(W,at),e(t,Dn),e(t,F),f(Ee,F,null),e(F,Mn),e(F,dt),e(F,Pn),e(F,ct),e(t,Ln),e(t,J),f(Ce,J,null),e(J,In),e(J,pt),e(J,jn),e(J,lt),e(t,Hn),e(t,N),f(ye,N,null),e(N,Vn),e(N,ut),e(N,Gn),e(N,mt),e(t,Sn),e(t,se),f(qe,se,null),e(se,Zn),e(se,ht),e(t,An),e(t,ie),f(De,ie,null),e(ie,Un),e(ie,gt),Et=!0},p(a,d){const R={};d&2&&(R.$$scope={dirty:d,ctx:a}),P.$set(R);const O={};d&2&&(O.$$scope={dirty:d,ctx:a}),ee.$set(O)},i(a){Et||(x(n.$$.fragment,a),x(u.$$.fragment,a),x(P.$$.fragment,a),x(c.$$.fragment,a),x(pe.$$.fragment,a),x(le.$$.fragment,a),x(ue.$$.fragment,a),x(me.$$.fragment,a),x(he.$$.fragment,a),x(ge.$$.fragment,a),x(fe.$$.fragment,a),x(xe.$$.fragment,a),x($e.$$.fragment,a),x(_e.$$.fragment,a),x(ee.$$.fragment,a),x(ve.$$.fragment,a),x(ke.$$.fragment,a),x(be.$$.fragment,a),x(Te.$$.fragment,a),x(we.$$.fragment,a),x(ze.$$.fragment,a),x(Ee.$$.fragment,a),x(Ce.$$.fragment,a),x(ye.$$.fragment,a),x(qe.$$.fragment,a),x(De.$$.fragment,a),Et=!0)},o(a){$(n.$$.fragment,a),$(u.$$.fragment,a),$(P.$$.fragment,a),$(c.$$.fragment,a),$(pe.$$.fragment,a),$(le.$$.fragment,a),$(ue.$$.fragment,a),$(me.$$.fragment,a),$(he.$$.fragment,a),$(ge.$$.fragment,a),$(fe.$$.fragment,a),$(xe.$$.fragment,a),$($e.$$.fragment,a),$(_e.$$.fragment,a),$(ee.$$.fragment,a),$(ve.$$.fragment,a),$(ke.$$.fragment,a),$(be.$$.fragment,a),$(Te.$$.fragment,a),$(we.$$.fragment,a),$(ze.$$.fragment,a),$(Ee.$$.fragment,a),$(Ce.$$.fragment,a),$(ye.$$.fragment,a),$(qe.$$.fragment,a),$(De.$$.fragment,a),Et=!1},d(a){a&&(l(m),l(t)),_(n,a),_(u),_(P),_(c),_(pe),_(le),_(ue),_(me),_(he),_(ge),_(fe),_(xe),_($e),_(_e),_(ee),_(ve),_(ke),_(be),_(Te),_(we),_(ze),_(Ee),_(Ce),_(ye),_(qe),_(De)}}}function Jr(E){let n,m;return n=new Wn({props:{$$slots:{default:[Fr]},$$scope:{ctx:E}}}),{c(){h(n.$$.fragment)},l(t){g(n.$$.fragment,t)},m(t,u){f(n,t,u),m=!0},p(t,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:t}),n.$set(k)},i(t){m||(x(n.$$.fragment,t),m=!0)},o(t){$(n.$$.fragment,t),m=!1},d(t){_(n,t)}}}function Nr(E){let n,m='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){n=s("p"),n.innerHTML=m},l(t){n=i(t,"P",{"data-svelte-h":!0}),p(n)!=="svelte-4ytcyb"&&(n.innerHTML=m)},m(t,u){q(t,n,u)},p:It,d(t){t&&l(n)}}}function Br(E){let n,m;return n=new Wn({props:{$$slots:{default:[Nr]},$$scope:{ctx:E}}}),{c(){h(n.$$.fragment)},l(t){g(n.$$.fragment,t)},m(t,u){f(n,t,u),m=!0},p(t,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:t}),n.$set(k)},i(t){m||(x(n.$$.fragment,t),m=!0)},o(t){$(n.$$.fragment,t),m=!1},d(t){_(n,t)}}}function Xr(E){let n,m="The node API has not been documented yet.";return{c(){n=s("p"),n.textContent=m},l(t){n=i(t,"P",{"data-svelte-h":!0}),p(n)!=="svelte-1mrchm6"&&(n.textContent=m)},m(t,u){q(t,n,u)},p:It,d(t){t&&l(n)}}}function Qr(E){let n,m;return n=new Wn({props:{$$slots:{default:[Xr]},$$scope:{ctx:E}}}),{c(){h(n.$$.fragment)},l(t){g(n.$$.fragment,t)},m(t,u){f(n,t,u),m=!0},p(t,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:t}),n.$set(k)},i(t){m||(x(n.$$.fragment,t),m=!0)},o(t){$(n.$$.fragment,t),m=!1},d(t){_(n,t)}}}function Rr(E){let n,m,t,u,k,v,D,Y,C,ce,j,P,X,y;return k=new Vr({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),D=new Dr({props:{title:"Encoding",local:"encoding",headingTag:"h1"}}),C=new Zr({props:{python:!0,rust:!0,node:!0,$$slots:{node:[Qr],rust:[Br],python:[Jr]},$$scope:{ctx:E}}}),j=new Gr({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/encoding.mdx"}}),{c(){n=s("meta"),m=r(),t=s("p"),u=r(),h(k.$$.fragment),v=r(),h(D.$$.fragment),Y=r(),h(C.$$.fragment),ce=r(),h(j.$$.fragment),P=r(),X=s("p"),this.h()},l(c){const z=jr("svelte-u9bgzb",document.head);n=i(z,"META",{name:!0,content:!0}),z.forEach(l),m=o(c),t=i(c,"P",{}),b(t).forEach(l),u=o(c),g(k.$$.fragment,c),v=o(c),g(D.$$.fragment,c),Y=o(c),g(C.$$.fragment,c),ce=o(c),g(j.$$.fragment,c),P=o(c),X=i(c,"P",{}),b(X).forEach(l),this.h()},h(){T(n,"name","hf:doc:metadata"),T(n,"content",Or)},m(c,z){e(document.head,n),q(c,m,z),q(c,t,z),q(c,u,z),f(k,c,z),q(c,v,z),f(D,c,z),q(c,Y,z),f(C,c,z),q(c,ce,z),f(j,c,z),q(c,P,z),q(c,X,z),y=!0},p(c,[z]){const Q={};z&2&&(Q.$$scope={dirty:z,ctx:c}),C.$set(Q)},i(c){y||(x(k.$$.fragment,c),x(D.$$.fragment,c),x(C.$$.fragment,c),x(j.$$.fragment,c),y=!0)},o(c){$(k.$$.fragment,c),$(D.$$.fragment,c),$(C.$$.fragment,c),$(j.$$.fragment,c),y=!1},d(c){c&&(l(m),l(t),l(u),l(v),l(Y),l(ce),l(P),l(X)),l(n),_(k,c),_(D,c),_(C,c),_(j,c)}}}const Or='{"title":"Encoding","local":"encoding","sections":[{"title":"Encoding","local":"tokenizers.Encoding","sections":[],"depth":2}],"depth":1}';function Yr(E){return Pr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ao extends Lr{constructor(n){super(),Ir(this,n,Yr,Rr,Mr,{})}}export{ao as component};