tokenizers/pr_1985/en/_app/immutable/nodes/3.0c6fb235.js

import{s as hs,o as ks,n as D}from"../chunks/scheduler.7c59faff.js";import{S as ys,i as Ms,e as b,s as c,c as $,h as bs,a as x,d as a,b as p,f as J,g as f,j as C,k as m,l as d,m as h,t as k,n as y,o as M,p as v}from"../chunks/index.09bb5655.js";import{C as xs,H as V,E as js}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e176492a.js";import{D as R}from"../chunks/Docstring.523eb8f6.js";import{C as A}from"../chunks/CodeBlock.6879f932.js";import{T as vs,M as At}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";import{E as K}from"../chunks/ExampleCodeBlock.36e90ade.js";function Ts(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5kZWNvZGVycyUyMGltcG9ydCUyMERlY29kZVN0cmVhbSUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ3B0MiUyMiklMEFzdHJlYW0lMjAlM0QlMjBEZWNvZGVTdHJlYW0oc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTBBJTIzJTIwU2ltdWxhdGUlMjBzdHJlYW1pbmclMjB0b2tlbi1ieS10b2tlbiUyMGdlbmVyYXRpb24lMEF0b2tlbl9pZHMlMjAlM0QlMjB0b2tlbml6ZXIuZW5jb2RlKCUyMkhlbGxvJTJDJTIwc3RyZWFtaW5nJTIwd29ybGQhJTIyKS5pZHMlMEFmb3IlMjB0b2tlbl9pZCUyMGluJTIwdG9rZW5faWRzJTNBJTBBJTIwJTIwJTIwJTIwY2h1bmslMjAlM0QlMjBzdHJlYW0uc3RlcCh0b2tlbml6ZXIlMkMlMjB0b2tlbl9pZCklMEElMjAlMjAlMjAlMjBpZiUyMGNodW5rJTIwaXMlMjBub3QlMjBOb25lJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoY2h1bmslMkMlMjBlbmQlM0QlMjIlMjIlMkMlMjBmbHVzaCUzRFRydWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> DecodeStream
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>stream = DecodeStream(skip_special_tokens=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Simulate streaming token-by-token generation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>token_ids = tokenizer.encode(<span class="hljs-string">&quot;Hello, streaming world!&quot;</span>).ids
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> token_id <span class="hljs-keyword">in</span> token_ids:
<span class="hljs-meta">... </span>    chunk = stream.step(tokenizer, token_id)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> chunk <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>        <span class="hljs-built_in">print</span>(chunk, end=<span class="hljs-string">&quot;&quot;</span>, flush=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function ws(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBCUEVEZWNvZGVyJTBBZGVjb2RlciUyMCUzRCUyMEJQRURlY29kZXIoKSUwQWRlY29kZXIuZGVjb2RlKCU1QiUyMkhlbGxvJTNDJTJGdyUzRSUyMiUyQyUyMCUyMndvcmxkJTNDJTJGdyUzRSUyMiU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> BPEDecoder
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = BPEDecoder()
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;Hello&lt;/w&gt;&quot;</span>, <span class="hljs-string">&quot;world&lt;/w&gt;&quot;</span>])
<span class="hljs-string">&#x27;Hello world&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function Js(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBCeXRlRmFsbGJhY2slMkMlMjBGdXNlJTJDJTIwU2VxdWVuY2UlMEFkZWNvZGVyJTIwJTNEJTIwU2VxdWVuY2UoJTVCQnl0ZUZhbGxiYWNrKCklMkMlMjBGdXNlKCklNUQpJTBBZGVjb2Rlci5kZWNvZGUoJTVCJTIyJTNDMHg0OCUzRSUyMiUyQyUyMCUyMiUzQzB4NjUlM0UlMjIlMkMlMjAlMjIlM0MweDZDJTNFJTIyJTJDJTIwJTIyJTNDMHg2QyUzRSUyMiUyQyUyMCUyMiUzQzB4NkYlM0UlMjIlNUQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> ByteFallback, Fuse, <span class="hljs-type">Sequence</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = <span class="hljs-type">Sequence</span>([ByteFallback(), Fuse()])
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;&lt;0x48&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;0x65&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;0x6C&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;0x6C&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;0x6F&gt;&quot;</span>])
<span class="hljs-string">&#x27;Hello&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function Cs(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBCeXRlTGV2ZWwlMEFkZWNvZGVyJTIwJTNEJTIwQnl0ZUxldmVsKCklMEFkZWNvZGVyLmRlY29kZSglNUIlMjIlQzQlQTBIZWxsbyUyMiUyQyUyMCUyMiVDNCVBMHdvcmxkJTIyJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> ByteLevel
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = ByteLevel()
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;ĠHello&quot;</span>, <span class="hljs-string">&quot;Ġworld&quot;</span>])
<span class="hljs-string">&#x27; Hello world&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function Us(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBDVEMlMEFkZWNvZGVyJTIwJTNEJTIwQ1RDKCklMEFkZWNvZGVyLmRlY29kZSglNUIlMjJoJTIyJTJDJTIwJTIyZSUyMiUyQyUyMCUyMmUlMjIlMkMlMjAlMjIlM0NwYWQlM0UlMjIlMkMlMjAlMjJsJTIyJTJDJTIwJTIybCUyMiUyQyUyMCUyMm8lMjIlMkMlMjAlMjIlN0MlMjIlMkMlMjAlMjJ3JTIyJTJDJTIwJTIybyUyMiUyQyUyMCUyMnIlMjIlMkMlMjAlMjJsJTIyJTJDJTIwJTIyZCUyMiU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> CTC
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = CTC()
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;e&quot;</span>, <span class="hljs-string">&quot;e&quot;</span>, <span class="hljs-string">&quot;&lt;pad&gt;&quot;</span>, <span class="hljs-string">&quot;l&quot;</span>, <span class="hljs-string">&quot;l&quot;</span>, <span class="hljs-string">&quot;o&quot;</span>, <span class="hljs-string">&quot;|&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>, <span class="hljs-string">&quot;o&quot;</span>, <span class="hljs-string">&quot;r&quot;</span>, <span class="hljs-string">&quot;l&quot;</span>, <span class="hljs-string">&quot;d&quot;</span>])
<span class="hljs-string">&#x27;hello world&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function Is(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBGdXNlJTBBZGVjb2RlciUyMCUzRCUyMEZ1c2UoKSUwQWRlY29kZXIuZGVjb2RlKCU1QiUyMkhlbGxvJTIyJTJDJTIwJTIyJTJDJTIyJTJDJTIwJTIyJTIwJTIyJTJDJTIwJTIyd29ybGQlMjIlMkMlMjAlMjIhJTIyJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> Fuse
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = Fuse()
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;Hello&quot;</span>, <span class="hljs-string">&quot;,&quot;</span>, <span class="hljs-string">&quot; &quot;</span>, <span class="hljs-string">&quot;world&quot;</span>, <span class="hljs-string">&quot;!&quot;</span>])
<span class="hljs-string">&#x27;Hello, world!&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function zs(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBNZXRhc3BhY2UlMEFkZWNvZGVyJTIwJTNEJTIwTWV0YXNwYWNlKCklMEFkZWNvZGVyLmRlY29kZSglNUIlMjIlRTIlOTYlODFIZWxsbyUyMiUyQyUyMCUyMiVFMiU5NiU4MW15JTIyJTJDJTIwJTIyJUUyJTk2JTgxZnJpZW5kJTIyJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> Metaspace
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = Metaspace()
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;▁Hello&quot;</span>, <span class="hljs-string">&quot;▁my&quot;</span>, <span class="hljs-string">&quot;▁friend&quot;</span>])
<span class="hljs-string">&#x27;Hello my friend&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function _s(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBSZXBsYWNlJTBBZGVjb2RlciUyMCUzRCUyMFJlcGxhY2UoJTIyJUUyJTk2JTgxJTIyJTJDJTIwJTIyJTIwJTIyKSUwQWRlY29kZXIuZGVjb2RlKCU1QiUyMiVFMiU5NiU4MUhlbGxvJTIyJTJDJTIwJTIyJUUyJTk2JTgxd29ybGQlMjIlNUQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> Replace
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = Replace(<span class="hljs-string">&quot;▁&quot;</span>, <span class="hljs-string">&quot; &quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;▁Hello&quot;</span>, <span class="hljs-string">&quot;▁world&quot;</span>])
<span class="hljs-string">&#x27; Hello world&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function Bs(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBCeXRlRmFsbGJhY2slMkMlMjBGdXNlJTJDJTIwTWV0YXNwYWNlJTJDJTIwU2VxdWVuY2UlMEFkZWNvZGVyJTIwJTNEJTIwU2VxdWVuY2UoJTVCQnl0ZUZhbGxiYWNrKCklMkMlMjBGdXNlKCklMkMlMjBNZXRhc3BhY2UoKSU1RCklMEFkZWNvZGVyLmRlY29kZSglNUIlMjIlRTIlOTYlODFIZWxsbyUyMiUyQyUyMCUyMiVFMiU5NiU4MXdvcmxkJTIyJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> ByteFallback, Fuse, Metaspace, <span class="hljs-type">Sequence</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = <span class="hljs-type">Sequence</span>([ByteFallback(), Fuse(), Metaspace()])
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;▁Hello&quot;</span>, <span class="hljs-string">&quot;▁world&quot;</span>])
<span class="hljs-string">&#x27;Hello world&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function qs(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBTdHJpcCUwQWRlY29kZXIlMjAlM0QlMjBTdHJpcChjb250ZW50JTNEJTIyJUUyJTk2JTgxJTIyJTJDJTIwbGVmdCUzRDEpJTBBZGVjb2Rlci5kZWNvZGUoJTVCJTIyJUUyJTk2JTgxSGVsbG8lMjIlMkMlMjAlMjIlRTIlOTYlODF3b3JsZCUyMiU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> Strip
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = Strip(content=<span class="hljs-string">&quot;▁&quot;</span>, left=<span class="hljs-number">1</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;▁Hello&quot;</span>, <span class="hljs-string">&quot;▁world&quot;</span>])
<span class="hljs-string">&#x27;Hello world&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function Zs(j){let s,i="Example:",n,l,o;return l=new A({props:{code:"ZnJvbSUyMHRva2VuaXplcnMuZGVjb2RlcnMlMjBpbXBvcnQlMjBXb3JkUGllY2UlMEFkZWNvZGVyJTIwJTNEJTIwV29yZFBpZWNlKCklMEFkZWNvZGVyLmRlY29kZSglNUIlMjJIZWxsbyUyMiUyQyUyMCUyMiUyQyUyMiUyQyUyMCUyMiUyMyUyM3dvcmxkJTIyJTJDJTIwJTIyISUyMiU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.decoders <span class="hljs-keyword">import</span> WordPiece
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder = WordPiece()
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder.decode([<span class="hljs-string">&quot;Hello&quot;</span>, <span class="hljs-string">&quot;,&quot;</span>, <span class="hljs-string">&quot;##world&quot;</span>, <span class="hljs-string">&quot;!&quot;</span>])
<span class="hljs-string">&#x27;Hello, world!&#x27;</span>`,wrap:!1}}),{c(){s=b("p"),s.textContent=i,n=c(),$(l.$$.fragment)},l(t){s=x(t,"P",{"data-svelte-h":!0}),v(s)!=="svelte-11lpom8"&&(s.textContent=i),n=p(t),f(l.$$.fragment,t)},m(t,u){d(t,s,u),d(t,n,u),h(l,t,u),o=!0},p:D,i(t){o||(k(l.$$.fragment,t),o=!0)},o(t){y(l.$$.fragment,t),o=!1},d(t){t&&(a(s),a(n)),M(l,t)}}}function Ds(j){let s,i,n,l,o,t,u=`Provides incremental decoding of token IDs as they are generated, yielding
decoded text chunks as soon as they are available.`,le,w,he=`Unlike batch decoding, streaming decode is designed for use with autoregressive
generation — tokens arrive one at a time and the decoder needs to handle
multi-byte sequences (e.g., UTF-8 characters split across token boundaries) and
byte-fallback tokens gracefully.`,S,E,ae=`The decoder internally buffers tokens until it can produce a valid UTF-8 string
chunk, then yields that chunk and advances its internal state. This means
individual calls to <code>step()</code> may return
<code>None</code> when the current token completes a partial sequence that cannot yet
be decoded.`,re,g,T,U,ke,vt,Se,Ot=`Add the next token ID (or list of IDs) to the stream and return the next
decoded text chunk if one is available.`,Tt,Ee,es=`Because some characters span multiple tokens (e.g. multi-byte UTF-8
sequences or byte-fallback tokens), this method may return <code>None</code>
when the provided token does not yet complete a decodable unit. Callers
should simply continue feeding tokens until a non-<code>None</code> value is
returned.`,nt,ye,lt,W,Me,wt,We,ts="BPEDecoder Decoder",Jt,oe,at,be,rt,I,xe,Ct,Fe,ss="ByteFallback Decoder",Ut,Ne,ns=`ByteFallback is a decoder that handles tokens representing raw bytes in the
<code>&lt;0xNN&gt;</code> format (e.g., <code>&lt;0x61&gt;</code> for the byte <code>0x61</code> = <code>&#39;a&#39;</code>). It converts
such tokens to their corresponding bytes and attempts to decode the resulting byte
sequence as UTF-8. This is used in LLaMA/SentencePiece models that use byte fallback
for unknown characters. Inconvertible byte tokens are replaced with the Unicode
replacement character (U+FFFD).`,It,ce,ot,je,ct,z,ve,zt,Pe,ls="ByteLevel Decoder",_t,Ge,as=`This decoder is to be used in tandem with the
<a href="/docs/tokenizers/pr_1985/en/api/pre-tokenizers#tokenizers.pre_tokenizers.ByteLevel">ByteLevel</a> pre-tokenizer. It reverses the
byte-to-unicode mapping applied during pre-tokenization, converting the special
Unicode characters back into the original bytes to reconstruct the original string.`,Bt,pe,pt,Te,dt,F,we,qt,Qe,rs="CTC Decoder",Zt,de,it,Je,mt,_,Ce,Dt,He,os="Fuse Decoder",Rt,Xe,cs=`Fuse simply concatenates every token into a single string without any separator.
This is typically the last step in a decoder chain when other decoders need to
operate on individual tokens before they are joined together.`,Vt,ie,ut,Ue,gt,N,Ie,St,Ye,ps="Metaspace Decoder",Et,me,$t,ze,ft,B,_e,Wt,Le,ds="Replace Decoder",Ft,Ke,is=`This decoder is to be used in tandem with the
<a href="/docs/tokenizers/pr_1985/en/api/normalizers#tokenizers.normalizers.Replace">Replace</a> normalizer or a similar replace operation.
It reverses a string replacement by substituting the replacement content back
with the original pattern.`,Nt,ue,ht,Be,kt,q,qe,Pt,Ae,ms="Sequence Decoder",Gt,Oe,us=`Chains multiple decoders together, applying them in order. Each decoder in the
sequence processes the output of the previous one, allowing complex decoding
pipelines to be built from simpler components.`,Qt,ge,yt,Ze,Mt,Z,De,Ht,et,gs="Strip Decoder",Xt,tt,$s=`Strips a given number of occurrences of a character from the left and/or right
side of each token. This is useful for removing padding characters or special
prefix/suffix markers added during tokenization.`,Yt,$e,bt,Re,xt,P,Ve,Lt,st,fs="WordPiece Decoder",Kt,fe,jt;return s=new V({props:{title:"DecodeStream",local:"tokenizers.decoders.DecodeStream",headingTag:"h2"}}),l=new R({props:{name:"class tokenizers.decoders.DecodeStream",anchor:"tokenizers.decoders.DecodeStream",parameters:[{name:"ids",val:" = None"},{name:"skip_special_tokens",val:" = False"}],parametersDescription:[{anchor:"tokenizers.decoders.DecodeStream.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to skip special tokens (e.g. <code>[CLS]</code>, <code>[SEP]</code>, <code>&lt;s&gt;</code>) when
decoding.`,name:"skip_special_tokens"}]}}),g=new K({props:{anchor:"tokenizers.decoders.DecodeStream.example",$$slots:{default:[Ts]},$$scope:{ctx:j}}}),ke=new R({props:{name:"step",anchor:"tokenizers.decoders.DecodeStream.step",parameters:[{name:"tokenizer",val:""},{name:"id",val:""}],parametersDescription:[{anchor:"tokenizers.decoders.DecodeStream.step.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/tokenizers/pr_1985/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>) &#x2014;
The tokenizer whose decoder pipeline will be used.`,name:"tokenizer"},{anchor:"tokenizers.decoders.DecodeStream.step.id",description:`<strong>id</strong> (<code>int</code> or <code>List[int]</code>) &#x2014;
The next token ID, or a list of token IDs to append to the stream.`,name:"id"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The next decoded text chunk if enough tokens have
accumulated, or <code>None</code> if more tokens are still needed.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Optional[str]</code></p>
`}}),ye=new V({props:{title:"BPEDecoder",local:"tokenizers.decoders.BPEDecoder",headingTag:"h2"}}),Me=new R({props:{name:"class tokenizers.decoders.BPEDecoder",anchor:"tokenizers.decoders.BPEDecoder",parameters:[{name:"suffix",val:" = '</w>'"}],parametersDescription:[{anchor:"tokenizers.decoders.BPEDecoder.suffix",description:`<strong>suffix</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&lt;/w&gt;</code>) &#x2014;
The suffix that was used to characterize an end-of-word. This suffix will
be replaced by whitespaces during the decoding`,name:"suffix"}]}}),oe=new K({props:{anchor:"tokenizers.decoders.BPEDecoder.example",$$slots:{default:[ws]},$$scope:{ctx:j}}}),be=new V({props:{title:"ByteFallback",local:"tokenizers.decoders.ByteFallback",headingTag:"h2"}}),xe=new R({props:{name:"class tokenizers.decoders.ByteFallback",anchor:"tokenizers.decoders.ByteFallback",parameters:[]}}),ce=new K({props:{anchor:"tokenizers.decoders.ByteFallback.example",$$slots:{default:[Js]},$$scope:{ctx:j}}}),je=new V({props:{title:"ByteLevel",local:"tokenizers.decoders.ByteLevel",headingTag:"h2"}}),ve=new R({props:{name:"class tokenizers.decoders.ByteLevel",anchor:"tokenizers.decoders.ByteLevel",parameters:[]}}),pe=new K({props:{anchor:"tokenizers.decoders.ByteLevel.example",$$slots:{default:[Cs]},$$scope:{ctx:j}}}),Te=new V({props:{title:"CTC",local:"tokenizers.decoders.CTC",headingTag:"h2"}}),we=new R({props:{name:"class tokenizers.decoders.CTC",anchor:"tokenizers.decoders.CTC",parameters:[{name:"pad_token",val:" = '<pad>'"},{name:"word_delimiter_token",val:" = '|'"},{name:"cleanup",val:" = True"}],parametersDescription:[{anchor:"tokenizers.decoders.CTC.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&lt;pad&gt;</code>) &#x2014;
The pad token used by CTC to delimit a new token.`,name:"pad_token"},{anchor:"tokenizers.decoders.CTC.word_delimiter_token",description:`<strong>word_delimiter_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>|</code>) &#x2014;
The word delimiter token. It will be replaced by a <space></space>`,name:"word_delimiter_token"},{anchor:"tokenizers.decoders.CTC.cleanup",description:`<strong>cleanup</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to cleanup some tokenization artifacts.
Mainly spaces before punctuation, and some abbreviated english forms.`,name:"cleanup"}]}}),de=new K({props:{anchor:"tokenizers.decoders.CTC.example",$$slots:{default:[Us]},$$scope:{ctx:j}}}),Je=new V({props:{title:"Fuse",local:"tokenizers.decoders.Fuse",headingTag:"h2"}}),Ce=new R({props:{name:"class tokenizers.decoders.Fuse",anchor:"tokenizers.decoders.Fuse",parameters:[]}}),ie=new K({props:{anchor:"tokenizers.decoders.Fuse.example",$$slots:{default:[Is]},$$scope:{ctx:j}}}),Ue=new V({props:{title:"Metaspace",local:"tokenizers.decoders.Metaspace",headingTag:"h2"}}),Ie=new R({props:{name:"class tokenizers.decoders.Metaspace",anchor:"tokenizers.decoders.Metaspace",parameters:"",parametersDescription:[{anchor:"tokenizers.decoders.Metaspace.replacement",description:`<strong>replacement</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&#x2581;</code>) &#x2014;
The replacement character. Must be exactly one character. By default we
use the <em>&#x2581;</em> (U+2581) meta symbol (Same as in SentencePiece).`,name:"replacement"},{anchor:"tokenizers.decoders.Metaspace.prepend_scheme",description:`<strong>prepend_scheme</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;always&quot;</code>) &#x2014;
Whether to add a space to the first word if there isn&#x2019;t already one. This
lets us treat <em>hello</em> exactly like <em>say hello</em>.
Choices: &#x201C;always&#x201D;, &#x201C;never&#x201D;, &#x201C;first&#x201D;. First means the space is only added on the first
token (relevant when special tokens are used or other pre_tokenizer are used).`,name:"prepend_scheme"}]}}),me=new K({props:{anchor:"tokenizers.decoders.Metaspace.example",$$slots:{default:[zs]},$$scope:{ctx:j}}}),ze=new V({props:{title:"Replace",local:"tokenizers.decoders.Replace",headingTag:"h2"}}),_e=new R({props:{name:"class tokenizers.decoders.Replace",anchor:"tokenizers.decoders.Replace",parameters:[{name:"pattern",val:""},{name:"content",val:""}],parametersDescription:[{anchor:"tokenizers.decoders.Replace.pattern",description:`<strong>pattern</strong> (<code>str</code> or <code>Regex</code>) &#x2014;
The pattern that was used as the replacement target during encoding.`,name:"pattern"},{anchor:"tokenizers.decoders.Replace.content",description:`<strong>content</strong> (<code>str</code>) &#x2014;
The string to replace each match of the pattern with during decoding.`,name:"content"}]}}),ue=new K({props:{anchor:"tokenizers.decoders.Replace.example",$$slots:{default:[_s]},$$scope:{ctx:j}}}),Be=new V({props:{title:"Sequence",local:"tokenizers.decoders.Sequence",headingTag:"h2"}}),qe=new R({props:{name:"class tokenizers.decoders.Sequence",anchor:"tokenizers.decoders.Sequence",parameters:[{name:"decoders",val:""}],parametersDescription:[{anchor:"tokenizers.decoders.Sequence.decoders",description:`<strong>decoders</strong> (<code>List[Decoder]</code>) &#x2014;
The list of decoders to chain together.`,name:"decoders"}]}}),ge=new K({props:{anchor:"tokenizers.decoders.Sequence.example",$$slots:{default:[Bs]},$$scope:{ctx:j}}}),Ze=new V({props:{title:"Strip",local:"tokenizers.decoders.Strip",headingTag:"h2"}}),De=new R({props:{name:"class tokenizers.decoders.Strip",anchor:"tokenizers.decoders.Strip",parameters:[{name:"content",val:" = ' '"},{name:"left",val:" = 0"},{name:"right",val:" = 0"}],parametersDescription:[{anchor:"tokenizers.decoders.Strip.content",description:`<strong>content</strong> (<code>str</code>, defaults to <code>&quot; &quot;</code>) &#x2014;
The character to strip from each token.`,name:"content"},{anchor:"tokenizers.decoders.Strip.left",description:`<strong>left</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The number of occurrences of <code>content</code> to remove from the left
side of each token.`,name:"left"},{anchor:"tokenizers.decoders.Strip.right",description:`<strong>right</strong> (<code>int</code>, defaults to <code>0</code>) &#x2014;
The number of occurrences of <code>content</code> to remove from the right
side of each token.`,name:"right"}]}}),$e=new K({props:{anchor:"tokenizers.decoders.Strip.example",$$slots:{default:[qs]},$$scope:{ctx:j}}}),Re=new V({props:{title:"WordPiece",local:"tokenizers.decoders.WordPiece",headingTag:"h2"}}),Ve=new R({props:{name:"class tokenizers.decoders.WordPiece",anchor:"tokenizers.decoders.WordPiece",parameters:[{name:"prefix",val:" = '##'"},{name:"cleanup",val:" = True"}],parametersDescription:[{anchor:"tokenizers.decoders.WordPiece.prefix",description:`<strong>prefix</strong> (<code>str</code>, <em>optional</em>, defaults to <code>##</code>) &#x2014;
The prefix to use for subwords that are not a beginning-of-word`,name:"prefix"},{anchor:"tokenizers.decoders.WordPiece.cleanup",description:`<strong>cleanup</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation,
and some abbreviated english forms.`,name:"cleanup"}]}}),fe=new K({props:{anchor:"tokenizers.decoders.WordPiece.example",$$slots:{default:[Zs]},$$scope:{ctx:j}}}),{c(){$(s.$$.fragment),i=c(),n=b("div"),$(l.$$.fragment),o=c(),t=b("p"),t.textContent=u,le=c(),w=b("p"),w.textContent=he,S=c(),E=b("p"),E.innerHTML=ae,re=c(),$(g.$$.fragment),T=c(),U=b("div"),$(ke.$$.fragment),vt=c(),Se=b("p"),Se.textContent=Ot,Tt=c(),Ee=b("p"),Ee.innerHTML=es,nt=c(),$(ye.$$.fragment),lt=c(),W=b("div"),$(Me.$$.fragment),wt=c(),We=b("p"),We.textContent=ts,Jt=c(),$(oe.$$.fragment),at=c(),$(be.$$.fragment),rt=c(),I=b("div"),$(xe.$$.fragment),Ct=c(),Fe=b("p"),Fe.textContent=ss,Ut=c(),Ne=b("p"),Ne.innerHTML=ns,It=c(),$(ce.$$.fragment),ot=c(),$(je.$$.fragment),ct=c(),z=b("div"),$(ve.$$.fragment),zt=c(),Pe=b("p"),Pe.textContent=ls,_t=c(),Ge=b("p"),Ge.innerHTML=as,Bt=c(),$(pe.$$.fragment),pt=c(),$(Te.$$.fragment),dt=c(),F=b("div"),$(we.$$.fragment),qt=c(),Qe=b("p"),Qe.textContent=rs,Zt=c(),$(de.$$.fragment),it=c(),$(Je.$$.fragment),mt=c(),_=b("div"),$(Ce.$$.fragment),Dt=c(),He=b("p"),He.textContent=os,Rt=c(),Xe=b("p"),Xe.textContent=cs,Vt=c(),$(ie.$$.fragment),ut=c(),$(Ue.$$.fragment),gt=c(),N=b("div"),$(Ie.$$.fragment),St=c(),Ye=b("p"),Ye.textContent=ps,Et=c(),$(me.$$.fragment),$t=c(),$(ze.$$.fragment),ft=c(),B=b("div"),$(_e.$$.fragment),Wt=c(),Le=b("p"),Le.textContent=ds,Ft=c(),Ke=b("p"),Ke.innerHTML=is,Nt=c(),$(ue.$$.fragment),ht=c(),$(Be.$$.fragment),kt=c(),q=b("div"),$(qe.$$.fragment),Pt=c(),Ae=b("p"),Ae.textContent=ms,Gt=c(),Oe=b("p"),Oe.textContent=us,Qt=c(),$(ge.$$.fragment),yt=c(),$(Ze.$$.fragment),Mt=c(),Z=b("div"),$(De.$$.fragment),Ht=c(),et=b("p"),et.textContent=gs,Xt=c(),tt=b("p"),tt.textContent=$s,Yt=c(),$($e.$$.fragment),bt=c(),$(Re.$$.fragment),xt=c(),P=b("div"),$(Ve.$$.fragment),Lt=c(),st=b("p"),st.textContent=fs,Kt=c(),$(fe.$$.fragment),this.h()},l(e){f(s.$$.fragment,e),i=p(e),n=x(e,"DIV",{class:!0});var r=J(n);f(l.$$.fragment,r),o=p(r),t=x(r,"P",{"data-svelte-h":!0}),v(t)!=="svelte-rvnabk"&&(t.textContent=u),le=p(r),w=x(r,"P",{"data-svelte-h":!0}),v(w)!=="svelte-skcln5"&&(w.textContent=he),S=p(r),E=x(r,"P",{"data-svelte-h":!0}),v(E)!=="svelte-xm2wrd"&&(E.innerHTML=ae),re=p(r),f(g.$$.fragment,r),T=p(r),U=x(r,"DIV",{class:!0});var O=J(U);f(ke.$$.fragment,O),vt=p(O),Se=x(O,"P",{"data-svelte-h":!0}),v(Se)!=="svelte-kz9dtj"&&(Se.textContent=Ot),Tt=p(O),Ee=x(O,"P",{"data-svelte-h":!0}),v(Ee)!=="svelte-1n3pjfc"&&(Ee.innerHTML=es),O.forEach(a),r.forEach(a),nt=p(e),f(ye.$$.fragment,e),lt=p(e),W=x(e,"DIV",{class:!0});var ee=J(W);f(Me.$$.fragment,ee),wt=p(ee),We=x(ee,"P",{"data-svelte-h":!0}),v(We)!=="svelte-1jys3a9"&&(We.textContent=ts),Jt=p(ee),f(oe.$$.fragment,ee),ee.forEach(a),at=p(e),f(be.$$.fragment,e),rt=p(e),I=x(e,"DIV",{class:!0});var G=J(I);f(xe.$$.fragment,G),Ct=p(G),Fe=x(G,"P",{"data-svelte-h":!0}),v(Fe)!=="svelte-1bpklro"&&(Fe.textContent=ss),Ut=p(G),Ne=x(G,"P",{"data-svelte-h":!0}),v(Ne)!=="svelte-1q7cx54"&&(Ne.innerHTML=ns),It=p(G),f(ce.$$.fragment,G),G.forEach(a),ot=p(e),f(je.$$.fragment,e),ct=p(e),z=x(e,"DIV",{class:!0});var Q=J(z);f(ve.$$.fragment,Q),zt=p(Q),Pe=x(Q,"P",{"data-svelte-h":!0}),v(Pe)!=="svelte-zzneow"&&(Pe.textContent=ls),_t=p(Q),Ge=x(Q,"P",{"data-svelte-h":!0}),v(Ge)!=="svelte-1871ip7"&&(Ge.innerHTML=as),Bt=p(Q),f(pe.$$.fragment,Q),Q.forEach(a),pt=p(e),f(Te.$$.fragment,e),dt=p(e),F=x(e,"DIV",{class:!0});var te=J(F);f(we.$$.fragment,te),qt=p(te),Qe=x(te,"P",{"data-svelte-h":!0}),v(Qe)!=="svelte-nhi5vo"&&(Qe.textContent=rs),Zt=p(te),f(de.$$.fragment,te),te.forEach(a),it=p(e),f(Je.$$.fragment,e),mt=p(e),_=x(e,"DIV",{class:!0});var H=J(_);f(Ce.$$.fragment,H),Dt=p(H),He=x(H,"P",{"data-svelte-h":!0}),v(He)!=="svelte-h62rxn"&&(He.textContent=os),Rt=p(H),Xe=x(H,"P",{"data-svelte-h":!0}),v(Xe)!=="svelte-1v3xkc7"&&(Xe.textContent=cs),Vt=p(H),f(ie.$$.fragment,H),H.forEach(a),ut=p(e),f(Ue.$$.fragment,e),gt=p(e),N=x(e,"DIV",{class:!0});var se=J(N);f(Ie.$$.fragment,se),St=p(se),Ye=x(se,"P",{"data-svelte-h":!0}),v(Ye)!=="svelte-1djf8ll"&&(Ye.textContent=ps),Et=p(se),f(me.$$.fragment,se),se.forEach(a),$t=p(e),f(ze.$$.fragment,e),ft=p(e),B=x(e,"DIV",{class:!0});var X=J(B);f(_e.$$.fragment,X),Wt=p(X),Le=x(X,"P",{"data-svelte-h":!0}),v(Le)!=="svelte-16onw60"&&(Le.textContent=ds),Ft=p(X),Ke=x(X,"P",{"data-svelte-h":!0}),v(Ke)!=="svelte-145fyj2"&&(Ke.innerHTML=is),Nt=p(X),f(ue.$$.fragment,X),X.forEach(a),ht=p(e),f(Be.$$.fragment,e),kt=p(e),q=x(e,"DIV",{class:!0});var Y=J(q);f(qe.$$.fragment,Y),Pt=p(Y),Ae=x(Y,"P",{"data-svelte-h":!0}),v(Ae)!=="svelte-ornwlv"&&(Ae.textContent=ms),Gt=p(Y),Oe=x(Y,"P",{"data-svelte-h":!0}),v(Oe)!=="svelte-12ou3fb"&&(Oe.textContent=us),Qt=p(Y),f(ge.$$.fragment,Y),Y.forEach(a),yt=p(e),f(Ze.$$.fragment,e),Mt=p(e),Z=x(e,"DIV",{class:!0});var L=J(Z);f(De.$$.fragment,L),Ht=p(L),et=x(L,"P",{"data-svelte-h":!0}),v(et)!=="svelte-1jebb36"&&(et.textContent=gs),Xt=p(L),tt=x(L,"P",{"data-svelte-h":!0}),v(tt)!=="svelte-1pwf5kr"&&(tt.textContent=$s),Yt=p(L),f($e.$$.fragment,L),L.forEach(a),bt=p(e),f(Re.$$.fragment,e),xt=p(e),P=x(e,"DIV",{class:!0});var ne=J(P);f(Ve.$$.fragment,ne),Lt=p(ne),st=x(ne,"P",{"data-svelte-h":!0}),v(st)!=="svelte-1k7tubq"&&(st.textContent=fs),Kt=p(ne),f(fe.$$.fragment,ne),ne.forEach(a),this.h()},h(){C(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(n,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){h(s,e,r),d(e,i,r),d(e,n,r),h(l,n,null),m(n,o),m(n,t),m(n,le),m(n,w),m(n,S),m(n,E),m(n,re),h(g,n,null),m(n,T),m(n,U),h(ke,U,null),m(U,vt),m(U,Se),m(U,Tt),m(U,Ee),d(e,nt,r),h(ye,e,r),d(e,lt,r),d(e,W,r),h(Me,W,null),m(W,wt),m(W,We),m(W,Jt),h(oe,W,null),d(e,at,r),h(be,e,r),d(e,rt,r),d(e,I,r),h(xe,I,null),m(I,Ct),m(I,Fe),m(I,Ut),m(I,Ne),m(I,It),h(ce,I,null),d(e,ot,r),h(je,e,r),d(e,ct,r),d(e,z,r),h(ve,z,null),m(z,zt),m(z,Pe),m(z,_t),m(z,Ge),m(z,Bt),h(pe,z,null),d(e,pt,r),h(Te,e,r),d(e,dt,r),d(e,F,r),h(we,F,null),m(F,qt),m(F,Qe),m(F,Zt),h(de,F,null),d(e,it,r),h(Je,e,r),d(e,mt,r),d(e,_,r),h(Ce,_,null),m(_,Dt),m(_,He),m(_,Rt),m(_,Xe),m(_,Vt),h(ie,_,null),d(e,ut,r),h(Ue,e,r),d(e,gt,r),d(e,N,r),h(Ie,N,null),m(N,St),m(N,Ye),m(N,Et),h(me,N,null),d(e,$t,r),h(ze,e,r),d(e,ft,r),d(e,B,r),h(_e,B,null),m(B,Wt),m(B,Le),m(B,Ft),m(B,Ke),m(B,Nt),h(ue,B,null),d(e,ht,r),h(Be,e,r),d(e,kt,r),d(e,q,r),h(qe,q,null),m(q,Pt),m(q,Ae),m(q,Gt),m(q,Oe),m(q,Qt),h(ge,q,null),d(e,yt,r),h(Ze,e,r),d(e,Mt,r),d(e,Z,r),h(De,Z,null),m(Z,Ht),m(Z,et),m(Z,Xt),m(Z,tt),m(Z,Yt),h($e,Z,null),d(e,bt,r),h(Re,e,r),d(e,xt,r),d(e,P,r),h(Ve,P,null),m(P,Lt),m(P,st),m(P,Kt),h(fe,P,null),jt=!0},p(e,r){const O={};r&2&&(O.$$scope={dirty:r,ctx:e}),g.$set(O);const ee={};r&2&&(ee.$$scope={dirty:r,ctx:e}),oe.$set(ee);const G={};r&2&&(G.$$scope={dirty:r,ctx:e}),ce.$set(G);const Q={};r&2&&(Q.$$scope={dirty:r,ctx:e}),pe.$set(Q);const te={};r&2&&(te.$$scope={dirty:r,ctx:e}),de.$set(te);const H={};r&2&&(H.$$scope={dirty:r,ctx:e}),ie.$set(H);const se={};r&2&&(se.$$scope={dirty:r,ctx:e}),me.$set(se);const X={};r&2&&(X.$$scope={dirty:r,ctx:e}),ue.$set(X);const Y={};r&2&&(Y.$$scope={dirty:r,ctx:e}),ge.$set(Y);const L={};r&2&&(L.$$scope={dirty:r,ctx:e}),$e.$set(L);const ne={};r&2&&(ne.$$scope={dirty:r,ctx:e}),fe.$set(ne)},i(e){jt||(k(s.$$.fragment,e),k(l.$$.fragment,e),k(g.$$.fragment,e),k(ke.$$.fragment,e),k(ye.$$.fragment,e),k(Me.$$.fragment,e),k(oe.$$.fragment,e),k(be.$$.fragment,e),k(xe.$$.fragment,e),k(ce.$$.fragment,e),k(je.$$.fragment,e),k(ve.$$.fragment,e),k(pe.$$.fragment,e),k(Te.$$.fragment,e),k(we.$$.fragment,e),k(de.$$.fragment,e),k(Je.$$.fragment,e),k(Ce.$$.fragment,e),k(ie.$$.fragment,e),k(Ue.$$.fragment,e),k(Ie.$$.fragment,e),k(me.$$.fragment,e),k(ze.$$.fragment,e),k(_e.$$.fragment,e),k(ue.$$.fragment,e),k(Be.$$.fragment,e),k(qe.$$.fragment,e),k(ge.$$.fragment,e),k(Ze.$$.fragment,e),k(De.$$.fragment,e),k($e.$$.fragment,e),k(Re.$$.fragment,e),k(Ve.$$.fragment,e),k(fe.$$.fragment,e),jt=!0)},o(e){y(s.$$.fragment,e),y(l.$$.fragment,e),y(g.$$.fragment,e),y(ke.$$.fragment,e),y(ye.$$.fragment,e),y(Me.$$.fragment,e),y(oe.$$.fragment,e),y(be.$$.fragment,e),y(xe.$$.fragment,e),y(ce.$$.fragment,e),y(je.$$.fragment,e),y(ve.$$.fragment,e),y(pe.$$.fragment,e),y(Te.$$.fragment,e),y(we.$$.fragment,e),y(de.$$.fragment,e),y(Je.$$.fragment,e),y(Ce.$$.fragment,e),y(ie.$$.fragment,e),y(Ue.$$.fragment,e),y(Ie.$$.fragment,e),y(me.$$.fragment,e),y(ze.$$.fragment,e),y(_e.$$.fragment,e),y(ue.$$.fragment,e),y(Be.$$.fragment,e),y(qe.$$.fragment,e),y(ge.$$.fragment,e),y(Ze.$$.fragment,e),y(De.$$.fragment,e),y($e.$$.fragment,e),y(Re.$$.fragment,e),y(Ve.$$.fragment,e),y(fe.$$.fragment,e),jt=!1},d(e){e&&(a(i),a(n),a(nt),a(lt),a(W),a(at),a(rt),a(I),a(ot),a(ct),a(z),a(pt),a(dt),a(F),a(it),a(mt),a(_),a(ut),a(gt),a(N),a($t),a(ft),a(B),a(ht),a(kt),a(q),a(yt),a(Mt),a(Z),a(bt),a(xt),a(P)),M(s,e),M(l),M(g),M(ke),M(ye,e),M(Me),M(oe),M(be,e),M(xe),M(ce),M(je,e),M(ve),M(pe),M(Te,e),M(we),M(de),M(Je,e),M(Ce),M(ie),M(Ue,e),M(Ie),M(me),M(ze,e),M(_e),M(ue),M(Be,e),M(qe),M(ge),M(Ze,e),M(De),M($e),M(Re,e),M(Ve),M(fe)}}}function Rs(j){let s,i;return s=new At({props:{$$slots:{default:[Ds]},$$scope:{ctx:j}}}),{c(){$(s.$$.fragment)},l(n){f(s.$$.fragment,n)},m(n,l){h(s,n,l),i=!0},p(n,l){const o={};l&2&&(o.$$scope={dirty:l,ctx:n}),s.$set(o)},i(n){i||(k(s.$$.fragment,n),i=!0)},o(n){y(s.$$.fragment,n),i=!1},d(n){M(s,n)}}}function Vs(j){let s,i='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){s=b("p"),s.innerHTML=i},l(n){s=x(n,"P",{"data-svelte-h":!0}),v(s)!=="svelte-4ytcyb"&&(s.innerHTML=i)},m(n,l){d(n,s,l)},p:D,d(n){n&&a(s)}}}function Ss(j){let s,i;return s=new At({props:{$$slots:{default:[Vs]},$$scope:{ctx:j}}}),{c(){$(s.$$.fragment)},l(n){f(s.$$.fragment,n)},m(n,l){h(s,n,l),i=!0},p(n,l){const o={};l&2&&(o.$$scope={dirty:l,ctx:n}),s.$set(o)},i(n){i||(k(s.$$.fragment,n),i=!0)},o(n){y(s.$$.fragment,n),i=!1},d(n){M(s,n)}}}function Es(j){let s,i="The node API has not been documented yet.";return{c(){s=b("p"),s.textContent=i},l(n){s=x(n,"P",{"data-svelte-h":!0}),v(s)!=="svelte-1mrchm6"&&(s.textContent=i)},m(n,l){d(n,s,l)},p:D,d(n){n&&a(s)}}}function Ws(j){let s,i;return s=new At({props:{$$slots:{default:[Es]},$$scope:{ctx:j}}}),{c(){$(s.$$.fragment)},l(n){f(s.$$.fragment,n)},m(n,l){h(s,n,l),i=!0},p(n,l){const o={};l&2&&(o.$$scope={dirty:l,ctx:n}),s.$set(o)},i(n){i||(k(s.$$.fragment,n),i=!0)},o(n){y(s.$$.fragment,n),i=!1},d(n){M(s,n)}}}function Fs(j){let s,i,n,l,o,t,u,le,w,he,S,E,ae,re;return o=new xs({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),u=new V({props:{title:"Decoders",local:"decoders",headingTag:"h1"}}),w=new vs({props:{python:!0,rust:!0,node:!0,$$slots:{node:[Ws],rust:[Ss],python:[Rs]},$$scope:{ctx:j}}}),S=new js({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/decoders.mdx"}}),{c(){s=b("meta"),i=c(),n=b("p"),l=c(),$(o.$$.fragment),t=c(),$(u.$$.fragment),le=c(),$(w.$$.fragment),he=c(),$(S.$$.fragment),E=c(),ae=b("p"),this.h()},l(g){const T=bs("svelte-u9bgzb",document.head);s=x(T,"META",{name:!0,content:!0}),T.forEach(a),i=p(g),n=x(g,"P",{}),J(n).forEach(a),l=p(g),f(o.$$.fragment,g),t=p(g),f(u.$$.fragment,g),le=p(g),f(w.$$.fragment,g),he=p(g),f(S.$$.fragment,g),E=p(g),ae=x(g,"P",{}),J(ae).forEach(a),this.h()},h(){C(s,"name","hf:doc:metadata"),C(s,"content",Ns)},m(g,T){m(document.head,s),d(g,i,T),d(g,n,T),d(g,l,T),h(o,g,T),d(g,t,T),h(u,g,T),d(g,le,T),h(w,g,T),d(g,he,T),h(S,g,T),d(g,E,T),d(g,ae,T),re=!0},p(g,[T]){const U={};T&2&&(U.$$scope={dirty:T,ctx:g}),w.$set(U)},i(g){re||(k(o.$$.fragment,g),k(u.$$.fragment,g),k(w.$$.fragment,g),k(S.$$.fragment,g),re=!0)},o(g){y(o.$$.fragment,g),y(u.$$.fragment,g),y(w.$$.fragment,g),y(S.$$.fragment,g),re=!1},d(g){g&&(a(i),a(n),a(l),a(t),a(le),a(he),a(E),a(ae)),a(s),M(o,g),M(u,g),M(w,g),M(S,g)}}}const Ns='{"title":"Decoders","local":"decoders","sections":[{"title":"DecodeStream","local":"tokenizers.decoders.DecodeStream","sections":[],"depth":2},{"title":"BPEDecoder","local":"tokenizers.decoders.BPEDecoder","sections":[],"depth":2},{"title":"ByteFallback","local":"tokenizers.decoders.ByteFallback","sections":[],"depth":2},{"title":"ByteLevel","local":"tokenizers.decoders.ByteLevel","sections":[],"depth":2},{"title":"CTC","local":"tokenizers.decoders.CTC","sections":[],"depth":2},{"title":"Fuse","local":"tokenizers.decoders.Fuse","sections":[],"depth":2},{"title":"Metaspace","local":"tokenizers.decoders.Metaspace","sections":[],"depth":2},{"title":"Replace","local":"tokenizers.decoders.Replace","sections":[],"depth":2},{"title":"Sequence","local":"tokenizers.decoders.Sequence","sections":[],"depth":2},{"title":"Strip","local":"tokenizers.decoders.Strip","sections":[],"depth":2},{"title":"WordPiece","local":"tokenizers.decoders.WordPiece","sections":[],"depth":2}],"depth":1}';function Ps(j){return ks(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class As extends ys{constructor(s){super(),Ms(this,s,Ps,Fs,hs,{})}}export{As as component};