tokenizers/pr_2102/en/_app/immutable/nodes/8.ea720432.js

import{s as wr,o as Mr,n as B}from"../chunks/scheduler.7c59faff.js";import{S as Cr,i as jr,e as g,s as a,c as f,h as _r,a as d,d as l,b as o,f as C,g as u,j,k as p,l as c,m as h,t as z,n as b,o as x,p as y}from"../chunks/index.09bb5655.js";import{C as Nr,H as F,E as Tr}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.9cb13003.js";import{D as _}from"../chunks/Docstring.52966e01.js";import{C as H}from"../chunks/CodeBlock.91906b9f.js";import{T as Ur,M as Rn}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";import{E as V}from"../chunks/ExampleCodeBlock.a97e0167.js";function Fr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBCeXRlTGV2ZWwlMEFub3JtYWxpemVyJTIwJTNEJTIwQnl0ZUxldmVsKCklMEFub3JtYWxpemVyLm5vcm1hbGl6ZV9zdHIoJTIyaGVsbG8lNUNud29ybGQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> ByteLevel
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = ByteLevel()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;hello\\nworld&quot;</span>)
<span class="hljs-string">&#x27;helloĊworld&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Br(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBMb3dlcmNhc2UlMEFub3JtYWxpemVyJTIwJTNEJTIwTG93ZXJjYXNlKCklMEFub3JtYWxpemVyLm5vcm1hbGl6ZV9zdHIoJTIySGVsbG8lMjBXb3JsZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> Lowercase
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = Lowercase()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;Hello World&quot;</span>)
<span class="hljs-string">&#x27;hello world&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Jr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBORkMlMEFub3JtYWxpemVyJTIwJTNEJTIwTkZDKCklMEFub3JtYWxpemVyLm5vcm1hbGl6ZV9zdHIoJTIyZSU1Q3UwMzAxJTIyKSUyMCUyMCUyMyUyMCdlJyUyMCUyQiUyMGNvbWJpbmluZyUyMGFjY2VudA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> NFC
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = NFC()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;e\\u0301&quot;</span>)  <span class="hljs-comment"># &#x27;e&#x27; + combining accent</span>
<span class="hljs-string">&#x27;é&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Sr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBORkQlMEFub3JtYWxpemVyJTIwJTNEJTIwTkZEKCklMEFub3JtYWxpemVyLm5vcm1hbGl6ZV9zdHIoJTIySCVDMyVBOWxsbyUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> NFD
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = NFD()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;Héllo&quot;</span>)
<span class="hljs-string">&#x27;He\\u0301llo&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Xr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBORktDJTBBbm9ybWFsaXplciUyMCUzRCUyME5GS0MoKSUwQW5vcm1hbGl6ZXIubm9ybWFsaXplX3N0ciglMjIlRUYlQUMlODFuZSUyMGNhZiU1Q3UwMGU5JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> NFKC
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = NFKC()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;fine caf\\u00e9&quot;</span>)
<span class="hljs-string">&#x27;fine café&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Ir(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBORktEJTBBbm9ybWFsaXplciUyMCUzRCUyME5GS0QoKSUwQW5vcm1hbGl6ZXIubm9ybWFsaXplX3N0ciglMjIlRUYlQUMlODFuZSUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> NFKD
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = NFKD()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;fine&quot;</span>)
<span class="hljs-string">&#x27;fine&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Wr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBObXQlMEFub3JtYWxpemVyJTIwJTNEJTIwTm10KCklMEFub3JtYWxpemVyLm5vcm1hbGl6ZV9zdHIoJTIySGVsbG8lNUN4MDBXb3JsZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> Nmt
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = Nmt()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;Hello\\x00World&quot;</span>)
<span class="hljs-string">&#x27;Hello World&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Dr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBSZWdleCUwQWZyb20lMjB0b2tlbml6ZXJzLm5vcm1hbGl6ZXJzJTIwaW1wb3J0JTIwUmVwbGFjZSUwQSUyMyUyMFJlcGxhY2UlMjBhJTIwbGl0ZXJhbCUyMHN0cmluZyUwQVJlcGxhY2UoJTIyLiUyMiUyQyUyMCUyMiUyMCUyMikubm9ybWFsaXplX3N0ciglMjJoZWxsby53b3JsZCUyMiklMEElMjMlMjBSZXBsYWNlJTIwdXNpbmclMjBhJTIwcmVnZXglMEFSZXBsYWNlKFJlZ2V4KHIlMjIlNUNzJTJCJTIyKSUyQyUyMCUyMiUyMCUyMikubm9ybWFsaXplX3N0ciglMjJoZWxsbyUyMCUyMCUyMHdvcmxkJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Regex
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> Replace
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Replace a literal string</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Replace(<span class="hljs-string">&quot;.&quot;</span>, <span class="hljs-string">&quot; &quot;</span>).normalize_str(<span class="hljs-string">&quot;hello.world&quot;</span>)
<span class="hljs-string">&#x27;hello world&#x27;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Replace using a regex</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Replace(Regex(<span class="hljs-string">r&quot;\\s+&quot;</span>), <span class="hljs-string">&quot; &quot;</span>).normalize_str(<span class="hljs-string">&quot;hello   world&quot;</span>)
<span class="hljs-string">&#x27;hello world&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Pr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBORkQlMkMlMjBMb3dlcmNhc2UlMkMlMjBTdHJpcEFjY2VudHMlMkMlMjBTZXF1ZW5jZSUwQW5vcm1hbGl6ZXIlMjAlM0QlMjBTZXF1ZW5jZSglNUJORkQoKSUyQyUyMExvd2VyY2FzZSgpJTJDJTIwU3RyaXBBY2NlbnRzKCklNUQpJTBBbm9ybWFsaXplci5ub3JtYWxpemVfc3RyKCUyMkglQzMlQTlsbG8lMjBXJUMzJUI2cmxkJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> NFD, Lowercase, StripAccents, <span class="hljs-type">Sequence</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = <span class="hljs-type">Sequence</span>([NFD(), Lowercase(), StripAccents()])
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;Héllo Wörld&quot;</span>)
<span class="hljs-string">&#x27;hello world&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function qr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBTdHJpcCUwQW5vcm1hbGl6ZXIlMjAlM0QlMjBTdHJpcCgpJTBBbm9ybWFsaXplci5ub3JtYWxpemVfc3RyKCUyMiUyMCUyMGhlbGxvJTIwd29ybGQlMjAlMjAlMjIpJTBBU3RyaXAocmlnaHQlM0RGYWxzZSkubm9ybWFsaXplX3N0ciglMjIlMjAlMjBoZWxsbyUyMCUyMCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> Strip
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = Strip()
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;  hello world  &quot;</span>)
<span class="hljs-string">&#x27;hello world&#x27;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Strip(right=<span class="hljs-literal">False</span>).normalize_str(<span class="hljs-string">&quot;  hello  &quot;</span>)
<span class="hljs-string">&#x27;hello  &#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Zr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBORkQlMkMlMjBTdHJpcEFjY2VudHMlMkMlMjBTZXF1ZW5jZSUwQW5vcm1hbGl6ZXIlMjAlM0QlMjBTZXF1ZW5jZSglNUJORkQoKSUyQyUyMFN0cmlwQWNjZW50cygpJTVEKSUwQW5vcm1hbGl6ZXIubm9ybWFsaXplX3N0ciglMjJjYWYlQzMlQTklMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> NFD, StripAccents, <span class="hljs-type">Sequence</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = <span class="hljs-type">Sequence</span>([NFD(), StripAccents()])
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;café&quot;</span>)
<span class="hljs-string">&#x27;cafe&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Lr(w){let n,$="Example:",r,s,m;return s=new H({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubm9ybWFsaXplcnMlMjBpbXBvcnQlMjBCZXJ0Tm9ybWFsaXplciUwQW5vcm1hbGl6ZXIlMjAlM0QlMjBCZXJ0Tm9ybWFsaXplcihsb3dlcmNhc2UlM0RUcnVlKSUwQW5vcm1hbGl6ZXIubm9ybWFsaXplX3N0ciglMjJIJUMzJUE5bGxvJTIwV09STEQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.normalizers <span class="hljs-keyword">import</span> BertNormalizer
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer = BertNormalizer(lowercase=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>normalizer.normalize_str(<span class="hljs-string">&quot;Héllo WORLD&quot;</span>)
<span class="hljs-string">&#x27;hello world&#x27;</span>`,lang:"python",wrap:!1}}),{c(){n=g("p"),n.textContent=$,r=a(),f(s.$$.fragment)},l(t){n=d(t,"P",{"data-svelte-h":!0}),y(n)!=="svelte-11lpom8"&&(n.textContent=$),r=o(t),u(s.$$.fragment,t)},m(t,v){c(t,n,v),c(t,r,v),h(s,t,v),m=!0},p:B,i(t){m||(z(s.$$.fragment,t),m=!0)},o(t){b(s.$$.fragment,t),m=!1},d(t){t&&(l(n),l(r)),x(s,t)}}}function Er(w){let n,$,r,s,m,t,v="Bytelevel Normalizer",me,N,je=`Converts all bytes in the input to their Unicode representation using the GPT-2
byte-to-unicode mapping. Every byte value (0–255) is mapped to a unique visible
character so that any arbitrary binary input can be tokenized without needing a
special unknown token.`,R,Q,ce=`This normalizer is used together with the
<a href="/docs/tokenizers/pr_2102/en/api/pre-tokenizers#tokenizers.pre_tokenizers.ByteLevel">ByteLevel</a> pre-tokenizer and
<a href="/docs/tokenizers/pr_2102/en/api/decoders#tokenizers.decoders.ByteLevel">ByteLevel</a> decoder.`,$e,k,M,re,Ut,J,_e,rn,tt,Qn="Lowercase Normalizer",sn,nt,Kn=`Converts all text to lowercase using Unicode-aware lowercasing. This is equivalent
to calling <code>str.lower</code> on the input.`,ln,ge,Ft,Ne,Bt,S,Te,an,rt,Gn="NFC Unicode Normalizer",on,st,An=`Applies Unicode NFC (Canonical Decomposition, followed by Canonical Composition)
normalization. First decomposes characters, then recomposes them using canonical
composition rules. This produces the canonical composed form.`,pn,de,Jt,Ue,St,T,Fe,mn,lt,Yn="NFD Unicode Normalizer",cn,at,On=`Applies Unicode NFD (Canonical Decomposition) normalization. Decomposes characters into
their canonical components. For example, accented characters like <code>é</code> (U+00E9) are
decomposed into <code>e</code> (U+0065) + combining accent (U+0301).`,$n,ot,er=`This is often used as a first step before stripping accents with
<a href="/docs/tokenizers/pr_2102/en/api/normalizers#tokenizers.normalizers.StripAccents">StripAccents</a>.`,gn,fe,Xt,Be,It,X,Je,dn,it,tr="NFKC Unicode Normalizer",fn,pt,nr=`Applies Unicode NFKC (Compatibility Decomposition, followed by Canonical Composition)
normalization. Like NFC but also maps compatibility characters to their canonical
equivalents. This is the normalization used by Python’s <code>str.casefold</code> and
by many NLP pipelines.`,un,ue,Wt,Se,Dt,I,Xe,hn,mt,rr="NFKD Unicode Normalizer",zn,ct,sr=`Applies Unicode NFKD (Compatibility Decomposition) normalization. Like NFD but also
decomposes compatibility characters. For example, the ligature <code>fi</code> (U+FB01) is
decomposed into <code>f</code> + <code>i</code>.`,bn,he,Pt,Ie,qt,W,We,xn,$t,lr="Nmt normalizer",vn,gt,ar=`Normalizer used in the Google NMT pipeline. It handles various text cleaning tasks
including removing control characters, normalizing whitespace, and replacing certain
Unicode characters. This is equivalent to the normalization done in the original
SentencePiece NMT preprocessing.`,kn,ze,Zt,De,Lt,U,Pe,yn,dt,or="Base class for all normalizers",wn,ft,ir=`This class is not supposed to be instantiated directly. Instead, any implementation of a
Normalizer will return an instance of this class when instantiated.`,Mn,se,qe,Cn,ut,pr="Normalize a <code>NormalizedString</code> in-place",jn,ht,mr=`This method allows to modify a <code>NormalizedString</code> to
keep track of the alignment information. If you just want to see the result
of the normalization on a raw string, you can use
<code>normalize_str()</code>`,_n,le,Ze,Nn,zt,cr="Normalize the given string",Tn,bt,$r=`This method provides a way to visualize the effect of a
<a href="/docs/tokenizers/pr_2102/en/api/normalizers#tokenizers.normalizers.Normalizer">Normalizer</a> but it does not keep track of the alignment
information. If you need to get/convert offsets, you can use
<code>normalize()</code>`,Et,Le,Vt,K,Ee,Un,xt,gr="Precompiled normalizer",Fn,vt,dr=`A normalizer that uses a precompiled character map built from a SentencePiece model.
This normalizer is automatically extracted from SentencePiece <code>.model</code> files and
should not be constructed manually — it is used internally for full compatibility
with SentencePiece-based tokenizers.`,Ht,Ve,Rt,D,He,Bn,kt,fr="Replace normalizer",Jn,yt,ur=`Replaces occurrences of a pattern in the input string with the given content.
The pattern can be either a plain string or a regular expression wrapped in
<code>Regex</code>.`,Sn,be,Qt,Re,Kt,G,Qe,Xn,wt,hr=`Allows concatenating multiple other Normalizer as a Sequence.
All the normalizers run in sequence in the given order`,In,xe,Gt,Ke,At,P,Ge,Wn,Mt,zr="Strip normalizer",Dn,Ct,br="Removes leading and/or trailing whitespace from the input string.",Pn,ve,Yt,Ae,Ot,q,Ye,qn,jt,xr="StripAccents normalizer",Zn,_t,vr=`Strips all accent marks (combining diacritical characters) from the input. This
normalizer should typically be used after applying <a href="/docs/tokenizers/pr_2102/en/api/normalizers#tokenizers.normalizers.NFD">NFD</a>
or <a href="/docs/tokenizers/pr_2102/en/api/normalizers#tokenizers.normalizers.NFKD">NFKD</a> decomposition, which separates base
characters from their combining accents.`,Ln,ke,en,Oe,tn,Z,et,En,Nt,kr="BertNormalizer",Vn,Tt,yr=`Takes care of normalizing raw text before giving it to a Bert model.
This includes cleaning the text, handling accents, chinese chars and lowercasing`,Hn,ye,nn;return n=new F({props:{title:"ByteLevel",local:"tokenizers.normalizers.ByteLevel",headingTag:"h2"}}),s=new _({props:{name:"class tokenizers.normalizers.ByteLevel",anchor:"tokenizers.normalizers.ByteLevel",parameters:[]}}),k=new V({props:{anchor:"tokenizers.normalizers.ByteLevel.example",$$slots:{default:[Fr]},$$scope:{ctx:w}}}),re=new F({props:{title:"Lowercase",local:"tokenizers.normalizers.Lowercase",headingTag:"h2"}}),_e=new _({props:{name:"class tokenizers.normalizers.Lowercase",anchor:"tokenizers.normalizers.Lowercase",parameters:[]}}),ge=new V({props:{anchor:"tokenizers.normalizers.Lowercase.example",$$slots:{default:[Br]},$$scope:{ctx:w}}}),Ne=new F({props:{title:"NFC",local:"tokenizers.normalizers.NFC",headingTag:"h2"}}),Te=new _({props:{name:"class tokenizers.normalizers.NFC",anchor:"tokenizers.normalizers.NFC",parameters:[]}}),de=new V({props:{anchor:"tokenizers.normalizers.NFC.example",$$slots:{default:[Jr]},$$scope:{ctx:w}}}),Ue=new F({props:{title:"NFD",local:"tokenizers.normalizers.NFD",headingTag:"h2"}}),Fe=new _({props:{name:"class tokenizers.normalizers.NFD",anchor:"tokenizers.normalizers.NFD",parameters:[]}}),fe=new V({props:{anchor:"tokenizers.normalizers.NFD.example",$$slots:{default:[Sr]},$$scope:{ctx:w}}}),Be=new F({props:{title:"NFKC",local:"tokenizers.normalizers.NFKC",headingTag:"h2"}}),Je=new _({props:{name:"class tokenizers.normalizers.NFKC",anchor:"tokenizers.normalizers.NFKC",parameters:[]}}),ue=new V({props:{anchor:"tokenizers.normalizers.NFKC.example",$$slots:{default:[Xr]},$$scope:{ctx:w}}}),Se=new F({props:{title:"NFKD",local:"tokenizers.normalizers.NFKD",headingTag:"h2"}}),Xe=new _({props:{name:"class tokenizers.normalizers.NFKD",anchor:"tokenizers.normalizers.NFKD",parameters:[]}}),he=new V({props:{anchor:"tokenizers.normalizers.NFKD.example",$$slots:{default:[Ir]},$$scope:{ctx:w}}}),Ie=new F({props:{title:"Nmt",local:"tokenizers.normalizers.Nmt",headingTag:"h2"}}),We=new _({props:{name:"class tokenizers.normalizers.Nmt",anchor:"tokenizers.normalizers.Nmt",parameters:[]}}),ze=new V({props:{anchor:"tokenizers.normalizers.Nmt.example",$$slots:{default:[Wr]},$$scope:{ctx:w}}}),De=new F({props:{title:"Normalizer",local:"tokenizers.normalizers.Normalizer",headingTag:"h2"}}),Pe=new _({props:{name:"class tokenizers.normalizers.Normalizer",anchor:"tokenizers.normalizers.Normalizer",parameters:[]}}),qe=new _({props:{name:"normalize",anchor:"tokenizers.normalizers.Normalizer.normalize",parameters:[{name:"normalized",val:""}],parametersDescription:[{anchor:"tokenizers.normalizers.Normalizer.normalize.normalized",description:`<strong>normalized</strong> (<code>NormalizedString</code>) &#x2014;
The normalized string on which to apply this
<a href="/docs/tokenizers/pr_2102/en/api/normalizers#tokenizers.normalizers.Normalizer">Normalizer</a>`,name:"normalized"}]}}),Ze=new _({props:{name:"normalize_str",anchor:"tokenizers.normalizers.Normalizer.normalize_str",parameters:[{name:"sequence",val:""}],parametersDescription:[{anchor:"tokenizers.normalizers.Normalizer.normalize_str.sequence",description:`<strong>sequence</strong> (<code>str</code>) &#x2014;
A string to normalize`,name:"sequence"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A string after normalization</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>str</code></p>
`}}),Le=new F({props:{title:"Precompiled",local:"tokenizers.normalizers.Precompiled",headingTag:"h2"}}),Ee=new _({props:{name:"class tokenizers.normalizers.Precompiled",anchor:"tokenizers.normalizers.Precompiled",parameters:[{name:"precompiled_charsmap",val:""}],parametersDescription:[{anchor:"tokenizers.normalizers.Precompiled.precompiled_charsmap",description:`<strong>precompiled_charsmap</strong> (<code>bytes</code>) &#x2014;
The raw bytes of the precompiled character map, as found inside a
SentencePiece <code>.model</code> file.`,name:"precompiled_charsmap"}]}}),Ve=new F({props:{title:"Replace",local:"tokenizers.normalizers.Replace",headingTag:"h2"}}),He=new _({props:{name:"class tokenizers.normalizers.Replace",anchor:"tokenizers.normalizers.Replace",parameters:[{name:"pattern",val:""},{name:"content",val:""}],parametersDescription:[{anchor:"tokenizers.normalizers.Replace.pattern",description:`<strong>pattern</strong> (<code>str</code> or <code>Regex</code>) &#x2014;
The pattern to search for. Use a plain string for literal replacement,
or wrap a regex pattern in <code>Regex</code> for regex replacement.`,name:"pattern"},{anchor:"tokenizers.normalizers.Replace.content",description:`<strong>content</strong> (<code>str</code>) &#x2014;
The string to replace each match with.`,name:"content"}]}}),be=new V({props:{anchor:"tokenizers.normalizers.Replace.example",$$slots:{default:[Dr]},$$scope:{ctx:w}}}),Re=new F({props:{title:"Sequence",local:"tokenizers.normalizers.Sequence",headingTag:"h2"}}),Qe=new _({props:{name:"class tokenizers.normalizers.Sequence",anchor:"tokenizers.normalizers.Sequence",parameters:[{name:"normalizers",val:""}],parametersDescription:[{anchor:"tokenizers.normalizers.Sequence.normalizers",description:`<strong>normalizers</strong> (<code>List[Normalizer]</code>) &#x2014;
A list of Normalizer to be run as a sequence`,name:"normalizers"}]}}),xe=new V({props:{anchor:"tokenizers.normalizers.Sequence.example",$$slots:{default:[Pr]},$$scope:{ctx:w}}}),Ke=new F({props:{title:"Strip",local:"tokenizers.normalizers.Strip",headingTag:"h2"}}),Ge=new _({props:{name:"class tokenizers.normalizers.Strip",anchor:"tokenizers.normalizers.Strip",parameters:[{name:"left",val:" = True"},{name:"right",val:" = True"}],parametersDescription:[{anchor:"tokenizers.normalizers.Strip.left",description:`<strong>left</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to strip leading (left) whitespace.`,name:"left"},{anchor:"tokenizers.normalizers.Strip.right",description:`<strong>right</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to strip trailing (right) whitespace.`,name:"right"}]}}),ve=new V({props:{anchor:"tokenizers.normalizers.Strip.example",$$slots:{default:[qr]},$$scope:{ctx:w}}}),Ae=new F({props:{title:"StripAccents",local:"tokenizers.normalizers.StripAccents",headingTag:"h2"}}),Ye=new _({props:{name:"class tokenizers.normalizers.StripAccents",anchor:"tokenizers.normalizers.StripAccents",parameters:[]}}),ke=new V({props:{anchor:"tokenizers.normalizers.StripAccents.example",$$slots:{default:[Zr]},$$scope:{ctx:w}}}),Oe=new F({props:{title:"BertNormalizer",local:"tokenizers.normalizers.BertNormalizer",headingTag:"h2"}}),et=new _({props:{name:"class tokenizers.normalizers.BertNormalizer",anchor:"tokenizers.normalizers.BertNormalizer",parameters:[{name:"clean_text",val:" = True"},{name:"handle_chinese_chars",val:" = True"},{name:"strip_accents",val:" = None"},{name:"lowercase",val:" = True"}],parametersDescription:[{anchor:"tokenizers.normalizers.BertNormalizer.clean_text",description:`<strong>clean_text</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to clean the text, by removing any control characters
and replacing all whitespaces by the classic one.`,name:"clean_text"},{anchor:"tokenizers.normalizers.BertNormalizer.handle_chinese_chars",description:`<strong>handle_chinese_chars</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to handle chinese chars by putting spaces around them.`,name:"handle_chinese_chars"},{anchor:"tokenizers.normalizers.BertNormalizer.strip_accents",description:`<strong>strip_accents</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to strip all accents. If this option is not specified (ie == None),
then it will be determined by the value for <em>lowercase</em> (as in the original Bert).`,name:"strip_accents"},{anchor:"tokenizers.normalizers.BertNormalizer.lowercase",description:`<strong>lowercase</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to lowercase.`,name:"lowercase"}]}}),ye=new V({props:{anchor:"tokenizers.normalizers.BertNormalizer.example",$$slots:{default:[Lr]},$$scope:{ctx:w}}}),{c(){f(n.$$.fragment),$=a(),r=g("div"),f(s.$$.fragment),m=a(),t=g("p"),t.textContent=v,me=a(),N=g("p"),N.textContent=je,R=a(),Q=g("p"),Q.innerHTML=ce,$e=a(),f(k.$$.fragment),M=a(),f(re.$$.fragment),Ut=a(),J=g("div"),f(_e.$$.fragment),rn=a(),tt=g("p"),tt.textContent=Qn,sn=a(),nt=g("p"),nt.innerHTML=Kn,ln=a(),f(ge.$$.fragment),Ft=a(),f(Ne.$$.fragment),Bt=a(),S=g("div"),f(Te.$$.fragment),an=a(),rt=g("p"),rt.textContent=Gn,on=a(),st=g("p"),st.textContent=An,pn=a(),f(de.$$.fragment),Jt=a(),f(Ue.$$.fragment),St=a(),T=g("div"),f(Fe.$$.fragment),mn=a(),lt=g("p"),lt.textContent=Yn,cn=a(),at=g("p"),at.innerHTML=On,$n=a(),ot=g("p"),ot.innerHTML=er,gn=a(),f(fe.$$.fragment),Xt=a(),f(Be.$$.fragment),It=a(),X=g("div"),f(Je.$$.fragment),dn=a(),it=g("p"),it.textContent=tr,fn=a(),pt=g("p"),pt.innerHTML=nr,un=a(),f(ue.$$.fragment),Wt=a(),f(Se.$$.fragment),Dt=a(),I=g("div"),f(Xe.$$.fragment),hn=a(),mt=g("p"),mt.textContent=rr,zn=a(),ct=g("p"),ct.innerHTML=sr,bn=a(),f(he.$$.fragment),Pt=a(),f(Ie.$$.fragment),qt=a(),W=g("div"),f(We.$$.fragment),xn=a(),$t=g("p"),$t.textContent=lr,vn=a(),gt=g("p"),gt.textContent=ar,kn=a(),f(ze.$$.fragment),Zt=a(),f(De.$$.fragment),Lt=a(),U=g("div"),f(Pe.$$.fragment),yn=a(),dt=g("p"),dt.textContent=or,wn=a(),ft=g("p"),ft.textContent=ir,Mn=a(),se=g("div"),f(qe.$$.fragment),Cn=a(),ut=g("p"),ut.innerHTML=pr,jn=a(),ht=g("p"),ht.innerHTML=mr,_n=a(),le=g("div"),f(Ze.$$.fragment),Nn=a(),zt=g("p"),zt.textContent=cr,Tn=a(),bt=g("p"),bt.innerHTML=$r,Et=a(),f(Le.$$.fragment),Vt=a(),K=g("div"),f(Ee.$$.fragment),Un=a(),xt=g("p"),xt.textContent=gr,Fn=a(),vt=g("p"),vt.innerHTML=dr,Ht=a(),f(Ve.$$.fragment),Rt=a(),D=g("div"),f(He.$$.fragment),Bn=a(),kt=g("p"),kt.textContent=fr,Jn=a(),yt=g("p"),yt.innerHTML=ur,Sn=a(),f(be.$$.fragment),Qt=a(),f(Re.$$.fragment),Kt=a(),G=g("div"),f(Qe.$$.fragment),Xn=a(),wt=g("p"),wt.textContent=hr,In=a(),f(xe.$$.fragment),Gt=a(),f(Ke.$$.fragment),At=a(),P=g("div"),f(Ge.$$.fragment),Wn=a(),Mt=g("p"),Mt.textContent=zr,Dn=a(),Ct=g("p"),Ct.textContent=br,Pn=a(),f(ve.$$.fragment),Yt=a(),f(Ae.$$.fragment),Ot=a(),q=g("div"),f(Ye.$$.fragment),qn=a(),jt=g("p"),jt.textContent=xr,Zn=a(),_t=g("p"),_t.innerHTML=vr,Ln=a(),f(ke.$$.fragment),en=a(),f(Oe.$$.fragment),tn=a(),Z=g("div"),f(et.$$.fragment),En=a(),Nt=g("p"),Nt.textContent=kr,Vn=a(),Tt=g("p"),Tt.textContent=yr,Hn=a(),f(ye.$$.fragment),this.h()},l(e){u(n.$$.fragment,e),$=o(e),r=d(e,"DIV",{class:!0});var i=C(r);u(s.$$.fragment,i),m=o(i),t=d(i,"P",{"data-svelte-h":!0}),y(t)!=="svelte-1g2mx95"&&(t.textContent=v),me=o(i),N=d(i,"P",{"data-svelte-h":!0}),y(N)!=="svelte-8y3avw"&&(N.textContent=je),R=o(i),Q=d(i,"P",{"data-svelte-h":!0}),y(Q)!=="svelte-1gtrcyr"&&(Q.innerHTML=ce),$e=o(i),u(k.$$.fragment,i),i.forEach(l),M=o(e),u(re.$$.fragment,e),Ut=o(e),J=d(e,"DIV",{class:!0});var A=C(J);u(_e.$$.fragment,A),rn=o(A),tt=d(A,"P",{"data-svelte-h":!0}),y(tt)!=="svelte-1ws8pdc"&&(tt.textContent=Qn),sn=o(A),nt=d(A,"P",{"data-svelte-h":!0}),y(nt)!=="svelte-1kqd1o"&&(nt.innerHTML=Kn),ln=o(A),u(ge.$$.fragment,A),A.forEach(l),Ft=o(e),u(Ne.$$.fragment,e),Bt=o(e),S=d(e,"DIV",{class:!0});var Y=C(S);u(Te.$$.fragment,Y),an=o(Y),rt=d(Y,"P",{"data-svelte-h":!0}),y(rt)!=="svelte-199ec7z"&&(rt.textContent=Gn),on=o(Y),st=d(Y,"P",{"data-svelte-h":!0}),y(st)!=="svelte-77slk0"&&(st.textContent=An),pn=o(Y),u(de.$$.fragment,Y),Y.forEach(l),Jt=o(e),u(Ue.$$.fragment,e),St=o(e),T=d(e,"DIV",{class:!0});var L=C(T);u(Fe.$$.fragment,L),mn=o(L),lt=d(L,"P",{"data-svelte-h":!0}),y(lt)!=="svelte-17uaov6"&&(lt.textContent=Yn),cn=o(L),at=d(L,"P",{"data-svelte-h":!0}),y(at)!=="svelte-s7mzcj"&&(at.innerHTML=On),$n=o(L),ot=d(L,"P",{"data-svelte-h":!0}),y(ot)!=="svelte-s4bthq"&&(ot.innerHTML=er),gn=o(L),u(fe.$$.fragment,L),L.forEach(l),Xt=o(e),u(Be.$$.fragment,e),It=o(e),X=d(e,"DIV",{class:!0});var O=C(X);u(Je.$$.fragment,O),dn=o(O),it=d(O,"P",{"data-svelte-h":!0}),y(it)!=="svelte-1mvgix8"&&(it.textContent=tr),fn=o(O),pt=d(O,"P",{"data-svelte-h":!0}),y(pt)!=="svelte-vwpc95"&&(pt.innerHTML=nr),un=o(O),u(ue.$$.fragment,O),O.forEach(l),Wt=o(e),u(Se.$$.fragment,e),Dt=o(e),I=d(e,"DIV",{class:!0});var ee=C(I);u(Xe.$$.fragment,ee),hn=o(ee),mt=d(ee,"P",{"data-svelte-h":!0}),y(mt)!=="svelte-dyixwr"&&(mt.textContent=rr),zn=o(ee),ct=d(ee,"P",{"data-svelte-h":!0}),y(ct)!=="svelte-tj31pi"&&(ct.innerHTML=sr),bn=o(ee),u(he.$$.fragment,ee),ee.forEach(l),Pt=o(e),u(Ie.$$.fragment,e),qt=o(e),W=d(e,"DIV",{class:!0});var te=C(W);u(We.$$.fragment,te),xn=o(te),$t=d(te,"P",{"data-svelte-h":!0}),y($t)!=="svelte-1isyure"&&($t.textContent=lr),vn=o(te),gt=d(te,"P",{"data-svelte-h":!0}),y(gt)!=="svelte-g6sfqa"&&(gt.textContent=ar),kn=o(te),u(ze.$$.fragment,te),te.forEach(l),Zt=o(e),u(De.$$.fragment,e),Lt=o(e),U=d(e,"DIV",{class:!0});var E=C(U);u(Pe.$$.fragment,E),yn=o(E),dt=d(E,"P",{"data-svelte-h":!0}),y(dt)!=="svelte-1jpqc6v"&&(dt.textContent=or),wn=o(E),ft=d(E,"P",{"data-svelte-h":!0}),y(ft)!=="svelte-ne1itq"&&(ft.textContent=ir),Mn=o(E),se=d(E,"DIV",{class:!0});var ae=C(se);u(qe.$$.fragment,ae),Cn=o(ae),ut=d(ae,"P",{"data-svelte-h":!0}),y(ut)!=="svelte-a9bu3y"&&(ut.innerHTML=pr),jn=o(ae),ht=d(ae,"P",{"data-svelte-h":!0}),y(ht)!=="svelte-599qjf"&&(ht.innerHTML=mr),ae.forEach(l),_n=o(E),le=d(E,"DIV",{class:!0});var oe=C(le);u(Ze.$$.fragment,oe),Nn=o(oe),zt=d(oe,"P",{"data-svelte-h":!0}),y(zt)!=="svelte-1hrxycy"&&(zt.textContent=cr),Tn=o(oe),bt=d(oe,"P",{"data-svelte-h":!0}),y(bt)!=="svelte-17ensox"&&(bt.innerHTML=$r),oe.forEach(l),E.forEach(l),Et=o(e),u(Le.$$.fragment,e),Vt=o(e),K=d(e,"DIV",{class:!0});var ie=C(K);u(Ee.$$.fragment,ie),Un=o(ie),xt=d(ie,"P",{"data-svelte-h":!0}),y(xt)!=="svelte-o7u5rr"&&(xt.textContent=gr),Fn=o(ie),vt=d(ie,"P",{"data-svelte-h":!0}),y(vt)!=="svelte-87qo20"&&(vt.innerHTML=dr),ie.forEach(l),Ht=o(e),u(Ve.$$.fragment,e),Rt=o(e),D=d(e,"DIV",{class:!0});var ne=C(D);u(He.$$.fragment,ne),Bn=o(ne),kt=d(ne,"P",{"data-svelte-h":!0}),y(kt)!=="svelte-1s88uyt"&&(kt.textContent=fr),Jn=o(ne),yt=d(ne,"P",{"data-svelte-h":!0}),y(yt)!=="svelte-qrmftq"&&(yt.innerHTML=ur),Sn=o(ne),u(be.$$.fragment,ne),ne.forEach(l),Qt=o(e),u(Re.$$.fragment,e),Kt=o(e),G=d(e,"DIV",{class:!0});var pe=C(G);u(Qe.$$.fragment,pe),Xn=o(pe),wt=d(pe,"P",{"data-svelte-h":!0}),y(wt)!=="svelte-105zdm9"&&(wt.textContent=hr),In=o(pe),u(xe.$$.fragment,pe),pe.forEach(l),Gt=o(e),u(Ke.$$.fragment,e),At=o(e),P=d(e,"DIV",{class:!0});var we=C(P);u(Ge.$$.fragment,we),Wn=o(we),Mt=d(we,"P",{"data-svelte-h":!0}),y(Mt)!=="svelte-1279gw7"&&(Mt.textContent=zr),Dn=o(we),Ct=d(we,"P",{"data-svelte-h":!0}),y(Ct)!=="svelte-iv4e6j"&&(Ct.textContent=br),Pn=o(we),u(ve.$$.fragment,we),we.forEach(l),Yt=o(e),u(Ae.$$.fragment,e),Ot=o(e),q=d(e,"DIV",{class:!0});var Me=C(q);u(Ye.$$.fragment,Me),qn=o(Me),jt=d(Me,"P",{"data-svelte-h":!0}),y(jt)!=="svelte-7kf6fw"&&(jt.textContent=xr),Zn=o(Me),_t=d(Me,"P",{"data-svelte-h":!0}),y(_t)!=="svelte-1ws3pmz"&&(_t.innerHTML=vr),Ln=o(Me),u(ke.$$.fragment,Me),Me.forEach(l),en=o(e),u(Oe.$$.fragment,e),tn=o(e),Z=d(e,"DIV",{class:!0});var Ce=C(Z);u(et.$$.fragment,Ce),En=o(Ce),Nt=d(Ce,"P",{"data-svelte-h":!0}),y(Nt)!=="svelte-kgfnlo"&&(Nt.textContent=kr),Vn=o(Ce),Tt=d(Ce,"P",{"data-svelte-h":!0}),y(Tt)!=="svelte-ma12ts"&&(Tt.textContent=yr),Hn=o(Ce),u(ye.$$.fragment,Ce),Ce.forEach(l),this.h()},h(){j(r,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,i){h(n,e,i),c(e,$,i),c(e,r,i),h(s,r,null),p(r,m),p(r,t),p(r,me),p(r,N),p(r,R),p(r,Q),p(r,$e),h(k,r,null),c(e,M,i),h(re,e,i),c(e,Ut,i),c(e,J,i),h(_e,J,null),p(J,rn),p(J,tt),p(J,sn),p(J,nt),p(J,ln),h(ge,J,null),c(e,Ft,i),h(Ne,e,i),c(e,Bt,i),c(e,S,i),h(Te,S,null),p(S,an),p(S,rt),p(S,on),p(S,st),p(S,pn),h(de,S,null),c(e,Jt,i),h(Ue,e,i),c(e,St,i),c(e,T,i),h(Fe,T,null),p(T,mn),p(T,lt),p(T,cn),p(T,at),p(T,$n),p(T,ot),p(T,gn),h(fe,T,null),c(e,Xt,i),h(Be,e,i),c(e,It,i),c(e,X,i),h(Je,X,null),p(X,dn),p(X,it),p(X,fn),p(X,pt),p(X,un),h(ue,X,null),c(e,Wt,i),h(Se,e,i),c(e,Dt,i),c(e,I,i),h(Xe,I,null),p(I,hn),p(I,mt),p(I,zn),p(I,ct),p(I,bn),h(he,I,null),c(e,Pt,i),h(Ie,e,i),c(e,qt,i),c(e,W,i),h(We,W,null),p(W,xn),p(W,$t),p(W,vn),p(W,gt),p(W,kn),h(ze,W,null),c(e,Zt,i),h(De,e,i),c(e,Lt,i),c(e,U,i),h(Pe,U,null),p(U,yn),p(U,dt),p(U,wn),p(U,ft),p(U,Mn),p(U,se),h(qe,se,null),p(se,Cn),p(se,ut),p(se,jn),p(se,ht),p(U,_n),p(U,le),h(Ze,le,null),p(le,Nn),p(le,zt),p(le,Tn),p(le,bt),c(e,Et,i),h(Le,e,i),c(e,Vt,i),c(e,K,i),h(Ee,K,null),p(K,Un),p(K,xt),p(K,Fn),p(K,vt),c(e,Ht,i),h(Ve,e,i),c(e,Rt,i),c(e,D,i),h(He,D,null),p(D,Bn),p(D,kt),p(D,Jn),p(D,yt),p(D,Sn),h(be,D,null),c(e,Qt,i),h(Re,e,i),c(e,Kt,i),c(e,G,i),h(Qe,G,null),p(G,Xn),p(G,wt),p(G,In),h(xe,G,null),c(e,Gt,i),h(Ke,e,i),c(e,At,i),c(e,P,i),h(Ge,P,null),p(P,Wn),p(P,Mt),p(P,Dn),p(P,Ct),p(P,Pn),h(ve,P,null),c(e,Yt,i),h(Ae,e,i),c(e,Ot,i),c(e,q,i),h(Ye,q,null),p(q,qn),p(q,jt),p(q,Zn),p(q,_t),p(q,Ln),h(ke,q,null),c(e,en,i),h(Oe,e,i),c(e,tn,i),c(e,Z,i),h(et,Z,null),p(Z,En),p(Z,Nt),p(Z,Vn),p(Z,Tt),p(Z,Hn),h(ye,Z,null),nn=!0},p(e,i){const A={};i&2&&(A.$$scope={dirty:i,ctx:e}),k.$set(A);const Y={};i&2&&(Y.$$scope={dirty:i,ctx:e}),ge.$set(Y);const L={};i&2&&(L.$$scope={dirty:i,ctx:e}),de.$set(L);const O={};i&2&&(O.$$scope={dirty:i,ctx:e}),fe.$set(O);const ee={};i&2&&(ee.$$scope={dirty:i,ctx:e}),ue.$set(ee);const te={};i&2&&(te.$$scope={dirty:i,ctx:e}),he.$set(te);const E={};i&2&&(E.$$scope={dirty:i,ctx:e}),ze.$set(E);const ae={};i&2&&(ae.$$scope={dirty:i,ctx:e}),be.$set(ae);const oe={};i&2&&(oe.$$scope={dirty:i,ctx:e}),xe.$set(oe);const ie={};i&2&&(ie.$$scope={dirty:i,ctx:e}),ve.$set(ie);const ne={};i&2&&(ne.$$scope={dirty:i,ctx:e}),ke.$set(ne);const pe={};i&2&&(pe.$$scope={dirty:i,ctx:e}),ye.$set(pe)},i(e){nn||(z(n.$$.fragment,e),z(s.$$.fragment,e),z(k.$$.fragment,e),z(re.$$.fragment,e),z(_e.$$.fragment,e),z(ge.$$.fragment,e),z(Ne.$$.fragment,e),z(Te.$$.fragment,e),z(de.$$.fragment,e),z(Ue.$$.fragment,e),z(Fe.$$.fragment,e),z(fe.$$.fragment,e),z(Be.$$.fragment,e),z(Je.$$.fragment,e),z(ue.$$.fragment,e),z(Se.$$.fragment,e),z(Xe.$$.fragment,e),z(he.$$.fragment,e),z(Ie.$$.fragment,e),z(We.$$.fragment,e),z(ze.$$.fragment,e),z(De.$$.fragment,e),z(Pe.$$.fragment,e),z(qe.$$.fragment,e),z(Ze.$$.fragment,e),z(Le.$$.fragment,e),z(Ee.$$.fragment,e),z(Ve.$$.fragment,e),z(He.$$.fragment,e),z(be.$$.fragment,e),z(Re.$$.fragment,e),z(Qe.$$.fragment,e),z(xe.$$.fragment,e),z(Ke.$$.fragment,e),z(Ge.$$.fragment,e),z(ve.$$.fragment,e),z(Ae.$$.fragment,e),z(Ye.$$.fragment,e),z(ke.$$.fragment,e),z(Oe.$$.fragment,e),z(et.$$.fragment,e),z(ye.$$.fragment,e),nn=!0)},o(e){b(n.$$.fragment,e),b(s.$$.fragment,e),b(k.$$.fragment,e),b(re.$$.fragment,e),b(_e.$$.fragment,e),b(ge.$$.fragment,e),b(Ne.$$.fragment,e),b(Te.$$.fragment,e),b(de.$$.fragment,e),b(Ue.$$.fragment,e),b(Fe.$$.fragment,e),b(fe.$$.fragment,e),b(Be.$$.fragment,e),b(Je.$$.fragment,e),b(ue.$$.fragment,e),b(Se.$$.fragment,e),b(Xe.$$.fragment,e),b(he.$$.fragment,e),b(Ie.$$.fragment,e),b(We.$$.fragment,e),b(ze.$$.fragment,e),b(De.$$.fragment,e),b(Pe.$$.fragment,e),b(qe.$$.fragment,e),b(Ze.$$.fragment,e),b(Le.$$.fragment,e),b(Ee.$$.fragment,e),b(Ve.$$.fragment,e),b(He.$$.fragment,e),b(be.$$.fragment,e),b(Re.$$.fragment,e),b(Qe.$$.fragment,e),b(xe.$$.fragment,e),b(Ke.$$.fragment,e),b(Ge.$$.fragment,e),b(ve.$$.fragment,e),b(Ae.$$.fragment,e),b(Ye.$$.fragment,e),b(ke.$$.fragment,e),b(Oe.$$.fragment,e),b(et.$$.fragment,e),b(ye.$$.fragment,e),nn=!1},d(e){e&&(l($),l(r),l(M),l(Ut),l(J),l(Ft),l(Bt),l(S),l(Jt),l(St),l(T),l(Xt),l(It),l(X),l(Wt),l(Dt),l(I),l(Pt),l(qt),l(W),l(Zt),l(Lt),l(U),l(Et),l(Vt),l(K),l(Ht),l(Rt),l(D),l(Qt),l(Kt),l(G),l(Gt),l(At),l(P),l(Yt),l(Ot),l(q),l(en),l(tn),l(Z)),x(n,e),x(s),x(k),x(re,e),x(_e),x(ge),x(Ne,e),x(Te),x(de),x(Ue,e),x(Fe),x(fe),x(Be,e),x(Je),x(ue),x(Se,e),x(Xe),x(he),x(Ie,e),x(We),x(ze),x(De,e),x(Pe),x(qe),x(Ze),x(Le,e),x(Ee),x(Ve,e),x(He),x(be),x(Re,e),x(Qe),x(xe),x(Ke,e),x(Ge),x(ve),x(Ae,e),x(Ye),x(ke),x(Oe,e),x(et),x(ye)}}}function Vr(w){let n,$;return n=new Rn({props:{$$slots:{default:[Er]},$$scope:{ctx:w}}}),{c(){f(n.$$.fragment)},l(r){u(n.$$.fragment,r)},m(r,s){h(n,r,s),$=!0},p(r,s){const m={};s&2&&(m.$$scope={dirty:s,ctx:r}),n.$set(m)},i(r){$||(z(n.$$.fragment,r),$=!0)},o(r){b(n.$$.fragment,r),$=!1},d(r){x(n,r)}}}function Hr(w){let n,$='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){n=g("p"),n.innerHTML=$},l(r){n=d(r,"P",{"data-svelte-h":!0}),y(n)!=="svelte-4ytcyb"&&(n.innerHTML=$)},m(r,s){c(r,n,s)},p:B,d(r){r&&l(n)}}}function Rr(w){let n,$;return n=new Rn({props:{$$slots:{default:[Hr]},$$scope:{ctx:w}}}),{c(){f(n.$$.fragment)},l(r){u(n.$$.fragment,r)},m(r,s){h(n,r,s),$=!0},p(r,s){const m={};s&2&&(m.$$scope={dirty:s,ctx:r}),n.$set(m)},i(r){$||(z(n.$$.fragment,r),$=!0)},o(r){b(n.$$.fragment,r),$=!1},d(r){x(n,r)}}}function Qr(w){let n,$="The node API has not been documented yet.";return{c(){n=g("p"),n.textContent=$},l(r){n=d(r,"P",{"data-svelte-h":!0}),y(n)!=="svelte-1mrchm6"&&(n.textContent=$)},m(r,s){c(r,n,s)},p:B,d(r){r&&l(n)}}}function Kr(w){let n,$;return n=new Rn({props:{$$slots:{default:[Qr]},$$scope:{ctx:w}}}),{c(){f(n.$$.fragment)},l(r){u(n.$$.fragment,r)},m(r,s){h(n,r,s),$=!0},p(r,s){const m={};s&2&&(m.$$scope={dirty:s,ctx:r}),n.$set(m)},i(r){$||(z(n.$$.fragment,r),$=!0)},o(r){b(n.$$.fragment,r),$=!1},d(r){x(n,r)}}}function Gr(w){let n,$,r,s,m,t,v,me,N,je,R,Q,ce,$e;return m=new Nr({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),v=new F({props:{title:"Normalizers",local:"normalizers",headingTag:"h1"}}),N=new Ur({props:{python:!0,rust:!0,node:!0,$$slots:{node:[Kr],rust:[Rr],python:[Vr]},$$scope:{ctx:w}}}),R=new Tr({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/normalizers.mdx"}}),{c(){n=g("meta"),$=a(),r=g("p"),s=a(),f(m.$$.fragment),t=a(),f(v.$$.fragment),me=a(),f(N.$$.fragment),je=a(),f(R.$$.fragment),Q=a(),ce=g("p"),this.h()},l(k){const M=_r("svelte-u9bgzb",document.head);n=d(M,"META",{name:!0,content:!0}),M.forEach(l),$=o(k),r=d(k,"P",{}),C(r).forEach(l),s=o(k),u(m.$$.fragment,k),t=o(k),u(v.$$.fragment,k),me=o(k),u(N.$$.fragment,k),je=o(k),u(R.$$.fragment,k),Q=o(k),ce=d(k,"P",{}),C(ce).forEach(l),this.h()},h(){j(n,"name","hf:doc:metadata"),j(n,"content",Ar)},m(k,M){p(document.head,n),c(k,$,M),c(k,r,M),c(k,s,M),h(m,k,M),c(k,t,M),h(v,k,M),c(k,me,M),h(N,k,M),c(k,je,M),h(R,k,M),c(k,Q,M),c(k,ce,M),$e=!0},p(k,[M]){const re={};M&2&&(re.$$scope={dirty:M,ctx:k}),N.$set(re)},i(k){$e||(z(m.$$.fragment,k),z(v.$$.fragment,k),z(N.$$.fragment,k),z(R.$$.fragment,k),$e=!0)},o(k){b(m.$$.fragment,k),b(v.$$.fragment,k),b(N.$$.fragment,k),b(R.$$.fragment,k),$e=!1},d(k){k&&(l($),l(r),l(s),l(t),l(me),l(je),l(Q),l(ce)),l(n),x(m,k),x(v,k),x(N,k),x(R,k)}}}const Ar='{"title":"Normalizers","local":"normalizers","sections":[{"title":"ByteLevel","local":"tokenizers.normalizers.ByteLevel","sections":[],"depth":2},{"title":"Lowercase","local":"tokenizers.normalizers.Lowercase","sections":[],"depth":2},{"title":"NFC","local":"tokenizers.normalizers.NFC","sections":[],"depth":2},{"title":"NFD","local":"tokenizers.normalizers.NFD","sections":[],"depth":2},{"title":"NFKC","local":"tokenizers.normalizers.NFKC","sections":[],"depth":2},{"title":"NFKD","local":"tokenizers.normalizers.NFKD","sections":[],"depth":2},{"title":"Nmt","local":"tokenizers.normalizers.Nmt","sections":[],"depth":2},{"title":"Normalizer","local":"tokenizers.normalizers.Normalizer","sections":[],"depth":2},{"title":"Precompiled","local":"tokenizers.normalizers.Precompiled","sections":[],"depth":2},{"title":"Replace","local":"tokenizers.normalizers.Replace","sections":[],"depth":2},{"title":"Sequence","local":"tokenizers.normalizers.Sequence","sections":[],"depth":2},{"title":"Strip","local":"tokenizers.normalizers.Strip","sections":[],"depth":2},{"title":"StripAccents","local":"tokenizers.normalizers.StripAccents","sections":[],"depth":2},{"title":"BertNormalizer","local":"tokenizers.normalizers.BertNormalizer","sections":[],"depth":2}],"depth":1}';function Yr(w){return Mr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class as extends Cr{constructor(n){super(),jr(this,n,Yr,Gr,wr,{})}}export{as as component};