tokenizers/pr_2139/en/_app/immutable/nodes/15.bf52bd49.js

import{s as B,n as O,o as I}from"../chunks/scheduler.7c59faff.js";import{S as J,i as K,e as h,s,c as C,h as N,a as d,d as n,b as o,f as R,g as E,p as q,j,k as Q,l as a,m as M,t as L,n as H,o as S}from"../chunks/index.09bb5655.js";import{C as V,H as A,E as W}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fb0d6fe2.js";function X(U){let i,k,$,_,r,v,l,x,m,D=`Fast State-of-the-art tokenizers, optimized for both research and
production`,z,f,F=`<a href="https://github.com/huggingface/tokenizers" rel="nofollow">🤗 Tokenizers</a> provides an
implementation of today’s most used tokenizers, with a focus on
performance and versatility. These tokenizers are also used in <a href="https://github.com/huggingface/transformers" rel="nofollow">🤗 Transformers</a>.`,b,p,y,u,G="<li>Train new vocabularies and tokenize, using today’s most used tokenizers.</li> <li>Extremely fast (both training and tokenization), thanks to the Rust implementation. Takes less than 20 seconds to tokenize a GB of text on a server’s CPU.</li> <li>Easy to use, but also extremely versatile.</li> <li>Designed for both research and production.</li> <li>Full alignment tracking. Even with destructive normalization, it’s always possible to get the part of the original sentence that corresponds to any token.</li> <li>Does all the pre-processing: Truncation, Padding, add the special tokens your model needs.</li>",T,c,w,g,P;return r=new V({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),l=new A({props:{title:"Tokenizers",local:"tokenizers",headingTag:"h1"}}),p=new A({props:{title:"Main features:",local:"main-features",headingTag:"h1"}}),c=new W({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/index.mdx"}}),{c(){i=h("meta"),k=s(),$=h("p"),_=s(),C(r.$$.fragment),v=s(),C(l.$$.fragment),x=s(),m=h("p"),m.textContent=D,z=s(),f=h("p"),f.innerHTML=F,b=s(),C(p.$$.fragment),y=s(),u=h("ul"),u.innerHTML=G,T=s(),C(c.$$.fragment),w=s(),g=h("p"),this.h()},l(e){const t=N("svelte-u9bgzb",document.head);i=d(t,"META",{name:!0,content:!0}),t.forEach(n),k=o(e),$=d(e,"P",{}),R($).forEach(n),_=o(e),E(r.$$.fragment,e),v=o(e),E(l.$$.fragment,e),x=o(e),m=d(e,"P",{"data-svelte-h":!0}),q(m)!=="svelte-hqcehv"&&(m.textContent=D),z=o(e),f=d(e,"P",{"data-svelte-h":!0}),q(f)!=="svelte-ydrho4"&&(f.innerHTML=F),b=o(e),E(p.$$.fragment,e),y=o(e),u=d(e,"UL",{"data-svelte-h":!0}),q(u)!=="svelte-1k61qc"&&(u.innerHTML=G),T=o(e),E(c.$$.fragment,e),w=o(e),g=d(e,"P",{}),R(g).forEach(n),this.h()},h(){j(i,"name","hf:doc:metadata"),j(i,"content",Y)},m(e,t){Q(document.head,i),a(e,k,t),a(e,$,t),a(e,_,t),M(r,e,t),a(e,v,t),M(l,e,t),a(e,x,t),a(e,m,t),a(e,z,t),a(e,f,t),a(e,b,t),M(p,e,t),a(e,y,t),a(e,u,t),a(e,T,t),M(c,e,t),a(e,w,t),a(e,g,t),P=!0},p:O,i(e){P||(L(r.$$.fragment,e),L(l.$$.fragment,e),L(p.$$.fragment,e),L(c.$$.fragment,e),P=!0)},o(e){H(r.$$.fragment,e),H(l.$$.fragment,e),H(p.$$.fragment,e),H(c.$$.fragment,e),P=!1},d(e){e&&(n(k),n($),n(_),n(v),n(x),n(m),n(z),n(f),n(b),n(y),n(u),n(T),n(w),n(g)),n(i),S(r,e),S(l,e),S(p,e),S(c,e)}}}const Y='{"title":"Tokenizers","local":"tokenizers","sections":[],"depth":1}';function Z(U){return I(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ae extends J{constructor(i){super(),K(this,i,Z,X,B,{})}}export{ae as component};