tokenizers/main/en/_app/chunks/TokenizersLanguageContent-hf-doc-builder.js

import{D as W,S as $,i as H,s as C,F as B,H as I,I as R,J as T,q as d,o as p,N as y,O as N,a as w,d as _,b as n,g as A,G as b,L as M,e as D,w as q,k as E,t as X,c as G,x as e1,m as V,h as t1,y as z,P as l1,n as Z,B as P,p as x,K as s1,T as a1,U as n1}from"./vendor-hf-doc-builder.js";const S=W("python");function r1(a){let e;const s=a[1].default,t=B(s,a,a[0],null);return{c(){t&&t.c()},l(l){t&&t.l(l)},m(l,i){t&&t.m(l,i),e=!0},p(l,[i]){t&&t.p&&(!e||i&1)&&I(t,s,l,l[0],e?T(s,l[0],i,null):R(l[0]),null)},i(l){e||(d(t,l),e=!0)},o(l){p(t,l),e=!1},d(l){t&&t.d(l)}}}function i1(a,e,s){let{$$slots:t={},$$scope:l}=e;return a.$$set=i=>{"$$scope"in i&&s(0,l=i.$$scope)},[l,t]}class y1 extends ${constructor(e){super();H(this,e,i1,r1,C,{})}}function o1(a){let e,s,t;return{c(){e=y("svg"),s=y("path"),t=y("path"),this.h()},l(l){e=N(l,"svg",{class:!0,xmlns:!0,"xmlns:xlink":!0,"aria-hidden":!0,focusable:!0,role:!0,width:!0,height:!0,preserveAspectRatio:!0,viewBox:!0});var i=w(e);s=N(i,"path",{d:!0,fill:!0}),w(s).forEach(_),t=N(i,"path",{d:!0,fill:!0}),w(t).forEach(_),i.forEach(_),this.h()},h(){n(s,"d","M15.84.5a16.4,16.4,0,0,0-3.57.32C9.1,1.39,8.53,2.53,8.53,4.64V7.48H16v1H5.77a4.73,4.73,0,0,0-4.7,3.74,14.82,14.82,0,0,0,0,7.54c.57,2.28,1.86,3.82,4,3.82h2.6V20.14a4.73,4.73,0,0,1,4.63-4.63h7.38a3.72,3.72,0,0,0,3.73-3.73V4.64A4.16,4.16,0,0,0,19.65.82,20.49,20.49,0,0,0,15.84.5ZM11.78,2.77a1.39,1.39,0,0,1,1.38,1.46,1.37,1.37,0,0,1-1.38,1.38A1.42,1.42,0,0,1,10.4,4.23,1.44,1.44,0,0,1,11.78,2.77Z"),n(s,"fill","#5a9fd4"),n(t,"d","M16.16,31.5a16.4,16.4,0,0,0,3.57-.32c3.17-.57,3.74-1.71,3.74-3.82V24.52H16v-1H26.23a4.73,4.73,0,0,0,4.7-3.74,14.82,14.82,0,0,0,0-7.54c-.57-2.28-1.86-3.82-4-3.82h-2.6v3.41a4.73,4.73,0,0,1-4.63,4.63H12.35a3.72,3.72,0,0,0-3.73,3.73v7.14a4.16,4.16,0,0,0,3.73,3.82A20.49,20.49,0,0,0,16.16,31.5Zm4.06-2.27a1.39,1.39,0,0,1-1.38-1.46,1.37,1.37,0,0,1,1.38-1.38,1.42,1.42,0,0,1,1.38,1.38A1.44,1.44,0,0,1,20.22,29.23Z"),n(t,"fill","#ffd43b"),n(e,"class",a[0]),n(e,"xmlns","http://www.w3.org/2000/svg"),n(e,"xmlns:xlink","http://www.w3.org/1999/xlink"),n(e,"aria-hidden","true"),n(e,"focusable","false"),n(e,"role","img"),n(e,"width","1em"),n(e,"height","1em"),n(e,"preserveAspectRatio","xMidYMid meet"),n(e,"viewBox","0 0 32 32")},m(l,i){A(l,e,i),b(e,s),b(e,t)},p(l,[i]){i&1&&n(e,"class",l[0])},i:M,o:M,d(l){l&&_(e)}}}function c1(a,e,s){let{classNames:t=""}=e;return a.$$set=l=>{"classNames"in l&&s(0,t=l.classNames)},[t]}class f1 extends ${constructor(e){super();H(this,e,c1,o1,C,{classNames:0})}}function u1(a){let e,s;return{c(){e=y("svg"),s=y("path"),this.h()},l(t){e=N(t,"svg",{class:!0,xmlns:!0,"xmlns:xlink":!0,"aria-hidden":!0,focusable:!0,role:!0,preserveAspectRatio:!0,width:!0,height:!0,viewBox:!0});var l=w(e);s=N(l,"path",{d:!0,fill:!0}),w(s).forEach(_),l.forEach(_),this.h()},h(){n(s,"d","M31.77,15.61l-1.34-.83c0-.13,0-.26,0-.39l1.16-1.08a.46.46,0,0,0,.14-.43.44.44,0,0,0-.29-.34L29.92,12l-.12-.38.92-1.28a.46.46,0,0,0,.06-.45.47.47,0,0,0-.36-.28l-1.55-.25L28.68,9l.66-1.44a.48.48,0,0,0,0-.45.46.46,0,0,0-.4-.2L27.32,7l-.25-.3.36-1.54a.46.46,0,0,0-.12-.43.46.46,0,0,0-.43-.13l-1.54.37L25,4.68l.06-1.58a.44.44,0,0,0-.21-.4.45.45,0,0,0-.45,0L23,3.32l-.35-.19L22.4,1.57a.46.46,0,0,0-.28-.35.48.48,0,0,0-.45.05l-1.28.92L20,2.08,19.46.6a.44.44,0,0,0-.34-.29.46.46,0,0,0-.43.14L17.62,1.6l-.39,0L16.39.22a.46.46,0,0,0-.78,0l-.83,1.34-.39,0L13.31.45a.46.46,0,0,0-.43-.14.44.44,0,0,0-.34.29L12,2.08l-.38.11-1.28-.92a.48.48,0,0,0-.45-.05.5.5,0,0,0-.28.35L9.35,3.13,9,3.32,7.57,2.66a.45.45,0,0,0-.45,0,.49.49,0,0,0-.21.4L7,4.68l-.31.25L5.13,4.56a.48.48,0,0,0-.44.13.46.46,0,0,0-.12.43l.36,1.54L4.68,7l-1.58,0a.46.46,0,0,0-.4.2.48.48,0,0,0,0,.45L3.32,9l-.19.35L1.57,9.6a.47.47,0,0,0-.35.28.48.48,0,0,0,.05.45l.92,1.28c0,.12-.07.25-.11.38L.6,12.54a.44.44,0,0,0-.29.34.46.46,0,0,0,.14.43L1.6,14.39l0,.39-1.35.83a.47.47,0,0,0,0,.78l1.35.84,0,.39L.45,18.69a.46.46,0,0,0-.14.43.44.44,0,0,0,.29.34L2.08,20c0,.13.07.26.11.39l-.92,1.28a.46.46,0,0,0-.05.44.45.45,0,0,0,.36.28l1.55.25.19.35-.65,1.44a.45.45,0,0,0,.43.65L4.68,25l.25.3-.36,1.54a.46.46,0,0,0,.12.43.48.48,0,0,0,.44.12l1.54-.36.3.25L6.91,28.9a.49.49,0,0,0,.21.4.48.48,0,0,0,.45,0L9,28.68l.35.19.26,1.56a.46.46,0,0,0,.27.35.48.48,0,0,0,.45-.05l1.28-.92.38.12.55,1.47a.47.47,0,0,0,.34.29.46.46,0,0,0,.43-.13l1.08-1.16.39,0,.83,1.34A.46.46,0,0,0,16,32a.47.47,0,0,0,.4-.22l.83-1.34.39,0,1.08,1.16a.46.46,0,0,0,.43.13.47.47,0,0,0,.34-.29L20,29.93l.38-.12,1.28.92a.48.48,0,0,0,.45.05.45.45,0,0,0,.27-.35l.26-1.56.35-.19,1.43.66a.48.48,0,0,0,.45,0,.49.49,0,0,0,.21-.4L25,27.32l.3-.25,1.54.36a.48.48,0,0,0,.44-.12.46.46,0,0,0,.12-.43l-.36-1.54.25-.3,1.58.05a.45.45,0,0,0,.43-.65L28.69,23l.19-.35,1.55-.25a.45.45,0,0,0,.36-.28.43.43,0,0,0-.06-.44l-.92-1.28.12-.39,1.48-.55a.44.44,0,0,0,.29-.34.46.46,0,0,0-.14-.43L30.4,17.62c0-.13,0-.26,0-.39l1.34-.84a.46.46,0,0,0,0-.78Zm-9,11.16A1,1,0,1,1,23.92,26a.95.95,0,0,1-1.14.73Zm-.45-3.09a.87.87,0,0,0-1,.67l-.48,2.22a11.74,11.74,0,0,1-9.75,0l-.48-2.23a.85.85,0,0,0-1-.66l-2,.42a14.67,14.67,0,0,1-1-1.2h9.58c.1,0,.18,0,.18-.12V19.35c0-.1-.08-.12-.18-.12h-2.8V17.08h3a1.9,1.9,0,0,1,1.86,1.62c.12.47.39,2,.57,2.5s.91,1.65,1.69,1.65h4.77l.17,0a11,11,0,0,1-1.08,1.27l-2-.43Zm-13.24,3A.94.94,0,0,1,8,26a1,1,0,1,1,1.13.73ZM5.45,12a1,1,0,0,1-1.74.77,1,1,0,0,1,.49-1.26A1,1,0,0,1,5.45,12ZM4.33,14.66l2.05-.91a.87.87,0,0,0,.44-1.15l-.42-.95H8.06v7.46H4.73a11.37,11.37,0,0,1-.45-3.21,10.41,10.41,0,0,1,.07-1.26Zm9-.73v-2.2h3.95c.2,0,1.44.24,1.44,1.16,0,.77-.95,1-1.73,1H13.32Zm14.34,2q0,.45,0,.87h-1.2c-.12,0-.17.08-.17.2v.55c0,1.3-.73,1.58-1.37,1.65s-1.29-.25-1.37-.63a4.13,4.13,0,0,0-1.91-3.21C22.79,14.59,24,13.49,24,12a3.76,3.76,0,0,0-1.83-3.09,5.22,5.22,0,0,0-2.52-.83H7.25a11.79,11.79,0,0,1,6.54-3.7l1.47,1.54a.87.87,0,0,0,1.22,0l1.64-1.57a11.69,11.69,0,0,1,8,5.72L25,12.64a.87.87,0,0,0,.44,1.14l2.16,1a11.46,11.46,0,0,1,.06,1.17ZM15.25,3.1a1,1,0,0,1,1.34,0,1,1,0,0,1,0,1.35,1,1,0,0,1-1.34,0,1,1,0,0,1,0-1.35Zm11.13,9a.94.94,0,0,1,1.25-.48,1,1,0,1,1-1.25.48Z"),n(s,"fill","currentColor"),n(e,"class",a[0]),n(e,"xmlns","http://www.w3.org/2000/svg"),n(e,"xmlns:xlink","http://www.w3.org/1999/xlink"),n(e,"aria-hidden","true"),n(e,"focusable","false"),n(e,"role","img"),n(e,"preserveAspectRatio","xMidYMid meet"),n(e,"width","1em"),n(e,"height","1em"),n(e,"viewBox","0 0 32 32")},m(t,l){A(t,e,l),b(e,s)},p(t,[l]){l&1&&n(e,"class",t[0])},i:M,o:M,d(t){t&&_(e)}}}function h1(a,e,s){let{classNames:t=""}=e;return a.$$set=l=>{"classNames"in l&&s(0,t=l.classNames)},[t]}class m1 extends ${constructor(e){super();H(this,e,h1,u1,C,{classNames:0})}}function d1(a){let e,s;return{c(){e=y("svg"),s=y("path"),this.h()},l(t){e=N(t,"svg",{class:!0,xmlns:!0,"xmlns:xlink":!0,"aria-hidden":!0,role:!0,width:!0,height:!0,preserveAspectRatio:!0,viewBox:!0});var l=w(e);s=N(l,"path",{fill:!0,d:!0}),w(s).forEach(_),l.forEach(_),this.h()},h(){n(s,"fill","#539E43"),n(s,"d","M128 288.464c-3.975 0-7.685-1.06-11.13-2.915l-35.247-20.936c-5.3-2.915-2.65-3.975-1.06-4.505c7.155-2.385 8.48-2.915 15.9-7.156c.796-.53 1.856-.265 2.65.265l27.032 16.166c1.06.53 2.385.53 3.18 0l105.74-61.217c1.06-.53 1.59-1.59 1.59-2.915V83.08c0-1.325-.53-2.385-1.59-2.915l-105.74-60.953c-1.06-.53-2.385-.53-3.18 0L20.405 80.166c-1.06.53-1.59 1.855-1.59 2.915v122.17c0 1.06.53 2.385 1.59 2.915l28.887 16.695c15.636 7.95 25.44-1.325 25.44-10.6V93.68c0-1.59 1.326-3.18 3.181-3.18h13.516c1.59 0 3.18 1.325 3.18 3.18v120.58c0 20.936-11.396 33.126-31.272 33.126c-6.095 0-10.865 0-24.38-6.625l-27.827-15.9C4.24 220.885 0 213.465 0 205.515V83.346C0 75.396 4.24 67.976 11.13 64L116.87 2.783c6.625-3.71 15.635-3.71 22.26 0L244.87 64C251.76 67.975 256 75.395 256 83.346v122.17c0 7.95-4.24 15.37-11.13 19.345L139.13 286.08c-3.445 1.59-7.42 2.385-11.13 2.385Zm32.596-84.009c-46.377 0-55.917-21.2-55.917-39.221c0-1.59 1.325-3.18 3.18-3.18h13.78c1.59 0 2.916 1.06 2.916 2.65c2.12 14.045 8.215 20.936 36.306 20.936c22.261 0 31.802-5.035 31.802-16.96c0-6.891-2.65-11.926-37.367-15.372c-28.886-2.915-46.907-9.275-46.907-32.33c0-21.467 18.02-34.187 48.232-34.187c33.921 0 50.617 11.66 52.737 37.101c0 .795-.265 1.59-.795 2.385c-.53.53-1.325 1.06-2.12 1.06h-13.78c-1.326 0-2.65-1.06-2.916-2.385c-3.18-14.575-11.395-19.345-33.126-19.345c-24.38 0-27.296 8.48-27.296 14.84c0 7.686 3.445 10.07 36.306 14.31c32.597 4.24 47.967 10.336 47.967 33.127c-.265 23.321-19.345 36.571-53.002 36.571Z"),n(e,"class",a[0]),n(e,"xmlns","http://www.w3.org/2000/svg"),n(e,"xmlns:xlink","http://www.w3.org/1999/xlink"),n(e,"aria-hidden","true"),n(e,"role","img"),n(e,"width","1em"),n(e,"height","1em"),n(e,"preserveAspectRatio","xMidYMid meet"),n(e,"viewBox","0 0 256 289")},m(t,l){A(t,e,l),b(e,s)},p(t,[l]){l&1&&n(e,"class",t[0])},i:M,o:M,d(t){t&&_(e)}}}function _1(a,e,s){let{classNames:t=""}=e;return a.$$set=l=>{"classNames"in l&&s(0,t=l.classNames)},[t]}class g1 extends ${constructor(e){super();H(this,e,_1,d1,C,{classNames:0})}}const p1=a=>({}),Y=a=>({}),v1=a=>({}),F=a=>({}),w1=a=>({}),O=a=>({});function U(a,e,s){const t=a.slice();return t[8]=e[s],t}function J(a){let e,s,t,l=a[8].label+"",i,v,g,u,k,h;var o=a[8].icon;function m(c){return{props:{classNames:"mr-1.5"}}}o&&(s=new o(m()));function r(){return a[7](a[8])}return{c(){e=D("div"),s&&q(s.$$.fragment),t=E(),i=X(l),v=E(),this.h()},l(c){e=G(c,"DIV",{class:!0});var f=w(e);s&&e1(s.$$.fragment,f),t=V(f),i=t1(f,l),v=V(f),f.forEach(_),this.h()},h(){n(e,"class",g="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd "+(a[3]===a[8].id?"border-gray-800 bg-black dark:bg-gray-700 text-white":"text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm"))},m(c,f){A(c,e,f),s&&z(s,e,null),b(e,t),b(e,i),b(e,v),u=!0,k||(h=l1(e,"click",r),k=!0)},p(c,f){if(a=c,o!==(o=a[8].icon)){if(s){Z();const L=s;p(L.$$.fragment,1,0,()=>{P(L,1)}),x()}o?(s=new o(m()),q(s.$$.fragment),d(s.$$.fragment,1),z(s,e,t)):s=null}(!u||f&8&&g!==(g="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd "+(a[3]===a[8].id?"border-gray-800 bg-black dark:bg-gray-700 text-white":"text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm")))&&n(e,"class",g)},i(c){u||(s&&d(s.$$.fragment,c),u=!0)},o(c){s&&p(s.$$.fragment,c),u=!1},d(c){c&&_(e),s&&P(s),k=!1,h()}}}function K(a){let e;const s=a[6].python,t=B(s,a,a[5],O);return{c(){t&&t.c()},l(l){t&&t.l(l)},m(l,i){t&&t.m(l,i),e=!0},p(l,i){t&&t.p&&(!e||i&32)&&I(t,s,l,l[5],e?T(s,l[5],i,w1):R(l[5]),O)},i(l){e||(d(t,l),e=!0)},o(l){p(t,l),e=!1},d(l){t&&t.d(l)}}}function j(a){let e;const s=a[6].rust,t=B(s,a,a[5],F);return{c(){t&&t.c()},l(l){t&&t.l(l)},m(l,i){t&&t.m(l,i),e=!0},p(l,i){t&&t.p&&(!e||i&32)&&I(t,s,l,l[5],e?T(s,l[5],i,v1):R(l[5]),F)},i(l){e||(d(t,l),e=!0)},o(l){p(t,l),e=!1},d(l){t&&t.d(l)}}}function Q(a){let e;const s=a[6].node,t=B(s,a,a[5],Y);return{c(){t&&t.c()},l(l){t&&t.l(l)},m(l,i){t&&t.m(l,i),e=!0},p(l,i){t&&t.p&&(!e||i&32)&&I(t,s,l,l[5],e?T(s,l[5],i,p1):R(l[5]),Y)},i(l){e||(d(t,l),e=!0)},o(l){p(t,l),e=!1},d(l){t&&t.d(l)}}}function L1(a){let e,s,t,l,i,v,g=a[4],u=[];for(let r=0;r<g.length;r+=1)u[r]=J(U(a,g,r));const k=r=>p(u[r],1,1,()=>{u[r]=null});let h=a[0]&&a[3]==="python"&&K(a),o=a[1]&&a[3]==="rust"&&j(a),m=a[2]&&a[3]==="node"&&Q(a);return{c(){e=D("div");for(let r=0;r<u.length;r+=1)u[r].c();s=E(),t=D("div"),h&&h.c(),l=E(),o&&o.c(),i=E(),m&&m.c(),this.h()},l(r){e=G(r,"DIV",{class:!0});var c=w(e);for(let L=0;L<u.length;L+=1)u[L].l(c);c.forEach(_),s=V(r),t=G(r,"DIV",{class:!0});var f=w(t);h&&h.l(f),l=V(f),o&&o.l(f),i=V(f),m&&m.l(f),f.forEach(_),this.h()},h(){n(e,"class","flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"),n(t,"class","language-select")},m(r,c){A(r,e,c);for(let f=0;f<u.length;f+=1)u[f].m(e,null);A(r,s,c),A(r,t,c),h&&h.m(t,null),b(t,l),o&&o.m(t,null),b(t,i),m&&m.m(t,null),v=!0},p(r,[c]){if(c&24){g=r[4];let f;for(f=0;f<g.length;f+=1){const L=U(r,g,f);u[f]?(u[f].p(L,c),d(u[f],1)):(u[f]=J(L),u[f].c(),d(u[f],1),u[f].m(e,null))}for(Z(),f=g.length;f<u.length;f+=1)k(f);x()}r[0]&&r[3]==="python"?h?(h.p(r,c),c&9&&d(h,1)):(h=K(r),h.c(),d(h,1),h.m(t,l)):h&&(Z(),p(h,1,1,()=>{h=null}),x()),r[1]&&r[3]==="rust"?o?(o.p(r,c),c&10&&d(o,1)):(o=j(r),o.c(),d(o,1),o.m(t,i)):o&&(Z(),p(o,1,1,()=>{o=null}),x()),r[2]&&r[3]==="node"?m?(m.p(r,c),c&12&&d(m,1)):(m=Q(r),m.c(),d(m,1),m.m(t,null)):m&&(Z(),p(m,1,1,()=>{m=null}),x())},i(r){if(!v){for(let c=0;c<g.length;c+=1)d(u[c]);d(h),d(o),d(m),v=!0}},o(r){u=u.filter(Boolean);for(let c=0;c<u.length;c+=1)p(u[c]);p(h),p(o),p(m),v=!1},d(r){r&&_(e),s1(u,r),r&&_(s),r&&_(t),h&&h.d(),o&&o.d(),m&&m.d()}}}function b1(a,e,s){let t;a1(a,S,o=>s(3,t=o));let{$$slots:l={},$$scope:i}=e;const v=[{id:"python",icon:f1,label:"Python"},{id:"rust",icon:m1,label:"Rust"},{id:"node",icon:g1,label:"Node"}];let{python:g=!1}=e,{rust:u=!1}=e,{node:k=!1}=e;const h=o=>n1(S,t=o.id,t);return a.$$set=o=>{"python"in o&&s(0,g=o.python),"rust"in o&&s(1,u=o.rust),"node"in o&&s(2,k=o.node),"$$scope"in o&&s(5,i=o.$$scope)},[g,u,k,t,v,i,l,h]}class N1 extends ${constructor(e){super();H(this,e,b1,L1,C,{python:0,rust:1,node:2})}}export{y1 as M,N1 as T};