tokenizers/pr_2139/en/_app/immutable/nodes/4.366e3a8b.js

import{s as fe,o as de,n as ee}from"../chunks/scheduler.7c59faff.js";import{S as $e,i as me,e as d,s as c,c as C,h as ke,a as $,d as s,b as f,f as ae,g as v,j as ce,k as Te,l as o,m as L,t as g,n as I,o as E,p as k}from"../chunks/index.09bb5655.js";import{C as ze,H as Z,E as xe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fb0d6fe2.js";import{T as _e,M as te}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";function he(T){let t,u=`These types represent all the different kinds of input that a <a href="/docs/tokenizers/pr_2139/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> accepts
when using <code>encode_batch()</code>.`,n,l,p,z,h="tokenizers.TextEncodeInput",P,m,b="Represents a textual input for encoding. Can be either:",x,_,q=`<li>A single sequence: <a href="/docs/tokenizers/api/input-sequences#tokenizers.TextInputSequence">TextInputSequence</a></li> <li>A pair of sequences:
<ul><li>A Tuple of <a href="/docs/tokenizers/api/input-sequences#tokenizers.TextInputSequence">TextInputSequence</a></li> <li>Or a List of <a href="/docs/tokenizers/api/input-sequences#tokenizers.TextInputSequence">TextInputSequence</a> of size 2</li></ul></li>`,w,i,a="alias of <code>Union[str, Tuple[str, str], List[str]]</code>.",M,H,F,y,ne="tokenizers.PreTokenizedEncodeInput",G,S,ie="Represents a pre-tokenized input for encoding. Can be either:",B,U,se=`<li>A single sequence: <a href="/docs/tokenizers/api/input-sequences#tokenizers.PreTokenizedInputSequence">PreTokenizedInputSequence</a></li> <li>A pair of sequences:
<ul><li>A Tuple of <a href="/docs/tokenizers/api/input-sequences#tokenizers.PreTokenizedInputSequence">PreTokenizedInputSequence</a></li> <li>Or a List of <a href="/docs/tokenizers/api/input-sequences#tokenizers.PreTokenizedInputSequence">PreTokenizedInputSequence</a> of size 2</li></ul></li>`,J,A,oe="alias of <code>Union[List[str], Tuple[str], Tuple[Union[List[str], Tuple[str]], Union[List[str], Tuple[str]]], List[Union[List[str], Tuple[str]]]]</code>.",K,O,N,R,le="tokenizers.EncodeInput",Q,j,re="Represents all the possible types of input for encoding. Can be:",V,D,ue='<li>When <code>is_pretokenized=False</code>: <a href="#tokenizers.TextEncodeInput">TextEncodeInput</a></li> <li>When <code>is_pretokenized=True</code>: <a href="#tokenizers.PreTokenizedEncodeInput">PreTokenizedEncodeInput</a></li>',X,W,pe="alias of <code>Union[str, Tuple[str, str], List[str], Tuple[str], Tuple[Union[List[str], Tuple[str]], Union[List[str], Tuple[str]]], List[Union[List[str], Tuple[str]]]]</code>.",Y;return l=new Z({props:{title:"TextEncodeInput[[[ tokenizers.TextEncodeInput ]]]",local:"textencodeinput-tokenizerstextencodeinput-",headingTag:"h2"}}),H=new Z({props:{title:"PreTokenizedEncodeInput[[[ tokenizers.PreTokenizedEncodeInput ]]]",local:"pretokenizedencodeinput-tokenizerspretokenizedencodeinput-",headingTag:"h2"}}),O=new Z({props:{title:"EncodeInput[[[ tokenizers.EncodeInput ]]]",local:"encodeinput-tokenizersencodeinput-",headingTag:"h2"}}),{c(){t=d("p"),t.innerHTML=u,n=c(),C(l.$$.fragment),p=c(),z=d("code"),z.textContent=h,P=c(),m=d("p"),m.textContent=b,x=c(),_=d("ul"),_.innerHTML=q,w=c(),i=d("p"),i.innerHTML=a,M=c(),C(H.$$.fragment),F=c(),y=d("code"),y.textContent=ne,G=c(),S=d("p"),S.textContent=ie,B=c(),U=d("ul"),U.innerHTML=se,J=c(),A=d("p"),A.innerHTML=oe,K=c(),C(O.$$.fragment),N=c(),R=d("code"),R.textContent=le,Q=c(),j=d("p"),j.textContent=re,V=c(),D=d("ul"),D.innerHTML=ue,X=c(),W=d("p"),W.innerHTML=pe},l(e){t=$(e,"P",{"data-svelte-h":!0}),k(t)!=="svelte-woay98"&&(t.innerHTML=u),n=f(e),v(l.$$.fragment,e),p=f(e),z=$(e,"CODE",{"data-svelte-h":!0}),k(z)!=="svelte-bnjy2z"&&(z.textContent=h),P=f(e),m=$(e,"P",{"data-svelte-h":!0}),k(m)!=="svelte-164mvbn"&&(m.textContent=b),x=f(e),_=$(e,"UL",{"data-svelte-h":!0}),k(_)!=="svelte-elpq78"&&(_.innerHTML=q),w=f(e),i=$(e,"P",{"data-svelte-h":!0}),k(i)!=="svelte-1psi3dh"&&(i.innerHTML=a),M=f(e),v(H.$$.fragment,e),F=f(e),y=$(e,"CODE",{"data-svelte-h":!0}),k(y)!=="svelte-eh2lya"&&(y.textContent=ne),G=f(e),S=$(e,"P",{"data-svelte-h":!0}),k(S)!=="svelte-1j9inir"&&(S.textContent=ie),B=f(e),U=$(e,"UL",{"data-svelte-h":!0}),k(U)!=="svelte-1xbjebs"&&(U.innerHTML=se),J=f(e),A=$(e,"P",{"data-svelte-h":!0}),k(A)!=="svelte-10oq2d4"&&(A.innerHTML=oe),K=f(e),v(O.$$.fragment,e),N=f(e),R=$(e,"CODE",{"data-svelte-h":!0}),k(R)!=="svelte-1swk3ti"&&(R.textContent=le),Q=f(e),j=$(e,"P",{"data-svelte-h":!0}),k(j)!=="svelte-gcdzsl"&&(j.textContent=re),V=f(e),D=$(e,"UL",{"data-svelte-h":!0}),k(D)!=="svelte-102yi99"&&(D.innerHTML=ue),X=f(e),W=$(e,"P",{"data-svelte-h":!0}),k(W)!=="svelte-1zddld"&&(W.innerHTML=pe)},m(e,r){o(e,t,r),o(e,n,r),L(l,e,r),o(e,p,r),o(e,z,r),o(e,P,r),o(e,m,r),o(e,x,r),o(e,_,r),o(e,w,r),o(e,i,r),o(e,M,r),L(H,e,r),o(e,F,r),o(e,y,r),o(e,G,r),o(e,S,r),o(e,B,r),o(e,U,r),o(e,J,r),o(e,A,r),o(e,K,r),L(O,e,r),o(e,N,r),o(e,R,r),o(e,Q,r),o(e,j,r),o(e,V,r),o(e,D,r),o(e,X,r),o(e,W,r),Y=!0},p:ee,i(e){Y||(g(l.$$.fragment,e),g(H.$$.fragment,e),g(O.$$.fragment,e),Y=!0)},o(e){I(l.$$.fragment,e),I(H.$$.fragment,e),I(O.$$.fragment,e),Y=!1},d(e){e&&(s(t),s(n),s(p),s(z),s(P),s(m),s(x),s(_),s(w),s(i),s(M),s(F),s(y),s(G),s(S),s(B),s(U),s(J),s(A),s(K),s(N),s(R),s(Q),s(j),s(V),s(D),s(X),s(W)),E(l,e),E(H,e),E(O,e)}}}function Ce(T){let t,u;return t=new te({props:{$$slots:{default:[he]},$$scope:{ctx:T}}}),{c(){C(t.$$.fragment)},l(n){v(t.$$.fragment,n)},m(n,l){L(t,n,l),u=!0},p(n,l){const p={};l&2&&(p.$$scope={dirty:l,ctx:n}),t.$set(p)},i(n){u||(g(t.$$.fragment,n),u=!0)},o(n){I(t.$$.fragment,n),u=!1},d(n){E(t,n)}}}function ve(T){let t,u='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){t=d("p"),t.innerHTML=u},l(n){t=$(n,"P",{"data-svelte-h":!0}),k(t)!=="svelte-4ytcyb"&&(t.innerHTML=u)},m(n,l){o(n,t,l)},p:ee,d(n){n&&s(t)}}}function Le(T){let t,u;return t=new te({props:{$$slots:{default:[ve]},$$scope:{ctx:T}}}),{c(){C(t.$$.fragment)},l(n){v(t.$$.fragment,n)},m(n,l){L(t,n,l),u=!0},p(n,l){const p={};l&2&&(p.$$scope={dirty:l,ctx:n}),t.$set(p)},i(n){u||(g(t.$$.fragment,n),u=!0)},o(n){I(t.$$.fragment,n),u=!1},d(n){E(t,n)}}}function ge(T){let t,u="The node API has not been documented yet.";return{c(){t=d("p"),t.textContent=u},l(n){t=$(n,"P",{"data-svelte-h":!0}),k(t)!=="svelte-1mrchm6"&&(t.textContent=u)},m(n,l){o(n,t,l)},p:ee,d(n){n&&s(t)}}}function Ie(T){let t,u;return t=new te({props:{$$slots:{default:[ge]},$$scope:{ctx:T}}}),{c(){C(t.$$.fragment)},l(n){v(t.$$.fragment,n)},m(n,l){L(t,n,l),u=!0},p(n,l){const p={};l&2&&(p.$$scope={dirty:l,ctx:n}),t.$set(p)},i(n){u||(g(t.$$.fragment,n),u=!0)},o(n){I(t.$$.fragment,n),u=!1},d(n){E(t,n)}}}function Ee(T){let t,u,n,l,p,z,h,P,m,b,x,_,q,w;return p=new ze({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new Z({props:{title:"Encode Inputs",local:"encode-inputs",headingTag:"h1"}}),m=new _e({props:{python:!0,rust:!0,node:!0,$$slots:{node:[Ie],rust:[Le],python:[Ce]},$$scope:{ctx:T}}}),x=new xe({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/encode-inputs.mdx"}}),{c(){t=d("meta"),u=c(),n=d("p"),l=c(),C(p.$$.fragment),z=c(),C(h.$$.fragment),P=c(),C(m.$$.fragment),b=c(),C(x.$$.fragment),_=c(),q=d("p"),this.h()},l(i){const a=ke("svelte-u9bgzb",document.head);t=$(a,"META",{name:!0,content:!0}),a.forEach(s),u=f(i),n=$(i,"P",{}),ae(n).forEach(s),l=f(i),v(p.$$.fragment,i),z=f(i),v(h.$$.fragment,i),P=f(i),v(m.$$.fragment,i),b=f(i),v(x.$$.fragment,i),_=f(i),q=$(i,"P",{}),ae(q).forEach(s),this.h()},h(){ce(t,"name","hf:doc:metadata"),ce(t,"content",Pe)},m(i,a){Te(document.head,t),o(i,u,a),o(i,n,a),o(i,l,a),L(p,i,a),o(i,z,a),L(h,i,a),o(i,P,a),L(m,i,a),o(i,b,a),L(x,i,a),o(i,_,a),o(i,q,a),w=!0},p(i,[a]){const M={};a&2&&(M.$$scope={dirty:a,ctx:i}),m.$set(M)},i(i){w||(g(p.$$.fragment,i),g(h.$$.fragment,i),g(m.$$.fragment,i),g(x.$$.fragment,i),w=!0)},o(i){I(p.$$.fragment,i),I(h.$$.fragment,i),I(m.$$.fragment,i),I(x.$$.fragment,i),w=!1},d(i){i&&(s(u),s(n),s(l),s(z),s(P),s(b),s(_),s(q)),s(t),E(p,i),E(h,i),E(m,i),E(x,i)}}}const Pe='{"title":"Encode Inputs","local":"encode-inputs","sections":[{"title":"TextEncodeInput[[[ tokenizers.TextEncodeInput ]]]","local":"textencodeinput-tokenizerstextencodeinput-","sections":[],"depth":2},{"title":"PreTokenizedEncodeInput[[[ tokenizers.PreTokenizedEncodeInput ]]]","local":"pretokenizedencodeinput-tokenizerspretokenizedencodeinput-","sections":[],"depth":2},{"title":"EncodeInput[[[ tokenizers.EncodeInput ]]]","local":"encodeinput-tokenizersencodeinput-","sections":[],"depth":2}],"depth":1}';function we(T){return de(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends $e{constructor(t){super(),me(this,t,we,Ee,fe,{})}}export{ye as component};