tokenizers/pr_2101/en/_app/immutable/nodes/6.475bdfd5.js

import{s as oe,o as re,n as Q}from"../chunks/scheduler.7c59faff.js";import{S as ue,i as ae,e as m,s as f,c as q,h as pe,a as d,d as l,b as c,f as le,g as w,j as ie,k as fe,l as i,m as S,t as I,n as L,o as P,p as _}from"../chunks/index.09bb5655.js";import{C as ce,H as N,E as $e}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.cd47c5a1.js";import{T as me,M as V}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";function de(x){let t,r=`These types represent all the different kinds of sequence that can be used as input of a Tokenizer.
Globally, any sequence can be either a string or a list of strings, according to the operating
mode of the tokenizer: <code>raw text</code> vs <code>pre-tokenized</code>.`,n,o,p,T,v="tokenizers.TextInputSequence",b,$,M="A <code>str</code> that represents an input sequence",k,g,C,h,s="tokenizers.PreTokenizedInputSequence",a,z,X="A pre-tokenized input sequence. Can be one of:",R,y,Y="<li>A <code>List</code> of <code>str</code></li> <li>A <code>Tuple</code> of <code>str</code></li>",G,H,Z="alias of <code>Union[List[str], Tuple[str]]</code>.",W,E,j,A,ee="tokenizers.InputSequence",F,U,te="Represents all the possible types of input sequences for encoding. Can be:",B,D,ne='<li>When <code>is_pretokenized=False</code>: <a href="#tokenizers.TextInputSequence">TextInputSequence</a></li> <li>When <code>is_pretokenized=True</code>: <a href="#tokenizers.PreTokenizedInputSequence">PreTokenizedInputSequence</a></li>',J,O,se="alias of <code>Union[str, List[str], Tuple[str]]</code>.",K;return o=new N({props:{title:"TextInputSequence",local:"tokenizers.TextInputSequence",headingTag:"h2"}}),g=new N({props:{title:"PreTokenizedInputSequence",local:"tokenizers.PreTokenizedInputSequence",headingTag:"h2"}}),E=new N({props:{title:"InputSequence",local:"tokenizers.InputSequence",headingTag:"h2"}}),{c(){t=m("p"),t.innerHTML=r,n=f(),q(o.$$.fragment),p=f(),T=m("code"),T.textContent=v,b=f(),$=m("p"),$.innerHTML=M,k=f(),q(g.$$.fragment),C=f(),h=m("code"),h.textContent=s,a=f(),z=m("p"),z.textContent=X,R=f(),y=m("ul"),y.innerHTML=Y,G=f(),H=m("p"),H.innerHTML=Z,W=f(),q(E.$$.fragment),j=f(),A=m("code"),A.textContent=ee,F=f(),U=m("p"),U.textContent=te,B=f(),D=m("ul"),D.innerHTML=ne,J=f(),O=m("p"),O.innerHTML=se},l(e){t=d(e,"P",{"data-svelte-h":!0}),_(t)!=="svelte-1yv07ao"&&(t.innerHTML=r),n=c(e),w(o.$$.fragment,e),p=c(e),T=d(e,"CODE",{"data-svelte-h":!0}),_(T)!=="svelte-4k9a7w"&&(T.textContent=v),b=c(e),$=d(e,"P",{"data-svelte-h":!0}),_($)!=="svelte-1offf9v"&&($.innerHTML=M),k=c(e),w(g.$$.fragment,e),C=c(e),h=d(e,"CODE",{"data-svelte-h":!0}),_(h)!=="svelte-12fgn95"&&(h.textContent=s),a=c(e),z=d(e,"P",{"data-svelte-h":!0}),_(z)!=="svelte-19l4c3n"&&(z.textContent=X),R=c(e),y=d(e,"UL",{"data-svelte-h":!0}),_(y)!=="svelte-nyklpi"&&(y.innerHTML=Y),G=c(e),H=d(e,"P",{"data-svelte-h":!0}),_(H)!=="svelte-d0i2on"&&(H.innerHTML=Z),W=c(e),w(E.$$.fragment,e),j=c(e),A=d(e,"CODE",{"data-svelte-h":!0}),_(A)!=="svelte-ct5svd"&&(A.textContent=ee),F=c(e),U=d(e,"P",{"data-svelte-h":!0}),_(U)!=="svelte-17pgza3"&&(U.textContent=te),B=c(e),D=d(e,"UL",{"data-svelte-h":!0}),_(D)!=="svelte-lavs5d"&&(D.innerHTML=ne),J=c(e),O=d(e,"P",{"data-svelte-h":!0}),_(O)!=="svelte-1b945hg"&&(O.innerHTML=se)},m(e,u){i(e,t,u),i(e,n,u),S(o,e,u),i(e,p,u),i(e,T,u),i(e,b,u),i(e,$,u),i(e,k,u),S(g,e,u),i(e,C,u),i(e,h,u),i(e,a,u),i(e,z,u),i(e,R,u),i(e,y,u),i(e,G,u),i(e,H,u),i(e,W,u),S(E,e,u),i(e,j,u),i(e,A,u),i(e,F,u),i(e,U,u),i(e,B,u),i(e,D,u),i(e,J,u),i(e,O,u),K=!0},p:Q,i(e){K||(I(o.$$.fragment,e),I(g.$$.fragment,e),I(E.$$.fragment,e),K=!0)},o(e){L(o.$$.fragment,e),L(g.$$.fragment,e),L(E.$$.fragment,e),K=!1},d(e){e&&(l(t),l(n),l(p),l(T),l(b),l($),l(k),l(C),l(h),l(a),l(z),l(R),l(y),l(G),l(H),l(W),l(j),l(A),l(F),l(U),l(B),l(D),l(J),l(O)),P(o,e),P(g,e),P(E,e)}}}function _e(x){let t,r;return t=new V({props:{$$slots:{default:[de]},$$scope:{ctx:x}}}),{c(){q(t.$$.fragment)},l(n){w(t.$$.fragment,n)},m(n,o){S(t,n,o),r=!0},p(n,o){const p={};o&2&&(p.$$scope={dirty:o,ctx:n}),t.$set(p)},i(n){r||(I(t.$$.fragment,n),r=!0)},o(n){L(t.$$.fragment,n),r=!1},d(n){P(t,n)}}}function xe(x){let t,r='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){t=m("p"),t.innerHTML=r},l(n){t=d(n,"P",{"data-svelte-h":!0}),_(t)!=="svelte-4ytcyb"&&(t.innerHTML=r)},m(n,o){i(n,t,o)},p:Q,d(n){n&&l(t)}}}function Te(x){let t,r;return t=new V({props:{$$slots:{default:[xe]},$$scope:{ctx:x}}}),{c(){q(t.$$.fragment)},l(n){w(t.$$.fragment,n)},m(n,o){S(t,n,o),r=!0},p(n,o){const p={};o&2&&(p.$$scope={dirty:o,ctx:n}),t.$set(p)},i(n){r||(I(t.$$.fragment,n),r=!0)},o(n){L(t.$$.fragment,n),r=!1},d(n){P(t,n)}}}function ke(x){let t,r="The node API has not been documented yet.";return{c(){t=m("p"),t.textContent=r},l(n){t=d(n,"P",{"data-svelte-h":!0}),_(t)!=="svelte-1mrchm6"&&(t.textContent=r)},m(n,o){i(n,t,o)},p:Q,d(n){n&&l(t)}}}function ge(x){let t,r;return t=new V({props:{$$slots:{default:[ke]},$$scope:{ctx:x}}}),{c(){q(t.$$.fragment)},l(n){w(t.$$.fragment,n)},m(n,o){S(t,n,o),r=!0},p(n,o){const p={};o&2&&(p.$$scope={dirty:o,ctx:n}),t.$set(p)},i(n){r||(I(t.$$.fragment,n),r=!0)},o(n){L(t.$$.fragment,n),r=!1},d(n){P(t,n)}}}function he(x){let t,r,n,o,p,T,v,b,$,M,k,g,C,h;return p=new ce({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),v=new N({props:{title:"Input Sequences",local:"input-sequences",headingTag:"h1"}}),$=new me({props:{python:!0,rust:!0,node:!0,$$slots:{node:[ge],rust:[Te],python:[_e]},$$scope:{ctx:x}}}),k=new $e({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/input-sequences.mdx"}}),{c(){t=m("meta"),r=f(),n=m("p"),o=f(),q(p.$$.fragment),T=f(),q(v.$$.fragment),b=f(),q($.$$.fragment),M=f(),q(k.$$.fragment),g=f(),C=m("p"),this.h()},l(s){const a=pe("svelte-u9bgzb",document.head);t=d(a,"META",{name:!0,content:!0}),a.forEach(l),r=c(s),n=d(s,"P",{}),le(n).forEach(l),o=c(s),w(p.$$.fragment,s),T=c(s),w(v.$$.fragment,s),b=c(s),w($.$$.fragment,s),M=c(s),w(k.$$.fragment,s),g=c(s),C=d(s,"P",{}),le(C).forEach(l),this.h()},h(){ie(t,"name","hf:doc:metadata"),ie(t,"content",ve)},m(s,a){fe(document.head,t),i(s,r,a),i(s,n,a),i(s,o,a),S(p,s,a),i(s,T,a),S(v,s,a),i(s,b,a),S($,s,a),i(s,M,a),S(k,s,a),i(s,g,a),i(s,C,a),h=!0},p(s,[a]){const z={};a&2&&(z.$$scope={dirty:a,ctx:s}),$.$set(z)},i(s){h||(I(p.$$.fragment,s),I(v.$$.fragment,s),I($.$$.fragment,s),I(k.$$.fragment,s),h=!0)},o(s){L(p.$$.fragment,s),L(v.$$.fragment,s),L($.$$.fragment,s),L(k.$$.fragment,s),h=!1},d(s){s&&(l(r),l(n),l(o),l(T),l(b),l(M),l(g),l(C)),l(t),P(p,s),P(v,s),P($,s),P(k,s)}}}const ve='{"title":"Input Sequences","local":"input-sequences","sections":[{"title":"TextInputSequence","local":"tokenizers.TextInputSequence","sections":[],"depth":2},{"title":"PreTokenizedInputSequence","local":"tokenizers.PreTokenizedInputSequence","sections":[],"depth":2},{"title":"InputSequence","local":"tokenizers.InputSequence","sections":[],"depth":2}],"depth":1}';function Ce(x){return re(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ie extends ue{constructor(t){super(),ae(this,t,Ce,he,oe,{})}}export{Ie as component};