tokenizers/pr_2138/en/_app/immutable/nodes/4.Db2CjEOb.js

import"../chunks/DsnmJJEf.js";import{i as P,h as q,C as E,H as p,E as _,s as v}from"../chunks/DPtJuB9X.js";import{p as L,o as b,s as n,f as T,a as s,b as S,c as r,n as f}from"../chunks/CFwIJwKg.js";import{T as $,M as a}from"../chunks/Dd6Po5Jz.js";const U='{"title":"Encode Inputs","local":"encode-inputs","sections":[{"title":"TextEncodeInput[[[ tokenizers.TextEncodeInput ]]]","local":"textencodeinput-tokenizerstextencodeinput-","sections":[],"depth":2},{"title":"PreTokenizedEncodeInput[[[ tokenizers.PreTokenizedEncodeInput ]]]","local":"pretokenizedencodeinput-tokenizerspretokenizedencodeinput-","sections":[],"depth":2},{"title":"EncodeInput[[[ tokenizers.EncodeInput ]]]","local":"encodeinput-tokenizersencodeinput-","sections":[],"depth":2}],"depth":1}';var w=r('<meta name="hf:doc:metadata"/>'),y=r(`<p>These types represent all the different kinds of input that a <a href="/docs/tokenizers/pr_2138/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a> accepts
when using <code>encode_batch()</code>.</p> <!> <code>tokenizers.TextEncodeInput</code> <p>Represents a textual input for encoding. Can be either:</p> <ul><li>A single sequence: <a href="/docs/tokenizers/api/input-sequences#tokenizers.TextInputSequence">TextInputSequence</a></li> <li>A pair of sequences: <ul><li>A Tuple of <a href="/docs/tokenizers/api/input-sequences#tokenizers.TextInputSequence">TextInputSequence</a></li> <li>Or a List of <a href="/docs/tokenizers/api/input-sequences#tokenizers.TextInputSequence">TextInputSequence</a> of size 2</li></ul></li></ul> <p>alias of <code>Union[str, Tuple[str, str], List[str]]</code>.</p> <!> <code>tokenizers.PreTokenizedEncodeInput</code> <p>Represents a pre-tokenized input for encoding. Can be either:</p> <ul><li>A single sequence: <a href="/docs/tokenizers/api/input-sequences#tokenizers.PreTokenizedInputSequence">PreTokenizedInputSequence</a></li> <li>A pair of sequences: <ul><li>A Tuple of <a href="/docs/tokenizers/api/input-sequences#tokenizers.PreTokenizedInputSequence">PreTokenizedInputSequence</a></li> <li>Or a List of <a href="/docs/tokenizers/api/input-sequences#tokenizers.PreTokenizedInputSequence">PreTokenizedInputSequence</a> of size 2</li></ul></li></ul> <p>alias of <code>Union[List[str], Tuple[str], Tuple[Union[List[str], Tuple[str]], Union[List[str], Tuple[str]]], List[Union[List[str], Tuple[str]]]]</code>.</p> <!> <code>tokenizers.EncodeInput</code> <p>Represents all the possible types of input for encoding. Can be:</p> <ul><li>When <code>is_pretokenized=False</code>: <a href="#tokenizers.TextEncodeInput">TextEncodeInput</a></li> <li>When <code>is_pretokenized=True</code>: <a href="#tokenizers.PreTokenizedEncodeInput">PreTokenizedEncodeInput</a></li></ul> <p>alias of <code>Union[str, Tuple[str, str], List[str], Tuple[str], Tuple[Union[List[str], Tuple[str]], Union[List[str], Tuple[str]]], List[Union[List[str], Tuple[str]]]]</code>.</p>`,1),A=r('<p>The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.</p>'),C=r("<p>The node API has not been documented yet.</p>"),R=r("<p></p> <!> <!> <!> <!> <p></p>",1);function D(I,g){L(g,!1),b(()=>{new URLSearchParams(window.location.search).get("fw")}),P();var d=R();q("3qdwlg",e=>{var o=w();v(o,"content",U),s(e,o)});var c=n(T(d),2);E(c,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var u=n(c,2);p(u,{title:"Encode Inputs",local:"encode-inputs",headingTag:"h1"});var l=n(u,2);$(l,{python:!0,rust:!0,node:!0,$$slots:{python:(e,o)=>{a(e,{children:(i,z)=>{var t=y(),k=n(T(t),2);p(k,{title:"TextEncodeInput[[[ tokenizers.TextEncodeInput ]]]",local:"textencodeinput-tokenizerstextencodeinput-",headingTag:"h2"});var h=n(k,10);p(h,{title:"PreTokenizedEncodeInput[[[ tokenizers.PreTokenizedEncodeInput ]]]",local:"pretokenizedencodeinput-tokenizerspretokenizedencodeinput-",headingTag:"h2"});var x=n(h,10);p(x,{title:"EncodeInput[[[ tokenizers.EncodeInput ]]]",local:"encodeinput-tokenizersencodeinput-",headingTag:"h2"}),f(8),s(i,t)}})},rust:(e,o)=>{a(e,{children:(i,z)=>{var t=A();s(i,t)}})},node:(e,o)=>{a(e,{children:(i,z)=>{var t=C();s(i,t)}})}}});var m=n(l,2);_(m,{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/encode-inputs.mdx"}),f(2),s(I,d),S()}export{D as component};