tokenizers/pr_2134/en/_app/immutable/nodes/2.DMJL-FSf.js

import"../chunks/DsnmJJEf.js";import{i as M,h as L,C as R,H as A,E as C,s as K,D as r}from"../chunks/DPtJuB9X.js";import{p as E,o as H,s as e,f as y,a as l,b as N,c,d,n,r as s}from"../chunks/CFwIJwKg.js";import{T as j,M as u}from"../chunks/Dd6Po5Jz.js";const O='{"title":"Added Tokens","local":"added-tokens","sections":[{"title":"AddedToken","local":"tokenizers.AddedToken","sections":[],"depth":2}],"depth":1}';var U=c('<meta name="hf:doc:metadata"/>'),Y=c(`<!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Represents a token that can be be added to a <a href="/docs/tokenizers/pr_2134/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>.
It can have special options that defines the way it should behave.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Get the content of this <code>AddedToken</code></p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Get the value of the <code>lstrip</code> option</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Get the value of the <code>normalized</code> option</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Get the value of the <code>rstrip</code> option</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Get the value of the <code>single_word</code> option</p></div></div>`,1),B=c('<p>The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.</p>'),J=c("<p>The node API has not been documented yet.</p>"),Q=c("<p></p> <!> <!> <!> <!> <p></p>",1);function ee(D,F){E(F,!1),H(()=>{new URLSearchParams(window.location.search).get("fw")}),M();var g=Q();L("1z02een",o=>{var a=U();K(a,"content",O),l(o,a)});var v=e(y(g),2);R(v,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var f=e(v,2);A(f,{title:"Added Tokens",local:"added-tokens",headingTag:"h1"});var b=e(f,2);j(b,{python:!0,rust:!0,node:!0,$$slots:{python:(o,a)=>{u(o,{children:(i,w)=>{var t=Y(),z=y(t);A(z,{title:"AddedToken",local:"tokenizers.AddedToken",headingTag:"h2"});var T=e(z,2),_=d(T);r(_,{name:"class tokenizers.AddedToken",anchor:"tokenizers.AddedToken",parameters:[{name:"content",val:" = None"},{name:"single_word",val:" = False"},{name:"lstrip",val:" = False"},{name:"rstrip",val:" = False"},{name:"normalized",val:" = True"},{name:"special",val:" = False"}],parametersDescription:[{anchor:"tokenizers.AddedToken.content",description:"<strong>content</strong> (<code>str</code>) &#x2014; The content of the token",name:"content"},{anchor:"tokenizers.AddedToken.single_word",description:`<strong>single_word</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Defines whether this token should only match single words. If <code>True</code>, this
token will never match inside of a word. For example the token <code>ing</code> would match
on <code>tokenizing</code> if this option is <code>False</code>, but not if it is <code>True</code>.
The notion of &#x201D;<em>inside of a word</em>&#x201D; is defined by the word boundaries pattern in
regular expressions (ie. the token should start and end with word boundaries).`,name:"single_word"},{anchor:"tokenizers.AddedToken.lstrip",description:`<strong>lstrip</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Defines whether this token should strip all potential whitespaces on its left side.
If <code>True</code>, this token will greedily match any whitespace on its left. For
example if we try to match the token <code>[MASK]</code> with <code>lstrip=True</code>, in the text
<code>&quot;I saw a [MASK]&quot;</code>, we would match on <code>&quot; [MASK]&quot;</code>. (Note the space on the left).`,name:"lstrip"},{anchor:"tokenizers.AddedToken.rstrip",description:`<strong>rstrip</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Defines whether this token should strip all potential whitespaces on its right
side. If <code>True</code>, this token will greedily match any whitespace on its right.
It works just like <code>lstrip</code> but on the right.`,name:"rstrip"},{anchor:"tokenizers.AddedToken.normalized",description:`<strong>normalized</strong> (<code>bool</code>, defaults to <code>True</code> with  &#x2014;meth:<em>~tokenizers.Tokenizer.add_tokens</em> and <code>False</code> with <code>add_special_tokens()</code>):
Defines whether this token should match against the normalized version of the input
text. For example, with the added token <code>&quot;yesterday&quot;</code>, and a normalizer in charge of
lowercasing the text, the token could be extract from the input <code>&quot;I saw a lion Yesterday&quot;</code>.`,name:"normalized"},{anchor:"tokenizers.AddedToken.special",description:`<strong>special</strong> (<code>bool</code>, defaults to <code>False</code> with  &#x2014;meth:<em>~tokenizers.Tokenizer.add_tokens</em> and <code>False</code> with <code>add_special_tokens()</code>):
Defines whether this token should be skipped when decoding.`,name:"special"}]});var p=e(_,4),G=d(p);r(G,{name:"content",anchor:"tokenizers.AddedToken.content",parameters:[],isGetSetDescriptor:!0}),n(2),s(p);var h=e(p,2),P=d(h);r(P,{name:"lstrip",anchor:"tokenizers.AddedToken.lstrip",parameters:[],isGetSetDescriptor:!0}),n(2),s(h);var m=e(h,2),S=d(m);r(S,{name:"normalized",anchor:"tokenizers.AddedToken.normalized",parameters:[],isGetSetDescriptor:!0}),n(2),s(m);var k=e(m,2),I=d(k);r(I,{name:"rstrip",anchor:"tokenizers.AddedToken.rstrip",parameters:[],isGetSetDescriptor:!0}),n(2),s(k);var x=e(k,2),q=d(x);r(q,{name:"single_word",anchor:"tokenizers.AddedToken.single_word",parameters:[],isGetSetDescriptor:!0}),n(2),s(x),s(T),l(i,t)}})},rust:(o,a)=>{u(o,{children:(i,w)=>{var t=B();l(i,t)}})},node:(o,a)=>{u(o,{children:(i,w)=>{var t=J();l(i,t)}})}}});var $=e(b,2);C($,{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/added-tokens.mdx"}),n(2),l(D,g),N()}export{ee as component};