tokenizers/pr_2139/en/_app/immutable/nodes/12.d9cd75af.js

import{s as be,o as we,n as ee}from"../chunks/scheduler.7c59faff.js";import{S as ve,i as ye,e as w,s as m,c as h,h as Me,a as v,d as l,b as d,f as H,g as u,j as Y,k as M,l as c,m as f,t as $,n as _,o as k,p as W}from"../chunks/index.09bb5655.js";import{C as Je,H as ne,E as xe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fb0d6fe2.js";import{D as te}from"../chunks/Docstring.1c25d4eb.js";import{C as ce}from"../chunks/CodeBlock.aaa4219f.js";import{T as ze,M as $e}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";import{E as pe}from"../chunks/ExampleCodeBlock.58a3c1c1.js";function Ue(b){let e,o="Example:",t,r,i;return r=new ce({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubW9kZWxzJTIwaW1wb3J0JTIwQlBFJTBBZnJvbSUyMHRva2VuaXplcnMudHJhaW5lcnMlMjBpbXBvcnQlMjBCcGVUcmFpbmVyJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRDMwMDAwJTJDJTBBJTIwJTIwJTIwJTIwc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlM0N1bmslM0UlMjIlMkMlMjAlMjIlM0NzJTNFJTIyJTJDJTIwJTIyJTNDJTJGcyUzRSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMG1pbl9mcmVxdWVuY3klM0QyJTJDJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplcihCUEUoKSklMEF0b2tlbml6ZXIudHJhaW4oJTVCJTIycGF0aCUyRnRvJTJGY29ycHVzLnR4dCUyMiU1RCUyQyUyMHRyYWluZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(
<span class="hljs-meta">... </span>    vocab_size=<span class="hljs-number">30000</span>,
<span class="hljs-meta">... </span>    special_tokens=[<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;s&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>],
<span class="hljs-meta">... </span>    min_frequency=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE())
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train([<span class="hljs-string">&quot;path/to/corpus.txt&quot;</span>], trainer)`,lang:"python",wrap:!1}}),{c(){e=w("p"),e.textContent=o,t=m(),h(r.$$.fragment)},l(n){e=v(n,"P",{"data-svelte-h":!0}),W(e)!=="svelte-11lpom8"&&(e.textContent=o),t=d(n),u(r.$$.fragment,n)},m(n,g){c(n,e,g),c(n,t,g),f(r,n,g),i=!0},p:ee,i(n){i||($(r.$$.fragment,n),i=!0)},o(n){_(r.$$.fragment,n),i=!1},d(n){n&&(l(e),l(t)),k(r,n)}}}function je(b){let e,o="Example:",t,r,i;return r=new ce({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubW9kZWxzJTIwaW1wb3J0JTIwVW5pZ3JhbSUwQWZyb20lMjB0b2tlbml6ZXJzLnRyYWluZXJzJTIwaW1wb3J0JTIwVW5pZ3JhbVRyYWluZXIlMEF0cmFpbmVyJTIwJTNEJTIwVW5pZ3JhbVRyYWluZXIoJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRDgwMDAlMkMlMEElMjAlMjAlMjAlMjBzcGVjaWFsX3Rva2VucyUzRCU1QiUyMiUzQ3VuayUzRSUyMiUyQyUyMCUyMiUzQ3MlM0UlMjIlMkMlMjAlMjIlM0MlMkZzJTNFJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdW5rX3Rva2VuJTNEJTIyJTNDdW5rJTNFJTIyJTJDJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplcihVbmlncmFtKCkpJTBBdG9rZW5pemVyLnRyYWluKCU1QiUyMnBhdGglMkZ0byUyRmNvcnB1cy50eHQlMjIlNUQlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> Unigram
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> UnigramTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = UnigramTrainer(
<span class="hljs-meta">... </span>    vocab_size=<span class="hljs-number">8000</span>,
<span class="hljs-meta">... </span>    special_tokens=[<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;s&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>],
<span class="hljs-meta">... </span>    unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(Unigram())
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train([<span class="hljs-string">&quot;path/to/corpus.txt&quot;</span>], trainer)`,lang:"python",wrap:!1}}),{c(){e=w("p"),e.textContent=o,t=m(),h(r.$$.fragment)},l(n){e=v(n,"P",{"data-svelte-h":!0}),W(e)!=="svelte-11lpom8"&&(e.textContent=o),t=d(n),u(r.$$.fragment,n)},m(n,g){c(n,e,g),c(n,t,g),f(r,n,g),i=!0},p:ee,i(n){i||($(r.$$.fragment,n),i=!0)},o(n){_(r.$$.fragment,n),i=!1},d(n){n&&(l(e),l(t)),k(r,n)}}}function Ie(b){let e,o="Example:",t,r,i;return r=new ce({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubW9kZWxzJTIwaW1wb3J0JTIwV29yZExldmVsJTBBZnJvbSUyMHRva2VuaXplcnMudHJhaW5lcnMlMjBpbXBvcnQlMjBXb3JkTGV2ZWxUcmFpbmVyJTBBdHJhaW5lciUyMCUzRCUyMFdvcmRMZXZlbFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRDEwMDAwJTJDJTBBJTIwJTIwJTIwJTIwc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlM0N1bmslM0UlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBtaW5fZnJlcXVlbmN5JTNEMSUyQyUwQSklMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoV29yZExldmVsKHVua190b2tlbiUzRCUyMiUzQ3VuayUzRSUyMikpJTBBdG9rZW5pemVyLnRyYWluKCU1QiUyMnBhdGglMkZ0byUyRmNvcnB1cy50eHQlMjIlNUQlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> WordLevel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> WordLevelTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = WordLevelTrainer(
<span class="hljs-meta">... </span>    vocab_size=<span class="hljs-number">10000</span>,
<span class="hljs-meta">... </span>    special_tokens=[<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>],
<span class="hljs-meta">... </span>    min_frequency=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(WordLevel(unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train([<span class="hljs-string">&quot;path/to/corpus.txt&quot;</span>], trainer)`,lang:"python",wrap:!1}}),{c(){e=w("p"),e.textContent=o,t=m(),h(r.$$.fragment)},l(n){e=v(n,"P",{"data-svelte-h":!0}),W(e)!=="svelte-11lpom8"&&(e.textContent=o),t=d(n),u(r.$$.fragment,n)},m(n,g){c(n,e,g),c(n,t,g),f(r,n,g),i=!0},p:ee,i(n){i||($(r.$$.fragment,n),i=!0)},o(n){_(r.$$.fragment,n),i=!1},d(n){n&&(l(e),l(t)),k(r,n)}}}function We(b){let e,o="Example:",t,r,i;return r=new ce({props:{code:"ZnJvbSUyMHRva2VuaXplcnMubW9kZWxzJTIwaW1wb3J0JTIwV29yZFBpZWNlJTBBZnJvbSUyMHRva2VuaXplcnMudHJhaW5lcnMlMjBpbXBvcnQlMjBXb3JkUGllY2VUcmFpbmVyJTBBdHJhaW5lciUyMCUzRCUyMFdvcmRQaWVjZVRyYWluZXIoJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRDMwMDAwJTJDJTBBJTIwJTIwJTIwJTIwc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEJTJDJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplcihXb3JkUGllY2UodW5rX3Rva2VuJTNEJTIyJTVCVU5LJTVEJTIyKSklMEF0b2tlbml6ZXIudHJhaW4oJTVCJTIycGF0aCUyRnRvJTJGY29ycHVzLnR4dCUyMiU1RCUyQyUyMHRyYWluZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> WordPiece
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> WordPieceTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = WordPieceTrainer(
<span class="hljs-meta">... </span>    vocab_size=<span class="hljs-number">30000</span>,
<span class="hljs-meta">... </span>    special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>],
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(WordPiece(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train([<span class="hljs-string">&quot;path/to/corpus.txt&quot;</span>], trainer)`,lang:"python",wrap:!1}}),{c(){e=w("p"),e.textContent=o,t=m(),h(r.$$.fragment)},l(n){e=v(n,"P",{"data-svelte-h":!0}),W(e)!=="svelte-11lpom8"&&(e.textContent=o),t=d(n),u(r.$$.fragment,n)},m(n,g){c(n,e,g),c(n,t,g),f(r,n,g),i=!0},p:ee,i(n){i||($(r.$$.fragment,n),i=!0)},o(n){_(r.$$.fragment,n),i=!1},d(n){n&&(l(e),l(t)),k(r,n)}}}function Be(b){let e,o,t,r,i,n,g="Trainer capable of training a BPE model",Z,y,P,J,x,C,z,s="Get the number of unique words after feeding the corpus",T,B,ae,U,X,me,G,_e="Trainer capable of training a Unigram model",de,E,re,A,se,j,L,ge,K,ke="Trainer capable of training a WordLevel model",he,N,oe,D,ie,I,S,ue,O,Te="Trainer capable of training a WordPiece model",fe,Q,le;return e=new ne({props:{title:"BpeTrainer",local:"tokenizers.trainers.BpeTrainer",headingTag:"h2"}}),r=new te({props:{name:"class tokenizers.trainers.BpeTrainer",anchor:"tokenizers.trainers.BpeTrainer",parameters:[{name:"vocab_size",val:" = 30000"},{name:"min_frequency",val:" = 0"},{name:"show_progress",val:" = True"},{name:"progress_format",val:" = 'indicatif'"},{name:"special_tokens",val:" = []"},{name:"limit_alphabet",val:" = None"},{name:"initial_alphabet",val:" = []"},{name:"continuing_subword_prefix",val:" = None"},{name:"end_of_word_suffix",val:" = None"},{name:"max_token_length",val:" = None"},{name:"words",val:" = {}"}],parametersDescription:[{anchor:"tokenizers.trainers.BpeTrainer.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the final vocabulary, including all tokens and alphabet.`,name:"vocab_size"},{anchor:"tokenizers.trainers.BpeTrainer.min_frequency",description:`<strong>min_frequency</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The minimum frequency a pair should have in order to be merged.`,name:"min_frequency"},{anchor:"tokenizers.trainers.BpeTrainer.show_progress",description:`<strong>show_progress</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to show progress bars while training.`,name:"show_progress"},{anchor:"tokenizers.trainers.BpeTrainer.special_tokens",description:`<strong>special_tokens</strong> (<code>List[Union[str, AddedToken]]</code>, <em>optional</em>) &#x2014;
A list of special tokens the model should know of.`,name:"special_tokens"},{anchor:"tokenizers.trainers.BpeTrainer.limit_alphabet",description:`<strong>limit_alphabet</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The maximum different characters to keep in the alphabet.`,name:"limit_alphabet"},{anchor:"tokenizers.trainers.BpeTrainer.initial_alphabet",description:`<strong>initial_alphabet</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of characters to include in the initial alphabet, even
if not seen in the training dataset.
If the strings contain more than one character, only the first one
is kept.`,name:"initial_alphabet"},{anchor:"tokenizers.trainers.BpeTrainer.continuing_subword_prefix",description:`<strong>continuing_subword_prefix</strong> (<code>str</code>, <em>optional</em>) &#x2014;
A prefix to be used for every subword that is not a beginning-of-word.`,name:"continuing_subword_prefix"},{anchor:"tokenizers.trainers.BpeTrainer.end_of_word_suffix",description:`<strong>end_of_word_suffix</strong> (<code>str</code>, <em>optional</em>) &#x2014;
A suffix to be used for every subword that is a end-of-word.`,name:"end_of_word_suffix"},{anchor:"tokenizers.trainers.BpeTrainer.max_token_length",description:`<strong>max_token_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Prevents creating tokens longer than the specified size.
This can help with reducing polluting your vocabulary with
highly repetitive tokens like <em>======</em> for wikipedia`,name:"max_token_length"}]}}),y=new pe({props:{anchor:"tokenizers.trainers.BpeTrainer.example",$$slots:{default:[Ue]},$$scope:{ctx:b}}}),x=new te({props:{name:"get_word_count",anchor:"tokenizers.trainers.BpeTrainer.get_word_count",parameters:[]}}),B=new ne({props:{title:"UnigramTrainer",local:"tokenizers.trainers.UnigramTrainer",headingTag:"h2"}}),X=new te({props:{name:"class tokenizers.trainers.UnigramTrainer",anchor:"tokenizers.trainers.UnigramTrainer",parameters:[{name:"vocab_size",val:" = 8000"},{name:"show_progress",val:" = True"},{name:"special_tokens",val:" = []"},{name:"initial_alphabet",val:" = []"},{name:"shrinking_factor",val:" = 0.75"},{name:"unk_token",val:" = None"},{name:"max_piece_length",val:" = 16"},{name:"n_sub_iterations",val:" = 2"}],parametersDescription:[{anchor:"tokenizers.trainers.UnigramTrainer.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>) &#x2014;
The size of the final vocabulary, including all tokens and alphabet.`,name:"vocab_size"},{anchor:"tokenizers.trainers.UnigramTrainer.show_progress",description:`<strong>show_progress</strong> (<code>bool</code>) &#x2014;
Whether to show progress bars while training.`,name:"show_progress"},{anchor:"tokenizers.trainers.UnigramTrainer.special_tokens",description:`<strong>special_tokens</strong> (<code>List[Union[str, AddedToken]]</code>) &#x2014;
A list of special tokens the model should know of.`,name:"special_tokens"},{anchor:"tokenizers.trainers.UnigramTrainer.initial_alphabet",description:`<strong>initial_alphabet</strong> (<code>List[str]</code>) &#x2014;
A list of characters to include in the initial alphabet, even
if not seen in the training dataset.
If the strings contain more than one character, only the first one
is kept.`,name:"initial_alphabet"},{anchor:"tokenizers.trainers.UnigramTrainer.shrinking_factor",description:`<strong>shrinking_factor</strong> (<code>float</code>) &#x2014;
The shrinking factor used at each step of the training to prune the
vocabulary.`,name:"shrinking_factor"},{anchor:"tokenizers.trainers.UnigramTrainer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>) &#x2014;
The token used for out-of-vocabulary tokens.`,name:"unk_token"},{anchor:"tokenizers.trainers.UnigramTrainer.max_piece_length",description:`<strong>max_piece_length</strong> (<code>int</code>) &#x2014;
The maximum length of a given token.`,name:"max_piece_length"},{anchor:"tokenizers.trainers.UnigramTrainer.n_sub_iterations",description:`<strong>n_sub_iterations</strong> (<code>int</code>) &#x2014;
The number of iterations of the EM algorithm to perform before
pruning the vocabulary.`,name:"n_sub_iterations"}]}}),E=new pe({props:{anchor:"tokenizers.trainers.UnigramTrainer.example",$$slots:{default:[je]},$$scope:{ctx:b}}}),A=new ne({props:{title:"WordLevelTrainer",local:"tokenizers.trainers.WordLevelTrainer",headingTag:"h2"}}),L=new te({props:{name:"class tokenizers.trainers.WordLevelTrainer",anchor:"tokenizers.trainers.WordLevelTrainer",parameters:[{name:"vocab_size",val:" = 30000"},{name:"min_frequency",val:" = 0"},{name:"show_progress",val:" = True"},{name:"special_tokens",val:" = []"}],parametersDescription:[{anchor:"tokenizers.trainers.WordLevelTrainer.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the final vocabulary, including all tokens and alphabet.`,name:"vocab_size"},{anchor:"tokenizers.trainers.WordLevelTrainer.min_frequency",description:`<strong>min_frequency</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The minimum frequency a pair should have in order to be merged.`,name:"min_frequency"},{anchor:"tokenizers.trainers.WordLevelTrainer.show_progress",description:`<strong>show_progress</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to show progress bars while training.`,name:"show_progress"},{anchor:"tokenizers.trainers.WordLevelTrainer.special_tokens",description:`<strong>special_tokens</strong> (<code>List[Union[str, AddedToken]]</code>) &#x2014;
A list of special tokens the model should know of.`,name:"special_tokens"}]}}),N=new pe({props:{anchor:"tokenizers.trainers.WordLevelTrainer.example",$$slots:{default:[Ie]},$$scope:{ctx:b}}}),D=new ne({props:{title:"WordPieceTrainer",local:"tokenizers.trainers.WordPieceTrainer",headingTag:"h2"}}),S=new te({props:{name:"class tokenizers.trainers.WordPieceTrainer",anchor:"tokenizers.trainers.WordPieceTrainer",parameters:[{name:"vocab_size",val:" = 30000"},{name:"min_frequency",val:" = 0"},{name:"show_progress",val:" = True"},{name:"special_tokens",val:" = []"},{name:"limit_alphabet",val:" = None"},{name:"initial_alphabet",val:" = []"},{name:"continuing_subword_prefix",val:" = '##'"},{name:"end_of_word_suffix",val:" = None"}],parametersDescription:[{anchor:"tokenizers.trainers.WordPieceTrainer.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The size of the final vocabulary, including all tokens and alphabet.`,name:"vocab_size"},{anchor:"tokenizers.trainers.WordPieceTrainer.min_frequency",description:`<strong>min_frequency</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The minimum frequency a pair should have in order to be merged.`,name:"min_frequency"},{anchor:"tokenizers.trainers.WordPieceTrainer.show_progress",description:`<strong>show_progress</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to show progress bars while training.`,name:"show_progress"},{anchor:"tokenizers.trainers.WordPieceTrainer.special_tokens",description:`<strong>special_tokens</strong> (<code>List[Union[str, AddedToken]]</code>, <em>optional</em>) &#x2014;
A list of special tokens the model should know of.`,name:"special_tokens"},{anchor:"tokenizers.trainers.WordPieceTrainer.limit_alphabet",description:`<strong>limit_alphabet</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The maximum different characters to keep in the alphabet.`,name:"limit_alphabet"},{anchor:"tokenizers.trainers.WordPieceTrainer.initial_alphabet",description:`<strong>initial_alphabet</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of characters to include in the initial alphabet, even
if not seen in the training dataset.
If the strings contain more than one character, only the first one
is kept.`,name:"initial_alphabet"},{anchor:"tokenizers.trainers.WordPieceTrainer.continuing_subword_prefix",description:`<strong>continuing_subword_prefix</strong> (<code>str</code>, <em>optional</em>) &#x2014;
A prefix to be used for every subword that is not a beginning-of-word.`,name:"continuing_subword_prefix"},{anchor:"tokenizers.trainers.WordPieceTrainer.end_of_word_suffix",description:`<strong>end_of_word_suffix</strong> (<code>str</code>, <em>optional</em>) &#x2014;
A suffix to be used for every subword that is a end-of-word.`,name:"end_of_word_suffix"}]}}),Q=new pe({props:{anchor:"tokenizers.trainers.WordPieceTrainer.example",$$slots:{default:[We]},$$scope:{ctx:b}}}),{c(){h(e.$$.fragment),o=m(),t=w("div"),h(r.$$.fragment),i=m(),n=w("p"),n.textContent=g,Z=m(),h(y.$$.fragment),P=m(),J=w("div"),h(x.$$.fragment),C=m(),z=w("p"),z.textContent=s,T=m(),h(B.$$.fragment),ae=m(),U=w("div"),h(X.$$.fragment),me=m(),G=w("p"),G.textContent=_e,de=m(),h(E.$$.fragment),re=m(),h(A.$$.fragment),se=m(),j=w("div"),h(L.$$.fragment),ge=m(),K=w("p"),K.textContent=ke,he=m(),h(N.$$.fragment),oe=m(),h(D.$$.fragment),ie=m(),I=w("div"),h(S.$$.fragment),ue=m(),O=w("p"),O.textContent=Te,fe=m(),h(Q.$$.fragment),this.h()},l(a){u(e.$$.fragment,a),o=d(a),t=v(a,"DIV",{class:!0});var p=H(t);u(r.$$.fragment,p),i=d(p),n=v(p,"P",{"data-svelte-h":!0}),W(n)!=="svelte-1iof5l3"&&(n.textContent=g),Z=d(p),u(y.$$.fragment,p),P=d(p),J=v(p,"DIV",{class:!0});var F=H(J);u(x.$$.fragment,F),C=d(F),z=v(F,"P",{"data-svelte-h":!0}),W(z)!=="svelte-im9d0w"&&(z.textContent=s),F.forEach(l),p.forEach(l),T=d(a),u(B.$$.fragment,a),ae=d(a),U=v(a,"DIV",{class:!0});var V=H(U);u(X.$$.fragment,V),me=d(V),G=v(V,"P",{"data-svelte-h":!0}),W(G)!=="svelte-ukd897"&&(G.textContent=_e),de=d(V),u(E.$$.fragment,V),V.forEach(l),re=d(a),u(A.$$.fragment,a),se=d(a),j=v(a,"DIV",{class:!0});var R=H(j);u(L.$$.fragment,R),ge=d(R),K=v(R,"P",{"data-svelte-h":!0}),W(K)!=="svelte-1ffzdis"&&(K.textContent=ke),he=d(R),u(N.$$.fragment,R),R.forEach(l),oe=d(a),u(D.$$.fragment,a),ie=d(a),I=v(a,"DIV",{class:!0});var q=H(I);u(S.$$.fragment,q),ue=d(q),O=v(q,"P",{"data-svelte-h":!0}),W(O)!=="svelte-1yngc5g"&&(O.textContent=Te),fe=d(q),u(Q.$$.fragment,q),q.forEach(l),this.h()},h(){Y(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Y(t,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Y(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Y(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Y(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(a,p){f(e,a,p),c(a,o,p),c(a,t,p),f(r,t,null),M(t,i),M(t,n),M(t,Z),f(y,t,null),M(t,P),M(t,J),f(x,J,null),M(J,C),M(J,z),c(a,T,p),f(B,a,p),c(a,ae,p),c(a,U,p),f(X,U,null),M(U,me),M(U,G),M(U,de),f(E,U,null),c(a,re,p),f(A,a,p),c(a,se,p),c(a,j,p),f(L,j,null),M(j,ge),M(j,K),M(j,he),f(N,j,null),c(a,oe,p),f(D,a,p),c(a,ie,p),c(a,I,p),f(S,I,null),M(I,ue),M(I,O),M(I,fe),f(Q,I,null),le=!0},p(a,p){const F={};p&2&&(F.$$scope={dirty:p,ctx:a}),y.$set(F);const V={};p&2&&(V.$$scope={dirty:p,ctx:a}),E.$set(V);const R={};p&2&&(R.$$scope={dirty:p,ctx:a}),N.$set(R);const q={};p&2&&(q.$$scope={dirty:p,ctx:a}),Q.$set(q)},i(a){le||($(e.$$.fragment,a),$(r.$$.fragment,a),$(y.$$.fragment,a),$(x.$$.fragment,a),$(B.$$.fragment,a),$(X.$$.fragment,a),$(E.$$.fragment,a),$(A.$$.fragment,a),$(L.$$.fragment,a),$(N.$$.fragment,a),$(D.$$.fragment,a),$(S.$$.fragment,a),$(Q.$$.fragment,a),le=!0)},o(a){_(e.$$.fragment,a),_(r.$$.fragment,a),_(y.$$.fragment,a),_(x.$$.fragment,a),_(B.$$.fragment,a),_(X.$$.fragment,a),_(E.$$.fragment,a),_(A.$$.fragment,a),_(L.$$.fragment,a),_(N.$$.fragment,a),_(D.$$.fragment,a),_(S.$$.fragment,a),_(Q.$$.fragment,a),le=!1},d(a){a&&(l(o),l(t),l(T),l(ae),l(U),l(re),l(se),l(j),l(oe),l(ie),l(I)),k(e,a),k(r),k(y),k(x),k(B,a),k(X),k(E),k(A,a),k(L),k(N),k(D,a),k(S),k(Q)}}}function Ce(b){let e,o;return e=new $e({props:{$$slots:{default:[Be]},$$scope:{ctx:b}}}),{c(){h(e.$$.fragment)},l(t){u(e.$$.fragment,t)},m(t,r){f(e,t,r),o=!0},p(t,r){const i={};r&2&&(i.$$scope={dirty:r,ctx:t}),e.$set(i)},i(t){o||($(e.$$.fragment,t),o=!0)},o(t){_(e.$$.fragment,t),o=!1},d(t){k(e,t)}}}function Ve(b){let e,o='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){e=w("p"),e.innerHTML=o},l(t){e=v(t,"P",{"data-svelte-h":!0}),W(e)!=="svelte-4ytcyb"&&(e.innerHTML=o)},m(t,r){c(t,e,r)},p:ee,d(t){t&&l(e)}}}function Re(b){let e,o;return e=new $e({props:{$$slots:{default:[Ve]},$$scope:{ctx:b}}}),{c(){h(e.$$.fragment)},l(t){u(e.$$.fragment,t)},m(t,r){f(e,t,r),o=!0},p(t,r){const i={};r&2&&(i.$$scope={dirty:r,ctx:t}),e.$set(i)},i(t){o||($(e.$$.fragment,t),o=!0)},o(t){_(e.$$.fragment,t),o=!1},d(t){k(e,t)}}}function qe(b){let e,o="The node API has not been documented yet.";return{c(){e=w("p"),e.textContent=o},l(t){e=v(t,"P",{"data-svelte-h":!0}),W(e)!=="svelte-1mrchm6"&&(e.textContent=o)},m(t,r){c(t,e,r)},p:ee,d(t){t&&l(e)}}}function Ze(b){let e,o;return e=new $e({props:{$$slots:{default:[qe]},$$scope:{ctx:b}}}),{c(){h(e.$$.fragment)},l(t){u(e.$$.fragment,t)},m(t,r){f(e,t,r),o=!0},p(t,r){const i={};r&2&&(i.$$scope={dirty:r,ctx:t}),e.$set(i)},i(t){o||($(e.$$.fragment,t),o=!0)},o(t){_(e.$$.fragment,t),o=!1},d(t){k(e,t)}}}function Pe(b){let e,o,t,r,i,n,g,Z,y,P,J,x,C,z;return i=new Je({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new ne({props:{title:"Trainers",local:"trainers",headingTag:"h1"}}),y=new ze({props:{python:!0,rust:!0,node:!0,$$slots:{node:[Ze],rust:[Re],python:[Ce]},$$scope:{ctx:b}}}),J=new xe({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/trainers.mdx"}}),{c(){e=w("meta"),o=m(),t=w("p"),r=m(),h(i.$$.fragment),n=m(),h(g.$$.fragment),Z=m(),h(y.$$.fragment),P=m(),h(J.$$.fragment),x=m(),C=w("p"),this.h()},l(s){const T=Me("svelte-u9bgzb",document.head);e=v(T,"META",{name:!0,content:!0}),T.forEach(l),o=d(s),t=v(s,"P",{}),H(t).forEach(l),r=d(s),u(i.$$.fragment,s),n=d(s),u(g.$$.fragment,s),Z=d(s),u(y.$$.fragment,s),P=d(s),u(J.$$.fragment,s),x=d(s),C=v(s,"P",{}),H(C).forEach(l),this.h()},h(){Y(e,"name","hf:doc:metadata"),Y(e,"content",Ee)},m(s,T){M(document.head,e),c(s,o,T),c(s,t,T),c(s,r,T),f(i,s,T),c(s,n,T),f(g,s,T),c(s,Z,T),f(y,s,T),c(s,P,T),f(J,s,T),c(s,x,T),c(s,C,T),z=!0},p(s,[T]){const B={};T&2&&(B.$$scope={dirty:T,ctx:s}),y.$set(B)},i(s){z||($(i.$$.fragment,s),$(g.$$.fragment,s),$(y.$$.fragment,s),$(J.$$.fragment,s),z=!0)},o(s){_(i.$$.fragment,s),_(g.$$.fragment,s),_(y.$$.fragment,s),_(J.$$.fragment,s),z=!1},d(s){s&&(l(o),l(t),l(r),l(n),l(Z),l(P),l(x),l(C)),l(e),k(i,s),k(g,s),k(y,s),k(J,s)}}}const Ee='{"title":"Trainers","local":"trainers","sections":[{"title":"BpeTrainer","local":"tokenizers.trainers.BpeTrainer","sections":[],"depth":2},{"title":"UnigramTrainer","local":"tokenizers.trainers.UnigramTrainer","sections":[],"depth":2},{"title":"WordLevelTrainer","local":"tokenizers.trainers.WordLevelTrainer","sections":[],"depth":2},{"title":"WordPieceTrainer","local":"tokenizers.trainers.WordPieceTrainer","sections":[],"depth":2}],"depth":1}';function Ne(b){return we(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class He extends ve{constructor(e){super(),ye(this,e,Ne,Pe,be,{})}}export{He as component};