Buckets:
| import{s as vc,o as yc,n as Ee}from"../chunks/scheduler.9991993c.js";import{S as wc,i as zc,g as r,s as n,r as l,A as $c,h as s,f as d,c as o,j as T,u as p,x as a,k as v,y as e,a as z,v as m,d as h,t as u,w as f}from"../chunks/index.7fc9a5e7.js";import{T as Tc}from"../chunks/Tip.9de92fc6.js";import{D as w}from"../chunks/Docstring.ef7d0149.js";import{C as Qo}from"../chunks/CodeBlock.e11cba92.js";import{E as Oo}from"../chunks/ExampleCodeBlock.0db1a011.js";import{H as Mr,E as Pc}from"../chunks/EditOnGithub.84ab7f0e.js";function Cc(W){let c,I="Examples:",k,_,P;return _=new Qo({props:{code:"JTIzJTIwTGV0J3MlMjBzZWUlMjBob3clMjB0byUyMGluY3JlYXNlJTIwdGhlJTIwdm9jYWJ1bGFyeSUyMG9mJTIwQmVydCUyMG1vZGVsJTIwYW5kJTIwdG9rZW5pemVyJTBBdG9rZW5pemVyJTIwJTNEJTIwQmVydFRva2VuaXplckZhc3QuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZS1iZXJ0JTJGYmVydC1iYXNlLXVuY2FzZWQlMjIpJTBBbW9kZWwlMjAlM0QlMjBCZXJ0TW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZS1iZXJ0JTJGYmVydC1iYXNlLXVuY2FzZWQlMjIpJTBBJTBBbnVtX2FkZGVkX3Rva3MlMjAlM0QlMjB0b2tlbml6ZXIuYWRkX3Rva2VucyglNUIlMjJuZXdfdG9rMSUyMiUyQyUyMCUyMm15X25ldy10b2syJTIyJTVEKSUwQXByaW50KCUyMldlJTIwaGF2ZSUyMGFkZGVkJTIyJTJDJTIwbnVtX2FkZGVkX3Rva3MlMkMlMjAlMjJ0b2tlbnMlMjIpJTBBJTIzJTIwTm90aWNlJTNBJTIwcmVzaXplX3Rva2VuX2VtYmVkZGluZ3MlMjBleHBlY3QlMjB0byUyMHJlY2VpdmUlMjB0aGUlMjBmdWxsJTIwc2l6ZSUyMG9mJTIwdGhlJTIwbmV3JTIwdm9jYWJ1bGFyeSUyQyUyMGkuZS4lMkMlMjB0aGUlMjBsZW5ndGglMjBvZiUyMHRoZSUyMHRva2VuaXplci4lMEFtb2RlbC5yZXNpemVfdG9rZW5fZW1iZWRkaW5ncyhsZW4odG9rZW5pemVyKSk=",highlighted:`<span class="hljs-comment"># Let's see how to increase the vocabulary of Bert model and tokenizer</span> | |
| tokenizer = BertTokenizerFast.from_pretrained(<span class="hljs-string">"google-bert/bert-base-uncased"</span>) | |
| model = BertModel.from_pretrained(<span class="hljs-string">"google-bert/bert-base-uncased"</span>) | |
| num_added_toks = tokenizer.add_tokens([<span class="hljs-string">"new_tok1"</span>, <span class="hljs-string">"my_new-tok2"</span>]) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"We have added"</span>, num_added_toks, <span class="hljs-string">"tokens"</span>) | |
| <span class="hljs-comment"># Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.</span> | |
| model.resize_token_embeddings(<span class="hljs-built_in">len</span>(tokenizer))`,wrap:!1}}),{c(){c=r("p"),c.textContent=I,k=n(),l(_.$$.fragment)},l(i){c=s(i,"P",{"data-svelte-h":!0}),a(c)!=="svelte-kvfsh7"&&(c.textContent=I),k=o(i),p(_.$$.fragment,i)},m(i,M){z(i,c,M),z(i,k,M),m(_,i,M),P=!0},p:Ee,i(i){P||(h(_.$$.fragment,i),P=!0)},o(i){u(_.$$.fragment,i),P=!1},d(i){i&&(d(c),d(k)),f(_,i)}}}function qc(W){let c,I="Examples:",k,_,P;return _=new Qo({props:{code:"JTIzJTIwTGV0J3MlMjBzZWUlMjBob3clMjB0byUyMGFkZCUyMGElMjBuZXclMjBjbGFzc2lmaWNhdGlvbiUyMHRva2VuJTIwdG8lMjBHUFQtMiUwQXRva2VuaXplciUyMCUzRCUyMEdQVDJUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyKSUwQW1vZGVsJTIwJTNEJTIwR1BUMk1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJvcGVuYWktY29tbXVuaXR5JTJGZ3B0MiUyMiklMEElMEFzcGVjaWFsX3Rva2Vuc19kaWN0JTIwJTNEJTIwJTdCJTIyY2xzX3Rva2VuJTIyJTNBJTIwJTIyJTNDQ0xTJTNFJTIyJTdEJTBBJTBBbnVtX2FkZGVkX3Rva3MlMjAlM0QlMjB0b2tlbml6ZXIuYWRkX3NwZWNpYWxfdG9rZW5zKHNwZWNpYWxfdG9rZW5zX2RpY3QpJTBBcHJpbnQoJTIyV2UlMjBoYXZlJTIwYWRkZWQlMjIlMkMlMjBudW1fYWRkZWRfdG9rcyUyQyUyMCUyMnRva2VucyUyMiklMEElMjMlMjBOb3RpY2UlM0ElMjByZXNpemVfdG9rZW5fZW1iZWRkaW5ncyUyMGV4cGVjdCUyMHRvJTIwcmVjZWl2ZSUyMHRoZSUyMGZ1bGwlMjBzaXplJTIwb2YlMjB0aGUlMjBuZXclMjB2b2NhYnVsYXJ5JTJDJTIwaS5lLiUyQyUyMHRoZSUyMGxlbmd0aCUyMG9mJTIwdGhlJTIwdG9rZW5pemVyLiUwQW1vZGVsLnJlc2l6ZV90b2tlbl9lbWJlZGRpbmdzKGxlbih0b2tlbml6ZXIpKSUwQSUwQWFzc2VydCUyMHRva2VuaXplci5jbHNfdG9rZW4lMjAlM0QlM0QlMjAlMjIlM0NDTFMlM0UlMjI=",highlighted:`<span class="hljs-comment"># Let's see how to add a new classification token to GPT-2</span> | |
| tokenizer = GPT2Tokenizer.from_pretrained(<span class="hljs-string">"openai-community/gpt2"</span>) | |
| model = GPT2Model.from_pretrained(<span class="hljs-string">"openai-community/gpt2"</span>) | |
| special_tokens_dict = {<span class="hljs-string">"cls_token"</span>: <span class="hljs-string">"<CLS>"</span>} | |
| num_added_toks = tokenizer.add_special_tokens(special_tokens_dict) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"We have added"</span>, num_added_toks, <span class="hljs-string">"tokens"</span>) | |
| <span class="hljs-comment"># Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.</span> | |
| model.resize_token_embeddings(<span class="hljs-built_in">len</span>(tokenizer)) | |
| <span class="hljs-keyword">assert</span> tokenizer.cls_token == <span class="hljs-string">"<CLS>"</span>`,wrap:!1}}),{c(){c=r("p"),c.textContent=I,k=n(),l(_.$$.fragment)},l(i){c=s(i,"P",{"data-svelte-h":!0}),a(c)!=="svelte-kvfsh7"&&(c.textContent=I),k=o(i),p(_.$$.fragment,i)},m(i,M){z(i,c,M),z(i,k,M),m(_,i,M),P=!0},p:Ee,i(i){P||(h(_.$$.fragment,i),P=!0)},o(i){u(_.$$.fragment,i),P=!1},d(i){i&&(d(c),d(k)),f(_,i)}}}function Mc(W){let c,I="Examples:",k,_,P;return _=new Qo({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUtYmVydCUyRmJlcnQtYmFzZS1jYXNlZCUyMiklMEElMEElMjMlMjBQdXNoJTIwdGhlJTIwdG9rZW5pemVyJTIwdG8lMjB5b3VyJTIwbmFtZXNwYWNlJTIwd2l0aCUyMHRoZSUyMG5hbWUlMjAlMjJteS1maW5ldHVuZWQtYmVydCUyMi4lMEF0b2tlbml6ZXIucHVzaF90b19odWIoJTIybXktZmluZXR1bmVkLWJlcnQlMjIpJTBBJTBBJTIzJTIwUHVzaCUyMHRoZSUyMHRva2VuaXplciUyMHRvJTIwYW4lMjBvcmdhbml6YXRpb24lMjB3aXRoJTIwdGhlJTIwbmFtZSUyMCUyMm15LWZpbmV0dW5lZC1iZXJ0JTIyLiUwQXRva2VuaXplci5wdXNoX3RvX2h1YiglMjJodWdnaW5nZmFjZSUyRm15LWZpbmV0dW5lZC1iZXJ0JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer | |
| tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google-bert/bert-base-cased"</span>) | |
| <span class="hljs-comment"># Push the tokenizer to your namespace with the name "my-finetuned-bert".</span> | |
| tokenizer.push_to_hub(<span class="hljs-string">"my-finetuned-bert"</span>) | |
| <span class="hljs-comment"># Push the tokenizer to an organization with the name "my-finetuned-bert".</span> | |
| tokenizer.push_to_hub(<span class="hljs-string">"huggingface/my-finetuned-bert"</span>)`,wrap:!1}}),{c(){c=r("p"),c.textContent=I,k=n(),l(_.$$.fragment)},l(i){c=s(i,"P",{"data-svelte-h":!0}),a(c)!=="svelte-kvfsh7"&&(c.textContent=I),k=o(i),p(_.$$.fragment,i)},m(i,M){z(i,c,M),z(i,k,M),m(_,i,M),P=!0},p:Ee,i(i){P||(h(_.$$.fragment,i),P=!0)},o(i){u(_.$$.fragment,i),P=!1},d(i){i&&(d(c),d(k)),f(_,i)}}}function Ic(W){let c,I=`This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put | |
| this inside your training loop.`;return{c(){c=r("p"),c.textContent=I},l(k){c=s(k,"P",{"data-svelte-h":!0}),a(c)!=="svelte-1yi8eve"&&(c.textContent=I)},m(k,_){z(k,c,_)},p:Ee,d(k){k&&d(c)}}}function Fc(W){let c,I="Examples:",k,_,P;return _=new Qo({props:{code:"JTIzJTIwTGV0J3MlMjBzZWUlMjBob3clMjB0byUyMGluY3JlYXNlJTIwdGhlJTIwdm9jYWJ1bGFyeSUyMG9mJTIwQmVydCUyMG1vZGVsJTIwYW5kJTIwdG9rZW5pemVyJTBBdG9rZW5pemVyJTIwJTNEJTIwQmVydFRva2VuaXplckZhc3QuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZS1iZXJ0JTJGYmVydC1iYXNlLXVuY2FzZWQlMjIpJTBBbW9kZWwlMjAlM0QlMjBCZXJ0TW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZS1iZXJ0JTJGYmVydC1iYXNlLXVuY2FzZWQlMjIpJTBBJTBBbnVtX2FkZGVkX3Rva3MlMjAlM0QlMjB0b2tlbml6ZXIuYWRkX3Rva2VucyglNUIlMjJuZXdfdG9rMSUyMiUyQyUyMCUyMm15X25ldy10b2syJTIyJTVEKSUwQXByaW50KCUyMldlJTIwaGF2ZSUyMGFkZGVkJTIyJTJDJTIwbnVtX2FkZGVkX3Rva3MlMkMlMjAlMjJ0b2tlbnMlMjIpJTBBJTIzJTIwTm90aWNlJTNBJTIwcmVzaXplX3Rva2VuX2VtYmVkZGluZ3MlMjBleHBlY3QlMjB0byUyMHJlY2VpdmUlMjB0aGUlMjBmdWxsJTIwc2l6ZSUyMG9mJTIwdGhlJTIwbmV3JTIwdm9jYWJ1bGFyeSUyQyUyMGkuZS4lMkMlMjB0aGUlMjBsZW5ndGglMjBvZiUyMHRoZSUyMHRva2VuaXplci4lMEFtb2RlbC5yZXNpemVfdG9rZW5fZW1iZWRkaW5ncyhsZW4odG9rZW5pemVyKSk=",highlighted:`<span class="hljs-comment"># Let's see how to increase the vocabulary of Bert model and tokenizer</span> | |
| tokenizer = BertTokenizerFast.from_pretrained(<span class="hljs-string">"google-bert/bert-base-uncased"</span>) | |
| model = BertModel.from_pretrained(<span class="hljs-string">"google-bert/bert-base-uncased"</span>) | |
| num_added_toks = tokenizer.add_tokens([<span class="hljs-string">"new_tok1"</span>, <span class="hljs-string">"my_new-tok2"</span>]) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"We have added"</span>, num_added_toks, <span class="hljs-string">"tokens"</span>) | |
| <span class="hljs-comment"># Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.</span> | |
| model.resize_token_embeddings(<span class="hljs-built_in">len</span>(tokenizer))`,wrap:!1}}),{c(){c=r("p"),c.textContent=I,k=n(),l(_.$$.fragment)},l(i){c=s(i,"P",{"data-svelte-h":!0}),a(c)!=="svelte-kvfsh7"&&(c.textContent=I),k=o(i),p(_.$$.fragment,i)},m(i,M){z(i,c,M),z(i,k,M),m(_,i,M),P=!0},p:Ee,i(i){P||(h(_.$$.fragment,i),P=!0)},o(i){u(_.$$.fragment,i),P=!1},d(i){i&&(d(c),d(k)),f(_,i)}}}function Lc(W){let c,I="Examples:",k,_,P;return _=new Qo({props:{code:"JTIzJTIwTGV0J3MlMjBzZWUlMjBob3clMjB0byUyMGFkZCUyMGElMjBuZXclMjBjbGFzc2lmaWNhdGlvbiUyMHRva2VuJTIwdG8lMjBHUFQtMiUwQXRva2VuaXplciUyMCUzRCUyMEdQVDJUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyKSUwQW1vZGVsJTIwJTNEJTIwR1BUMk1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJvcGVuYWktY29tbXVuaXR5JTJGZ3B0MiUyMiklMEElMEFzcGVjaWFsX3Rva2Vuc19kaWN0JTIwJTNEJTIwJTdCJTIyY2xzX3Rva2VuJTIyJTNBJTIwJTIyJTNDQ0xTJTNFJTIyJTdEJTBBJTBBbnVtX2FkZGVkX3Rva3MlMjAlM0QlMjB0b2tlbml6ZXIuYWRkX3NwZWNpYWxfdG9rZW5zKHNwZWNpYWxfdG9rZW5zX2RpY3QpJTBBcHJpbnQoJTIyV2UlMjBoYXZlJTIwYWRkZWQlMjIlMkMlMjBudW1fYWRkZWRfdG9rcyUyQyUyMCUyMnRva2VucyUyMiklMEElMjMlMjBOb3RpY2UlM0ElMjByZXNpemVfdG9rZW5fZW1iZWRkaW5ncyUyMGV4cGVjdCUyMHRvJTIwcmVjZWl2ZSUyMHRoZSUyMGZ1bGwlMjBzaXplJTIwb2YlMjB0aGUlMjBuZXclMjB2b2NhYnVsYXJ5JTJDJTIwaS5lLiUyQyUyMHRoZSUyMGxlbmd0aCUyMG9mJTIwdGhlJTIwdG9rZW5pemVyLiUwQW1vZGVsLnJlc2l6ZV90b2tlbl9lbWJlZGRpbmdzKGxlbih0b2tlbml6ZXIpKSUwQSUwQWFzc2VydCUyMHRva2VuaXplci5jbHNfdG9rZW4lMjAlM0QlM0QlMjAlMjIlM0NDTFMlM0UlMjI=",highlighted:`<span class="hljs-comment"># Let's see how to add a new classification token to GPT-2</span> | |
| tokenizer = GPT2Tokenizer.from_pretrained(<span class="hljs-string">"openai-community/gpt2"</span>) | |
| model = GPT2Model.from_pretrained(<span class="hljs-string">"openai-community/gpt2"</span>) | |
| special_tokens_dict = {<span class="hljs-string">"cls_token"</span>: <span class="hljs-string">"<CLS>"</span>} | |
| num_added_toks = tokenizer.add_special_tokens(special_tokens_dict) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"We have added"</span>, num_added_toks, <span class="hljs-string">"tokens"</span>) | |
| <span class="hljs-comment"># Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.</span> | |
| model.resize_token_embeddings(<span class="hljs-built_in">len</span>(tokenizer)) | |
| <span class="hljs-keyword">assert</span> tokenizer.cls_token == <span class="hljs-string">"<CLS>"</span>`,wrap:!1}}),{c(){c=r("p"),c.textContent=I,k=n(),l(_.$$.fragment)},l(i){c=s(i,"P",{"data-svelte-h":!0}),a(c)!=="svelte-kvfsh7"&&(c.textContent=I),k=o(i),p(_.$$.fragment,i)},m(i,M){z(i,c,M),z(i,k,M),m(_,i,M),P=!0},p:Ee,i(i){P||(h(_.$$.fragment,i),P=!0)},o(i){u(_.$$.fragment,i),P=!1},d(i){i&&(d(c),d(k)),f(_,i)}}}function jc(W){let c,I="Examples:",k,_,P;return _=new Qo({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUtYmVydCUyRmJlcnQtYmFzZS1jYXNlZCUyMiklMEElMEElMjMlMjBQdXNoJTIwdGhlJTIwdG9rZW5pemVyJTIwdG8lMjB5b3VyJTIwbmFtZXNwYWNlJTIwd2l0aCUyMHRoZSUyMG5hbWUlMjAlMjJteS1maW5ldHVuZWQtYmVydCUyMi4lMEF0b2tlbml6ZXIucHVzaF90b19odWIoJTIybXktZmluZXR1bmVkLWJlcnQlMjIpJTBBJTBBJTIzJTIwUHVzaCUyMHRoZSUyMHRva2VuaXplciUyMHRvJTIwYW4lMjBvcmdhbml6YXRpb24lMjB3aXRoJTIwdGhlJTIwbmFtZSUyMCUyMm15LWZpbmV0dW5lZC1iZXJ0JTIyLiUwQXRva2VuaXplci5wdXNoX3RvX2h1YiglMjJodWdnaW5nZmFjZSUyRm15LWZpbmV0dW5lZC1iZXJ0JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer | |
| tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google-bert/bert-base-cased"</span>) | |
| <span class="hljs-comment"># Push the tokenizer to your namespace with the name "my-finetuned-bert".</span> | |
| tokenizer.push_to_hub(<span class="hljs-string">"my-finetuned-bert"</span>) | |
| <span class="hljs-comment"># Push the tokenizer to an organization with the name "my-finetuned-bert".</span> | |
| tokenizer.push_to_hub(<span class="hljs-string">"huggingface/my-finetuned-bert"</span>)`,wrap:!1}}),{c(){c=r("p"),c.textContent=I,k=n(),l(_.$$.fragment)},l(i){c=s(i,"P",{"data-svelte-h":!0}),a(c)!=="svelte-kvfsh7"&&(c.textContent=I),k=o(i),p(_.$$.fragment,i)},m(i,M){z(i,c,M),z(i,k,M),m(_,i,M),P=!0},p:Ee,i(i){P||(h(_.$$.fragment,i),P=!0)},o(i){u(_.$$.fragment,i),P=!1},d(i){i&&(d(c),d(k)),f(_,i)}}}function Wc(W){let c,I=`This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put | |
| this inside your training loop.`;return{c(){c=r("p"),c.textContent=I},l(k){c=s(k,"P",{"data-svelte-h":!0}),a(c)!=="svelte-1yi8eve"&&(c.textContent=I)},m(k,_){z(k,c,_)},p:Ee,d(k){k&&d(c)}}}function Jc(W){let c,I,k,_,P,i,M,xi='tokenizer负责准备输入以供模型使用。该库包含所有模型的tokenizer。大多数tokenizer都有两种版本：一个是完全的 Python 实现，另一个是基于 Rust 库 <a href="https://github.com/huggingface/tokenizers" rel="nofollow">🤗 Tokenizers</a> 的“Fast”实现。“Fast” 实现允许：',er,De,Ti="<li>在批量分词时显著提速</li> <li>在原始字符串（字符和单词）和token空间之间进行映射的其他方法（例如，获取包含给定字符的token的索引或与给定token对应的字符范围）。</li>",tr,Ze,vi='基类 [PreTrainedTokenizer] 和 [PreTrained TokenizerFast] 实现了在模型输入中编码字符串输入的常用方法（见下文），并从本地文件或目录或从库提供的预训练的 tokenizer（从 HuggingFace 的 AWS S3 存储库下载）实例化/保存 python 和“Fast” tokenizer。它们都依赖于包含常用方法的 <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase">PreTrainedTokenizerBase</a>和<a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.SpecialTokensMixin">SpecialTokensMixin</a>。',nr,He,yi='因此，<a href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> 和 <a href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> 实现了使用所有tokenizers的主要方法：',or,Ge,wi="<li>分词（将字符串拆分为子词标记字符串），将tokens字符串转换为id并转换回来，以及编码/解码（即标记化并转换为整数）。</li> <li>以独立于底层结构（BPE、SentencePiece……）的方式向词汇表中添加新tokens。</li> <li>管理特殊tokens（如mask、句首等）：添加它们，将它们分配给tokenizer中的属性以便于访问，并确保它们在标记过程中不会被分割。</li>",rr,Ae,zi='<a href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.BatchEncoding">BatchEncoding</a> 包含 <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase">PreTrainedTokenizerBase</a> 的编码方法（<code>__call__</code>、<code>encode_plus</code> 和 <code>batch_encode_plus</code>）的输出，并且是从 Python 字典派生的。当tokenizer是纯 Python tokenizer时，此类的行为就像标准的 Python 字典一样，并保存这些方法计算的各种模型输入（<code>input_ids</code>、<code>attention_mask</code> 等）。当分词器是“Fast”分词器时（即由 HuggingFace 的 <a href="https://github.com/huggingface/tokenizers" rel="nofollow">tokenizers 库</a> 支持），此类还提供了几种高级对齐方法，可用于在原始字符串（字符和单词）与token空间之间进行映射（例如，获取包含给定字符的token的索引或与给定token对应的字符范围）。',sr,Re,ar,b,Se,Ir,Gt,$i="Base class for all slow tokenizers.",Fr,At,Pi='Inherits from <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase">PreTrainedTokenizerBase</a>.',Lr,Rt,Ci=`Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading | |
| pretrained tokenizers as well as adding tokens to the vocabulary.`,jr,St,qi=`This class also contain the added tokens in a unified way on top of all tokenizers so we don’t have to handle the | |
| specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece…).`,Wr,Xt,Mi="Class attributes (overridden by derived classes)",Jr,Yt,Ii=`<li><strong>vocab_files_names</strong> (<code>Dict[str, str]</code>) — A dictionary with, as keys, the <code>__init__</code> keyword name of each | |
| vocabulary file required by the model, and as associated values, the filename for saving the associated file | |
| (string).</li> <li><strong>pretrained_vocab_files_map</strong> (<code>Dict[str, Dict[str, str]]</code>) — A dictionary of dictionaries, with the | |
| high-level keys being the <code>__init__</code> keyword name of each vocabulary file required by the model, the | |
| low-level being the <code>short-cut-names</code> of the pretrained models with, as associated values, the <code>url</code> to the | |
| associated pretrained vocabulary file.</li> <li><strong>model_input_names</strong> (<code>List[str]</code>) — A list of inputs expected in the forward pass of the model.</li> <li><strong>padding_side</strong> (<code>str</code>) — The default value for the side on which the model should have padding applied. | |
| Should be <code>'right'</code> or <code>'left'</code>.</li> <li><strong>truncation_side</strong> (<code>str</code>) — The default value for the side on which the model should have truncation | |
| applied. Should be <code>'right'</code> or <code>'left'</code>.</li>`,Ur,fe,Xe,Br,Ot,Fi=`Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of | |
| sequences.`,Nr,V,Ye,Vr,Qt,Li=`Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to | |
| it with indices starting from length of the current vocabulary and and will be isolated before the tokenization | |
| algorithm is applied. Added tokens and tokens from the vocabulary of the tokenization algorithm are therefore | |
| not treated in the same way.`,Er,Kt,ji=`Note, when adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix | |
| of the model so that its embedding matrix matches the tokenizer.`,Dr,en,Wi='In order to do that, please use the <a href="/docs/transformers/pr_34786/zh/main_classes/model#transformers.PreTrainedModel.resize_token_embeddings">resize_token_embeddings()</a> method.',Zr,_e,Hr,F,Oe,Gr,tn,Ji=`Add a dictionary of special tokens (eos, pad, cls, etc.) to the encoder and link them to class attributes. If | |
| special tokens are NOT in the vocabulary, they are added to it (indexed starting from the last index of the | |
| current vocabulary).`,Ar,nn,Ui=`When adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix of the | |
| model so that its embedding matrix matches the tokenizer.`,Rr,on,Bi='In order to do that, please use the <a href="/docs/transformers/pr_34786/zh/main_classes/model#transformers.PreTrainedModel.resize_token_embeddings">resize_token_embeddings()</a> method.',Sr,rn,Ni="Using <code>add_special_tokens</code> will ensure your special tokens can be used in several ways:",Xr,sn,Vi=`<li>Special tokens can be skipped when decoding using <code>skip_special_tokens = True</code>.</li> <li>Special tokens are carefully handled by the tokenizer (they are never split), similar to <code>AddedTokens</code>.</li> <li>You can easily refer to special tokens using tokenizer class attributes like <code>tokenizer.cls_token</code>. This | |
| makes it easy to develop model-agnostic training and fine-tuning scripts.</li>`,Yr,an,Ei=`When possible, special tokens are already registered for provided pretrained models (for instance | |
| <code>BertTokenizer</code> <code>cls_token</code> is already registered to be :obj<em>’[CLS]’</em> and XLM’s one is also registered to be | |
| <code>'</s>'</code>).`,Or,ge,Qr,ke,Qe,Kr,dn,Di=`Converts a list of dictionaries with <code>"role"</code> and <code>"content"</code> keys to a list of token | |
| ids. This method is intended for use with chat models, and will read the tokenizer’s chat_template attribute to | |
| determine the format and control tokens to use when converting.`,es,be,Ke,ts,cn,Zi="Convert a list of lists of token ids into a list of strings by calling decode.",ns,Y,et,os,ln,Hi=`Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special | |
| tokens and clean up tokenization spaces.`,rs,pn,Gi="Similar to doing <code>self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))</code>.",ss,O,tt,as,mn,Ai="Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary.",is,hn,Ri="Same as doing <code>self.convert_tokens_to_ids(self.tokenize(text))</code>.",ds,Q,nt,cs,un,Si="Upload the tokenizer files to the 🤗 Model Hub.",ls,xe,ps,Te,ot,ms,fn,Xi=`Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and | |
| added tokens.`,hs,ve,rt,us,_n,Yi=`Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the | |
| vocabulary.`,fs,ye,st,_s,gn,Oi=`Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from | |
| the fast call because for now we always add the tokens even if they are already in the vocabulary. This is | |
| something we should change.`,gs,K,at,ks,kn,Qi="Returns the number of added tokens when encoding a sequence with special tokens.",bs,we,xs,ee,it,Ts,bn,Ki="Performs any necessary transformations before tokenization.",vs,xn,ed=`This method should pop the arguments from kwargs and return the remaining <code>kwargs</code> as well. We test the | |
| <code>kwargs</code> at the end of the encoding process to be sure all the arguments have been used.`,ys,te,dt,ws,Tn,td="Converts a string into a sequence of tokens, using the tokenizer.",zs,vn,nd=`Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies | |
| (BPE/SentencePieces/WordPieces). Takes care of added tokens.`,ir,ct,dr,lt,od='<a href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> 依赖于 <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">tokenizers</a> 库。可以非常简单地将从 🤗 tokenizers 库获取的tokenizers加载到 🤗 transformers 中。查看 <a href="../fast_tokenizers">使用 🤗 tokenizers 的分词器</a> 页面以了解如何执行此操作。',cr,x,pt,$s,yn,rd="Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).",Ps,wn,sd='Inherits from <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase">PreTrainedTokenizerBase</a>.',Cs,zn,ad=`Handles all the shared methods for tokenization and special tokens, as well as methods for | |
| downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary.`,qs,$n,id=`This class also contains the added tokens in a unified way on top of all tokenizers so we don’t have to handle the | |
| specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece…).`,Ms,Pn,dd="Class attributes (overridden by derived classes)",Is,Cn,cd=`<li><strong>vocab_files_names</strong> (<code>Dict[str, str]</code>) — A dictionary with, as keys, the <code>__init__</code> keyword name of each | |
| vocabulary file required by the model, and as associated values, the filename for saving the associated file | |
| (string).</li> <li><strong>pretrained_vocab_files_map</strong> (<code>Dict[str, Dict[str, str]]</code>) — A dictionary of dictionaries, with the | |
| high-level keys being the <code>__init__</code> keyword name of each vocabulary file required by the model, the | |
| low-level being the <code>short-cut-names</code> of the pretrained models with, as associated values, the <code>url</code> to the | |
| associated pretrained vocabulary file.</li> <li><strong>model_input_names</strong> (<code>List[str]</code>) — A list of inputs expected in the forward pass of the model.</li> <li><strong>padding_side</strong> (<code>str</code>) — The default value for the side on which the model should have padding applied. | |
| Should be <code>'right'</code> or <code>'left'</code>.</li> <li><strong>truncation_side</strong> (<code>str</code>) — The default value for the side on which the model should have truncation | |
| applied. Should be <code>'right'</code> or <code>'left'</code>.</li>`,Fs,ze,mt,Ls,qn,ld=`Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of | |
| sequences.`,js,E,ht,Ws,Mn,pd=`Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to | |
| it with indices starting from length of the current vocabulary and and will be isolated before the tokenization | |
| algorithm is applied. Added tokens and tokens from the vocabulary of the tokenization algorithm are therefore | |
| not treated in the same way.`,Js,In,md=`Note, when adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix | |
| of the model so that its embedding matrix matches the tokenizer.`,Us,Fn,hd='In order to do that, please use the <a href="/docs/transformers/pr_34786/zh/main_classes/model#transformers.PreTrainedModel.resize_token_embeddings">resize_token_embeddings()</a> method.',Bs,$e,Ns,L,ut,Vs,Ln,ud=`Add a dictionary of special tokens (eos, pad, cls, etc.) to the encoder and link them to class attributes. If | |
| special tokens are NOT in the vocabulary, they are added to it (indexed starting from the last index of the | |
| current vocabulary).`,Es,jn,fd=`When adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix of the | |
| model so that its embedding matrix matches the tokenizer.`,Ds,Wn,_d='In order to do that, please use the <a href="/docs/transformers/pr_34786/zh/main_classes/model#transformers.PreTrainedModel.resize_token_embeddings">resize_token_embeddings()</a> method.',Zs,Jn,gd="Using <code>add_special_tokens</code> will ensure your special tokens can be used in several ways:",Hs,Un,kd=`<li>Special tokens can be skipped when decoding using <code>skip_special_tokens = True</code>.</li> <li>Special tokens are carefully handled by the tokenizer (they are never split), similar to <code>AddedTokens</code>.</li> <li>You can easily refer to special tokens using tokenizer class attributes like <code>tokenizer.cls_token</code>. This | |
| makes it easy to develop model-agnostic training and fine-tuning scripts.</li>`,Gs,Bn,bd=`When possible, special tokens are already registered for provided pretrained models (for instance | |
| <code>BertTokenizer</code> <code>cls_token</code> is already registered to be :obj<em>’[CLS]’</em> and XLM’s one is also registered to be | |
| <code>'</s>'</code>).`,As,Pe,Rs,Ce,ft,Ss,Nn,xd=`Converts a list of dictionaries with <code>"role"</code> and <code>"content"</code> keys to a list of token | |
| ids. This method is intended for use with chat models, and will read the tokenizer’s chat_template attribute to | |
| determine the format and control tokens to use when converting.`,Xs,qe,_t,Ys,Vn,Td="Convert a list of lists of token ids into a list of strings by calling decode.",Os,ne,gt,Qs,En,vd=`Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special | |
| tokens and clean up tokenization spaces.`,Ks,Dn,yd="Similar to doing <code>self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))</code>.",ea,oe,kt,ta,Zn,wd="Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary.",na,Hn,zd="Same as doing <code>self.convert_tokens_to_ids(self.tokenize(text))</code>.",oa,re,bt,ra,Gn,$d="Upload the tokenizer files to the 🤗 Model Hub.",sa,Me,aa,Ie,xt,ia,An,Pd=`Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and | |
| added tokens.`,da,Fe,Tt,ca,Rn,Cd=`Converts a token string (or a sequence of tokens) in a single integer id (or a Iterable of ids), using the | |
| vocabulary.`,la,Le,vt,pa,Sn,qd="Returns the added tokens in the vocabulary as a dictionary of token to index.",ma,se,yt,ha,Xn,Md="Returns the number of added tokens when encoding a sequence with special tokens.",ua,je,fa,ae,wt,_a,Yn,Id=`Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers | |
| library) and restore the tokenizer settings afterwards.`,ga,On,Fd=`The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a | |
| padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed | |
| section.`,ka,We,zt,ba,Qn,Ld=`Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline) | |
| as the current one.`,lr,$t,pr,C,Pt,xa,Kn,jd=`Holds the output of the <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__"><strong>call</strong>()</a>, | |
| <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode_plus">encode_plus()</a> and | |
| <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.batch_encode_plus">batch_encode_plus()</a> methods (tokens, attention_masks, etc).`,Ta,eo,Wd=`This class is derived from a python dictionary and can be used as a dictionary. In addition, this class exposes | |
| utility methods to map from word/character space to token space.`,va,D,Ct,ya,to,Jd=`Get the index of the token in the encoded output comprising a character in the original string for a sequence | |
| of the batch.`,wa,no,Ud="Can be called as:",za,oo,Bd="<li><code>self.char_to_token(char_index)</code> if batch size is 1</li> <li><code>self.char_to_token(batch_index, char_index)</code> if batch size is greater or equal to 1</li>",$a,ro,Nd=`This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words | |
| are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized | |
| words.`,Pa,Z,qt,Ca,so,Vd=`Get the word in the original string corresponding to a character in the original string of a sequence of the | |
| batch.`,qa,ao,Ed="Can be called as:",Ma,io,Dd="<li><code>self.char_to_word(char_index)</code> if batch size is 1</li> <li><code>self.char_to_word(batch_index, char_index)</code> if batch size is greater than 1</li>",Ia,co,Zd=`This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words | |
| are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized | |
| words.`,Fa,Je,Mt,La,lo,Hd="Convert the inner content to tensors.",ja,ie,It,Wa,po,Gd="Return a list mapping the tokens to the id of their original sentences:",Ja,mo,Ad=`<li><code>None</code> for special tokens added around or between sequences,</li> <li><code>0</code> for tokens corresponding to words in the first sequence,</li> <li><code>1</code> for tokens corresponding to words in the second sequence when a pair of sequences was jointly | |
| encoded.</li>`,Ua,Ue,Ft,Ba,ho,Rd="Send all values to device by calling <code>v.to(device)</code> (PyTorch only).",Na,U,Lt,Va,uo,Sd="Get the character span corresponding to an encoded token in a sequence of the batch.",Ea,fo,Xd='Character spans are returned as a <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.CharSpan">CharSpan</a> with:',Da,_o,Yd=`<li><strong>start</strong> — Index of the first character in the original string associated to the token.</li> <li><strong>end</strong> — Index of the character following the last character in the original string associated to the | |
| token.</li>`,Za,go,Od="Can be called as:",Ha,ko,Qd="<li><code>self.token_to_chars(token_index)</code> if batch size is 1</li> <li><code>self.token_to_chars(batch_index, token_index)</code> if batch size is greater or equal to 1</li>",Ga,H,jt,Aa,bo,Kd=`Get the index of the sequence represented by the given token. In the general use case, this method returns <code>0</code> | |
| for a single sequence or the first sequence of a pair, and <code>1</code> for the second sequence of a pair`,Ra,xo,ec="Can be called as:",Sa,To,tc="<li><code>self.token_to_sequence(token_index)</code> if batch size is 1</li> <li><code>self.token_to_sequence(batch_index, token_index)</code> if batch size is greater than 1</li>",Xa,vo,nc=`This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e., | |
| words are defined by the user). In this case it allows to easily associate encoded tokens with provided | |
| tokenized words.`,Ya,G,Wt,Oa,yo,oc="Get the index of the word corresponding (i.e. comprising) to an encoded token in a sequence of the batch.",Qa,wo,rc="Can be called as:",Ka,zo,sc="<li><code>self.token_to_word(token_index)</code> if batch size is 1</li> <li><code>self.token_to_word(batch_index, token_index)</code> if batch size is greater than 1</li>",ei,$o,ac=`This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e., | |
| words are defined by the user). In this case it allows to easily associate encoded tokens with provided | |
| tokenized words.`,ti,Be,Jt,ni,Po,ic=`Return the list of tokens (sub-parts of the input strings after word/subword splitting and before conversion to | |
| integer indices) at a given batch index (only works for the output of a fast tokenizer).`,oi,Ne,Ut,ri,Co,dc="Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.",si,B,Bt,ai,qo,cc="Get the character span in the original string corresponding to given word in a sequence of the batch.",ii,Mo,lc="Character spans are returned as a CharSpan NamedTuple with:",di,Io,pc="<li>start: index of the first character in the original string</li> <li>end: index of the character following the last character in the original string</li>",ci,Fo,mc="Can be called as:",li,Lo,hc="<li><code>self.word_to_chars(word_index)</code> if batch size is 1</li> <li><code>self.word_to_chars(batch_index, word_index)</code> if batch size is greater or equal to 1</li>",pi,J,Nt,mi,jo,uc="Get the encoded token span corresponding to a word in a sequence of the batch.",hi,Wo,fc='Token spans are returned as a <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.TokenSpan">TokenSpan</a> with:',ui,Jo,_c="<li><strong>start</strong> — Index of the first token.</li> <li><strong>end</strong> — Index of the token following the last token.</li>",fi,Uo,gc="Can be called as:",_i,Bo,kc=`<li><code>self.word_to_tokens(word_index, sequence_index: int = 0)</code> if batch size is 1</li> <li><code>self.word_to_tokens(batch_index, word_index, sequence_index: int = 0)</code> if batch size is greater or equal to | |
| 1</li>`,gi,No,bc=`This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words | |
| are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized | |
| words.`,ki,Ve,Vt,bi,Vo,xc="Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.",mr,Et,hr,Ko,ur;return P=new Mr({props:{title:"Tokenizer",local:"tokenizer",headingTag:"h1"}}),Re=new Mr({props:{title:"PreTrainedTokenizer",local:"transformers.PreTrainedTokenizer",headingTag:"h2"}}),Se=new w({props:{name:"class transformers.PreTrainedTokenizer",anchor:"transformers.PreTrainedTokenizer",parameters:[{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.model_max_length",description:`<strong>model_max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| The maximum length (in number of tokens) for the inputs to the transformer model. When the tokenizer is | |
| loaded with <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.from_pretrained">from_pretrained()</a>, this will be set to the | |
| value stored for the associated model in <code>max_model_input_sizes</code> (see above). If no value is provided, will | |
| default to VERY_LARGE_INTEGER (<code>int(1e30)</code>).`,name:"model_max_length"},{anchor:"transformers.PreTrainedTokenizer.padding_side",description:`<strong>padding_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have padding applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"padding_side"},{anchor:"transformers.PreTrainedTokenizer.truncation_side",description:`<strong>truncation_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have truncation applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"truncation_side"},{anchor:"transformers.PreTrainedTokenizer.chat_template",description:`<strong>chat_template</strong> (<code>str</code>, <em>optional</em>) — | |
| A Jinja template string that will be used to format lists of chat messages. See | |
| <a href="https://huggingface.co/docs/transformers/chat_templating" rel="nofollow">https://huggingface.co/docs/transformers/chat_templating</a> for a full description.`,name:"chat_template"},{anchor:"transformers.PreTrainedTokenizer.model_input_names",description:`<strong>model_input_names</strong> (<code>List[string]</code>, <em>optional</em>) — | |
| The list of inputs accepted by the forward pass of the model (like <code>"token_type_ids"</code> or | |
| <code>"attention_mask"</code>). Default value is picked from the class attribute of the same name.`,name:"model_input_names"},{anchor:"transformers.PreTrainedTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing the beginning of a sentence. Will be associated to <code>self.bos_token</code> and | |
| <code>self.bos_token_id</code>.`,name:"bos_token"},{anchor:"transformers.PreTrainedTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing the end of a sentence. Will be associated to <code>self.eos_token</code> and | |
| <code>self.eos_token_id</code>.`,name:"eos_token"},{anchor:"transformers.PreTrainedTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing an out-of-vocabulary token. Will be associated to <code>self.unk_token</code> and | |
| <code>self.unk_token_id</code>.`,name:"unk_token"},{anchor:"transformers.PreTrainedTokenizer.sep_token",description:`<strong>sep_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token separating two different sentences in the same input (used by BERT for instance). Will be | |
| associated to <code>self.sep_token</code> and <code>self.sep_token_id</code>.`,name:"sep_token"},{anchor:"transformers.PreTrainedTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by | |
| attention mechanisms or loss computation. Will be associated to <code>self.pad_token</code> and <code>self.pad_token_id</code>.`,name:"pad_token"},{anchor:"transformers.PreTrainedTokenizer.cls_token",description:`<strong>cls_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing the class of the input (used by BERT for instance). Will be associated to | |
| <code>self.cls_token</code> and <code>self.cls_token_id</code>.`,name:"cls_token"},{anchor:"transformers.PreTrainedTokenizer.mask_token",description:`<strong>mask_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing a masked token (used by masked-language modeling pretraining objectives, like | |
| BERT). Will be associated to <code>self.mask_token</code> and <code>self.mask_token_id</code>.`,name:"mask_token"},{anchor:"transformers.PreTrainedTokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (tuple or list of <code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A tuple or a list of additional special tokens. Add them here to ensure they are skipped when decoding with | |
| <code>skip_special_tokens</code> is set to True. If they are not part of the vocabulary, they will be added at the end | |
| of the vocabulary.`,name:"additional_special_tokens"},{anchor:"transformers.PreTrainedTokenizer.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not the model should cleanup the spaces that were added when splitting the input text during the | |
| tokenization process.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.PreTrainedTokenizer.split_special_tokens",description:`<strong>split_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the special tokens should be split during the tokenization process. Passing will affect the | |
| internal state of the tokenizer. The default behavior is to not split special tokens. This means that if | |
| <code><s></code> is the <code>bos_token</code>, then <code>tokenizer.tokenize("<s>") = ['<s></code>]. Otherwise, if | |
| <code>split_special_tokens=True</code>, then <code>tokenizer.tokenize("<s>")</code> will be give <code>['<','s', '>']</code>.`,name:"split_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils.py#L407"}}),Xe=new w({props:{name:"__call__",anchor:"transformers.PreTrainedTokenizer.__call__",parameters:[{name:"text",val:": Union = None"},{name:"text_pair",val:": Union = None"},{name:"text_target",val:": Union = None"},{name:"text_pair_target",val:": Union = None"},{name:"add_special_tokens",val:": bool = True"},{name:"padding",val:": Union = False"},{name:"truncation",val:": Union = None"},{name:"max_length",val:": Optional = None"},{name:"stride",val:": int = 0"},{name:"is_split_into_words",val:": bool = False"},{name:"pad_to_multiple_of",val:": Optional = None"},{name:"padding_side",val:": Optional = None"},{name:"return_tensors",val:": Union = None"},{name:"return_token_type_ids",val:": Optional = None"},{name:"return_attention_mask",val:": Optional = None"},{name:"return_overflowing_tokens",val:": bool = False"},{name:"return_special_tokens_mask",val:": bool = False"},{name:"return_offsets_mapping",val:": bool = False"},{name:"return_length",val:": bool = False"},{name:"verbose",val:": bool = True"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.__call__.text",description:`<strong>text</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings | |
| (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set | |
| <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text"},{anchor:"transformers.PreTrainedTokenizer.__call__.text_pair",description:`<strong>text_pair</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings | |
| (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set | |
| <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text_pair"},{anchor:"transformers.PreTrainedTokenizer.__call__.text_target",description:`<strong>text_target</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a | |
| list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), | |
| you must set <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text_target"},{anchor:"transformers.PreTrainedTokenizer.__call__.text_pair_target",description:`<strong>text_pair_target</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a | |
| list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), | |
| you must set <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text_pair_target"},{anchor:"transformers.PreTrainedTokenizer.__call__.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to add special tokens when encoding the sequences. This will use the underlying | |
| <code>PretrainedTokenizerBase.build_inputs_with_special_tokens</code> function, which defines which tokens are | |
| automatically added to the input ids. This is usefull if you want to add <code>bos</code> or <code>eos</code> tokens | |
| automatically.`,name:"add_special_tokens"},{anchor:"transformers.PreTrainedTokenizer.__call__.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls padding. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest'</code>: Pad to the longest sequence in the batch (or no padding if only a single | |
| sequence if provided).</li> | |
| <li><code>'max_length'</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum | |
| acceptable input length for the model if that argument is not provided.</li> | |
| <li><code>False</code> or <code>'do_not_pad'</code> (default): No padding (i.e., can output a batch with sequences of different | |
| lengths).</li> | |
| </ul>`,name:"padding"},{anchor:"transformers.PreTrainedTokenizer.__call__.truncation",description:`<strong>truncation</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.tokenization_utils_base.TruncationStrategy">TruncationStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls truncation. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or | |
| to the maximum acceptable input length for the model if that argument is not provided. This will | |
| truncate token by token, removing a token from the longest sequence in the pair if a pair of | |
| sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_second'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>False</code> or <code>'do_not_truncate'</code> (default): No truncation (i.e., can output batch with sequence lengths | |
| greater than the model maximum admissible input size).</li> | |
| </ul>`,name:"truncation"},{anchor:"transformers.PreTrainedTokenizer.__call__.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Controls the maximum length to use by one of the truncation/padding parameters.</p> | |
| <p>If left unset or set to <code>None</code>, this will use the predefined model maximum length if a maximum length | |
| is required by one of the truncation/padding parameters. If the model has no specific maximum input | |
| length (like XLNet) truncation/padding to a maximum length will be deactivated.`,name:"max_length"},{anchor:"transformers.PreTrainedTokenizer.__call__.stride",description:`<strong>stride</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If set to a number along with <code>max_length</code>, the overflowing tokens returned when | |
| <code>return_overflowing_tokens=True</code> will contain some tokens from the end of the truncated sequence | |
| returned to provide some overlap between truncated and overflowing sequences. The value of this | |
| argument defines the number of overlapping tokens.`,name:"stride"},{anchor:"transformers.PreTrainedTokenizer.__call__.is_split_into_words",description:`<strong>is_split_into_words</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the input is already pre-tokenized (e.g., split into words). If set to <code>True</code>, the | |
| tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) | |
| which it will tokenize. This is useful for NER or token classification.`,name:"is_split_into_words"},{anchor:"transformers.PreTrainedTokenizer.__call__.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value. Requires <code>padding</code> to be activated. | |
| This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability | |
| <code>>= 7.5</code> (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.PreTrainedTokenizer.__call__.padding_side",description:`<strong>padding_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have padding applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"padding_side"},{anchor:"transformers.PreTrainedTokenizer.__call__.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) — | |
| If set, will return tensors instead of list of python integers. Acceptable values are:</p> | |
| <ul> | |
| <li><code>'tf'</code>: Return TensorFlow <code>tf.constant</code> objects.</li> | |
| <li><code>'pt'</code>: Return PyTorch <code>torch.Tensor</code> objects.</li> | |
| <li><code>'np'</code>: Return Numpy <code>np.ndarray</code> objects.</li> | |
| </ul>`,name:"return_tensors"},{anchor:"transformers.PreTrainedTokenizer.__call__.return_token_type_ids",description:`<strong>return_token_type_ids</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to return token type IDs. If left to the default, will return the token type IDs according to | |
| the specific tokenizer’s default, defined by the <code>return_outputs</code> attribute.</p> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"return_token_type_ids"},{anchor:"transformers.PreTrainedTokenizer.__call__.return_attention_mask",description:`<strong>return_attention_mask</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to return the attention mask. If left to the default, will return the attention mask according | |
| to the specific tokenizer’s default, defined by the <code>return_outputs</code> attribute.</p> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"return_attention_mask"},{anchor:"transformers.PreTrainedTokenizer.__call__.return_overflowing_tokens",description:`<strong>return_overflowing_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch | |
| of pairs) is provided with <code>truncation_strategy = longest_first</code> or <code>True</code>, an error is raised instead | |
| of returning overflowing tokens.`,name:"return_overflowing_tokens"},{anchor:"transformers.PreTrainedTokenizer.__call__.return_special_tokens_mask",description:`<strong>return_special_tokens_mask</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return special tokens mask information.`,name:"return_special_tokens_mask"},{anchor:"transformers.PreTrainedTokenizer.__call__.return_offsets_mapping",description:`<strong>return_offsets_mapping</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return <code>(char_start, char_end)</code> for each token.</p> | |
| <p>This is only available on fast tokenizers inheriting from <a href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>, if using | |
| Python’s tokenizer, this method will raise <code>NotImplementedError</code>.`,name:"return_offsets_mapping"},{anchor:"transformers.PreTrainedTokenizer.__call__.return_length",description:`<strong>return_length</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return the lengths of the encoded inputs.`,name:"return_length"},{anchor:"transformers.PreTrainedTokenizer.__call__.verbose",description:`<strong>verbose</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to print more information and warnings. | |
| **kwargs — passed to the <code>self.tokenize()</code> method`,name:"verbose"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L2764",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.BatchEncoding" | |
| >BatchEncoding</a> with the following fields:</p> | |
| <ul> | |
| <li> | |
| <p><strong>input_ids</strong> — List of token ids to be fed to a model.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a></p> | |
| </li> | |
| <li> | |
| <p><strong>token_type_ids</strong> — List of token type ids to be fed to a model (when <code>return_token_type_ids=True</code> or | |
| if <em>“token_type_ids”</em> is in <code>self.model_input_names</code>).</p> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a></p> | |
| </li> | |
| <li> | |
| <p><strong>attention_mask</strong> — List of indices specifying which tokens should be attended to by the model (when | |
| <code>return_attention_mask=True</code> or if <em>“attention_mask”</em> is in <code>self.model_input_names</code>).</p> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a></p> | |
| </li> | |
| <li> | |
| <p><strong>overflowing_tokens</strong> — List of overflowing tokens sequences (when a <code>max_length</code> is specified and | |
| <code>return_overflowing_tokens=True</code>).</p> | |
| </li> | |
| <li> | |
| <p><strong>num_truncated_tokens</strong> — Number of tokens truncated (when a <code>max_length</code> is specified and | |
| <code>return_overflowing_tokens=True</code>).</p> | |
| </li> | |
| <li> | |
| <p><strong>special_tokens_mask</strong> — List of 0s and 1s, with 1 specifying added special tokens and 0 specifying | |
| regular sequence tokens (when <code>add_special_tokens=True</code> and <code>return_special_tokens_mask=True</code>).</p> | |
| </li> | |
| <li> | |
| <p><strong>length</strong> — The length of the inputs (when <code>return_length=True</code>)</p> | |
| </li> | |
| </ul> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.BatchEncoding" | |
| >BatchEncoding</a></p> | |
| `}}),Ye=new w({props:{name:"add_tokens",anchor:"transformers.PreTrainedTokenizer.add_tokens",parameters:[{name:"new_tokens",val:": Union"},{name:"special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.add_tokens.new_tokens",description:`<strong>new_tokens</strong> (<code>str</code>, <code>tokenizers.AddedToken</code> or a list of <em>str</em> or <code>tokenizers.AddedToken</code>) — | |
| Tokens are only added if they are not already in the vocabulary. <code>tokenizers.AddedToken</code> wraps a string | |
| token to let you personalize its behavior: whether this token should only match against a single word, | |
| whether this token should strip all potential whitespaces on the left side, whether this token should | |
| strip all potential whitespaces on the right side, etc.`,name:"new_tokens"},{anchor:"transformers.PreTrainedTokenizer.add_tokens.special_tokens",description:`<strong>special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Can be used to specify if the token is a special token. This mostly change the normalization behavior | |
| (special tokens like CLS or [MASK] are usually not lower-cased for instance).</p> | |
| <p>See details for <code>tokenizers.AddedToken</code> in HuggingFace tokenizers library.`,name:"special_tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L998",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Number of tokens added to the vocabulary.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),_e=new Oo({props:{anchor:"transformers.PreTrainedTokenizer.add_tokens.example",$$slots:{default:[Cc]},$$scope:{ctx:W}}}),Oe=new w({props:{name:"add_special_tokens",anchor:"transformers.PreTrainedTokenizer.add_special_tokens",parameters:[{name:"special_tokens_dict",val:": Dict"},{name:"replace_additional_special_tokens",val:" = True"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.add_special_tokens.special_tokens_dict",description:`<strong>special_tokens_dict</strong> (dictionary <em>str</em> to <em>str</em> or <code>tokenizers.AddedToken</code>) — | |
| Keys should be in the list of predefined special attributes: [<code>bos_token</code>, <code>eos_token</code>, <code>unk_token</code>, | |
| <code>sep_token</code>, <code>pad_token</code>, <code>cls_token</code>, <code>mask_token</code>, <code>additional_special_tokens</code>].</p> | |
| <p>Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer | |
| assign the index of the <code>unk_token</code> to them).`,name:"special_tokens_dict"},{anchor:"transformers.PreTrainedTokenizer.add_special_tokens.replace_additional_special_tokens",description:`<strong>replace_additional_special_tokens</strong> (<code>bool</code>, <em>optional</em>,, defaults to <code>True</code>) — | |
| If <code>True</code>, the existing list of additional special tokens will be replaced by the list provided in | |
| <code>special_tokens_dict</code>. Otherwise, <code>self._special_tokens_map["additional_special_tokens"]</code> is just extended. In the former | |
| case, the tokens will NOT be removed from the tokenizer’s full vocabulary - they are only being flagged | |
| as non-special tokens. Remember, this only affects which tokens are skipped during decoding, not the | |
| <code>added_tokens_encoder</code> and <code>added_tokens_decoder</code>. This means that the previous | |
| <code>additional_special_tokens</code> are still added tokens, and will not be split by the model.`,name:"replace_additional_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L896",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Number of tokens added to the vocabulary.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),ge=new Oo({props:{anchor:"transformers.PreTrainedTokenizer.add_special_tokens.example",$$slots:{default:[qc]},$$scope:{ctx:W}}}),Qe=new w({props:{name:"apply_chat_template",anchor:"transformers.PreTrainedTokenizer.apply_chat_template",parameters:[{name:"conversation",val:": Union"},{name:"tools",val:": Optional = None"},{name:"documents",val:": Optional = None"},{name:"chat_template",val:": Optional = None"},{name:"add_generation_prompt",val:": bool = False"},{name:"continue_final_message",val:": bool = False"},{name:"tokenize",val:": bool = True"},{name:"padding",val:": bool = False"},{name:"truncation",val:": bool = False"},{name:"max_length",val:": Optional = None"},{name:"return_tensors",val:": Union = None"},{name:"return_dict",val:": bool = False"},{name:"return_assistant_tokens_mask",val:": bool = False"},{name:"tokenizer_kwargs",val:": Optional = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.conversation",description:`<strong>conversation</strong> (Union[List[Dict[str, str]], List[List[Dict[str, str]]]]) — A list of dicts | |
| with “role” and “content” keys, representing the chat history so far.`,name:"conversation"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.tools",description:`<strong>tools</strong> (<code>List[Dict]</code>, <em>optional</em>) — | |
| A list of tools (callable functions) that will be accessible to the model. If the template does not | |
| support function calling, this argument will have no effect. Each tool should be passed as a JSON Schema, | |
| giving the name, description and argument types for the tool. See our | |
| <a href="https://huggingface.co/docs/transformers/main/en/chat_templating#automated-function-conversion-for-tool-use" rel="nofollow">chat templating guide</a> | |
| for more information.`,name:"tools"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.documents",description:`<strong>documents</strong> (<code>List[Dict[str, str]]</code>, <em>optional</em>) — | |
| A list of dicts representing documents that will be accessible to the model if it is performing RAG | |
| (retrieval-augmented generation). If the template does not support RAG, this argument will have no | |
| effect. We recommend that each document should be a dict containing “title” and “text” keys. Please | |
| see the RAG section of the <a href="https://huggingface.co/docs/transformers/main/en/chat_templating#arguments-for-RAG" rel="nofollow">chat templating guide</a> | |
| for examples of passing documents with chat templates.`,name:"documents"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.chat_template",description:`<strong>chat_template</strong> (<code>str</code>, <em>optional</em>) — | |
| A Jinja template to use for this conversion. It is usually not necessary to pass anything to this | |
| argument, as the model’s template will be used by default.`,name:"chat_template"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.add_generation_prompt",description:`<strong>add_generation_prompt</strong> (bool, <em>optional</em>) — | |
| If this is set, a prompt with the token(s) that indicate | |
| the start of an assistant message will be appended to the formatted output. This is useful when you want to generate a response from the model. | |
| Note that this argument will be passed to the chat template, and so it must be supported in the | |
| template for this argument to have any effect.`,name:"add_generation_prompt"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.continue_final_message",description:`<strong>continue_final_message</strong> (bool, <em>optional</em>) — | |
| If this is set, the chat will be formatted so that the final | |
| message in the chat is open-ended, without any EOS tokens. The model will continue this message | |
| rather than starting a new one. This allows you to “prefill” part of | |
| the model’s response for it. Cannot be used at the same time as <code>add_generation_prompt</code>.`,name:"continue_final_message"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.tokenize",description:`<strong>tokenize</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Whether to tokenize the output. If <code>False</code>, the output will be a string.`,name:"tokenize"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.padding",description:`<strong>padding</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to pad sequences to the maximum length. Has no effect if tokenize is <code>False</code>.`,name:"padding"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.truncation",description:`<strong>truncation</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to truncate sequences at the maximum length. Has no effect if tokenize is <code>False</code>.`,name:"truncation"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Maximum length (in tokens) to use for padding or truncation. Has no effect if tokenize is <code>False</code>. If | |
| not specified, the tokenizer’s <code>max_length</code> attribute will be used as a default.`,name:"max_length"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) — | |
| If set, will return tensors of a particular framework. Has no effect if tokenize is <code>False</code>. Acceptable | |
| values are:<ul> | |
| <li><code>'tf'</code>: Return TensorFlow <code>tf.Tensor</code> objects.</li> | |
| <li><code>'pt'</code>: Return PyTorch <code>torch.Tensor</code> objects.</li> | |
| <li><code>'np'</code>: Return NumPy <code>np.ndarray</code> objects.</li> | |
| <li><code>'jax'</code>: Return JAX <code>jnp.ndarray</code> objects.</li> | |
| </ul>`,name:"return_tensors"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to return a dictionary with named outputs. Has no effect if tokenize is <code>False</code>.`,name:"return_dict"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.tokenizer_kwargs",description:"<strong>tokenizer_kwargs</strong> (<code>Dict[str -- Any]</code>, <em>optional</em>): Additional kwargs to pass to the tokenizer.",name:"tokenizer_kwargs"},{anchor:"transformers.PreTrainedTokenizer.apply_chat_template.return_assistant_tokens_mask",description:`<strong>return_assistant_tokens_mask</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to return a mask of the assistant generated tokens. For tokens generated by the assistant, | |
| the mask will contain 1. For user and system tokens, the mask will contain 0. | |
| This functionality is only available for chat templates that support it via the <code>{% generation %}</code> keyword. | |
| **kwargs — Additional kwargs to pass to the template renderer. Will be accessible by the chat template.`,name:"return_assistant_tokens_mask"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L1522",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A list of token ids representing the tokenized chat so far, including control tokens. This | |
| output is ready to pass to the model, either directly or via methods like <code>generate()</code>. If <code>return_dict</code> is | |
| set, will return a dict of tokenizer outputs instead.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Union[List[int], Dict]</code></p> | |
| `}}),Ke=new w({props:{name:"batch_decode",anchor:"transformers.PreTrainedTokenizer.batch_decode",parameters:[{name:"sequences",val:": Union"},{name:"skip_special_tokens",val:": bool = False"},{name:"clean_up_tokenization_spaces",val:": bool = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.batch_decode.sequences",description:`<strong>sequences</strong> (<code>Union[List[int], List[List[int]], np.ndarray, torch.Tensor, tf.Tensor]</code>) — | |
| List of tokenized input ids. Can be obtained using the <code>__call__</code> method.`,name:"sequences"},{anchor:"transformers.PreTrainedTokenizer.batch_decode.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to remove special tokens in the decoding.`,name:"skip_special_tokens"},{anchor:"transformers.PreTrainedTokenizer.batch_decode.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to clean up the tokenization spaces. If <code>None</code>, will default to | |
| <code>self.clean_up_tokenization_spaces</code>.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.PreTrainedTokenizer.batch_decode.kwargs",description:`<strong>kwargs</strong> (additional keyword arguments, <em>optional</em>) — | |
| Will be passed to the underlying model specific decode method.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L3760",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The list of decoded sentences.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[str]</code></p> | |
| `}}),et=new w({props:{name:"decode",anchor:"transformers.PreTrainedTokenizer.decode",parameters:[{name:"token_ids",val:": Union"},{name:"skip_special_tokens",val:": bool = False"},{name:"clean_up_tokenization_spaces",val:": bool = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.decode.token_ids",description:`<strong>token_ids</strong> (<code>Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]</code>) — | |
| List of tokenized input ids. Can be obtained using the <code>__call__</code> method.`,name:"token_ids"},{anchor:"transformers.PreTrainedTokenizer.decode.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to remove special tokens in the decoding.`,name:"skip_special_tokens"},{anchor:"transformers.PreTrainedTokenizer.decode.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to clean up the tokenization spaces. If <code>None</code>, will default to | |
| <code>self.clean_up_tokenization_spaces</code>.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.PreTrainedTokenizer.decode.kwargs",description:`<strong>kwargs</strong> (additional keyword arguments, <em>optional</em>) — | |
| Will be passed to the underlying model specific decode method.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L3794",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The decoded sentence.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>str</code></p> | |
| `}}),tt=new w({props:{name:"encode",anchor:"transformers.PreTrainedTokenizer.encode",parameters:[{name:"text",val:": Union"},{name:"text_pair",val:": Union = None"},{name:"add_special_tokens",val:": bool = True"},{name:"padding",val:": Union = False"},{name:"truncation",val:": Union = None"},{name:"max_length",val:": Optional = None"},{name:"stride",val:": int = 0"},{name:"padding_side",val:": Optional = None"},{name:"return_tensors",val:": Union = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.encode.text",description:`<strong>text</strong> (<code>str</code>, <code>List[str]</code> or <code>List[int]</code>) — | |
| The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the | |
| <code>tokenize</code> method) or a list of integers (tokenized string ids using the <code>convert_tokens_to_ids</code> | |
| method).`,name:"text"},{anchor:"transformers.PreTrainedTokenizer.encode.text_pair",description:`<strong>text_pair</strong> (<code>str</code>, <code>List[str]</code> or <code>List[int]</code>, <em>optional</em>) — | |
| Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using | |
| the <code>tokenize</code> method) or a list of integers (tokenized string ids using the <code>convert_tokens_to_ids</code> | |
| method).`,name:"text_pair"},{anchor:"transformers.PreTrainedTokenizer.encode.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to add special tokens when encoding the sequences. This will use the underlying | |
| <code>PretrainedTokenizerBase.build_inputs_with_special_tokens</code> function, which defines which tokens are | |
| automatically added to the input ids. This is usefull if you want to add <code>bos</code> or <code>eos</code> tokens | |
| automatically.`,name:"add_special_tokens"},{anchor:"transformers.PreTrainedTokenizer.encode.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls padding. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest'</code>: Pad to the longest sequence in the batch (or no padding if only a single | |
| sequence if provided).</li> | |
| <li><code>'max_length'</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum | |
| acceptable input length for the model if that argument is not provided.</li> | |
| <li><code>False</code> or <code>'do_not_pad'</code> (default): No padding (i.e., can output a batch with sequences of different | |
| lengths).</li> | |
| </ul>`,name:"padding"},{anchor:"transformers.PreTrainedTokenizer.encode.truncation",description:`<strong>truncation</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.tokenization_utils_base.TruncationStrategy">TruncationStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls truncation. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or | |
| to the maximum acceptable input length for the model if that argument is not provided. This will | |
| truncate token by token, removing a token from the longest sequence in the pair if a pair of | |
| sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_second'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>False</code> or <code>'do_not_truncate'</code> (default): No truncation (i.e., can output batch with sequence lengths | |
| greater than the model maximum admissible input size).</li> | |
| </ul>`,name:"truncation"},{anchor:"transformers.PreTrainedTokenizer.encode.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Controls the maximum length to use by one of the truncation/padding parameters.</p> | |
| <p>If left unset or set to <code>None</code>, this will use the predefined model maximum length if a maximum length | |
| is required by one of the truncation/padding parameters. If the model has no specific maximum input | |
| length (like XLNet) truncation/padding to a maximum length will be deactivated.`,name:"max_length"},{anchor:"transformers.PreTrainedTokenizer.encode.stride",description:`<strong>stride</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If set to a number along with <code>max_length</code>, the overflowing tokens returned when | |
| <code>return_overflowing_tokens=True</code> will contain some tokens from the end of the truncated sequence | |
| returned to provide some overlap between truncated and overflowing sequences. The value of this | |
| argument defines the number of overlapping tokens.`,name:"stride"},{anchor:"transformers.PreTrainedTokenizer.encode.is_split_into_words",description:`<strong>is_split_into_words</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the input is already pre-tokenized (e.g., split into words). If set to <code>True</code>, the | |
| tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) | |
| which it will tokenize. This is useful for NER or token classification.`,name:"is_split_into_words"},{anchor:"transformers.PreTrainedTokenizer.encode.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value. Requires <code>padding</code> to be activated. | |
| This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability | |
| <code>>= 7.5</code> (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.PreTrainedTokenizer.encode.padding_side",description:`<strong>padding_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have padding applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"padding_side"},{anchor:"transformers.PreTrainedTokenizer.encode.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) — | |
| If set, will return tensors instead of list of python integers. Acceptable values are:</p> | |
| <ul> | |
| <li><code>'tf'</code>: Return TensorFlow <code>tf.constant</code> objects.</li> | |
| <li><code>'pt'</code>: Return PyTorch <code>torch.Tensor</code> objects.</li> | |
| <li><code>'np'</code>: Return Numpy <code>np.ndarray</code> objects.</li> | |
| </ul> | |
| <p>**kwargs — Passed along to the <code>.tokenize()</code> method.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L2570",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The tokenized ids of the text.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[int]</code>, <code>torch.Tensor</code>, <code>tf.Tensor</code> or <code>np.ndarray</code></p> | |
| `}}),nt=new w({props:{name:"push_to_hub",anchor:"transformers.PreTrainedTokenizer.push_to_hub",parameters:[{name:"repo_id",val:": str"},{name:"use_temp_dir",val:": Optional = None"},{name:"commit_message",val:": Optional = None"},{name:"private",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"max_shard_size",val:": Union = '5GB'"},{name:"create_pr",val:": bool = False"},{name:"safe_serialization",val:": bool = True"},{name:"revision",val:": str = None"},{name:"commit_description",val:": str = None"},{name:"tags",val:": Optional = None"},{name:"**deprecated_kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.push_to_hub.repo_id",description:`<strong>repo_id</strong> (<code>str</code>) — | |
| The name of the repository you want to push your tokenizer to. It should contain your organization name | |
| when pushing to a given organization.`,name:"repo_id"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.use_temp_dir",description:`<strong>use_temp_dir</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to use a temporary directory to store the files saved before they are pushed to the Hub. | |
| Will default to <code>True</code> if there is no directory named like <code>repo_id</code>, <code>False</code> otherwise.`,name:"use_temp_dir"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.commit_message",description:`<strong>commit_message</strong> (<code>str</code>, <em>optional</em>) — | |
| Message to commit while pushing. Will default to <code>"Upload tokenizer"</code>.`,name:"commit_message"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.private",description:`<strong>private</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not the repository created should be private.`,name:"private"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.token",description:`<strong>token</strong> (<code>bool</code> or <code>str</code>, <em>optional</em>) — | |
| The token to use as HTTP bearer authorization for remote files. If <code>True</code>, will use the token generated | |
| when running <code>huggingface-cli login</code> (stored in <code>~/.huggingface</code>). Will default to <code>True</code> if <code>repo_url</code> | |
| is not specified.`,name:"token"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.max_shard_size",description:`<strong>max_shard_size</strong> (<code>int</code> or <code>str</code>, <em>optional</em>, defaults to <code>"5GB"</code>) — | |
| Only applicable for models. The maximum size for a checkpoint before being sharded. Checkpoints shard | |
| will then be each of size lower than this size. If expressed as a string, needs to be digits followed | |
| by a unit (like <code>"5MB"</code>). We default it to <code>"5GB"</code> so that users can easily load models on free-tier | |
| Google Colab instances without any CPU OOM issues.`,name:"max_shard_size"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.create_pr",description:`<strong>create_pr</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to create a PR with the uploaded files or directly commit.`,name:"create_pr"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.safe_serialization",description:`<strong>safe_serialization</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to convert the model weights in safetensors format for safer serialization.`,name:"safe_serialization"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.revision",description:`<strong>revision</strong> (<code>str</code>, <em>optional</em>) — | |
| Branch to push the uploaded files to.`,name:"revision"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.commit_description",description:`<strong>commit_description</strong> (<code>str</code>, <em>optional</em>) — | |
| The description of the commit that will be created`,name:"commit_description"},{anchor:"transformers.PreTrainedTokenizer.push_to_hub.tags",description:`<strong>tags</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| List of tags to push on the Hub.`,name:"tags"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/utils/hub.py#L828"}}),xe=new Oo({props:{anchor:"transformers.PreTrainedTokenizer.push_to_hub.example",$$slots:{default:[Mc]},$$scope:{ctx:W}}}),ot=new w({props:{name:"convert_ids_to_tokens",anchor:"transformers.PreTrainedTokenizer.convert_ids_to_tokens",parameters:[{name:"ids",val:": Union"},{name:"skip_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.convert_ids_to_tokens.ids",description:`<strong>ids</strong> (<code>int</code> or <code>List[int]</code>) — | |
| The token id (or token ids) to convert to tokens.`,name:"ids"},{anchor:"transformers.PreTrainedTokenizer.convert_ids_to_tokens.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to remove special tokens in the decoding.`,name:"skip_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils.py#L1043",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The decoded token(s).</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>str</code> or <code>List[str]</code></p> | |
| `}}),rt=new w({props:{name:"convert_tokens_to_ids",anchor:"transformers.PreTrainedTokenizer.convert_tokens_to_ids",parameters:[{name:"tokens",val:": Union"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.convert_tokens_to_ids.tokens",description:"<strong>tokens</strong> (<code>str</code> or <code>List[str]</code>) — One or several token(s) to convert to token id(s).",name:"tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils.py#L711",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The token id or list of token ids.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code> or <code>List[int]</code></p> | |
| `}}),st=new w({props:{name:"get_added_vocab",anchor:"transformers.PreTrainedTokenizer.get_added_vocab",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils.py#L488",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The added tokens.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Dict[str, int]</code></p> | |
| `}}),at=new w({props:{name:"num_special_tokens_to_add",anchor:"transformers.PreTrainedTokenizer.num_special_tokens_to_add",parameters:[{name:"pair",val:": bool = False"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.num_special_tokens_to_add.pair",description:`<strong>pair</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether the number of added tokens should be computed in the case of a sequence pair or a single | |
| sequence.`,name:"pair"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils.py#L599",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Number of special tokens added to sequences.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),we=new Tc({props:{$$slots:{default:[Ic]},$$scope:{ctx:W}}}),it=new w({props:{name:"prepare_for_tokenization",anchor:"transformers.PreTrainedTokenizer.prepare_for_tokenization",parameters:[{name:"text",val:": str"},{name:"is_split_into_words",val:": bool = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.prepare_for_tokenization.text",description:`<strong>text</strong> (<code>str</code>) — | |
| The text to prepare.`,name:"text"},{anchor:"transformers.PreTrainedTokenizer.prepare_for_tokenization.is_split_into_words",description:`<strong>is_split_into_words</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the input is already pre-tokenized (e.g., split into words). If set to <code>True</code>, the | |
| tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) | |
| which it will tokenize. This is useful for NER or token classification.`,name:"is_split_into_words"},{anchor:"transformers.PreTrainedTokenizer.prepare_for_tokenization.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) — | |
| Keyword arguments to use for the tokenization.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils.py#L983",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The prepared text and the unused kwargs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Tuple[str, Dict[str, Any]]</code></p> | |
| `}}),dt=new w({props:{name:"tokenize",anchor:"transformers.PreTrainedTokenizer.tokenize",parameters:[{name:"text",val:": str"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizer.tokenize.text",description:`<strong>text</strong> (<code>str</code>) — | |
| The sequence to be encoded.`,name:"text"},{anchor:"transformers.PreTrainedTokenizer.tokenize.*kwargs",description:`*<strong>*kwargs</strong> (additional keyword arguments) — | |
| Passed along to the model-specific <code>prepare_for_tokenization</code> preprocessing method.`,name:"*kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils.py#L622",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The list of tokens.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[str]</code></p> | |
| `}}),ct=new Mr({props:{title:"PreTrainedTokenizerFast",local:"transformers.PreTrainedTokenizerFast",headingTag:"h2"}}),pt=new w({props:{name:"class transformers.PreTrainedTokenizerFast",anchor:"transformers.PreTrainedTokenizerFast",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.model_max_length",description:`<strong>model_max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| The maximum length (in number of tokens) for the inputs to the transformer model. When the tokenizer is | |
| loaded with <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.from_pretrained">from_pretrained()</a>, this will be set to the | |
| value stored for the associated model in <code>max_model_input_sizes</code> (see above). If no value is provided, will | |
| default to VERY_LARGE_INTEGER (<code>int(1e30)</code>).`,name:"model_max_length"},{anchor:"transformers.PreTrainedTokenizerFast.padding_side",description:`<strong>padding_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have padding applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"padding_side"},{anchor:"transformers.PreTrainedTokenizerFast.truncation_side",description:`<strong>truncation_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have truncation applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"truncation_side"},{anchor:"transformers.PreTrainedTokenizerFast.chat_template",description:`<strong>chat_template</strong> (<code>str</code>, <em>optional</em>) — | |
| A Jinja template string that will be used to format lists of chat messages. See | |
| <a href="https://huggingface.co/docs/transformers/chat_templating" rel="nofollow">https://huggingface.co/docs/transformers/chat_templating</a> for a full description.`,name:"chat_template"},{anchor:"transformers.PreTrainedTokenizerFast.model_input_names",description:`<strong>model_input_names</strong> (<code>List[string]</code>, <em>optional</em>) — | |
| The list of inputs accepted by the forward pass of the model (like <code>"token_type_ids"</code> or | |
| <code>"attention_mask"</code>). Default value is picked from the class attribute of the same name.`,name:"model_input_names"},{anchor:"transformers.PreTrainedTokenizerFast.bos_token",description:`<strong>bos_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing the beginning of a sentence. Will be associated to <code>self.bos_token</code> and | |
| <code>self.bos_token_id</code>.`,name:"bos_token"},{anchor:"transformers.PreTrainedTokenizerFast.eos_token",description:`<strong>eos_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing the end of a sentence. Will be associated to <code>self.eos_token</code> and | |
| <code>self.eos_token_id</code>.`,name:"eos_token"},{anchor:"transformers.PreTrainedTokenizerFast.unk_token",description:`<strong>unk_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing an out-of-vocabulary token. Will be associated to <code>self.unk_token</code> and | |
| <code>self.unk_token_id</code>.`,name:"unk_token"},{anchor:"transformers.PreTrainedTokenizerFast.sep_token",description:`<strong>sep_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token separating two different sentences in the same input (used by BERT for instance). Will be | |
| associated to <code>self.sep_token</code> and <code>self.sep_token_id</code>.`,name:"sep_token"},{anchor:"transformers.PreTrainedTokenizerFast.pad_token",description:`<strong>pad_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by | |
| attention mechanisms or loss computation. Will be associated to <code>self.pad_token</code> and <code>self.pad_token_id</code>.`,name:"pad_token"},{anchor:"transformers.PreTrainedTokenizerFast.cls_token",description:`<strong>cls_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing the class of the input (used by BERT for instance). Will be associated to | |
| <code>self.cls_token</code> and <code>self.cls_token_id</code>.`,name:"cls_token"},{anchor:"transformers.PreTrainedTokenizerFast.mask_token",description:`<strong>mask_token</strong> (<code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A special token representing a masked token (used by masked-language modeling pretraining objectives, like | |
| BERT). Will be associated to <code>self.mask_token</code> and <code>self.mask_token_id</code>.`,name:"mask_token"},{anchor:"transformers.PreTrainedTokenizerFast.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (tuple or list of <code>str</code> or <code>tokenizers.AddedToken</code>, <em>optional</em>) — | |
| A tuple or a list of additional special tokens. Add them here to ensure they are skipped when decoding with | |
| <code>skip_special_tokens</code> is set to True. If they are not part of the vocabulary, they will be added at the end | |
| of the vocabulary.`,name:"additional_special_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not the model should cleanup the spaces that were added when splitting the input text during the | |
| tokenization process.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.PreTrainedTokenizerFast.split_special_tokens",description:`<strong>split_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the special tokens should be split during the tokenization process. Passing will affect the | |
| internal state of the tokenizer. The default behavior is to not split special tokens. This means that if | |
| <code><s></code> is the <code>bos_token</code>, then <code>tokenizer.tokenize("<s>") = ['<s></code>]. Otherwise, if | |
| <code>split_special_tokens=True</code>, then <code>tokenizer.tokenize("<s>")</code> will be give <code>['<','s', '>']</code>.`,name:"split_special_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.tokenizer_object",description:`<strong>tokenizer_object</strong> (<code>tokenizers.Tokenizer</code>) — | |
| A <code>tokenizers.Tokenizer</code> object from 🤗 tokenizers to instantiate from. See <a href="../fast_tokenizers">Using tokenizers from 🤗 | |
| tokenizers</a> for more information.`,name:"tokenizer_object"},{anchor:"transformers.PreTrainedTokenizerFast.tokenizer_file",description:`<strong>tokenizer_file</strong> (<code>str</code>) — | |
| A path to a local JSON file representing a previously serialized <code>tokenizers.Tokenizer</code> object from 🤗 | |
| tokenizers.`,name:"tokenizer_file"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_fast.py#L81"}}),mt=new w({props:{name:"__call__",anchor:"transformers.PreTrainedTokenizerFast.__call__",parameters:[{name:"text",val:": Union = None"},{name:"text_pair",val:": Union = None"},{name:"text_target",val:": Union = None"},{name:"text_pair_target",val:": Union = None"},{name:"add_special_tokens",val:": bool = True"},{name:"padding",val:": Union = False"},{name:"truncation",val:": Union = None"},{name:"max_length",val:": Optional = None"},{name:"stride",val:": int = 0"},{name:"is_split_into_words",val:": bool = False"},{name:"pad_to_multiple_of",val:": Optional = None"},{name:"padding_side",val:": Optional = None"},{name:"return_tensors",val:": Union = None"},{name:"return_token_type_ids",val:": Optional = None"},{name:"return_attention_mask",val:": Optional = None"},{name:"return_overflowing_tokens",val:": bool = False"},{name:"return_special_tokens_mask",val:": bool = False"},{name:"return_offsets_mapping",val:": bool = False"},{name:"return_length",val:": bool = False"},{name:"verbose",val:": bool = True"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.__call__.text",description:`<strong>text</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings | |
| (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set | |
| <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.text_pair",description:`<strong>text_pair</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings | |
| (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set | |
| <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text_pair"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.text_target",description:`<strong>text_target</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a | |
| list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), | |
| you must set <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text_target"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.text_pair_target",description:`<strong>text_pair_target</strong> (<code>str</code>, <code>List[str]</code>, <code>List[List[str]]</code>, <em>optional</em>) — | |
| The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a | |
| list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), | |
| you must set <code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).`,name:"text_pair_target"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to add special tokens when encoding the sequences. This will use the underlying | |
| <code>PretrainedTokenizerBase.build_inputs_with_special_tokens</code> function, which defines which tokens are | |
| automatically added to the input ids. This is usefull if you want to add <code>bos</code> or <code>eos</code> tokens | |
| automatically.`,name:"add_special_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls padding. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest'</code>: Pad to the longest sequence in the batch (or no padding if only a single | |
| sequence if provided).</li> | |
| <li><code>'max_length'</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum | |
| acceptable input length for the model if that argument is not provided.</li> | |
| <li><code>False</code> or <code>'do_not_pad'</code> (default): No padding (i.e., can output a batch with sequences of different | |
| lengths).</li> | |
| </ul>`,name:"padding"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.truncation",description:`<strong>truncation</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.tokenization_utils_base.TruncationStrategy">TruncationStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls truncation. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or | |
| to the maximum acceptable input length for the model if that argument is not provided. This will | |
| truncate token by token, removing a token from the longest sequence in the pair if a pair of | |
| sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_second'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>False</code> or <code>'do_not_truncate'</code> (default): No truncation (i.e., can output batch with sequence lengths | |
| greater than the model maximum admissible input size).</li> | |
| </ul>`,name:"truncation"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Controls the maximum length to use by one of the truncation/padding parameters.</p> | |
| <p>If left unset or set to <code>None</code>, this will use the predefined model maximum length if a maximum length | |
| is required by one of the truncation/padding parameters. If the model has no specific maximum input | |
| length (like XLNet) truncation/padding to a maximum length will be deactivated.`,name:"max_length"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.stride",description:`<strong>stride</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If set to a number along with <code>max_length</code>, the overflowing tokens returned when | |
| <code>return_overflowing_tokens=True</code> will contain some tokens from the end of the truncated sequence | |
| returned to provide some overlap between truncated and overflowing sequences. The value of this | |
| argument defines the number of overlapping tokens.`,name:"stride"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.is_split_into_words",description:`<strong>is_split_into_words</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the input is already pre-tokenized (e.g., split into words). If set to <code>True</code>, the | |
| tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) | |
| which it will tokenize. This is useful for NER or token classification.`,name:"is_split_into_words"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value. Requires <code>padding</code> to be activated. | |
| This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability | |
| <code>>= 7.5</code> (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.padding_side",description:`<strong>padding_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have padding applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"padding_side"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) — | |
| If set, will return tensors instead of list of python integers. Acceptable values are:</p> | |
| <ul> | |
| <li><code>'tf'</code>: Return TensorFlow <code>tf.constant</code> objects.</li> | |
| <li><code>'pt'</code>: Return PyTorch <code>torch.Tensor</code> objects.</li> | |
| <li><code>'np'</code>: Return Numpy <code>np.ndarray</code> objects.</li> | |
| </ul>`,name:"return_tensors"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.return_token_type_ids",description:`<strong>return_token_type_ids</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to return token type IDs. If left to the default, will return the token type IDs according to | |
| the specific tokenizer’s default, defined by the <code>return_outputs</code> attribute.</p> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"return_token_type_ids"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.return_attention_mask",description:`<strong>return_attention_mask</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to return the attention mask. If left to the default, will return the attention mask according | |
| to the specific tokenizer’s default, defined by the <code>return_outputs</code> attribute.</p> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"return_attention_mask"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.return_overflowing_tokens",description:`<strong>return_overflowing_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch | |
| of pairs) is provided with <code>truncation_strategy = longest_first</code> or <code>True</code>, an error is raised instead | |
| of returning overflowing tokens.`,name:"return_overflowing_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.return_special_tokens_mask",description:`<strong>return_special_tokens_mask</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return special tokens mask information.`,name:"return_special_tokens_mask"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.return_offsets_mapping",description:`<strong>return_offsets_mapping</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return <code>(char_start, char_end)</code> for each token.</p> | |
| <p>This is only available on fast tokenizers inheriting from <a href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>, if using | |
| Python’s tokenizer, this method will raise <code>NotImplementedError</code>.`,name:"return_offsets_mapping"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.return_length",description:`<strong>return_length</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to return the lengths of the encoded inputs.`,name:"return_length"},{anchor:"transformers.PreTrainedTokenizerFast.__call__.verbose",description:`<strong>verbose</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to print more information and warnings. | |
| **kwargs — passed to the <code>self.tokenize()</code> method`,name:"verbose"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L2764",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.BatchEncoding" | |
| >BatchEncoding</a> with the following fields:</p> | |
| <ul> | |
| <li> | |
| <p><strong>input_ids</strong> — List of token ids to be fed to a model.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a></p> | |
| </li> | |
| <li> | |
| <p><strong>token_type_ids</strong> — List of token type ids to be fed to a model (when <code>return_token_type_ids=True</code> or | |
| if <em>“token_type_ids”</em> is in <code>self.model_input_names</code>).</p> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a></p> | |
| </li> | |
| <li> | |
| <p><strong>attention_mask</strong> — List of indices specifying which tokens should be attended to by the model (when | |
| <code>return_attention_mask=True</code> or if <em>“attention_mask”</em> is in <code>self.model_input_names</code>).</p> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a></p> | |
| </li> | |
| <li> | |
| <p><strong>overflowing_tokens</strong> — List of overflowing tokens sequences (when a <code>max_length</code> is specified and | |
| <code>return_overflowing_tokens=True</code>).</p> | |
| </li> | |
| <li> | |
| <p><strong>num_truncated_tokens</strong> — Number of tokens truncated (when a <code>max_length</code> is specified and | |
| <code>return_overflowing_tokens=True</code>).</p> | |
| </li> | |
| <li> | |
| <p><strong>special_tokens_mask</strong> — List of 0s and 1s, with 1 specifying added special tokens and 0 specifying | |
| regular sequence tokens (when <code>add_special_tokens=True</code> and <code>return_special_tokens_mask=True</code>).</p> | |
| </li> | |
| <li> | |
| <p><strong>length</strong> — The length of the inputs (when <code>return_length=True</code>)</p> | |
| </li> | |
| </ul> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.BatchEncoding" | |
| >BatchEncoding</a></p> | |
| `}}),ht=new w({props:{name:"add_tokens",anchor:"transformers.PreTrainedTokenizerFast.add_tokens",parameters:[{name:"new_tokens",val:": Union"},{name:"special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.add_tokens.new_tokens",description:`<strong>new_tokens</strong> (<code>str</code>, <code>tokenizers.AddedToken</code> or a list of <em>str</em> or <code>tokenizers.AddedToken</code>) — | |
| Tokens are only added if they are not already in the vocabulary. <code>tokenizers.AddedToken</code> wraps a string | |
| token to let you personalize its behavior: whether this token should only match against a single word, | |
| whether this token should strip all potential whitespaces on the left side, whether this token should | |
| strip all potential whitespaces on the right side, etc.`,name:"new_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.add_tokens.special_tokens",description:`<strong>special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Can be used to specify if the token is a special token. This mostly change the normalization behavior | |
| (special tokens like CLS or [MASK] are usually not lower-cased for instance).</p> | |
| <p>See details for <code>tokenizers.AddedToken</code> in HuggingFace tokenizers library.`,name:"special_tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L998",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Number of tokens added to the vocabulary.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),$e=new Oo({props:{anchor:"transformers.PreTrainedTokenizerFast.add_tokens.example",$$slots:{default:[Fc]},$$scope:{ctx:W}}}),ut=new w({props:{name:"add_special_tokens",anchor:"transformers.PreTrainedTokenizerFast.add_special_tokens",parameters:[{name:"special_tokens_dict",val:": Dict"},{name:"replace_additional_special_tokens",val:" = True"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.add_special_tokens.special_tokens_dict",description:`<strong>special_tokens_dict</strong> (dictionary <em>str</em> to <em>str</em> or <code>tokenizers.AddedToken</code>) — | |
| Keys should be in the list of predefined special attributes: [<code>bos_token</code>, <code>eos_token</code>, <code>unk_token</code>, | |
| <code>sep_token</code>, <code>pad_token</code>, <code>cls_token</code>, <code>mask_token</code>, <code>additional_special_tokens</code>].</p> | |
| <p>Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer | |
| assign the index of the <code>unk_token</code> to them).`,name:"special_tokens_dict"},{anchor:"transformers.PreTrainedTokenizerFast.add_special_tokens.replace_additional_special_tokens",description:`<strong>replace_additional_special_tokens</strong> (<code>bool</code>, <em>optional</em>,, defaults to <code>True</code>) — | |
| If <code>True</code>, the existing list of additional special tokens will be replaced by the list provided in | |
| <code>special_tokens_dict</code>. Otherwise, <code>self._special_tokens_map["additional_special_tokens"]</code> is just extended. In the former | |
| case, the tokens will NOT be removed from the tokenizer’s full vocabulary - they are only being flagged | |
| as non-special tokens. Remember, this only affects which tokens are skipped during decoding, not the | |
| <code>added_tokens_encoder</code> and <code>added_tokens_decoder</code>. This means that the previous | |
| <code>additional_special_tokens</code> are still added tokens, and will not be split by the model.`,name:"replace_additional_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L896",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Number of tokens added to the vocabulary.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),Pe=new Oo({props:{anchor:"transformers.PreTrainedTokenizerFast.add_special_tokens.example",$$slots:{default:[Lc]},$$scope:{ctx:W}}}),ft=new w({props:{name:"apply_chat_template",anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template",parameters:[{name:"conversation",val:": Union"},{name:"tools",val:": Optional = None"},{name:"documents",val:": Optional = None"},{name:"chat_template",val:": Optional = None"},{name:"add_generation_prompt",val:": bool = False"},{name:"continue_final_message",val:": bool = False"},{name:"tokenize",val:": bool = True"},{name:"padding",val:": bool = False"},{name:"truncation",val:": bool = False"},{name:"max_length",val:": Optional = None"},{name:"return_tensors",val:": Union = None"},{name:"return_dict",val:": bool = False"},{name:"return_assistant_tokens_mask",val:": bool = False"},{name:"tokenizer_kwargs",val:": Optional = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.conversation",description:`<strong>conversation</strong> (Union[List[Dict[str, str]], List[List[Dict[str, str]]]]) — A list of dicts | |
| with “role” and “content” keys, representing the chat history so far.`,name:"conversation"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.tools",description:`<strong>tools</strong> (<code>List[Dict]</code>, <em>optional</em>) — | |
| A list of tools (callable functions) that will be accessible to the model. If the template does not | |
| support function calling, this argument will have no effect. Each tool should be passed as a JSON Schema, | |
| giving the name, description and argument types for the tool. See our | |
| <a href="https://huggingface.co/docs/transformers/main/en/chat_templating#automated-function-conversion-for-tool-use" rel="nofollow">chat templating guide</a> | |
| for more information.`,name:"tools"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.documents",description:`<strong>documents</strong> (<code>List[Dict[str, str]]</code>, <em>optional</em>) — | |
| A list of dicts representing documents that will be accessible to the model if it is performing RAG | |
| (retrieval-augmented generation). If the template does not support RAG, this argument will have no | |
| effect. We recommend that each document should be a dict containing “title” and “text” keys. Please | |
| see the RAG section of the <a href="https://huggingface.co/docs/transformers/main/en/chat_templating#arguments-for-RAG" rel="nofollow">chat templating guide</a> | |
| for examples of passing documents with chat templates.`,name:"documents"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.chat_template",description:`<strong>chat_template</strong> (<code>str</code>, <em>optional</em>) — | |
| A Jinja template to use for this conversion. It is usually not necessary to pass anything to this | |
| argument, as the model’s template will be used by default.`,name:"chat_template"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.add_generation_prompt",description:`<strong>add_generation_prompt</strong> (bool, <em>optional</em>) — | |
| If this is set, a prompt with the token(s) that indicate | |
| the start of an assistant message will be appended to the formatted output. This is useful when you want to generate a response from the model. | |
| Note that this argument will be passed to the chat template, and so it must be supported in the | |
| template for this argument to have any effect.`,name:"add_generation_prompt"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.continue_final_message",description:`<strong>continue_final_message</strong> (bool, <em>optional</em>) — | |
| If this is set, the chat will be formatted so that the final | |
| message in the chat is open-ended, without any EOS tokens. The model will continue this message | |
| rather than starting a new one. This allows you to “prefill” part of | |
| the model’s response for it. Cannot be used at the same time as <code>add_generation_prompt</code>.`,name:"continue_final_message"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.tokenize",description:`<strong>tokenize</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Whether to tokenize the output. If <code>False</code>, the output will be a string.`,name:"tokenize"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.padding",description:`<strong>padding</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to pad sequences to the maximum length. Has no effect if tokenize is <code>False</code>.`,name:"padding"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.truncation",description:`<strong>truncation</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to truncate sequences at the maximum length. Has no effect if tokenize is <code>False</code>.`,name:"truncation"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Maximum length (in tokens) to use for padding or truncation. Has no effect if tokenize is <code>False</code>. If | |
| not specified, the tokenizer’s <code>max_length</code> attribute will be used as a default.`,name:"max_length"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) — | |
| If set, will return tensors of a particular framework. Has no effect if tokenize is <code>False</code>. Acceptable | |
| values are:<ul> | |
| <li><code>'tf'</code>: Return TensorFlow <code>tf.Tensor</code> objects.</li> | |
| <li><code>'pt'</code>: Return PyTorch <code>torch.Tensor</code> objects.</li> | |
| <li><code>'np'</code>: Return NumPy <code>np.ndarray</code> objects.</li> | |
| <li><code>'jax'</code>: Return JAX <code>jnp.ndarray</code> objects.</li> | |
| </ul>`,name:"return_tensors"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to return a dictionary with named outputs. Has no effect if tokenize is <code>False</code>.`,name:"return_dict"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.tokenizer_kwargs",description:"<strong>tokenizer_kwargs</strong> (<code>Dict[str -- Any]</code>, <em>optional</em>): Additional kwargs to pass to the tokenizer.",name:"tokenizer_kwargs"},{anchor:"transformers.PreTrainedTokenizerFast.apply_chat_template.return_assistant_tokens_mask",description:`<strong>return_assistant_tokens_mask</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to return a mask of the assistant generated tokens. For tokens generated by the assistant, | |
| the mask will contain 1. For user and system tokens, the mask will contain 0. | |
| This functionality is only available for chat templates that support it via the <code>{% generation %}</code> keyword. | |
| **kwargs — Additional kwargs to pass to the template renderer. Will be accessible by the chat template.`,name:"return_assistant_tokens_mask"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L1522",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A list of token ids representing the tokenized chat so far, including control tokens. This | |
| output is ready to pass to the model, either directly or via methods like <code>generate()</code>. If <code>return_dict</code> is | |
| set, will return a dict of tokenizer outputs instead.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Union[List[int], Dict]</code></p> | |
| `}}),_t=new w({props:{name:"batch_decode",anchor:"transformers.PreTrainedTokenizerFast.batch_decode",parameters:[{name:"sequences",val:": Union"},{name:"skip_special_tokens",val:": bool = False"},{name:"clean_up_tokenization_spaces",val:": bool = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.batch_decode.sequences",description:`<strong>sequences</strong> (<code>Union[List[int], List[List[int]], np.ndarray, torch.Tensor, tf.Tensor]</code>) — | |
| List of tokenized input ids. Can be obtained using the <code>__call__</code> method.`,name:"sequences"},{anchor:"transformers.PreTrainedTokenizerFast.batch_decode.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to remove special tokens in the decoding.`,name:"skip_special_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.batch_decode.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to clean up the tokenization spaces. If <code>None</code>, will default to | |
| <code>self.clean_up_tokenization_spaces</code>.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.PreTrainedTokenizerFast.batch_decode.kwargs",description:`<strong>kwargs</strong> (additional keyword arguments, <em>optional</em>) — | |
| Will be passed to the underlying model specific decode method.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L3760",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The list of decoded sentences.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[str]</code></p> | |
| `}}),gt=new w({props:{name:"decode",anchor:"transformers.PreTrainedTokenizerFast.decode",parameters:[{name:"token_ids",val:": Union"},{name:"skip_special_tokens",val:": bool = False"},{name:"clean_up_tokenization_spaces",val:": bool = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.decode.token_ids",description:`<strong>token_ids</strong> (<code>Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]</code>) — | |
| List of tokenized input ids. Can be obtained using the <code>__call__</code> method.`,name:"token_ids"},{anchor:"transformers.PreTrainedTokenizerFast.decode.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to remove special tokens in the decoding.`,name:"skip_special_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.decode.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to clean up the tokenization spaces. If <code>None</code>, will default to | |
| <code>self.clean_up_tokenization_spaces</code>.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.PreTrainedTokenizerFast.decode.kwargs",description:`<strong>kwargs</strong> (additional keyword arguments, <em>optional</em>) — | |
| Will be passed to the underlying model specific decode method.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L3794",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The decoded sentence.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>str</code></p> | |
| `}}),kt=new w({props:{name:"encode",anchor:"transformers.PreTrainedTokenizerFast.encode",parameters:[{name:"text",val:": Union"},{name:"text_pair",val:": Union = None"},{name:"add_special_tokens",val:": bool = True"},{name:"padding",val:": Union = False"},{name:"truncation",val:": Union = None"},{name:"max_length",val:": Optional = None"},{name:"stride",val:": int = 0"},{name:"padding_side",val:": Optional = None"},{name:"return_tensors",val:": Union = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.encode.text",description:`<strong>text</strong> (<code>str</code>, <code>List[str]</code> or <code>List[int]</code>) — | |
| The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the | |
| <code>tokenize</code> method) or a list of integers (tokenized string ids using the <code>convert_tokens_to_ids</code> | |
| method).`,name:"text"},{anchor:"transformers.PreTrainedTokenizerFast.encode.text_pair",description:`<strong>text_pair</strong> (<code>str</code>, <code>List[str]</code> or <code>List[int]</code>, <em>optional</em>) — | |
| Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using | |
| the <code>tokenize</code> method) or a list of integers (tokenized string ids using the <code>convert_tokens_to_ids</code> | |
| method).`,name:"text_pair"},{anchor:"transformers.PreTrainedTokenizerFast.encode.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to add special tokens when encoding the sequences. This will use the underlying | |
| <code>PretrainedTokenizerBase.build_inputs_with_special_tokens</code> function, which defines which tokens are | |
| automatically added to the input ids. This is usefull if you want to add <code>bos</code> or <code>eos</code> tokens | |
| automatically.`,name:"add_special_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.encode.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls padding. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest'</code>: Pad to the longest sequence in the batch (or no padding if only a single | |
| sequence if provided).</li> | |
| <li><code>'max_length'</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum | |
| acceptable input length for the model if that argument is not provided.</li> | |
| <li><code>False</code> or <code>'do_not_pad'</code> (default): No padding (i.e., can output a batch with sequences of different | |
| lengths).</li> | |
| </ul>`,name:"padding"},{anchor:"transformers.PreTrainedTokenizerFast.encode.truncation",description:`<strong>truncation</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.tokenization_utils_base.TruncationStrategy">TruncationStrategy</a>, <em>optional</em>, defaults to <code>False</code>) — | |
| Activates and controls truncation. Accepts the following values:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or | |
| to the maximum acceptable input length for the model if that argument is not provided. This will | |
| truncate token by token, removing a token from the longest sequence in the pair if a pair of | |
| sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_first'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>'only_second'</code>: Truncate to a maximum length specified with the argument <code>max_length</code> or to the | |
| maximum acceptable input length for the model if that argument is not provided. This will only | |
| truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.</li> | |
| <li><code>False</code> or <code>'do_not_truncate'</code> (default): No truncation (i.e., can output batch with sequence lengths | |
| greater than the model maximum admissible input size).</li> | |
| </ul>`,name:"truncation"},{anchor:"transformers.PreTrainedTokenizerFast.encode.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Controls the maximum length to use by one of the truncation/padding parameters.</p> | |
| <p>If left unset or set to <code>None</code>, this will use the predefined model maximum length if a maximum length | |
| is required by one of the truncation/padding parameters. If the model has no specific maximum input | |
| length (like XLNet) truncation/padding to a maximum length will be deactivated.`,name:"max_length"},{anchor:"transformers.PreTrainedTokenizerFast.encode.stride",description:`<strong>stride</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If set to a number along with <code>max_length</code>, the overflowing tokens returned when | |
| <code>return_overflowing_tokens=True</code> will contain some tokens from the end of the truncated sequence | |
| returned to provide some overlap between truncated and overflowing sequences. The value of this | |
| argument defines the number of overlapping tokens.`,name:"stride"},{anchor:"transformers.PreTrainedTokenizerFast.encode.is_split_into_words",description:`<strong>is_split_into_words</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the input is already pre-tokenized (e.g., split into words). If set to <code>True</code>, the | |
| tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) | |
| which it will tokenize. This is useful for NER or token classification.`,name:"is_split_into_words"},{anchor:"transformers.PreTrainedTokenizerFast.encode.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value. Requires <code>padding</code> to be activated. | |
| This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability | |
| <code>>= 7.5</code> (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.PreTrainedTokenizerFast.encode.padding_side",description:`<strong>padding_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have padding applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"padding_side"},{anchor:"transformers.PreTrainedTokenizerFast.encode.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) — | |
| If set, will return tensors instead of list of python integers. Acceptable values are:</p> | |
| <ul> | |
| <li><code>'tf'</code>: Return TensorFlow <code>tf.constant</code> objects.</li> | |
| <li><code>'pt'</code>: Return PyTorch <code>torch.Tensor</code> objects.</li> | |
| <li><code>'np'</code>: Return Numpy <code>np.ndarray</code> objects.</li> | |
| </ul> | |
| <p>**kwargs — Passed along to the <code>.tokenize()</code> method.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L2570",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The tokenized ids of the text.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[int]</code>, <code>torch.Tensor</code>, <code>tf.Tensor</code> or <code>np.ndarray</code></p> | |
| `}}),bt=new w({props:{name:"push_to_hub",anchor:"transformers.PreTrainedTokenizerFast.push_to_hub",parameters:[{name:"repo_id",val:": str"},{name:"use_temp_dir",val:": Optional = None"},{name:"commit_message",val:": Optional = None"},{name:"private",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"max_shard_size",val:": Union = '5GB'"},{name:"create_pr",val:": bool = False"},{name:"safe_serialization",val:": bool = True"},{name:"revision",val:": str = None"},{name:"commit_description",val:": str = None"},{name:"tags",val:": Optional = None"},{name:"**deprecated_kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.repo_id",description:`<strong>repo_id</strong> (<code>str</code>) — | |
| The name of the repository you want to push your tokenizer to. It should contain your organization name | |
| when pushing to a given organization.`,name:"repo_id"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.use_temp_dir",description:`<strong>use_temp_dir</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to use a temporary directory to store the files saved before they are pushed to the Hub. | |
| Will default to <code>True</code> if there is no directory named like <code>repo_id</code>, <code>False</code> otherwise.`,name:"use_temp_dir"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.commit_message",description:`<strong>commit_message</strong> (<code>str</code>, <em>optional</em>) — | |
| Message to commit while pushing. Will default to <code>"Upload tokenizer"</code>.`,name:"commit_message"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.private",description:`<strong>private</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not the repository created should be private.`,name:"private"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.token",description:`<strong>token</strong> (<code>bool</code> or <code>str</code>, <em>optional</em>) — | |
| The token to use as HTTP bearer authorization for remote files. If <code>True</code>, will use the token generated | |
| when running <code>huggingface-cli login</code> (stored in <code>~/.huggingface</code>). Will default to <code>True</code> if <code>repo_url</code> | |
| is not specified.`,name:"token"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.max_shard_size",description:`<strong>max_shard_size</strong> (<code>int</code> or <code>str</code>, <em>optional</em>, defaults to <code>"5GB"</code>) — | |
| Only applicable for models. The maximum size for a checkpoint before being sharded. Checkpoints shard | |
| will then be each of size lower than this size. If expressed as a string, needs to be digits followed | |
| by a unit (like <code>"5MB"</code>). We default it to <code>"5GB"</code> so that users can easily load models on free-tier | |
| Google Colab instances without any CPU OOM issues.`,name:"max_shard_size"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.create_pr",description:`<strong>create_pr</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to create a PR with the uploaded files or directly commit.`,name:"create_pr"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.safe_serialization",description:`<strong>safe_serialization</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to convert the model weights in safetensors format for safer serialization.`,name:"safe_serialization"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.revision",description:`<strong>revision</strong> (<code>str</code>, <em>optional</em>) — | |
| Branch to push the uploaded files to.`,name:"revision"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.commit_description",description:`<strong>commit_description</strong> (<code>str</code>, <em>optional</em>) — | |
| The description of the commit that will be created`,name:"commit_description"},{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.tags",description:`<strong>tags</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| List of tags to push on the Hub.`,name:"tags"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/utils/hub.py#L828"}}),Me=new Oo({props:{anchor:"transformers.PreTrainedTokenizerFast.push_to_hub.example",$$slots:{default:[jc]},$$scope:{ctx:W}}}),xt=new w({props:{name:"convert_ids_to_tokens",anchor:"transformers.PreTrainedTokenizerFast.convert_ids_to_tokens",parameters:[{name:"ids",val:": Union"},{name:"skip_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.convert_ids_to_tokens.ids",description:`<strong>ids</strong> (<code>int</code> or <code>List[int]</code>) — | |
| The token id (or token ids) to convert to tokens.`,name:"ids"},{anchor:"transformers.PreTrainedTokenizerFast.convert_ids_to_tokens.skip_special_tokens",description:`<strong>skip_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to remove special tokens in the decoding.`,name:"skip_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_fast.py#L381",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The decoded token(s).</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>str</code> or <code>List[str]</code></p> | |
| `}}),Tt=new w({props:{name:"convert_tokens_to_ids",anchor:"transformers.PreTrainedTokenizerFast.convert_tokens_to_ids",parameters:[{name:"tokens",val:": Union"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.convert_tokens_to_ids.tokens",description:"<strong>tokens</strong> (<code>str</code> or <code>Iterable[str]</code>) — One or several token(s) to convert to token id(s).",name:"tokens"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_fast.py#L329",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The token id or list of token ids.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code> or <code>List[int]</code></p> | |
| `}}),vt=new w({props:{name:"get_added_vocab",anchor:"transformers.PreTrainedTokenizerFast.get_added_vocab",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_fast.py#L253",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The added tokens.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Dict[str, int]</code></p> | |
| `}}),yt=new w({props:{name:"num_special_tokens_to_add",anchor:"transformers.PreTrainedTokenizerFast.num_special_tokens_to_add",parameters:[{name:"pair",val:": bool = False"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.num_special_tokens_to_add.pair",description:`<strong>pair</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether the number of added tokens should be computed in the case of a sequence pair or a single | |
| sequence.`,name:"pair"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_fast.py#L360",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Number of special tokens added to sequences.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),je=new Tc({props:{$$slots:{default:[Wc]},$$scope:{ctx:W}}}),wt=new w({props:{name:"set_truncation_and_padding",anchor:"transformers.PreTrainedTokenizerFast.set_truncation_and_padding",parameters:[{name:"padding_strategy",val:": PaddingStrategy"},{name:"truncation_strategy",val:": TruncationStrategy"},{name:"max_length",val:": int"},{name:"stride",val:": int"},{name:"pad_to_multiple_of",val:": Optional"},{name:"padding_side",val:": Optional"}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.set_truncation_and_padding.padding_strategy",description:`<strong>padding_strategy</strong> (<a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>) — | |
| The kind of padding that will be applied to the input`,name:"padding_strategy"},{anchor:"transformers.PreTrainedTokenizerFast.set_truncation_and_padding.truncation_strategy",description:`<strong>truncation_strategy</strong> (<a href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.tokenization_utils_base.TruncationStrategy">TruncationStrategy</a>) — | |
| The kind of truncation that will be applied to the input`,name:"truncation_strategy"},{anchor:"transformers.PreTrainedTokenizerFast.set_truncation_and_padding.max_length",description:`<strong>max_length</strong> (<code>int</code>) — | |
| The maximum size of a sequence.`,name:"max_length"},{anchor:"transformers.PreTrainedTokenizerFast.set_truncation_and_padding.stride",description:`<strong>stride</strong> (<code>int</code>) — | |
| The stride to use when handling overflow.`,name:"stride"},{anchor:"transformers.PreTrainedTokenizerFast.set_truncation_and_padding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value. This is especially useful to enable | |
| the use of Tensor Cores on NVIDIA hardware with compute capability <code>>= 7.5</code> (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.PreTrainedTokenizerFast.set_truncation_and_padding.padding_side",description:`<strong>padding_side</strong> (<code>str</code>, <em>optional</em>) — | |
| The side on which the model should have padding applied. Should be selected between [‘right’, ‘left’]. | |
| Default value is picked from the class attribute of the same name.`,name:"padding_side"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_fast.py#L410"}}),zt=new w({props:{name:"train_new_from_iterator",anchor:"transformers.PreTrainedTokenizerFast.train_new_from_iterator",parameters:[{name:"text_iterator",val:""},{name:"vocab_size",val:""},{name:"length",val:" = None"},{name:"new_special_tokens",val:" = None"},{name:"special_tokens_map",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.PreTrainedTokenizerFast.train_new_from_iterator.text_iterator",description:`<strong>text_iterator</strong> (generator of <code>List[str]</code>) — | |
| The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts | |
| if you have everything in memory.`,name:"text_iterator"},{anchor:"transformers.PreTrainedTokenizerFast.train_new_from_iterator.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>) — | |
| The size of the vocabulary you want for your tokenizer.`,name:"vocab_size"},{anchor:"transformers.PreTrainedTokenizerFast.train_new_from_iterator.length",description:`<strong>length</strong> (<code>int</code>, <em>optional</em>) — | |
| The total number of sequences in the iterator. This is used to provide meaningful progress tracking`,name:"length"},{anchor:"transformers.PreTrainedTokenizerFast.train_new_from_iterator.new_special_tokens",description:`<strong>new_special_tokens</strong> (list of <code>str</code> or <code>AddedToken</code>, <em>optional</em>) — | |
| A list of new special tokens to add to the tokenizer you are training.`,name:"new_special_tokens"},{anchor:"transformers.PreTrainedTokenizerFast.train_new_from_iterator.special_tokens_map",description:`<strong>special_tokens_map</strong> (<code>Dict[str, str]</code>, <em>optional</em>) — | |
| If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special | |
| token name to new special token name in this argument.`,name:"special_tokens_map"},{anchor:"transformers.PreTrainedTokenizerFast.train_new_from_iterator.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) — | |
| Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_fast.py#L713",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A new tokenizer of the same type as the original one, trained on | |
| <code>text_iterator</code>.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast" | |
| >PreTrainedTokenizerFast</a></p> | |
| `}}),$t=new Mr({props:{title:"BatchEncoding",local:"transformers.BatchEncoding",headingTag:"h2"}}),Pt=new w({props:{name:"class transformers.BatchEncoding",anchor:"transformers.BatchEncoding",parameters:[{name:"data",val:": Optional = None"},{name:"encoding",val:": Union = None"},{name:"tensor_type",val:": Union = None"},{name:"prepend_batch_axis",val:": bool = False"},{name:"n_sequences",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.BatchEncoding.data",description:`<strong>data</strong> (<code>dict</code>, <em>optional</em>) — | |
| Dictionary of lists/arrays/tensors returned by the <code>__call__</code>/<code>encode_plus</code>/<code>batch_encode_plus</code> methods | |
| (‘input_ids’, ‘attention_mask’, etc.).`,name:"data"},{anchor:"transformers.BatchEncoding.encoding",description:`<strong>encoding</strong> (<code>tokenizers.Encoding</code> or <code>Sequence[tokenizers.Encoding]</code>, <em>optional</em>) — | |
| If the tokenizer is a fast tokenizer which outputs additional information like mapping from word/character | |
| space to token space the <code>tokenizers.Encoding</code> instance or list of instance (for batches) hold this | |
| information.`,name:"encoding"},{anchor:"transformers.BatchEncoding.tensor_type",description:`<strong>tensor_type</strong> (<code>Union[None, str, TensorType]</code>, <em>optional</em>) — | |
| You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at | |
| initialization.`,name:"tensor_type"},{anchor:"transformers.BatchEncoding.prepend_batch_axis",description:`<strong>prepend_batch_axis</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to add a batch axis when converting to tensors (see <code>tensor_type</code> above). Note that this | |
| parameter has an effect if the parameter <code>tensor_type</code> is set, <em>otherwise has no effect</em>.`,name:"prepend_batch_axis"},{anchor:"transformers.BatchEncoding.n_sequences",description:`<strong>n_sequences</strong> (<code>Optional[int]</code>, <em>optional</em>) — | |
| You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at | |
| initialization.`,name:"n_sequences"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L192"}}),Ct=new w({props:{name:"char_to_token",anchor:"transformers.BatchEncoding.char_to_token",parameters:[{name:"batch_or_char_index",val:": int"},{name:"char_index",val:": Optional = None"},{name:"sequence_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.char_to_token.batch_or_char_index",description:`<strong>batch_or_char_index</strong> (<code>int</code>) — | |
| Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of | |
| the word in the sequence`,name:"batch_or_char_index"},{anchor:"transformers.BatchEncoding.char_to_token.char_index",description:`<strong>char_index</strong> (<code>int</code>, <em>optional</em>) — | |
| If a batch index is provided in <em>batch_or_token_index</em>, this can be the index of the word in the | |
| sequence.`,name:"char_index"},{anchor:"transformers.BatchEncoding.char_to_token.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0 | |
| or 1) the provided character index belongs to.`,name:"sequence_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L572",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Index of the token, or None if the char index refers to a whitespace only token and whitespace is | |
| trimmed with <code>trim_offsets=True</code>.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),qt=new w({props:{name:"char_to_word",anchor:"transformers.BatchEncoding.char_to_word",parameters:[{name:"batch_or_char_index",val:": int"},{name:"char_index",val:": Optional = None"},{name:"sequence_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.char_to_word.batch_or_char_index",description:`<strong>batch_or_char_index</strong> (<code>int</code>) — | |
| Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of | |
| the character in the original string.`,name:"batch_or_char_index"},{anchor:"transformers.BatchEncoding.char_to_word.char_index",description:`<strong>char_index</strong> (<code>int</code>, <em>optional</em>) — | |
| If a batch index is provided in <em>batch_or_token_index</em>, this can be the index of the character in the | |
| original string.`,name:"char_index"},{anchor:"transformers.BatchEncoding.char_to_word.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0 | |
| or 1) the provided character index belongs to.`,name:"sequence_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L659",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Index or indices of the associated encoded token(s).</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code> or <code>List[int]</code></p> | |
| `}}),Mt=new w({props:{name:"convert_to_tensors",anchor:"transformers.BatchEncoding.convert_to_tensors",parameters:[{name:"tensor_type",val:": Union = None"},{name:"prepend_batch_axis",val:": bool = False"}],parametersDescription:[{anchor:"transformers.BatchEncoding.convert_to_tensors.tensor_type",description:`<strong>tensor_type</strong> (<code>str</code> or <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) — | |
| The type of tensors to use. If <code>str</code>, should be one of the values of the enum <a href="/docs/transformers/pr_34786/zh/internal/file_utils#transformers.TensorType">TensorType</a>. If | |
| <code>None</code>, no modification is done.`,name:"tensor_type"},{anchor:"transformers.BatchEncoding.convert_to_tensors.prepend_batch_axis",description:`<strong>prepend_batch_axis</strong> (<code>int</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to add the batch dimension during the conversion.`,name:"prepend_batch_axis"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L698"}}),It=new w({props:{name:"sequence_ids",anchor:"transformers.BatchEncoding.sequence_ids",parameters:[{name:"batch_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.sequence_ids.batch_index",description:"<strong>batch_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — The index to access in the batch.",name:"batch_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L336",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A list indicating the sequence id corresponding to each token. Special tokens added | |
| by the tokenizer are mapped to <code>None</code> and other tokens are mapped to the index of their corresponding | |
| sequence.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[Optional[int]]</code></p> | |
| `}}),Ft=new w({props:{name:"to",anchor:"transformers.BatchEncoding.to",parameters:[{name:"device",val:": Union"}],parametersDescription:[{anchor:"transformers.BatchEncoding.to.device",description:"<strong>device</strong> (<code>str</code> or <code>torch.device</code>) — The device to put the tensors on.",name:"device"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L801",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same instance after modification.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_34786/zh/main_classes/tokenizer#transformers.BatchEncoding" | |
| >BatchEncoding</a></p> | |
| `}}),Lt=new w({props:{name:"token_to_chars",anchor:"transformers.BatchEncoding.token_to_chars",parameters:[{name:"batch_or_token_index",val:": int"},{name:"token_index",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.BatchEncoding.token_to_chars.batch_or_token_index",description:`<strong>batch_or_token_index</strong> (<code>int</code>) — | |
| Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of | |
| the token in the sequence.`,name:"batch_or_token_index"},{anchor:"transformers.BatchEncoding.token_to_chars.token_index",description:`<strong>token_index</strong> (<code>int</code>, <em>optional</em>) — | |
| If a batch index is provided in <em>batch_or_token_index</em>, this can be the index of the token or tokens in | |
| the sequence.`,name:"token_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L533",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Span of characters in the original string, or None, if the token | |
| (e.g. <s>, </s>) doesn’t correspond to any chars in the origin string.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.CharSpan" | |
| >CharSpan</a></p> | |
| `}}),jt=new w({props:{name:"token_to_sequence",anchor:"transformers.BatchEncoding.token_to_sequence",parameters:[{name:"batch_or_token_index",val:": int"},{name:"token_index",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.BatchEncoding.token_to_sequence.batch_or_token_index",description:`<strong>batch_or_token_index</strong> (<code>int</code>) — | |
| Index of the sequence in the batch. If the batch only comprises one sequence, this can be the index of | |
| the token in the sequence.`,name:"batch_or_token_index"},{anchor:"transformers.BatchEncoding.token_to_sequence.token_index",description:`<strong>token_index</strong> (<code>int</code>, <em>optional</em>) — | |
| If a batch index is provided in <em>batch_or_token_index</em>, this can be the index of the token in the | |
| sequence.`,name:"token_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L403",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Index of the word in the input sequence.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),Wt=new w({props:{name:"token_to_word",anchor:"transformers.BatchEncoding.token_to_word",parameters:[{name:"batch_or_token_index",val:": int"},{name:"token_index",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.BatchEncoding.token_to_word.batch_or_token_index",description:`<strong>batch_or_token_index</strong> (<code>int</code>) — | |
| Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of | |
| the token in the sequence.`,name:"batch_or_token_index"},{anchor:"transformers.BatchEncoding.token_to_word.token_index",description:`<strong>token_index</strong> (<code>int</code>, <em>optional</em>) — | |
| If a batch index is provided in <em>batch_or_token_index</em>, this can be the index of the token in the | |
| sequence.`,name:"token_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L442",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Index of the word in the input sequence.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>int</code></p> | |
| `}}),Jt=new w({props:{name:"tokens",anchor:"transformers.BatchEncoding.tokens",parameters:[{name:"batch_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.tokens.batch_index",description:"<strong>batch_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — The index to access in the batch.",name:"batch_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L318",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The list of tokens at that index.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[str]</code></p> | |
| `}}),Ut=new w({props:{name:"word_ids",anchor:"transformers.BatchEncoding.word_ids",parameters:[{name:"batch_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.word_ids.batch_index",description:"<strong>batch_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — The index to access in the batch.",name:"batch_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L384",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A list indicating the word corresponding to each token. Special tokens added by the | |
| tokenizer are mapped to <code>None</code> and other tokens are mapped to the index of their corresponding word | |
| (several tokens will be mapped to the same word index if they are parts of that word).</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[Optional[int]]</code></p> | |
| `}}),Bt=new w({props:{name:"word_to_chars",anchor:"transformers.BatchEncoding.word_to_chars",parameters:[{name:"batch_or_word_index",val:": int"},{name:"word_index",val:": Optional = None"},{name:"sequence_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.word_to_chars.batch_or_word_index",description:`<strong>batch_or_word_index</strong> (<code>int</code>) — | |
| Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of | |
| the word in the sequence`,name:"batch_or_word_index"},{anchor:"transformers.BatchEncoding.word_to_chars.word_index",description:`<strong>word_index</strong> (<code>int</code>, <em>optional</em>) — | |
| If a batch index is provided in <em>batch_or_token_index</em>, this can be the index of the word in the | |
| sequence.`,name:"word_index"},{anchor:"transformers.BatchEncoding.word_to_chars.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0 | |
| or 1) the provided word index belongs to.`,name:"sequence_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L614",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Span(s) of the associated character or characters in the string. CharSpan | |
| are NamedTuple with:</p> | |
| <ul> | |
| <li>start: index of the first character associated to the token in the original string</li> | |
| <li>end: index of the character following the last character associated to the token in the original | |
| string</li> | |
| </ul> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>CharSpan</code> or <code>List[CharSpan]</code></p> | |
| `}}),Nt=new w({props:{name:"word_to_tokens",anchor:"transformers.BatchEncoding.word_to_tokens",parameters:[{name:"batch_or_word_index",val:": int"},{name:"word_index",val:": Optional = None"},{name:"sequence_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.word_to_tokens.batch_or_word_index",description:`<strong>batch_or_word_index</strong> (<code>int</code>) — | |
| Index of the sequence in the batch. If the batch only comprises one sequence, this can be the index of | |
| the word in the sequence.`,name:"batch_or_word_index"},{anchor:"transformers.BatchEncoding.word_to_tokens.word_index",description:`<strong>word_index</strong> (<code>int</code>, <em>optional</em>) — | |
| If a batch index is provided in <em>batch_or_token_index</em>, this can be the index of the word in the | |
| sequence.`,name:"word_index"},{anchor:"transformers.BatchEncoding.word_to_tokens.sequence_index",description:`<strong>sequence_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0 | |
| or 1) the provided word index belongs to.`,name:"sequence_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L480",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>Span of tokens in the encoded sequence. Returns | |
| <code>None</code> if no tokens correspond to the word. This can happen especially when the token is a special token | |
| that has been used to format the tokenization. For example when we add a class token at the very beginning | |
| of the tokenization.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>(<a | |
| href="/docs/transformers/pr_34786/zh/internal/tokenization_utils#transformers.TokenSpan" | |
| >TokenSpan</a>, <em>optional</em>)</p> | |
| `}}),Vt=new w({props:{name:"words",anchor:"transformers.BatchEncoding.words",parameters:[{name:"batch_index",val:": int = 0"}],parametersDescription:[{anchor:"transformers.BatchEncoding.words.batch_index",description:"<strong>batch_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — The index to access in the batch.",name:"batch_index"}],source:"https://github.com/huggingface/transformers/blob/vr_34786/src/transformers/tokenization_utils_base.py#L360",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A list indicating the word corresponding to each token. Special tokens added by the | |
| tokenizer are mapped to <code>None</code> and other tokens are mapped to the index of their corresponding word | |
| (several tokens will be mapped to the same word index if they are parts of that word).</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>List[Optional[int]]</code></p> | |
| `}}),Et=new Pc({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/main_classes/tokenizer.md"}}),{c(){c=r("meta"),I=n(),k=r("p"),_=n(),l(P.$$.fragment),i=n(),M=r("p"),M.innerHTML=xi,er=n(),De=r("ol"),De.innerHTML=Ti,tr=n(),Ze=r("p"),Ze.innerHTML=vi,nr=n(),He=r("p"),He.innerHTML=yi,or=n(),Ge=r("ul"),Ge.innerHTML=wi,rr=n(),Ae=r("p"),Ae.innerHTML=zi,sr=n(),l(Re.$$.fragment),ar=n(),b=r("div"),l(Se.$$.fragment),Ir=n(),Gt=r("p"),Gt.textContent=$i,Fr=n(),At=r("p"),At.innerHTML=Pi,Lr=n(),Rt=r("p"),Rt.textContent=Ci,jr=n(),St=r("p"),St.textContent=qi,Wr=n(),Xt=r("p"),Xt.textContent=Mi,Jr=n(),Yt=r("ul"),Yt.innerHTML=Ii,Ur=n(),fe=r("div"),l(Xe.$$.fragment),Br=n(),Ot=r("p"),Ot.textContent=Fi,Nr=n(),V=r("div"),l(Ye.$$.fragment),Vr=n(),Qt=r("p"),Qt.textContent=Li,Er=n(),Kt=r("p"),Kt.textContent=ji,Dr=n(),en=r("p"),en.innerHTML=Wi,Zr=n(),l(_e.$$.fragment),Hr=n(),F=r("div"),l(Oe.$$.fragment),Gr=n(),tn=r("p"),tn.textContent=Ji,Ar=n(),nn=r("p"),nn.textContent=Ui,Rr=n(),on=r("p"),on.innerHTML=Bi,Sr=n(),rn=r("p"),rn.innerHTML=Ni,Xr=n(),sn=r("ul"),sn.innerHTML=Vi,Yr=n(),an=r("p"),an.innerHTML=Ei,Or=n(),l(ge.$$.fragment),Qr=n(),ke=r("div"),l(Qe.$$.fragment),Kr=n(),dn=r("p"),dn.innerHTML=Di,es=n(),be=r("div"),l(Ke.$$.fragment),ts=n(),cn=r("p"),cn.textContent=Zi,ns=n(),Y=r("div"),l(et.$$.fragment),os=n(),ln=r("p"),ln.textContent=Hi,rs=n(),pn=r("p"),pn.innerHTML=Gi,ss=n(),O=r("div"),l(tt.$$.fragment),as=n(),mn=r("p"),mn.textContent=Ai,is=n(),hn=r("p"),hn.innerHTML=Ri,ds=n(),Q=r("div"),l(nt.$$.fragment),cs=n(),un=r("p"),un.textContent=Si,ls=n(),l(xe.$$.fragment),ps=n(),Te=r("div"),l(ot.$$.fragment),ms=n(),fn=r("p"),fn.textContent=Xi,hs=n(),ve=r("div"),l(rt.$$.fragment),us=n(),_n=r("p"),_n.textContent=Yi,fs=n(),ye=r("div"),l(st.$$.fragment),_s=n(),gn=r("p"),gn.textContent=Oi,gs=n(),K=r("div"),l(at.$$.fragment),ks=n(),kn=r("p"),kn.textContent=Qi,bs=n(),l(we.$$.fragment),xs=n(),ee=r("div"),l(it.$$.fragment),Ts=n(),bn=r("p"),bn.textContent=Ki,vs=n(),xn=r("p"),xn.innerHTML=ed,ys=n(),te=r("div"),l(dt.$$.fragment),ws=n(),Tn=r("p"),Tn.textContent=td,zs=n(),vn=r("p"),vn.textContent=nd,ir=n(),l(ct.$$.fragment),dr=n(),lt=r("p"),lt.innerHTML=od,cr=n(),x=r("div"),l(pt.$$.fragment),$s=n(),yn=r("p"),yn.textContent=rd,Ps=n(),wn=r("p"),wn.innerHTML=sd,Cs=n(),zn=r("p"),zn.textContent=ad,qs=n(),$n=r("p"),$n.textContent=id,Ms=n(),Pn=r("p"),Pn.textContent=dd,Is=n(),Cn=r("ul"),Cn.innerHTML=cd,Fs=n(),ze=r("div"),l(mt.$$.fragment),Ls=n(),qn=r("p"),qn.textContent=ld,js=n(),E=r("div"),l(ht.$$.fragment),Ws=n(),Mn=r("p"),Mn.textContent=pd,Js=n(),In=r("p"),In.textContent=md,Us=n(),Fn=r("p"),Fn.innerHTML=hd,Bs=n(),l($e.$$.fragment),Ns=n(),L=r("div"),l(ut.$$.fragment),Vs=n(),Ln=r("p"),Ln.textContent=ud,Es=n(),jn=r("p"),jn.textContent=fd,Ds=n(),Wn=r("p"),Wn.innerHTML=_d,Zs=n(),Jn=r("p"),Jn.innerHTML=gd,Hs=n(),Un=r("ul"),Un.innerHTML=kd,Gs=n(),Bn=r("p"),Bn.innerHTML=bd,As=n(),l(Pe.$$.fragment),Rs=n(),Ce=r("div"),l(ft.$$.fragment),Ss=n(),Nn=r("p"),Nn.innerHTML=xd,Xs=n(),qe=r("div"),l(_t.$$.fragment),Ys=n(),Vn=r("p"),Vn.textContent=Td,Os=n(),ne=r("div"),l(gt.$$.fragment),Qs=n(),En=r("p"),En.textContent=vd,Ks=n(),Dn=r("p"),Dn.innerHTML=yd,ea=n(),oe=r("div"),l(kt.$$.fragment),ta=n(),Zn=r("p"),Zn.textContent=wd,na=n(),Hn=r("p"),Hn.innerHTML=zd,oa=n(),re=r("div"),l(bt.$$.fragment),ra=n(),Gn=r("p"),Gn.textContent=$d,sa=n(),l(Me.$$.fragment),aa=n(),Ie=r("div"),l(xt.$$.fragment),ia=n(),An=r("p"),An.textContent=Pd,da=n(),Fe=r("div"),l(Tt.$$.fragment),ca=n(),Rn=r("p"),Rn.textContent=Cd,la=n(),Le=r("div"),l(vt.$$.fragment),pa=n(),Sn=r("p"),Sn.textContent=qd,ma=n(),se=r("div"),l(yt.$$.fragment),ha=n(),Xn=r("p"),Xn.textContent=Md,ua=n(),l(je.$$.fragment),fa=n(),ae=r("div"),l(wt.$$.fragment),_a=n(),Yn=r("p"),Yn.textContent=Id,ga=n(),On=r("p"),On.textContent=Fd,ka=n(),We=r("div"),l(zt.$$.fragment),ba=n(),Qn=r("p"),Qn.textContent=Ld,lr=n(),l($t.$$.fragment),pr=n(),C=r("div"),l(Pt.$$.fragment),xa=n(),Kn=r("p"),Kn.innerHTML=jd,Ta=n(),eo=r("p"),eo.textContent=Wd,va=n(),D=r("div"),l(Ct.$$.fragment),ya=n(),to=r("p"),to.textContent=Jd,wa=n(),no=r("p"),no.textContent=Ud,za=n(),oo=r("ul"),oo.innerHTML=Bd,$a=n(),ro=r("p"),ro.textContent=Nd,Pa=n(),Z=r("div"),l(qt.$$.fragment),Ca=n(),so=r("p"),so.textContent=Vd,qa=n(),ao=r("p"),ao.textContent=Ed,Ma=n(),io=r("ul"),io.innerHTML=Dd,Ia=n(),co=r("p"),co.textContent=Zd,Fa=n(),Je=r("div"),l(Mt.$$.fragment),La=n(),lo=r("p"),lo.textContent=Hd,ja=n(),ie=r("div"),l(It.$$.fragment),Wa=n(),po=r("p"),po.textContent=Gd,Ja=n(),mo=r("ul"),mo.innerHTML=Ad,Ua=n(),Ue=r("div"),l(Ft.$$.fragment),Ba=n(),ho=r("p"),ho.innerHTML=Rd,Na=n(),U=r("div"),l(Lt.$$.fragment),Va=n(),uo=r("p"),uo.textContent=Sd,Ea=n(),fo=r("p"),fo.innerHTML=Xd,Da=n(),_o=r("ul"),_o.innerHTML=Yd,Za=n(),go=r("p"),go.textContent=Od,Ha=n(),ko=r("ul"),ko.innerHTML=Qd,Ga=n(),H=r("div"),l(jt.$$.fragment),Aa=n(),bo=r("p"),bo.innerHTML=Kd,Ra=n(),xo=r("p"),xo.textContent=ec,Sa=n(),To=r("ul"),To.innerHTML=tc,Xa=n(),vo=r("p"),vo.textContent=nc,Ya=n(),G=r("div"),l(Wt.$$.fragment),Oa=n(),yo=r("p"),yo.textContent=oc,Qa=n(),wo=r("p"),wo.textContent=rc,Ka=n(),zo=r("ul"),zo.innerHTML=sc,ei=n(),$o=r("p"),$o.textContent=ac,ti=n(),Be=r("div"),l(Jt.$$.fragment),ni=n(),Po=r("p"),Po.textContent=ic,oi=n(),Ne=r("div"),l(Ut.$$.fragment),ri=n(),Co=r("p"),Co.textContent=dc,si=n(),B=r("div"),l(Bt.$$.fragment),ai=n(),qo=r("p"),qo.textContent=cc,ii=n(),Mo=r("p"),Mo.textContent=lc,di=n(),Io=r("ul"),Io.innerHTML=pc,ci=n(),Fo=r("p"),Fo.textContent=mc,li=n(),Lo=r("ul"),Lo.innerHTML=hc,pi=n(),J=r("div"),l(Nt.$$.fragment),mi=n(),jo=r("p"),jo.textContent=uc,hi=n(),Wo=r("p"),Wo.innerHTML=fc,ui=n(),Jo=r("ul"),Jo.innerHTML=_c,fi=n(),Uo=r("p"),Uo.textContent=gc,_i=n(),Bo=r("ul"),Bo.innerHTML=kc,gi=n(),No=r("p"),No.textContent=bc,ki=n(),Ve=r("div"),l(Vt.$$.fragment),bi=n(),Vo=r("p"),Vo.textContent=xc,mr=n(),l(Et.$$.fragment),hr=n(),Ko=r("p"),this.h()},l(t){const g=$c("svelte-u9bgzb",document.head);c=s(g,"META",{name:!0,content:!0}),g.forEach(d),I=o(t),k=s(t,"P",{}),T(k).forEach(d),_=o(t),p(P.$$.fragment,t),i=o(t),M=s(t,"P",{"data-svelte-h":!0}),a(M)!=="svelte-1lgw8ut"&&(M.innerHTML=xi),er=o(t),De=s(t,"OL",{"data-svelte-h":!0}),a(De)!=="svelte-1n20hjv"&&(De.innerHTML=Ti),tr=o(t),Ze=s(t,"P",{"data-svelte-h":!0}),a(Ze)!=="svelte-1bbvs5u"&&(Ze.innerHTML=vi),nr=o(t),He=s(t,"P",{"data-svelte-h":!0}),a(He)!=="svelte-1xcdv9u"&&(He.innerHTML=yi),or=o(t),Ge=s(t,"UL",{"data-svelte-h":!0}),a(Ge)!=="svelte-m9xzbe"&&(Ge.innerHTML=wi),rr=o(t),Ae=s(t,"P",{"data-svelte-h":!0}),a(Ae)!=="svelte-b0xcoj"&&(Ae.innerHTML=zi),sr=o(t),p(Re.$$.fragment,t),ar=o(t),b=s(t,"DIV",{class:!0});var y=T(b);p(Se.$$.fragment,y),Ir=o(y),Gt=s(y,"P",{"data-svelte-h":!0}),a(Gt)!=="svelte-1vieurq"&&(Gt.textContent=$i),Fr=o(y),At=s(y,"P",{"data-svelte-h":!0}),a(At)!=="svelte-1xcwogg"&&(At.innerHTML=Pi),Lr=o(y),Rt=s(y,"P",{"data-svelte-h":!0}),a(Rt)!=="svelte-y6enlf"&&(Rt.textContent=Ci),jr=o(y),St=s(y,"P",{"data-svelte-h":!0}),a(St)!=="svelte-abl5qq"&&(St.textContent=qi),Wr=o(y),Xt=s(y,"P",{"data-svelte-h":!0}),a(Xt)!=="svelte-1ixo79u"&&(Xt.textContent=Mi),Jr=o(y),Yt=s(y,"UL",{"data-svelte-h":!0}),a(Yt)!=="svelte-1y0k6z5"&&(Yt.innerHTML=Ii),Ur=o(y),fe=s(y,"DIV",{class:!0});var Dt=T(fe);p(Xe.$$.fragment,Dt),Br=o(Dt),Ot=s(Dt,"P",{"data-svelte-h":!0}),a(Ot)!=="svelte-kpxj0c"&&(Ot.textContent=Fi),Dt.forEach(d),Nr=o(y),V=s(y,"DIV",{class:!0});var R=T(V);p(Ye.$$.fragment,R),Vr=o(R),Qt=s(R,"P",{"data-svelte-h":!0}),a(Qt)!=="svelte-14rotd7"&&(Qt.textContent=Li),Er=o(R),Kt=s(R,"P",{"data-svelte-h":!0}),a(Kt)!=="svelte-j0w5r1"&&(Kt.textContent=ji),Dr=o(R),en=s(R,"P",{"data-svelte-h":!0}),a(en)!=="svelte-1enyojm"&&(en.innerHTML=Wi),Zr=o(R),p(_e.$$.fragment,R),R.forEach(d),Hr=o(y),F=s(y,"DIV",{class:!0});var j=T(F);p(Oe.$$.fragment,j),Gr=o(j),tn=s(j,"P",{"data-svelte-h":!0}),a(tn)!=="svelte-1j8s0i5"&&(tn.textContent=Ji),Ar=o(j),nn=s(j,"P",{"data-svelte-h":!0}),a(nn)!=="svelte-1w3ayx9"&&(nn.textContent=Ui),Rr=o(j),on=s(j,"P",{"data-svelte-h":!0}),a(on)!=="svelte-1enyojm"&&(on.innerHTML=Bi),Sr=o(j),rn=s(j,"P",{"data-svelte-h":!0}),a(rn)!=="svelte-5hxtpc"&&(rn.innerHTML=Ni),Xr=o(j),sn=s(j,"UL",{"data-svelte-h":!0}),a(sn)!=="svelte-1pes0uj"&&(sn.innerHTML=Vi),Yr=o(j),an=s(j,"P",{"data-svelte-h":!0}),a(an)!=="svelte-1butrf6"&&(an.innerHTML=Ei),Or=o(j),p(ge.$$.fragment,j),j.forEach(d),Qr=o(y),ke=s(y,"DIV",{class:!0});var Zt=T(ke);p(Qe.$$.fragment,Zt),Kr=o(Zt),dn=s(Zt,"P",{"data-svelte-h":!0}),a(dn)!=="svelte-j87b6t"&&(dn.innerHTML=Di),Zt.forEach(d),es=o(y),be=s(y,"DIV",{class:!0});var Ht=T(be);p(Ke.$$.fragment,Ht),ts=o(Ht),cn=s(Ht,"P",{"data-svelte-h":!0}),a(cn)!=="svelte-1deng2j"&&(cn.textContent=Zi),Ht.forEach(d),ns=o(y),Y=s(y,"DIV",{class:!0});var he=T(Y);p(et.$$.fragment,he),os=o(he),ln=s(he,"P",{"data-svelte-h":!0}),a(ln)!=="svelte-vbfkpu"&&(ln.textContent=Hi),rs=o(he),pn=s(he,"P",{"data-svelte-h":!0}),a(pn)!=="svelte-125uxon"&&(pn.innerHTML=Gi),he.forEach(d),ss=o(y),O=s(y,"DIV",{class:!0});var ue=T(O);p(tt.$$.fragment,ue),as=o(ue),mn=s(ue,"P",{"data-svelte-h":!0}),a(mn)!=="svelte-12b8hzo"&&(mn.textContent=Ai),is=o(ue),hn=s(ue,"P",{"data-svelte-h":!0}),a(hn)!=="svelte-1kyhveh"&&(hn.innerHTML=Ri),ue.forEach(d),ds=o(y),Q=s(y,"DIV",{class:!0});var Eo=T(Q);p(nt.$$.fragment,Eo),cs=o(Eo),un=s(Eo,"P",{"data-svelte-h":!0}),a(un)!=="svelte-tpmkl3"&&(un.textContent=Si),ls=o(Eo),p(xe.$$.fragment,Eo),Eo.forEach(d),ps=o(y),Te=s(y,"DIV",{class:!0});var fr=T(Te);p(ot.$$.fragment,fr),ms=o(fr),fn=s(fr,"P",{"data-svelte-h":!0}),a(fn)!=="svelte-cx157h"&&(fn.textContent=Xi),fr.forEach(d),hs=o(y),ve=s(y,"DIV",{class:!0});var _r=T(ve);p(rt.$$.fragment,_r),us=o(_r),_n=s(_r,"P",{"data-svelte-h":!0}),a(_n)!=="svelte-1urz5jj"&&(_n.textContent=Yi),_r.forEach(d),fs=o(y),ye=s(y,"DIV",{class:!0});var gr=T(ye);p(st.$$.fragment,gr),_s=o(gr),gn=s(gr,"P",{"data-svelte-h":!0}),a(gn)!=="svelte-2yfcci"&&(gn.textContent=Oi),gr.forEach(d),gs=o(y),K=s(y,"DIV",{class:!0});var Do=T(K);p(at.$$.fragment,Do),ks=o(Do),kn=s(Do,"P",{"data-svelte-h":!0}),a(kn)!=="svelte-1oqr1g4"&&(kn.textContent=Qi),bs=o(Do),p(we.$$.fragment,Do),Do.forEach(d),xs=o(y),ee=s(y,"DIV",{class:!0});var Zo=T(ee);p(it.$$.fragment,Zo),Ts=o(Zo),bn=s(Zo,"P",{"data-svelte-h":!0}),a(bn)!=="svelte-suiszn"&&(bn.textContent=Ki),vs=o(Zo),xn=s(Zo,"P",{"data-svelte-h":!0}),a(xn)!=="svelte-1bw0rb5"&&(xn.innerHTML=ed),Zo.forEach(d),ys=o(y),te=s(y,"DIV",{class:!0});var Ho=T(te);p(dt.$$.fragment,Ho),ws=o(Ho),Tn=s(Ho,"P",{"data-svelte-h":!0}),a(Tn)!=="svelte-sso1qb"&&(Tn.textContent=td),zs=o(Ho),vn=s(Ho,"P",{"data-svelte-h":!0}),a(vn)!=="svelte-1i4xsf5"&&(vn.textContent=nd),Ho.forEach(d),y.forEach(d),ir=o(t),p(ct.$$.fragment,t),dr=o(t),lt=s(t,"P",{"data-svelte-h":!0}),a(lt)!=="svelte-1j52689"&&(lt.innerHTML=od),cr=o(t),x=s(t,"DIV",{class:!0});var $=T(x);p(pt.$$.fragment,$),$s=o($),yn=s($,"P",{"data-svelte-h":!0}),a(yn)!=="svelte-1e1i5yj"&&(yn.textContent=rd),Ps=o($),wn=s($,"P",{"data-svelte-h":!0}),a(wn)!=="svelte-1xcwogg"&&(wn.innerHTML=sd),Cs=o($),zn=s($,"P",{"data-svelte-h":!0}),a(zn)!=="svelte-99yswb"&&(zn.textContent=ad),qs=o($),$n=s($,"P",{"data-svelte-h":!0}),a($n)!=="svelte-1y9tnev"&&($n.textContent=id),Ms=o($),Pn=s($,"P",{"data-svelte-h":!0}),a(Pn)!=="svelte-1ixo79u"&&(Pn.textContent=dd),Is=o($),Cn=s($,"UL",{"data-svelte-h":!0}),a(Cn)!=="svelte-1y0k6z5"&&(Cn.innerHTML=cd),Fs=o($),ze=s($,"DIV",{class:!0});var kr=T(ze);p(mt.$$.fragment,kr),Ls=o(kr),qn=s(kr,"P",{"data-svelte-h":!0}),a(qn)!=="svelte-kpxj0c"&&(qn.textContent=ld),kr.forEach(d),js=o($),E=s($,"DIV",{class:!0});var de=T(E);p(ht.$$.fragment,de),Ws=o(de),Mn=s(de,"P",{"data-svelte-h":!0}),a(Mn)!=="svelte-14rotd7"&&(Mn.textContent=pd),Js=o(de),In=s(de,"P",{"data-svelte-h":!0}),a(In)!=="svelte-j0w5r1"&&(In.textContent=md),Us=o(de),Fn=s(de,"P",{"data-svelte-h":!0}),a(Fn)!=="svelte-1enyojm"&&(Fn.innerHTML=hd),Bs=o(de),p($e.$$.fragment,de),de.forEach(d),Ns=o($),L=s($,"DIV",{class:!0});var N=T(L);p(ut.$$.fragment,N),Vs=o(N),Ln=s(N,"P",{"data-svelte-h":!0}),a(Ln)!=="svelte-1j8s0i5"&&(Ln.textContent=ud),Es=o(N),jn=s(N,"P",{"data-svelte-h":!0}),a(jn)!=="svelte-1w3ayx9"&&(jn.textContent=fd),Ds=o(N),Wn=s(N,"P",{"data-svelte-h":!0}),a(Wn)!=="svelte-1enyojm"&&(Wn.innerHTML=_d),Zs=o(N),Jn=s(N,"P",{"data-svelte-h":!0}),a(Jn)!=="svelte-5hxtpc"&&(Jn.innerHTML=gd),Hs=o(N),Un=s(N,"UL",{"data-svelte-h":!0}),a(Un)!=="svelte-1pes0uj"&&(Un.innerHTML=kd),Gs=o(N),Bn=s(N,"P",{"data-svelte-h":!0}),a(Bn)!=="svelte-1butrf6"&&(Bn.innerHTML=bd),As=o(N),p(Pe.$$.fragment,N),N.forEach(d),Rs=o($),Ce=s($,"DIV",{class:!0});var br=T(Ce);p(ft.$$.fragment,br),Ss=o(br),Nn=s(br,"P",{"data-svelte-h":!0}),a(Nn)!=="svelte-j87b6t"&&(Nn.innerHTML=xd),br.forEach(d),Xs=o($),qe=s($,"DIV",{class:!0});var xr=T(qe);p(_t.$$.fragment,xr),Ys=o(xr),Vn=s(xr,"P",{"data-svelte-h":!0}),a(Vn)!=="svelte-1deng2j"&&(Vn.textContent=Td),xr.forEach(d),Os=o($),ne=s($,"DIV",{class:!0});var Go=T(ne);p(gt.$$.fragment,Go),Qs=o(Go),En=s(Go,"P",{"data-svelte-h":!0}),a(En)!=="svelte-vbfkpu"&&(En.textContent=vd),Ks=o(Go),Dn=s(Go,"P",{"data-svelte-h":!0}),a(Dn)!=="svelte-125uxon"&&(Dn.innerHTML=yd),Go.forEach(d),ea=o($),oe=s($,"DIV",{class:!0});var Ao=T(oe);p(kt.$$.fragment,Ao),ta=o(Ao),Zn=s(Ao,"P",{"data-svelte-h":!0}),a(Zn)!=="svelte-12b8hzo"&&(Zn.textContent=wd),na=o(Ao),Hn=s(Ao,"P",{"data-svelte-h":!0}),a(Hn)!=="svelte-1kyhveh"&&(Hn.innerHTML=zd),Ao.forEach(d),oa=o($),re=s($,"DIV",{class:!0});var Ro=T(re);p(bt.$$.fragment,Ro),ra=o(Ro),Gn=s(Ro,"P",{"data-svelte-h":!0}),a(Gn)!=="svelte-tpmkl3"&&(Gn.textContent=$d),sa=o(Ro),p(Me.$$.fragment,Ro),Ro.forEach(d),aa=o($),Ie=s($,"DIV",{class:!0});var Tr=T(Ie);p(xt.$$.fragment,Tr),ia=o(Tr),An=s(Tr,"P",{"data-svelte-h":!0}),a(An)!=="svelte-cx157h"&&(An.textContent=Pd),Tr.forEach(d),da=o($),Fe=s($,"DIV",{class:!0});var vr=T(Fe);p(Tt.$$.fragment,vr),ca=o(vr),Rn=s(vr,"P",{"data-svelte-h":!0}),a(Rn)!=="svelte-1t879z4"&&(Rn.textContent=Cd),vr.forEach(d),la=o($),Le=s($,"DIV",{class:!0});var yr=T(Le);p(vt.$$.fragment,yr),pa=o(yr),Sn=s(yr,"P",{"data-svelte-h":!0}),a(Sn)!=="svelte-17grjhy"&&(Sn.textContent=qd),yr.forEach(d),ma=o($),se=s($,"DIV",{class:!0});var So=T(se);p(yt.$$.fragment,So),ha=o(So),Xn=s(So,"P",{"data-svelte-h":!0}),a(Xn)!=="svelte-1oqr1g4"&&(Xn.textContent=Md),ua=o(So),p(je.$$.fragment,So),So.forEach(d),fa=o($),ae=s($,"DIV",{class:!0});var Xo=T(ae);p(wt.$$.fragment,Xo),_a=o(Xo),Yn=s(Xo,"P",{"data-svelte-h":!0}),a(Yn)!=="svelte-wj61ov"&&(Yn.textContent=Id),ga=o(Xo),On=s(Xo,"P",{"data-svelte-h":!0}),a(On)!=="svelte-1423b5j"&&(On.textContent=Fd),Xo.forEach(d),ka=o($),We=s($,"DIV",{class:!0});var wr=T(We);p(zt.$$.fragment,wr),ba=o(wr),Qn=s(wr,"P",{"data-svelte-h":!0}),a(Qn)!=="svelte-dk6kyv"&&(Qn.textContent=Ld),wr.forEach(d),$.forEach(d),lr=o(t),p($t.$$.fragment,t),pr=o(t),C=s(t,"DIV",{class:!0});var q=T(C);p(Pt.$$.fragment,q),xa=o(q),Kn=s(q,"P",{"data-svelte-h":!0}),a(Kn)!=="svelte-ba095q"&&(Kn.innerHTML=jd),Ta=o(q),eo=s(q,"P",{"data-svelte-h":!0}),a(eo)!=="svelte-1df8ukr"&&(eo.textContent=Wd),va=o(q),D=s(q,"DIV",{class:!0});var ce=T(D);p(Ct.$$.fragment,ce),ya=o(ce),to=s(ce,"P",{"data-svelte-h":!0}),a(to)!=="svelte-1gpnz9t"&&(to.textContent=Jd),wa=o(ce),no=s(ce,"P",{"data-svelte-h":!0}),a(no)!=="svelte-hoi93q"&&(no.textContent=Ud),za=o(ce),oo=s(ce,"UL",{"data-svelte-h":!0}),a(oo)!=="svelte-i7p8st"&&(oo.innerHTML=Bd),$a=o(ce),ro=s(ce,"P",{"data-svelte-h":!0}),a(ro)!=="svelte-udqgev"&&(ro.textContent=Nd),ce.forEach(d),Pa=o(q),Z=s(q,"DIV",{class:!0});var le=T(Z);p(qt.$$.fragment,le),Ca=o(le),so=s(le,"P",{"data-svelte-h":!0}),a(so)!=="svelte-mtqdue"&&(so.textContent=Vd),qa=o(le),ao=s(le,"P",{"data-svelte-h":!0}),a(ao)!=="svelte-hoi93q"&&(ao.textContent=Ed),Ma=o(le),io=s(le,"UL",{"data-svelte-h":!0}),a(io)!=="svelte-2vo0y6"&&(io.innerHTML=Dd),Ia=o(le),co=s(le,"P",{"data-svelte-h":!0}),a(co)!=="svelte-udqgev"&&(co.textContent=Zd),le.forEach(d),Fa=o(q),Je=s(q,"DIV",{class:!0});var zr=T(Je);p(Mt.$$.fragment,zr),La=o(zr),lo=s(zr,"P",{"data-svelte-h":!0}),a(lo)!=="svelte-pxfh9u"&&(lo.textContent=Hd),zr.forEach(d),ja=o(q),ie=s(q,"DIV",{class:!0});var Yo=T(ie);p(It.$$.fragment,Yo),Wa=o(Yo),po=s(Yo,"P",{"data-svelte-h":!0}),a(po)!=="svelte-1839ko1"&&(po.textContent=Gd),Ja=o(Yo),mo=s(Yo,"UL",{"data-svelte-h":!0}),a(mo)!=="svelte-1ap7xk7"&&(mo.innerHTML=Ad),Yo.forEach(d),Ua=o(q),Ue=s(q,"DIV",{class:!0});var $r=T(Ue);p(Ft.$$.fragment,$r),Ba=o($r),ho=s($r,"P",{"data-svelte-h":!0}),a(ho)!=="svelte-8ewnfk"&&(ho.innerHTML=Rd),$r.forEach(d),Na=o(q),U=s(q,"DIV",{class:!0});var S=T(U);p(Lt.$$.fragment,S),Va=o(S),uo=s(S,"P",{"data-svelte-h":!0}),a(uo)!=="svelte-xav75j"&&(uo.textContent=Sd),Ea=o(S),fo=s(S,"P",{"data-svelte-h":!0}),a(fo)!=="svelte-1yptnn2"&&(fo.innerHTML=Xd),Da=o(S),_o=s(S,"UL",{"data-svelte-h":!0}),a(_o)!=="svelte-4fpj7o"&&(_o.innerHTML=Yd),Za=o(S),go=s(S,"P",{"data-svelte-h":!0}),a(go)!=="svelte-hoi93q"&&(go.textContent=Od),Ha=o(S),ko=s(S,"UL",{"data-svelte-h":!0}),a(ko)!=="svelte-10t7hv1"&&(ko.innerHTML=Qd),S.forEach(d),Ga=o(q),H=s(q,"DIV",{class:!0});var pe=T(H);p(jt.$$.fragment,pe),Aa=o(pe),bo=s(pe,"P",{"data-svelte-h":!0}),a(bo)!=="svelte-gouuon"&&(bo.innerHTML=Kd),Ra=o(pe),xo=s(pe,"P",{"data-svelte-h":!0}),a(xo)!=="svelte-hoi93q"&&(xo.textContent=ec),Sa=o(pe),To=s(pe,"UL",{"data-svelte-h":!0}),a(To)!=="svelte-q52scs"&&(To.innerHTML=tc),Xa=o(pe),vo=s(pe,"P",{"data-svelte-h":!0}),a(vo)!=="svelte-1fj0zmd"&&(vo.textContent=nc),pe.forEach(d),Ya=o(q),G=s(q,"DIV",{class:!0});var me=T(G);p(Wt.$$.fragment,me),Oa=o(me),yo=s(me,"P",{"data-svelte-h":!0}),a(yo)!=="svelte-a0a2o0"&&(yo.textContent=oc),Qa=o(me),wo=s(me,"P",{"data-svelte-h":!0}),a(wo)!=="svelte-hoi93q"&&(wo.textContent=rc),Ka=o(me),zo=s(me,"UL",{"data-svelte-h":!0}),a(zo)!=="svelte-vbk04m"&&(zo.innerHTML=sc),ei=o(me),$o=s(me,"P",{"data-svelte-h":!0}),a($o)!=="svelte-1fj0zmd"&&($o.textContent=ac),me.forEach(d),ti=o(q),Be=s(q,"DIV",{class:!0});var Pr=T(Be);p(Jt.$$.fragment,Pr),ni=o(Pr),Po=s(Pr,"P",{"data-svelte-h":!0}),a(Po)!=="svelte-or6vgj"&&(Po.textContent=ic),Pr.forEach(d),oi=o(q),Ne=s(q,"DIV",{class:!0});var Cr=T(Ne);p(Ut.$$.fragment,Cr),ri=o(Cr),Co=s(Cr,"P",{"data-svelte-h":!0}),a(Co)!=="svelte-1xfwjqh"&&(Co.textContent=dc),Cr.forEach(d),si=o(q),B=s(q,"DIV",{class:!0});var X=T(B);p(Bt.$$.fragment,X),ai=o(X),qo=s(X,"P",{"data-svelte-h":!0}),a(qo)!=="svelte-1p4j7cm"&&(qo.textContent=cc),ii=o(X),Mo=s(X,"P",{"data-svelte-h":!0}),a(Mo)!=="svelte-1m5vplt"&&(Mo.textContent=lc),di=o(X),Io=s(X,"UL",{"data-svelte-h":!0}),a(Io)!=="svelte-tivk04"&&(Io.innerHTML=pc),ci=o(X),Fo=s(X,"P",{"data-svelte-h":!0}),a(Fo)!=="svelte-hoi93q"&&(Fo.textContent=mc),li=o(X),Lo=s(X,"UL",{"data-svelte-h":!0}),a(Lo)!=="svelte-rgo9bl"&&(Lo.innerHTML=hc),X.forEach(d),pi=o(q),J=s(q,"DIV",{class:!0});var A=T(J);p(Nt.$$.fragment,A),mi=o(A),jo=s(A,"P",{"data-svelte-h":!0}),a(jo)!=="svelte-p6ca34"&&(jo.textContent=uc),hi=o(A),Wo=s(A,"P",{"data-svelte-h":!0}),a(Wo)!=="svelte-1xt4sho"&&(Wo.innerHTML=fc),ui=o(A),Jo=s(A,"UL",{"data-svelte-h":!0}),a(Jo)!=="svelte-adw1ie"&&(Jo.innerHTML=_c),fi=o(A),Uo=s(A,"P",{"data-svelte-h":!0}),a(Uo)!=="svelte-hoi93q"&&(Uo.textContent=gc),_i=o(A),Bo=s(A,"UL",{"data-svelte-h":!0}),a(Bo)!=="svelte-11iunx9"&&(Bo.innerHTML=kc),gi=o(A),No=s(A,"P",{"data-svelte-h":!0}),a(No)!=="svelte-udqgev"&&(No.textContent=bc),A.forEach(d),ki=o(q),Ve=s(q,"DIV",{class:!0});var qr=T(Ve);p(Vt.$$.fragment,qr),bi=o(qr),Vo=s(qr,"P",{"data-svelte-h":!0}),a(Vo)!=="svelte-1xfwjqh"&&(Vo.textContent=xc),qr.forEach(d),q.forEach(d),mr=o(t),p(Et.$$.fragment,t),hr=o(t),Ko=s(t,"P",{}),T(Ko).forEach(d),this.h()},h(){v(c,"name","hf:doc:metadata"),v(c,"content",Uc),v(fe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ke,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ee,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ze,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Ce,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(qe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(oe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(re,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Fe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ae,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(We,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Je,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,g){e(document.head,c),z(t,I,g),z(t,k,g),z(t,_,g),m(P,t,g),z(t,i,g),z(t,M,g),z(t,er,g),z(t,De,g),z(t,tr,g),z(t,Ze,g),z(t,nr,g),z(t,He,g),z(t,or,g),z(t,Ge,g),z(t,rr,g),z(t,Ae,g),z(t,sr,g),m(Re,t,g),z(t,ar,g),z(t,b,g),m(Se,b,null),e(b,Ir),e(b,Gt),e(b,Fr),e(b,At),e(b,Lr),e(b,Rt),e(b,jr),e(b,St),e(b,Wr),e(b,Xt),e(b,Jr),e(b,Yt),e(b,Ur),e(b,fe),m(Xe,fe,null),e(fe,Br),e(fe,Ot),e(b,Nr),e(b,V),m(Ye,V,null),e(V,Vr),e(V,Qt),e(V,Er),e(V,Kt),e(V,Dr),e(V,en),e(V,Zr),m(_e,V,null),e(b,Hr),e(b,F),m(Oe,F,null),e(F,Gr),e(F,tn),e(F,Ar),e(F,nn),e(F,Rr),e(F,on),e(F,Sr),e(F,rn),e(F,Xr),e(F,sn),e(F,Yr),e(F,an),e(F,Or),m(ge,F,null),e(b,Qr),e(b,ke),m(Qe,ke,null),e(ke,Kr),e(ke,dn),e(b,es),e(b,be),m(Ke,be,null),e(be,ts),e(be,cn),e(b,ns),e(b,Y),m(et,Y,null),e(Y,os),e(Y,ln),e(Y,rs),e(Y,pn),e(b,ss),e(b,O),m(tt,O,null),e(O,as),e(O,mn),e(O,is),e(O,hn),e(b,ds),e(b,Q),m(nt,Q,null),e(Q,cs),e(Q,un),e(Q,ls),m(xe,Q,null),e(b,ps),e(b,Te),m(ot,Te,null),e(Te,ms),e(Te,fn),e(b,hs),e(b,ve),m(rt,ve,null),e(ve,us),e(ve,_n),e(b,fs),e(b,ye),m(st,ye,null),e(ye,_s),e(ye,gn),e(b,gs),e(b,K),m(at,K,null),e(K,ks),e(K,kn),e(K,bs),m(we,K,null),e(b,xs),e(b,ee),m(it,ee,null),e(ee,Ts),e(ee,bn),e(ee,vs),e(ee,xn),e(b,ys),e(b,te),m(dt,te,null),e(te,ws),e(te,Tn),e(te,zs),e(te,vn),z(t,ir,g),m(ct,t,g),z(t,dr,g),z(t,lt,g),z(t,cr,g),z(t,x,g),m(pt,x,null),e(x,$s),e(x,yn),e(x,Ps),e(x,wn),e(x,Cs),e(x,zn),e(x,qs),e(x,$n),e(x,Ms),e(x,Pn),e(x,Is),e(x,Cn),e(x,Fs),e(x,ze),m(mt,ze,null),e(ze,Ls),e(ze,qn),e(x,js),e(x,E),m(ht,E,null),e(E,Ws),e(E,Mn),e(E,Js),e(E,In),e(E,Us),e(E,Fn),e(E,Bs),m($e,E,null),e(x,Ns),e(x,L),m(ut,L,null),e(L,Vs),e(L,Ln),e(L,Es),e(L,jn),e(L,Ds),e(L,Wn),e(L,Zs),e(L,Jn),e(L,Hs),e(L,Un),e(L,Gs),e(L,Bn),e(L,As),m(Pe,L,null),e(x,Rs),e(x,Ce),m(ft,Ce,null),e(Ce,Ss),e(Ce,Nn),e(x,Xs),e(x,qe),m(_t,qe,null),e(qe,Ys),e(qe,Vn),e(x,Os),e(x,ne),m(gt,ne,null),e(ne,Qs),e(ne,En),e(ne,Ks),e(ne,Dn),e(x,ea),e(x,oe),m(kt,oe,null),e(oe,ta),e(oe,Zn),e(oe,na),e(oe,Hn),e(x,oa),e(x,re),m(bt,re,null),e(re,ra),e(re,Gn),e(re,sa),m(Me,re,null),e(x,aa),e(x,Ie),m(xt,Ie,null),e(Ie,ia),e(Ie,An),e(x,da),e(x,Fe),m(Tt,Fe,null),e(Fe,ca),e(Fe,Rn),e(x,la),e(x,Le),m(vt,Le,null),e(Le,pa),e(Le,Sn),e(x,ma),e(x,se),m(yt,se,null),e(se,ha),e(se,Xn),e(se,ua),m(je,se,null),e(x,fa),e(x,ae),m(wt,ae,null),e(ae,_a),e(ae,Yn),e(ae,ga),e(ae,On),e(x,ka),e(x,We),m(zt,We,null),e(We,ba),e(We,Qn),z(t,lr,g),m($t,t,g),z(t,pr,g),z(t,C,g),m(Pt,C,null),e(C,xa),e(C,Kn),e(C,Ta),e(C,eo),e(C,va),e(C,D),m(Ct,D,null),e(D,ya),e(D,to),e(D,wa),e(D,no),e(D,za),e(D,oo),e(D,$a),e(D,ro),e(C,Pa),e(C,Z),m(qt,Z,null),e(Z,Ca),e(Z,so),e(Z,qa),e(Z,ao),e(Z,Ma),e(Z,io),e(Z,Ia),e(Z,co),e(C,Fa),e(C,Je),m(Mt,Je,null),e(Je,La),e(Je,lo),e(C,ja),e(C,ie),m(It,ie,null),e(ie,Wa),e(ie,po),e(ie,Ja),e(ie,mo),e(C,Ua),e(C,Ue),m(Ft,Ue,null),e(Ue,Ba),e(Ue,ho),e(C,Na),e(C,U),m(Lt,U,null),e(U,Va),e(U,uo),e(U,Ea),e(U,fo),e(U,Da),e(U,_o),e(U,Za),e(U,go),e(U,Ha),e(U,ko),e(C,Ga),e(C,H),m(jt,H,null),e(H,Aa),e(H,bo),e(H,Ra),e(H,xo),e(H,Sa),e(H,To),e(H,Xa),e(H,vo),e(C,Ya),e(C,G),m(Wt,G,null),e(G,Oa),e(G,yo),e(G,Qa),e(G,wo),e(G,Ka),e(G,zo),e(G,ei),e(G,$o),e(C,ti),e(C,Be),m(Jt,Be,null),e(Be,ni),e(Be,Po),e(C,oi),e(C,Ne),m(Ut,Ne,null),e(Ne,ri),e(Ne,Co),e(C,si),e(C,B),m(Bt,B,null),e(B,ai),e(B,qo),e(B,ii),e(B,Mo),e(B,di),e(B,Io),e(B,ci),e(B,Fo),e(B,li),e(B,Lo),e(C,pi),e(C,J),m(Nt,J,null),e(J,mi),e(J,jo),e(J,hi),e(J,Wo),e(J,ui),e(J,Jo),e(J,fi),e(J,Uo),e(J,_i),e(J,Bo),e(J,gi),e(J,No),e(C,ki),e(C,Ve),m(Vt,Ve,null),e(Ve,bi),e(Ve,Vo),z(t,mr,g),m(Et,t,g),z(t,hr,g),z(t,Ko,g),ur=!0},p(t,[g]){const y={};g&2&&(y.$$scope={dirty:g,ctx:t}),_e.$set(y);const Dt={};g&2&&(Dt.$$scope={dirty:g,ctx:t}),ge.$set(Dt);const R={};g&2&&(R.$$scope={dirty:g,ctx:t}),xe.$set(R);const j={};g&2&&(j.$$scope={dirty:g,ctx:t}),we.$set(j);const Zt={};g&2&&(Zt.$$scope={dirty:g,ctx:t}),$e.$set(Zt);const Ht={};g&2&&(Ht.$$scope={dirty:g,ctx:t}),Pe.$set(Ht);const he={};g&2&&(he.$$scope={dirty:g,ctx:t}),Me.$set(he);const ue={};g&2&&(ue.$$scope={dirty:g,ctx:t}),je.$set(ue)},i(t){ur||(h(P.$$.fragment,t),h(Re.$$.fragment,t),h(Se.$$.fragment,t),h(Xe.$$.fragment,t),h(Ye.$$.fragment,t),h(_e.$$.fragment,t),h(Oe.$$.fragment,t),h(ge.$$.fragment,t),h(Qe.$$.fragment,t),h(Ke.$$.fragment,t),h(et.$$.fragment,t),h(tt.$$.fragment,t),h(nt.$$.fragment,t),h(xe.$$.fragment,t),h(ot.$$.fragment,t),h(rt.$$.fragment,t),h(st.$$.fragment,t),h(at.$$.fragment,t),h(we.$$.fragment,t),h(it.$$.fragment,t),h(dt.$$.fragment,t),h(ct.$$.fragment,t),h(pt.$$.fragment,t),h(mt.$$.fragment,t),h(ht.$$.fragment,t),h($e.$$.fragment,t),h(ut.$$.fragment,t),h(Pe.$$.fragment,t),h(ft.$$.fragment,t),h(_t.$$.fragment,t),h(gt.$$.fragment,t),h(kt.$$.fragment,t),h(bt.$$.fragment,t),h(Me.$$.fragment,t),h(xt.$$.fragment,t),h(Tt.$$.fragment,t),h(vt.$$.fragment,t),h(yt.$$.fragment,t),h(je.$$.fragment,t),h(wt.$$.fragment,t),h(zt.$$.fragment,t),h($t.$$.fragment,t),h(Pt.$$.fragment,t),h(Ct.$$.fragment,t),h(qt.$$.fragment,t),h(Mt.$$.fragment,t),h(It.$$.fragment,t),h(Ft.$$.fragment,t),h(Lt.$$.fragment,t),h(jt.$$.fragment,t),h(Wt.$$.fragment,t),h(Jt.$$.fragment,t),h(Ut.$$.fragment,t),h(Bt.$$.fragment,t),h(Nt.$$.fragment,t),h(Vt.$$.fragment,t),h(Et.$$.fragment,t),ur=!0)},o(t){u(P.$$.fragment,t),u(Re.$$.fragment,t),u(Se.$$.fragment,t),u(Xe.$$.fragment,t),u(Ye.$$.fragment,t),u(_e.$$.fragment,t),u(Oe.$$.fragment,t),u(ge.$$.fragment,t),u(Qe.$$.fragment,t),u(Ke.$$.fragment,t),u(et.$$.fragment,t),u(tt.$$.fragment,t),u(nt.$$.fragment,t),u(xe.$$.fragment,t),u(ot.$$.fragment,t),u(rt.$$.fragment,t),u(st.$$.fragment,t),u(at.$$.fragment,t),u(we.$$.fragment,t),u(it.$$.fragment,t),u(dt.$$.fragment,t),u(ct.$$.fragment,t),u(pt.$$.fragment,t),u(mt.$$.fragment,t),u(ht.$$.fragment,t),u($e.$$.fragment,t),u(ut.$$.fragment,t),u(Pe.$$.fragment,t),u(ft.$$.fragment,t),u(_t.$$.fragment,t),u(gt.$$.fragment,t),u(kt.$$.fragment,t),u(bt.$$.fragment,t),u(Me.$$.fragment,t),u(xt.$$.fragment,t),u(Tt.$$.fragment,t),u(vt.$$.fragment,t),u(yt.$$.fragment,t),u(je.$$.fragment,t),u(wt.$$.fragment,t),u(zt.$$.fragment,t),u($t.$$.fragment,t),u(Pt.$$.fragment,t),u(Ct.$$.fragment,t),u(qt.$$.fragment,t),u(Mt.$$.fragment,t),u(It.$$.fragment,t),u(Ft.$$.fragment,t),u(Lt.$$.fragment,t),u(jt.$$.fragment,t),u(Wt.$$.fragment,t),u(Jt.$$.fragment,t),u(Ut.$$.fragment,t),u(Bt.$$.fragment,t),u(Nt.$$.fragment,t),u(Vt.$$.fragment,t),u(Et.$$.fragment,t),ur=!1},d(t){t&&(d(I),d(k),d(_),d(i),d(M),d(er),d(De),d(tr),d(Ze),d(nr),d(He),d(or),d(Ge),d(rr),d(Ae),d(sr),d(ar),d(b),d(ir),d(dr),d(lt),d(cr),d(x),d(lr),d(pr),d(C),d(mr),d(hr),d(Ko)),d(c),f(P,t),f(Re,t),f(Se),f(Xe),f(Ye),f(_e),f(Oe),f(ge),f(Qe),f(Ke),f(et),f(tt),f(nt),f(xe),f(ot),f(rt),f(st),f(at),f(we),f(it),f(dt),f(ct,t),f(pt),f(mt),f(ht),f($e),f(ut),f(Pe),f(ft),f(_t),f(gt),f(kt),f(bt),f(Me),f(xt),f(Tt),f(vt),f(yt),f(je),f(wt),f(zt),f($t,t),f(Pt),f(Ct),f(qt),f(Mt),f(It),f(Ft),f(Lt),f(jt),f(Wt),f(Jt),f(Ut),f(Bt),f(Nt),f(Vt),f(Et,t)}}}const Uc='{"title":"Tokenizer","local":"tokenizer","sections":[{"title":"PreTrainedTokenizer","local":"transformers.PreTrainedTokenizer","sections":[],"depth":2},{"title":"PreTrainedTokenizerFast","local":"transformers.PreTrainedTokenizerFast","sections":[],"depth":2},{"title":"BatchEncoding","local":"transformers.BatchEncoding","sections":[],"depth":2}],"depth":1}';function Bc(W){return yc(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ac extends wc{constructor(c){super(),zc(this,c,Bc,Jc,vc,{})}}export{Ac as component}; | |
Xet Storage Details
- Size:
- 201 kB
- Xet hash:
- 07b3052fcedf29410b5b6c021413ec86e5e927ed4e759a7000e288529d75ef1e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.